LCOV - code coverage report
Current view: top level - backends/glass - glass_database.h (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 954b5873a738 Lines: 6 6 100.0 %
Date: 2019-06-30 05:20:33 Functions: 3 3 100.0 %
Branches: 2 2 100.0 %

           Branch data     Line data    Source code
       1                 :            : /** @file glass_database.h
       2                 :            :  * @brief C++ class definition for glass database
       3                 :            :  */
       4                 :            : /* Copyright 1999,2000,2001 BrightStation PLC
       5                 :            :  * Copyright 2002 Ananova Ltd
       6                 :            :  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2019 Olly Betts
       7                 :            :  * Copyright 2008 Lemur Consulting Ltd
       8                 :            :  *
       9                 :            :  * This program is free software; you can redistribute it and/or
      10                 :            :  * modify it under the terms of the GNU General Public License as
      11                 :            :  * published by the Free Software Foundation; either version 2 of the
      12                 :            :  * License, or (at your option) any later version.
      13                 :            :  *
      14                 :            :  * This program is distributed in the hope that it will be useful,
      15                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      16                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      17                 :            :  * GNU General Public License for more details.
      18                 :            :  *
      19                 :            :  * You should have received a copy of the GNU General Public License
      20                 :            :  * along with this program; if not, write to the Free Software
      21                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
      22                 :            :  * USA
      23                 :            :  */
      24                 :            : 
      25                 :            : #ifndef XAPIAN_INCLUDED_GLASS_DATABASE_H
      26                 :            : #define XAPIAN_INCLUDED_GLASS_DATABASE_H
      27                 :            : 
      28                 :            : #include "backends/backends.h"
      29                 :            : #include "backends/databaseinternal.h"
      30                 :            : #include "glass_changes.h"
      31                 :            : #include "glass_docdata.h"
      32                 :            : #include "glass_inverter.h"
      33                 :            : #include "glass_positionlist.h"
      34                 :            : #include "glass_postlist.h"
      35                 :            : #include "glass_spelling.h"
      36                 :            : #include "glass_synonym.h"
      37                 :            : #include "glass_termlisttable.h"
      38                 :            : #include "glass_values.h"
      39                 :            : #include "glass_version.h"
      40                 :            : #include "../flint_lock.h"
      41                 :            : #include "glass_defs.h"
      42                 :            : #include "backends/valuestats.h"
      43                 :            : 
      44                 :            : #include "xapian/compactor.h"
      45                 :            : #include "xapian/constants.h"
      46                 :            : 
      47                 :            : #include <map>
      48                 :            : 
      49                 :            : class GlassTermList;
      50                 :            : class GlassAllDocsPostList;
      51                 :            : class HoneyDatabase;
      52                 :            : class RemoteConnection;
      53                 :            : 
      54                 :            : /** A backend designed for efficient indexing and retrieval, using
      55                 :            :  *  compressed posting lists and a btree storage scheme.
      56                 :            :  */
      57                 :            : class GlassDatabase : public Xapian::Database::Internal {
      58                 :            :     friend class GlassWritableDatabase;
      59                 :            :     friend class GlassTermList;
      60                 :            :     friend class GlassPostList;
      61                 :            :     friend class GlassAllTermsList;
      62                 :            :     friend class GlassAllDocsPostList;
      63                 :            :     friend class GlassSpellingWordsList;
      64                 :            :     friend class GlassSynonymTermList;
      65                 :            :     friend class HoneyDatabase; // For compacting to convert.
      66                 :            : 
      67                 :            :   private:
      68                 :            :     /** Directory to store databases in.
      69                 :            :      */
      70                 :            :     std::string db_dir;
      71                 :            : 
      72                 :            :     /** Whether the database is readonly.
      73                 :            :      */
      74                 :            :     bool readonly;
      75                 :            : 
      76                 :            :     /** The file describing the Glass database.
      77                 :            :      *  This file has information about the format of the database
      78                 :            :      *  which can't easily be stored in any of the individual tables.
      79                 :            :      */
      80                 :            :     GlassVersion version_file;
      81                 :            : 
      82                 :            :     /** Table storing posting lists.
      83                 :            :      *
      84                 :            :      *  Whenever an update is performed, this table is the first to be
      85                 :            :      *  updated: therefore, its most recent revision number is the most
      86                 :            :      *  recent anywhere in the database.
      87                 :            :      */
      88                 :            :     mutable GlassPostListTable postlist_table;
      89                 :            : 
      90                 :            :     /** Table storing position lists.
      91                 :            :      */
      92                 :            :     mutable GlassPositionListTable position_table;
      93                 :            : 
      94                 :            :     /** Table storing term lists.
      95                 :            :      */
      96                 :            :     GlassTermListTable termlist_table;
      97                 :            : 
      98                 :            :     /** Value manager. */
      99                 :            :     mutable GlassValueManager value_manager;
     100                 :            : 
     101                 :            :     /** Table storing synonym data.
     102                 :            :      */
     103                 :            :     mutable GlassSynonymTable synonym_table;
     104                 :            : 
     105                 :            :     /** Table storing spelling correction data.
     106                 :            :      */
     107                 :            :     mutable GlassSpellingTable spelling_table;
     108                 :            : 
     109                 :            :     /** Table storing document data.
     110                 :            :      */
     111                 :            :     GlassDocDataTable docdata_table;
     112                 :            : 
     113                 :            :     /// Lock object.
     114                 :            :     FlintLock lock;
     115                 :            : 
     116                 :            :     /// Replication changesets.
     117                 :            :     GlassChanges changes;
     118                 :            : 
     119                 :            :     /** Return true if a database exists at the path specified for this
     120                 :            :      *  database.
     121                 :            :      */
     122                 :            :     bool database_exists();
     123                 :            : 
     124                 :            :     /** Create new tables, and open them.
     125                 :            :      *  Any existing tables will be removed first.
     126                 :            :      */
     127                 :            :     void create_and_open_tables(int flags, unsigned int blocksize);
     128                 :            : 
     129                 :            :     /** Open all tables at most recent revision.
     130                 :            :      *
     131                 :            :      *  @exception Xapian::DatabaseCorruptError is thrown if a problem is
     132                 :            :      *  found with the database's format.
     133                 :            :      *
     134                 :            :      *  @return false if the tables were already open at the most recent
     135                 :            :      *  revision.
     136                 :            :      */
     137                 :            :     bool open_tables(int flags);
     138                 :            : 
     139                 :            :     /** Get a write lock on the database, or throw an
     140                 :            :      *  Xapian::DatabaseLockError if failure.
     141                 :            :      *
     142                 :            :      *  @param flags Bit-wise or of zero or more Xapian::DB_* constants
     143                 :            :      *
     144                 :            :      *  @param creating true if the database is in the process of being
     145                 :            :      *  created - if false, will throw a DatabaseOpening error if the lock
     146                 :            :      *  can't be acquired and the database doesn't exist.
     147                 :            :      */
     148                 :            :     void get_database_write_lock(int flags, bool creating);
     149                 :            : 
     150                 :            :     /** Get an object holding the next revision number which should be
     151                 :            :      *  used in the tables.
     152                 :            :      *
     153                 :            :      *  @return the next revision number.
     154                 :            :      */
     155                 :            :     glass_revision_number_t get_next_revision_number() const;
     156                 :            : 
     157                 :            :     /** Set the revision number in the tables.
     158                 :            :      *
     159                 :            :      *  This updates the disk tables so that the currently open revision
     160                 :            :      *  becomes the specified revision number.
     161                 :            :      *
     162                 :            :      *  @param new_revision The new revision number to store.  This must
     163                 :            :      *          be greater than the current revision number.  FIXME: If
     164                 :            :      *          we support rewinding to a previous revision, maybe this
     165                 :            :      *          needs to be greater than any previously used revision.
     166                 :            :      */
     167                 :            :     void set_revision_number(int flags, glass_revision_number_t new_revision);
     168                 :            : 
     169                 :            :     /** Re-open tables to recover from an overwritten condition,
     170                 :            :      *  or just get most up-to-date version.
     171                 :            :      */
     172                 :            :     bool reopen();
     173                 :            : 
     174                 :            :     /** Close all the tables permanently.
     175                 :            :      */
     176                 :            :     void close();
     177                 :            : 
     178                 :            :     /** Called if a modifications fail.
     179                 :            :      *
     180                 :            :      *  @param msg is a string description of the exception that was
     181                 :            :      *  raised when the modifications failed.
     182                 :            :      */
     183                 :            :     void modifications_failed(glass_revision_number_t new_revision,
     184                 :            :                               const std::string & msg);
     185                 :            : 
     186                 :            :     /** Apply any outstanding changes to the tables.
     187                 :            :      *
     188                 :            :      *  If an error occurs during this operation, this will be signalled
     189                 :            :      *  by an exception being thrown.  In this case the contents of the
     190                 :            :      *  tables on disk will be left in an unmodified state (though possibly
     191                 :            :      *  with increased revision numbers), and the outstanding changes will
     192                 :            :      *  be lost.
     193                 :            :      */
     194                 :            :     void apply();
     195                 :            : 
     196                 :            :     /** Cancel any outstanding changes to the tables.
     197                 :            :      */
     198                 :            :     void cancel();
     199                 :            : 
     200                 :            :     /** Send a set of messages which transfer the whole database.
     201                 :            :      */
     202                 :            :     void send_whole_database(RemoteConnection & conn, double end_time);
     203                 :            : 
     204                 :            :     /** Get the revision stored in a changeset.
     205                 :            :      */
     206                 :            :     void get_changeset_revisions(const string & path,
     207                 :            :                                  glass_revision_number_t * startrev,
     208                 :            :                                  glass_revision_number_t * endrev) const;
     209                 :            : 
     210                 :            :   public:
     211                 :            :     /** Create and open a glass database.
     212                 :            :      *
     213                 :            :      *  @exception Xapian::DatabaseCorruptError is thrown if a problem is
     214                 :            :      *         found with the database's format.
     215                 :            :      *
     216                 :            :      *  @exception Xapian::DatabaseOpeningError thrown if database can't
     217                 :            :      *             be opened.
     218                 :            :      *
     219                 :            :      *  @exception Xapian::DatabaseVersionError thrown if database is in an
     220                 :            :      *             unsupported format.  This implies that the database was
     221                 :            :      *             created by an older or newer version of Xapian.
     222                 :            :      *
     223                 :            :      *  @param dbdir directory holding glass tables
     224                 :            :      *
     225                 :            :      *  @param block_size Block size, in bytes, to use when creating
     226                 :            :      *                    tables.  This is only important, and has the
     227                 :            :      *                    correct value, when the database is being
     228                 :            :      *                    created.
     229                 :            :      */
     230                 :            :     explicit GlassDatabase(const string& db_dir_,
     231                 :            :                            int flags = Xapian::DB_READONLY_,
     232                 :            :                            unsigned int block_size = 0u);
     233                 :            : 
     234                 :            :     explicit GlassDatabase(int fd);
     235                 :            : 
     236                 :            :     ~GlassDatabase();
     237                 :            : 
     238                 :            :     /// Get a postlist table cursor (used by GlassValueList).
     239                 :      27567 :     GlassCursor * get_postlist_cursor() const {
     240                 :      27567 :         return postlist_table.cursor_get();
     241                 :            :     }
     242                 :            : 
     243                 :            :     /** Virtual methods of Database::Internal. */
     244                 :            :     //@{
     245                 :            :     Xapian::doccount get_doccount() const;
     246                 :            :     Xapian::docid get_lastdocid() const;
     247                 :            :     Xapian::totallength get_total_length() const;
     248                 :            :     Xapian::termcount get_doclength(Xapian::docid did) const;
     249                 :            :     Xapian::termcount get_unique_terms(Xapian::docid did) const;
     250                 :            :     void get_freqs(const string & term,
     251                 :            :                    Xapian::doccount * termfreq_ptr,
     252                 :            :                    Xapian::termcount * collfreq_ptr) const;
     253                 :            :     Xapian::doccount get_value_freq(Xapian::valueno slot) const;
     254                 :            :     std::string get_value_lower_bound(Xapian::valueno slot) const;
     255                 :            :     std::string get_value_upper_bound(Xapian::valueno slot) const;
     256                 :            :     Xapian::termcount get_doclength_lower_bound() const;
     257                 :            :     Xapian::termcount get_doclength_upper_bound() const;
     258                 :            :     Xapian::termcount get_wdf_upper_bound(const string & term) const;
     259                 :            :     Xapian::termcount get_unique_terms_lower_bound() const;
     260                 :            :     bool term_exists(const string & tname) const;
     261                 :            :     bool has_positions() const;
     262                 :            : 
     263                 :            :     PostList * open_post_list(const string & tname) const;
     264                 :            :     LeafPostList* open_leaf_post_list(const string& term,
     265                 :            :                                       bool need_read_pos) const;
     266                 :            :     ValueList * open_value_list(Xapian::valueno slot) const;
     267                 :            :     Xapian::Document::Internal* open_document(Xapian::docid did,
     268                 :            :                                               bool lazy) const;
     269                 :            : 
     270                 :            :     virtual void read_position_list(GlassRePositionList* pos_list,
     271                 :            :                                     Xapian::docid did,
     272                 :            :                                     const string& term) const;
     273                 :            :     virtual Xapian::termcount positionlist_count(Xapian::docid did,
     274                 :            :                                                  const string& term) const;
     275                 :            :     PositionList* open_position_list(Xapian::docid did,
     276                 :            :                                      const string& term) const;
     277                 :            :     TermList * open_term_list(Xapian::docid did) const;
     278                 :            :     TermList * open_term_list_direct(Xapian::docid did) const;
     279                 :            :     TermList * open_allterms(const string & prefix) const;
     280                 :            : 
     281                 :            :     TermList * open_spelling_termlist(const string & word) const;
     282                 :            :     TermList * open_spelling_wordlist() const;
     283                 :            :     Xapian::doccount get_spelling_frequency(const string & word) const;
     284                 :            : 
     285                 :            :     TermList * open_synonym_termlist(const string & term) const;
     286                 :            :     TermList * open_synonym_keylist(const string & prefix) const;
     287                 :            : 
     288                 :            :     string get_metadata(const string & key) const;
     289                 :            :     TermList * open_metadata_keylist(const std::string &prefix) const;
     290                 :            :     void write_changesets_to_fd(int fd,
     291                 :            :                                 const string & start_revision,
     292                 :            :                                 bool need_whole_db,
     293                 :            :                                 Xapian::ReplicationInfo * info);
     294                 :            :     /** Get the revision number which the tables are opened at.
     295                 :            :      *
     296                 :            :      *  @return the current revision number.
     297                 :            :      */
     298                 :            :     Xapian::rev get_revision() const;
     299                 :            :     string get_uuid() const;
     300                 :            : 
     301                 :            :     void request_document(Xapian::docid /*did*/) const;
     302                 :            :     void readahead_for_query(const Xapian::Query &query) const;
     303                 :            :     //@}
     304                 :            : 
     305                 :            :     [[noreturn]]
     306                 :            :     void throw_termlist_table_close_exception() const;
     307                 :            : 
     308                 :     123555 :     int get_backend_info(string * path) const {
     309         [ +  + ]:     123555 :         if (path) *path = db_dir;
     310                 :     123555 :         return BACKEND_GLASS;
     311                 :            :     }
     312                 :            : 
     313                 :       5998 :     bool single_file() const { return version_file.single_file(); }
     314                 :            : 
     315                 :            :     void get_used_docid_range(Xapian::docid & first,
     316                 :            :                               Xapian::docid & last) const;
     317                 :            : 
     318                 :            :     /** Return true if there are uncommitted changes. */
     319                 :            :     virtual bool has_uncommitted_changes() const;
     320                 :            : 
     321                 :            :     bool locked() const;
     322                 :            : 
     323                 :            :     static void compact(Xapian::Compactor * compactor,
     324                 :            :                         const char * destdir,
     325                 :            :                         int fd,
     326                 :            :                         const std::vector<const Xapian::Database::Internal*>& sources,
     327                 :            :                         const std::vector<Xapian::docid> & offset,
     328                 :            :                         size_t block_size,
     329                 :            :                         Xapian::Compactor::compaction_level compaction,
     330                 :            :                         unsigned flags,
     331                 :            :                         Xapian::docid last_docid);
     332                 :            : 
     333                 :            :     std::string get_description() const;
     334                 :            : };
     335                 :            : 
     336                 :            : /** A writable glass database.
     337                 :            :  */
     338                 :            : class GlassWritableDatabase : public GlassDatabase {
     339                 :            :     mutable Inverter inverter;
     340                 :            : 
     341                 :            :     mutable map<Xapian::valueno, ValueStats> value_stats;
     342                 :            : 
     343                 :            :     /** The number of documents added, deleted, or replaced since the last
     344                 :            :      *  flush.
     345                 :            :      */
     346                 :            :     mutable Xapian::doccount change_count;
     347                 :            : 
     348                 :            :     /// If change_count reaches this threshold we automatically flush.
     349                 :            :     Xapian::doccount flush_threshold;
     350                 :            : 
     351                 :            :     /** A pointer to the last document which was returned by
     352                 :            :      *  open_document(), or NULL if there is no such valid document.  This
     353                 :            :      *  is used purely for comparing with a supplied document to help with
     354                 :            :      *  optimising replace_document.  When the document internals are
     355                 :            :      *  deleted, this pointer gets set to NULL.
     356                 :            :      */
     357                 :            :     mutable Xapian::Document::Internal * modify_shortcut_document;
     358                 :            : 
     359                 :            :     /** The document ID for the last document returned by open_document().
     360                 :            :      */
     361                 :            :     mutable Xapian::docid modify_shortcut_docid;
     362                 :            : 
     363                 :            :     /** Check if we should autoflush.
     364                 :            :      *
     365                 :            :      *  Called at the end of each document changing operation.
     366                 :            :      */
     367                 :            :     void check_flush_threshold();
     368                 :            : 
     369                 :            :     /// Flush any unflushed postlist changes, but don't commit them.
     370                 :            :     void flush_postlist_changes() const;
     371                 :            : 
     372                 :            :     /// Close all the tables permanently.
     373                 :            :     void close();
     374                 :            : 
     375                 :            :     /// Apply changes.
     376                 :            :     void apply();
     377                 :            : 
     378                 :            :     //@{
     379                 :            :     /** Implementation of virtual methods: see Database::Internal for
     380                 :            :      *  details.
     381                 :            :      */
     382                 :            :     void commit();
     383                 :            : 
     384                 :            :     /** Cancel pending modifications to the database. */
     385                 :            :     void cancel();
     386                 :            : 
     387                 :            :     Xapian::docid add_document(const Xapian::Document& document);
     388                 :            :     Xapian::docid add_document_(Xapian::docid did,
     389                 :            :                                 const Xapian::Document& document);
     390                 :            :     // Stop the default implementation of delete_document(term) and
     391                 :            :     // replace_document(term) from being hidden.  This isn't really
     392                 :            :     // a problem as we only try to call them through the base class
     393                 :            :     // (where they aren't hidden) but some compilers generate a warning
     394                 :            :     // about the hiding.
     395                 :            :     using Xapian::Database::Internal::delete_document;
     396                 :            :     using Xapian::Database::Internal::replace_document;
     397                 :            :     void delete_document(Xapian::docid did);
     398                 :            :     void replace_document(Xapian::docid did, const Xapian::Document & document);
     399                 :            : 
     400                 :            :     Xapian::Document::Internal * open_document(Xapian::docid did,
     401                 :            :                                                bool lazy) const;
     402                 :            : 
     403                 :            :     //@}
     404                 :            : 
     405                 :            :   public:
     406                 :            :     /** Create and open a writable glass database.
     407                 :            :      *
     408                 :            :      *  @exception Xapian::DatabaseOpeningError thrown if database can't
     409                 :            :      *             be opened.
     410                 :            :      *
     411                 :            :      *  @exception Xapian::DatabaseVersionError thrown if database is in an
     412                 :            :      *             unsupported format.  This implies that the database was
     413                 :            :      *             created by an older or newer version of Xapian.
     414                 :            :      *
     415                 :            :      *  @param dir directory holding glass tables
     416                 :            :      */
     417                 :            :     GlassWritableDatabase(const string &dir, int flags, int block_size);
     418                 :            : 
     419                 :            :     ~GlassWritableDatabase();
     420                 :            : 
     421                 :            :     /** Virtual methods of Database::Internal. */
     422                 :            :     //@{
     423                 :            :     Xapian::termcount get_doclength(Xapian::docid did) const;
     424                 :            :     Xapian::termcount get_unique_terms(Xapian::docid did) const;
     425                 :            :     void get_freqs(const string & term,
     426                 :            :                    Xapian::doccount * termfreq_ptr,
     427                 :            :                    Xapian::termcount * collfreq_ptr) const;
     428                 :            :     Xapian::doccount get_value_freq(Xapian::valueno slot) const;
     429                 :            :     std::string get_value_lower_bound(Xapian::valueno slot) const;
     430                 :            :     std::string get_value_upper_bound(Xapian::valueno slot) const;
     431                 :            :     bool term_exists(const string & tname) const;
     432                 :            :     bool has_positions() const;
     433                 :            : 
     434                 :            :     PostList * open_post_list(const string & tname) const;
     435                 :            :     LeafPostList* open_leaf_post_list(const string& term,
     436                 :            :                                       bool need_read_pos) const;
     437                 :            :     ValueList * open_value_list(Xapian::valueno slot) const;
     438                 :            : 
     439                 :            :     void read_position_list(GlassRePositionList* pos_list,
     440                 :            :                             Xapian::docid did,
     441                 :            :                             const string& term) const;
     442                 :            :     Xapian::termcount positionlist_count(Xapian::docid did,
     443                 :            :                                          const string& term) const;
     444                 :            :     PositionList* open_position_list(Xapian::docid did,
     445                 :            :                                      const string& term) const;
     446                 :            :     TermList * open_allterms(const string & prefix) const;
     447                 :            : 
     448                 :            :     void add_spelling(const string & word, Xapian::termcount freqinc) const;
     449                 :            :     Xapian::termcount remove_spelling(const string & word,
     450                 :            :                                       Xapian::termcount freqdec) const;
     451                 :            :     TermList * open_spelling_wordlist() const;
     452                 :            : 
     453                 :            :     TermList * open_synonym_keylist(const string & prefix) const;
     454                 :            :     void add_synonym(const string & word, const string & synonym) const;
     455                 :            :     void remove_synonym(const string & word, const string & synonym) const;
     456                 :            :     void clear_synonyms(const string & word) const;
     457                 :            : 
     458                 :            :     void set_metadata(const string & key, const string & value);
     459                 :            :     void invalidate_doc_object(Xapian::Document::Internal * obj) const;
     460                 :            :     //@}
     461                 :            : 
     462                 :            :     /** Return true if there are uncommitted changes. */
     463                 :            :     bool has_uncommitted_changes() const;
     464                 :            : };
     465                 :            : 
     466                 :            : #ifdef DISABLE_GPL_LIBXAPIAN
     467                 :            : # error GPL source we cannot relicense included in libxapian
     468                 :            : #endif
     469                 :            : 
     470                 :            : #endif /* XAPIAN_INCLUDED_GLASS_DATABASE_H */

Generated by: LCOV version 1.11