LCOV - code coverage report
Current view: top level - backends - dbcheck.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 7028d852e609 Lines: 94 136 69.1 %
Date: 2019-02-17 14:59:59 Functions: 7 7 100.0 %
Branches: 87 287 30.3 %

           Branch data     Line data    Source code
       1                 :            : /** @file dbcheck.cc
       2                 :            :  * @brief Check the consistency of a database or table.
       3                 :            :  */
       4                 :            : /* Copyright 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 Olly Betts
       5                 :            :  *
       6                 :            :  * This program is free software; you can redistribute it and/or
       7                 :            :  * modify it under the terms of the GNU General Public License as
       8                 :            :  * published by the Free Software Foundation; either version 2 of the
       9                 :            :  * License, or (at your option) any later version.
      10                 :            :  *
      11                 :            :  * This program is distributed in the hope that it will be useful,
      12                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      13                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14                 :            :  * GNU General Public License for more details.
      15                 :            :  *
      16                 :            :  * You should have received a copy of the GNU General Public License
      17                 :            :  * along with this program; if not, write to the Free Software
      18                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
      19                 :            :  * USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : #include "xapian/database.h"
      24                 :            : 
      25                 :            : #include "xapian/constants.h"
      26                 :            : #include "xapian/error.h"
      27                 :            : 
      28                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      29                 :            : #include "glass/glass_changes.h"
      30                 :            : #include "glass/glass_database.h"
      31                 :            : #include "glass/glass_dbcheck.h"
      32                 :            : #include "glass/glass_defs.h"
      33                 :            : #include "glass/glass_version.h"
      34                 :            : #endif
      35                 :            : 
      36                 :            : #include "filetests.h"
      37                 :            : #include "omassert.h"
      38                 :            : #include "posixy_wrapper.h"
      39                 :            : #include "stringutils.h"
      40                 :            : 
      41                 :            : #include <ostream>
      42                 :            : #include <stdexcept>
      43                 :            : 
      44                 :            : using namespace std;
      45                 :            : 
      46                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      47                 :            : // Tables to check for a glass database.  Note: it's important to check
      48                 :            : // termlist before postlist so that we can cross-check the document lengths.
      49                 :            : static const struct { char name[9]; } glass_tables[] = {
      50                 :            :     { "docdata" },
      51                 :            :     { "termlist" },
      52                 :            :     { "postlist" },
      53                 :            :     { "position" },
      54                 :            :     { "spelling" },
      55                 :            :     { "synonym" }
      56                 :            : };
      57                 :            : #endif
      58                 :            : 
      59                 :            : static bool
      60                 :          5 : check_if_single_file_db(const struct stat & sb, const string & path,
      61                 :            :                         int * fd_ptr = NULL)
      62                 :            : {
      63                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      64         [ -  + ]:          5 :     if (!S_ISREG(sb.st_mode)) return false;
      65                 :            :     // Look at the size as a clue - if it's 0 or not a multiple of
      66                 :            :     // GLASS_MIN_BLOCKSIZE, then it's not a single-file glass database.  If it
      67                 :            :     // is, peek at the start of the file to determine which it is.
      68 [ +  - ][ +  + ]:          5 :     if (sb.st_size == 0 || sb.st_size % GLASS_MIN_BLOCKSIZE != 0)
      69                 :          1 :         return false;
      70                 :          4 :     int fd = posixy_open(path.c_str(), O_RDONLY|O_BINARY);
      71         [ +  - ]:          4 :     if (fd != -1) {
      72                 :            :         char magic_buf[14];
      73                 :            :         // FIXME: Don't duplicate magic check here...
      74 [ +  - ][ +  - ]:         12 :         if (io_read(fd, magic_buf, 14) == 14 &&
                 [ +  + ]
      75 [ +  - ][ +  - ]:          8 :             (!fd_ptr || lseek(fd, 0, SEEK_SET) == 0) &&
                 [ +  + ]
      76                 :          4 :             memcmp(magic_buf, "\x0f\x0dXapian Glass", 14) == 0) {
      77         [ +  - ]:          3 :             if (fd_ptr) {
      78                 :          3 :                 *fd_ptr = fd;
      79                 :            :             } else {
      80         [ #  # ]:          0 :                 ::close(fd);
      81                 :            :             }
      82                 :          3 :             return true;
      83                 :            :         }
      84         [ +  - ]:          1 :         ::close(fd);
      85                 :            :     }
      86                 :            : #else
      87                 :            :     (void)sb;
      88                 :            :     (void)path;
      89                 :            :     (void)fd_ptr;
      90                 :            : #endif
      91                 :          5 :     return false;
      92                 :            : }
      93                 :            : 
      94                 :            : // FIXME: We don't currently cross-check wdf between postlist and termlist.
      95                 :            : // It's hard to see how to efficiently.  We do cross-check doclens, but that
      96                 :            : // "only" requires (4 * last_docid()) bytes.
      97                 :            : 
      98                 :            : #if defined XAPIAN_HAS_GLASS_BACKEND
      99                 :            : static void
     100                 :         10 : reserve_doclens(vector<Xapian::termcount>& doclens, Xapian::docid last_docid,
     101                 :            :                 ostream * out)
     102                 :            : {
     103         [ -  + ]:         10 :     if (last_docid >= 0x40000000ul / sizeof(Xapian::termcount)) {
     104                 :            :         // The memory block needed by the vector would be >= 1GB.
     105         [ #  # ]:          0 :         if (out)
     106                 :            :             *out << "Cross-checking document lengths between the postlist and "
     107                 :            :                     "termlist tables would use more than 1GB of memory, so "
     108                 :          0 :                     "skipping that check" << endl;
     109                 :         10 :         return;
     110                 :            :     }
     111                 :            :     try {
     112         [ +  - ]:         10 :         doclens.reserve(last_docid + 1);
     113                 :          0 :     } catch (const std::bad_alloc &) {
     114                 :            :         // Failed to allocate the required memory.
     115         [ #  # ]:          0 :         if (out)
     116                 :            :             *out << "Couldn't allocate enough memory for cross-checking document "
     117                 :            :                     "lengths between the postlist and termlist tables, so "
     118   [ #  #  #  # ]:          0 :                     "skipping that check" << endl;
     119      [ #  #  # ]:          0 :     } catch (const std::length_error &) {
     120                 :            :         // There are too many elements for the vector to handle!
     121         [ #  # ]:          0 :         if (out)
     122                 :            :             *out << "Couldn't allocate enough elements for cross-checking document "
     123                 :            :                     "lengths between the postlist and termlist tables, so "
     124   [ #  #  #  # ]:          0 :                     "skipping that check" << endl;
     125                 :            :     }
     126                 :            : }
     127                 :            : #endif
     128                 :            : 
     129                 :            : static size_t
     130                 :          6 : check_db_dir(const string & path, int opts, std::ostream *out)
     131                 :            : {
     132                 :            :     struct stat sb;
     133 [ +  - ][ +  - ]:          6 :     if (stat((path + "/iamglass").c_str(), &sb) == 0) {
     134                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
     135                 :            :         (void)opts;
     136                 :            :         (void)out;
     137                 :            :         throw Xapian::FeatureUnavailableError("Glass database support isn't enabled");
     138                 :            : #else
     139                 :            :         // Check a whole glass database directory.
     140                 :          6 :         vector<Xapian::termcount> doclens;
     141                 :          6 :         size_t errors = 0;
     142                 :            : 
     143                 :            :         try {
     144                 :            :             // Check if the database can actually be opened.
     145         [ +  - ]:          6 :             Xapian::Database db(path);
     146         [ #  # ]:          0 :         } catch (const Xapian::Error & e) {
     147                 :            :             // Continue - we can still usefully look at how it is broken.
     148         [ #  # ]:          0 :             if (out)
     149         [ #  # ]:          0 :                 *out << "Database couldn't be opened for reading: "
     150   [ #  #  #  # ]:          0 :                      << e.get_description()
     151   [ #  #  #  # ]:          0 :                      << "\nContinuing check anyway" << endl;
     152                 :          0 :             ++errors;
     153                 :            :         }
     154                 :            : 
     155         [ +  - ]:         12 :         GlassVersion version_file(path);
     156         [ +  - ]:          6 :         version_file.read();
     157         [ +  + ]:         14 :         for (glass_revision_number_t r = version_file.get_revision(); r != 0; --r) {
     158         [ +  - ]:          8 :             string changes_file = path;
     159         [ +  - ]:          8 :             changes_file += "/changes";
     160 [ +  - ][ +  - ]:          8 :             changes_file += str(r);
     161         [ -  + ]:          8 :             if (file_exists(changes_file))
     162         [ #  # ]:          0 :                 GlassChanges::check(changes_file);
     163                 :          8 :         }
     164                 :            : 
     165                 :          6 :         Xapian::docid doccount = version_file.get_doccount();
     166                 :          6 :         Xapian::docid db_last_docid = version_file.get_last_docid();
     167         [ -  + ]:          6 :         if (db_last_docid < doccount) {
     168         [ #  # ]:          0 :             if (out)
     169 [ #  # ][ #  # ]:          0 :                 *out << "last_docid = " << db_last_docid << " < doccount = "
                 [ #  # ]
     170 [ #  # ][ #  # ]:          0 :                      << doccount << endl;
     171                 :          0 :             ++errors;
     172                 :            :         }
     173         [ +  - ]:          6 :         reserve_doclens(doclens, db_last_docid, out);
     174                 :            : 
     175                 :            :         // Check all the tables.
     176         [ +  + ]:         42 :         for (auto t : glass_tables) {
     177                 :            :             errors += check_glass_table(t.name, path, version_file, opts,
     178         [ +  - ]:         36 :                                         doclens, out);
     179                 :            :         }
     180                 :          6 :         return errors;
     181                 :            : #endif
     182                 :            :     }
     183                 :            : 
     184 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iamchert").c_str(), &sb) == 0) {
     185                 :            :         // Chert is no longer supported as of Xapian 1.5.0.
     186 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Chert database support was removed in Xapian 1.5.0");
                 [ #  # ]
     187                 :            :     }
     188                 :            : 
     189 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iamflint").c_str(), &sb) == 0) {
     190                 :            :         // Flint is no longer supported as of Xapian 1.3.0.
     191 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Flint database support was removed in Xapian 1.3.0");
                 [ #  # ]
     192                 :            :     }
     193                 :            : 
     194 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iambrass").c_str(), &sb) == 0) {
     195                 :            :         // Brass was renamed to glass as of Xapian 1.3.2.
     196 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Brass database support was removed in Xapian 1.3.2");
                 [ #  # ]
     197                 :            :     }
     198                 :            : 
     199 [ #  # ][ #  # ]:          0 :     if (stat((path + "/record_DB").c_str(), &sb) == 0) {
     200                 :            :         // Quartz is no longer supported as of Xapian 1.1.0.
     201 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Quartz database support was removed in Xapian 1.1.0");
                 [ #  # ]
     202                 :            :     }
     203                 :            : 
     204                 :            :     throw Xapian::DatabaseOpeningError(
     205 [ #  # ][ #  # ]:          0 :             "Directory does not contain a Xapian database");
                 [ #  # ]
     206                 :            : }
     207                 :            : 
     208                 :            : typedef enum { UNKNOWN, OLD, GLASS } backend_type;
     209                 :            : 
     210                 :            : /** Check a database table.
     211                 :            :  *
     212                 :            :  *  @param filename     The filename of the table (only used to get the directory and
     213                 :            :  *  @param opts         Xapian::check() options
     214                 :            :  *  @param out          std::ostream to write messages to (or NULL for no messages)
     215                 :            :  *  @param backend      Backend type
     216                 :            :  */
     217                 :            : static size_t
     218                 :          3 : check_db_table_(const string & filename, int opts, std::ostream *out,
     219                 :            :                 backend_type backend)
     220                 :            : {
     221                 :          3 :     size_t p = filename.find_last_of(DIR_SEPS);
     222                 :            :     // If we found a directory separator, advance p to the next character.  If
     223                 :            :     // we didn't, incrementing string::npos will give us 0, which is what we
     224                 :            :     // want.
     225                 :          3 :     ++p;
     226                 :            : 
     227         [ +  - ]:          3 :     string dir(filename, 0, p);
     228                 :            : 
     229         [ +  - ]:          6 :     string tablename;
     230         [ +  + ]:         26 :     while (p != filename.size()) {
     231                 :         25 :         char ch = filename[p++];
     232         [ +  + ]:         25 :         if (ch == '.') break;
     233         [ +  - ]:         23 :         tablename += C_tolower(ch);
     234                 :            :     }
     235                 :            : 
     236                 :            : #if defined XAPIAN_HAS_GLASS_BACKEND
     237                 :          6 :     vector<Xapian::termcount> doclens;
     238                 :            : #else
     239                 :            :     (void)opts;
     240                 :            :     (void)out;
     241                 :            : #endif
     242                 :            : 
     243         [ +  - ]:          3 :     if (backend == GLASS) {
     244                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
     245                 :            :         throw Xapian::FeatureUnavailableError("Glass database support isn't enabled");
     246                 :            : #else
     247         [ +  - ]:          3 :         GlassVersion version_file(dir);
     248         [ +  - ]:          3 :         version_file.read();
     249                 :            :         return check_glass_table(tablename.c_str(), dir, version_file, opts,
     250         [ +  - ]:          6 :                                  doclens, out);
     251                 :            : #endif
     252                 :            :     }
     253                 :            : 
     254                 :            :     Assert(backend == OLD);
     255                 :            :     // Chert, flint and brass all used the extension ".DB", so check which
     256                 :            :     // to give an appropriate error.
     257                 :            :     struct stat sb;
     258 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iamchert").c_str(), &sb) == 0) {
     259                 :            :         // Chert is no longer supported as of Xapian 1.5.0.
     260 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Chert database support was removed in Xapian 1.5.0");
                 [ #  # ]
     261                 :            :     }
     262 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iamflint").c_str(), &sb) == 0) {
     263                 :            :         // Flint is no longer supported as of Xapian 1.3.0.
     264 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Flint database support was removed in Xapian 1.3.0");
                 [ #  # ]
     265                 :            :     }
     266 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iambrass").c_str(), &sb) == 0) {
     267                 :            :         // Brass was renamed to glass as of Xapian 1.3.2.
     268 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Brass database support was removed in Xapian 1.3.2");
                 [ #  # ]
     269                 :            :     }
     270                 :            :     // Unaccompanied .DB file.
     271 [ #  # ][ #  # ]:          3 :     throw Xapian::FeatureUnavailableError("Flint, chert and brass database support have all been removed");
                 [ #  # ]
     272                 :            : }
     273                 :            : 
     274                 :            : static size_t
     275                 :          2 : check_db_table(const string & filename, int opts, std::ostream *out)
     276                 :            : {
     277                 :          2 :     backend_type backend = UNKNOWN;
     278                 :            :     // Just check a single Btree, given the correct filename.  Look at the
     279                 :            :     // extension to determine the type.
     280         [ -  + ]:          2 :     if (endswith(filename, ".DB")) {
     281                 :          0 :         backend = OLD;
     282         [ +  + ]:          2 :     } else if (endswith(filename, ".glass")) {
     283                 :          1 :         backend = GLASS;
     284                 :            :     } else {
     285                 :            :         throw Xapian::DatabaseOpeningError(
     286 [ +  - ][ +  - ]:          1 :                 "File is not a Xapian database or database table");
                 [ +  - ]
     287                 :            :     }
     288                 :            : 
     289                 :          1 :     return check_db_table_(filename, opts, out, backend);
     290                 :            : }
     291                 :            : 
     292                 :            : /** Check a single file DB from an fd.
     293                 :            :  *
     294                 :            :  *  Closes the fd (via GlassVersion doing so in its destructor).
     295                 :            :  */
     296                 :            : static size_t
     297                 :          4 : check_db_fd(int fd, int opts, std::ostream *out)
     298                 :            : {
     299                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
     300                 :            :     (void)opts;
     301                 :            :     (void)out;
     302                 :            :     ::close(fd);
     303                 :            :     throw Xapian::FeatureUnavailableError("Glass database support isn't enabled");
     304                 :            : #else
     305                 :            :     // Check a single-file glass database.
     306         [ +  - ]:          4 :     GlassVersion version_file(fd);
     307         [ +  - ]:          4 :     version_file.read();
     308                 :            : 
     309                 :          4 :     size_t errors = 0;
     310                 :          4 :     Xapian::docid doccount = version_file.get_doccount();
     311                 :          4 :     Xapian::docid db_last_docid = version_file.get_last_docid();
     312         [ -  + ]:          4 :     if (db_last_docid < doccount) {
     313         [ #  # ]:          0 :         if (out)
     314 [ #  # ][ #  # ]:          0 :             *out << "last_docid = " << db_last_docid << " < doccount = "
                 [ #  # ]
     315 [ #  # ][ #  # ]:          0 :                  << doccount << endl;
     316                 :          0 :         ++errors;
     317                 :            :     }
     318                 :          8 :     vector<Xapian::termcount> doclens;
     319         [ +  - ]:          4 :     reserve_doclens(doclens, db_last_docid, out);
     320                 :            : 
     321                 :            :     // Check all the tables.
     322         [ +  + ]:         28 :     for (auto t : glass_tables) {
     323                 :            :         errors += check_glass_table(t.name, fd, version_file.get_offset(),
     324                 :            :                                     version_file, opts, doclens,
     325         [ +  - ]:         24 :                                     out);
     326                 :            :     }
     327                 :          4 :     return errors;
     328                 :            : #endif
     329                 :            : }
     330                 :            : 
     331                 :            : namespace Xapian {
     332                 :            : 
     333                 :            : size_t
     334                 :         15 : Database::check_(const string * path_ptr, int fd, int opts, std::ostream *out)
     335                 :            : {
     336         [ +  + ]:         15 :     if (!out) {
     337                 :            :         // If we have nowhere to write output, then disable all the options
     338                 :            :         // which only affect what we output.
     339                 :          3 :         opts &= Xapian::DBCHECK_FIX;
     340                 :            :     }
     341                 :            : 
     342         [ +  + ]:         15 :     if (path_ptr == NULL) {
     343         [ +  - ]:          1 :         return check_db_fd(fd, opts, out);
     344                 :            :     }
     345                 :            : 
     346                 :         14 :     const string & path = *path_ptr;
     347                 :            :     struct stat sb;
     348         [ +  + ]:         14 :     if (stat(path.c_str(), &sb) == 0) {
     349         [ +  + ]:         11 :         if (S_ISDIR(sb.st_mode)) {
     350         [ +  - ]:          6 :             return check_db_dir(path, opts, out);
     351                 :            :         }
     352                 :            : 
     353         [ +  - ]:          5 :         if (S_ISREG(sb.st_mode)) {
     354 [ +  - ][ +  + ]:          5 :             if (check_if_single_file_db(sb, path, &fd)) {
     355         [ +  - ]:          3 :                 return check_db_fd(fd, opts, out);
     356                 :            :             }
     357         [ +  + ]:          2 :             return check_db_table(path, opts, out);
     358                 :            :         }
     359                 :            : 
     360 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseOpeningError("Not a regular file or directory");
                 [ #  # ]
     361                 :            :     }
     362                 :            : 
     363                 :            :     // The filename passed doesn't exist - see if it's the basename of the
     364                 :            :     // table (perhaps with "." after it), so the user can do xapian-check on
     365                 :            :     // "foo/termlist" or "foo/termlist." (which you would get from filename
     366                 :            :     // completion with older backends).
     367         [ +  - ]:          3 :     string filename = path;
     368         [ +  + ]:          3 :     if (endswith(filename, '.')) {
     369         [ +  - ]:          1 :         filename.resize(filename.size() - 1);
     370                 :            :     }
     371                 :            : 
     372                 :          3 :     backend_type backend = UNKNOWN;
     373 [ +  - ][ -  + ]:          3 :     if (stat((filename + ".DB").c_str(), &sb) == 0) {
     374                 :            :         // Could be chert, flint or brass - we check which below.
     375                 :          0 :         backend = OLD;
     376 [ +  - ][ +  + ]:          3 :     } else if (stat((filename + ".glass").c_str(), &sb) == 0) {
     377                 :          2 :         backend = GLASS;
     378                 :            :     } else {
     379                 :            :         throw Xapian::DatabaseOpeningError(
     380 [ +  - ][ +  - ]:          1 :                 "Couldn't find Xapian database or table to check", ENOENT);
     381                 :            :     }
     382                 :            : 
     383         [ +  - ]:         14 :     return check_db_table_(path, opts, out, backend);
     384                 :            : }
     385                 :            : 
     386                 :            : }

Generated by: LCOV version 1.11