LCOV - code coverage report
Current view: top level - backends - dbcheck.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 954b5873a738 Lines: 102 155 65.8 %
Date: 2019-06-30 05:20:33 Functions: 10 12 83.3 %
Branches: 88 308 28.6 %

           Branch data     Line data    Source code
       1                 :            : /** @file dbcheck.cc
       2                 :            :  * @brief Check the consistency of a database or table.
       3                 :            :  */
       4                 :            : /* Copyright 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2019 Olly Betts
       5                 :            :  *
       6                 :            :  * This program is free software; you can redistribute it and/or
       7                 :            :  * modify it under the terms of the GNU General Public License as
       8                 :            :  * published by the Free Software Foundation; either version 2 of the
       9                 :            :  * License, or (at your option) any later version.
      10                 :            :  *
      11                 :            :  * This program is distributed in the hope that it will be useful,
      12                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      13                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14                 :            :  * GNU General Public License for more details.
      15                 :            :  *
      16                 :            :  * You should have received a copy of the GNU General Public License
      17                 :            :  * along with this program; if not, write to the Free Software
      18                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
      19                 :            :  * USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : #include "xapian/database.h"
      24                 :            : 
      25                 :            : #include "xapian/constants.h"
      26                 :            : #include "xapian/error.h"
      27                 :            : 
      28                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      29                 :            : #include "glass/glass_changes.h"
      30                 :            : #include "glass/glass_database.h"
      31                 :            : #include "glass/glass_dbcheck.h"
      32                 :            : #include "glass/glass_defs.h"
      33                 :            : #include "glass/glass_version.h"
      34                 :            : #endif
      35                 :            : 
      36                 :            : #include "backends.h"
      37                 :            : #include "databasehelpers.h"
      38                 :            : #include "filetests.h"
      39                 :            : #include "omassert.h"
      40                 :            : #include "stringutils.h"
      41                 :            : 
      42                 :            : #include <ostream>
      43                 :            : #include <stdexcept>
      44                 :            : 
      45                 :            : using namespace std;
      46                 :            : 
      47                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      48                 :            : // Tables to check for a glass database.  Note: it's important to check
      49                 :            : // termlist before postlist so that we can cross-check the document lengths.
      50                 :            : static const struct { char name[9]; } glass_tables[] = {
      51                 :            :     { "docdata" },
      52                 :            :     { "termlist" },
      53                 :            :     { "postlist" },
      54                 :            :     { "position" },
      55                 :            :     { "spelling" },
      56                 :            :     { "synonym" }
      57                 :            : };
      58                 :            : #endif
      59                 :            : 
      60                 :            : // FIXME: We don't currently cross-check wdf between postlist and termlist.
      61                 :            : // It's hard to see how to efficiently.  We do cross-check doclens, but that
      62                 :            : // "only" requires (4 * last_docid()) bytes.
      63                 :            : 
      64                 :            : #if defined XAPIAN_HAS_GLASS_BACKEND
      65                 :            : static void
      66                 :         14 : reserve_doclens(vector<Xapian::termcount>& doclens, Xapian::docid last_docid,
      67                 :            :                 ostream * out)
      68                 :            : {
      69         [ -  + ]:         14 :     if (last_docid >= 0x40000000ul / sizeof(Xapian::termcount)) {
      70                 :            :         // The memory block needed by the vector would be >= 1GB.
      71         [ #  # ]:          0 :         if (out)
      72                 :            :             *out << "Cross-checking document lengths between the postlist and "
      73                 :            :                     "termlist tables would use more than 1GB of memory, so "
      74                 :          0 :                     "skipping that check" << endl;
      75                 :         14 :         return;
      76                 :            :     }
      77                 :            :     try {
      78         [ +  - ]:         14 :         doclens.reserve(last_docid + 1);
      79                 :          0 :     } catch (const std::bad_alloc &) {
      80                 :            :         // Failed to allocate the required memory.
      81         [ #  # ]:          0 :         if (out)
      82                 :            :             *out << "Couldn't allocate enough memory for cross-checking document "
      83                 :            :                     "lengths between the postlist and termlist tables, so "
      84   [ #  #  #  # ]:          0 :                     "skipping that check" << endl;
      85      [ #  #  # ]:          0 :     } catch (const std::length_error &) {
      86                 :            :         // There are too many elements for the vector to handle!
      87         [ #  # ]:          0 :         if (out)
      88                 :            :             *out << "Couldn't allocate enough elements for cross-checking document "
      89                 :            :                     "lengths between the postlist and termlist tables, so "
      90   [ #  #  #  # ]:          0 :                     "skipping that check" << endl;
      91                 :            :     }
      92                 :            : }
      93                 :            : #endif
      94                 :            : 
      95                 :            : static size_t
      96                 :         10 : check_db_dir(const string & path, int opts, std::ostream *out)
      97                 :            : {
      98                 :            :     struct stat sb;
      99 [ +  - ][ +  + ]:         10 :     if (stat((path + "/iamglass").c_str(), &sb) == 0) {
     100                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
     101                 :            :         (void)opts;
     102                 :            :         (void)out;
     103                 :            :         throw Xapian::FeatureUnavailableError("Glass database support isn't enabled");
     104                 :            : #else
     105                 :            :         // Check a whole glass database directory.
     106                 :          9 :         vector<Xapian::termcount> doclens;
     107                 :          9 :         size_t errors = 0;
     108                 :            : 
     109                 :            :         try {
     110                 :            :             // Check if the database can actually be opened.
     111         [ +  - ]:          9 :             Xapian::Database db(path);
     112   [ #  #  #  # ]:          0 :         } catch (const Xapian::Error & e) {
     113                 :            :             // Continue - we can still usefully look at how it is broken.
     114         [ #  # ]:          0 :             if (out)
     115         [ #  # ]:          0 :                 *out << "Database couldn't be opened for reading: "
     116   [ #  #  #  # ]:          0 :                      << e.get_description()
     117   [ #  #  #  # ]:          0 :                      << "\nContinuing check anyway" << endl;
     118                 :          0 :             ++errors;
     119                 :            :         }
     120                 :            : 
     121         [ +  - ]:         18 :         GlassVersion version_file(path);
     122         [ +  - ]:          9 :         version_file.read();
     123         [ +  + ]:         20 :         for (glass_revision_number_t r = version_file.get_revision(); r != 0; --r) {
     124         [ +  - ]:         11 :             string changes_file = path;
     125         [ +  - ]:         11 :             changes_file += "/changes";
     126 [ +  - ][ +  - ]:         11 :             changes_file += str(r);
     127         [ -  + ]:         11 :             if (file_exists(changes_file))
     128         [ #  # ]:          0 :                 GlassChanges::check(changes_file);
     129                 :         11 :         }
     130                 :            : 
     131                 :          9 :         Xapian::docid doccount = version_file.get_doccount();
     132                 :          9 :         Xapian::docid db_last_docid = version_file.get_last_docid();
     133         [ -  + ]:          9 :         if (db_last_docid < doccount) {
     134         [ #  # ]:          0 :             if (out)
     135 [ #  # ][ #  # ]:          0 :                 *out << "last_docid = " << db_last_docid << " < doccount = "
                 [ #  # ]
     136 [ #  # ][ #  # ]:          0 :                      << doccount << endl;
     137                 :          0 :             ++errors;
     138                 :            :         }
     139         [ +  - ]:          9 :         reserve_doclens(doclens, db_last_docid, out);
     140                 :            : 
     141                 :            :         // Check all the tables.
     142         [ +  + ]:         63 :         for (auto t : glass_tables) {
     143                 :            :             errors += check_glass_table(t.name, path, version_file, opts,
     144         [ +  - ]:         54 :                                         doclens, out);
     145                 :            :         }
     146                 :          9 :         return errors;
     147                 :            : #endif
     148                 :            :     }
     149                 :            : 
     150 [ +  - ][ +  - ]:          1 :     if (stat((path + "/iamhoney").c_str(), &sb) == 0) {
     151                 :            : #ifndef XAPIAN_HAS_HONEY_BACKEND
     152                 :            :         (void)opts;
     153                 :            :         (void)out;
     154                 :            :         auto msg = "Honey database support isn't enabled";
     155                 :            :         throw Xapian::FeatureUnavailableError(msg);
     156                 :            : #else
     157                 :            :         (void)opts;
     158                 :            :         (void)out;
     159                 :          1 :         auto msg = "Honey database checking not implemented";
     160 [ +  - ][ +  - ]:          1 :         throw Xapian::UnimplementedError(msg);
                 [ +  - ]
     161                 :            : #endif
     162                 :            :     }
     163                 :            : 
     164 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iamchert").c_str(), &sb) == 0) {
     165                 :            :         // Chert is no longer supported as of Xapian 1.5.0.
     166 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Chert database support was removed in Xapian 1.5.0");
                 [ #  # ]
     167                 :            :     }
     168                 :            : 
     169 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iamflint").c_str(), &sb) == 0) {
     170                 :            :         // Flint is no longer supported as of Xapian 1.3.0.
     171 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Flint database support was removed in Xapian 1.3.0");
                 [ #  # ]
     172                 :            :     }
     173                 :            : 
     174 [ #  # ][ #  # ]:          0 :     if (stat((path + "/iambrass").c_str(), &sb) == 0) {
     175                 :            :         // Brass was renamed to glass as of Xapian 1.3.2.
     176 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Brass database support was removed in Xapian 1.3.2");
                 [ #  # ]
     177                 :            :     }
     178                 :            : 
     179 [ #  # ][ #  # ]:          0 :     if (stat((path + "/record_DB").c_str(), &sb) == 0) {
     180                 :            :         // Quartz is no longer supported as of Xapian 1.1.0.
     181 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Quartz database support was removed in Xapian 1.1.0");
                 [ #  # ]
     182                 :            :     }
     183                 :            : 
     184                 :            :     throw Xapian::DatabaseOpeningError(
     185 [ #  # ][ #  # ]:          0 :             "Directory does not contain a Xapian database");
                 [ #  # ]
     186                 :            : }
     187                 :            : 
     188                 :            : /** Check a database table.
     189                 :            :  *
     190                 :            :  *  @param filename     The filename of the table (only used to get the directory and
     191                 :            :  *  @param opts         Xapian::check() options
     192                 :            :  *  @param out          std::ostream to write messages to (or NULL for no messages)
     193                 :            :  *  @param backend      Backend type (a BACKEND_XXX constant)
     194                 :            :  */
     195                 :            : static size_t
     196                 :          3 : check_db_table(const string& filename, int opts, std::ostream* out, int backend)
     197                 :            : {
     198                 :          3 :     size_t p = filename.find_last_of(DIR_SEPS);
     199                 :            :     // If we found a directory separator, advance p to the next character.  If
     200                 :            :     // we didn't, incrementing string::npos will give us 0, which is what we
     201                 :            :     // want.
     202                 :          3 :     ++p;
     203                 :            : 
     204         [ +  - ]:          3 :     string dir(filename, 0, p);
     205                 :            : 
     206         [ +  - ]:          6 :     string tablename;
     207         [ +  + ]:         26 :     while (p != filename.size()) {
     208                 :         25 :         char ch = filename[p++];
     209         [ +  + ]:         25 :         if (ch == '.') break;
     210         [ +  - ]:         23 :         tablename += C_tolower(ch);
     211                 :            :     }
     212                 :            : 
     213                 :            : #if defined XAPIAN_HAS_GLASS_BACKEND
     214                 :          6 :     vector<Xapian::termcount> doclens;
     215                 :            : #else
     216                 :            :     (void)opts;
     217                 :            :     (void)out;
     218                 :            : #endif
     219                 :            : 
     220   [ +  -  -  - ]:          3 :     switch (backend) {
     221                 :            :       case BACKEND_GLASS: {
     222                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
     223                 :            :         auto msg = "Glass database support isn't enabled";
     224                 :            :         throw Xapian::FeatureUnavailableError(msg);
     225                 :            : #else
     226         [ +  - ]:          3 :         GlassVersion version_file(dir);
     227         [ +  - ]:          3 :         version_file.read();
     228                 :            :         return check_glass_table(tablename.c_str(), dir, version_file, opts,
     229         [ +  - ]:          6 :                                  doclens, out);
     230                 :            : #endif
     231                 :            :       }
     232                 :            : 
     233                 :            :       case BACKEND_HONEY: {
     234                 :            : #ifndef XAPIAN_HAS_HONEY_BACKEND
     235                 :            :         auto msg = "Honey database support isn't enabled";
     236                 :            :         throw Xapian::FeatureUnavailableError(msg);
     237                 :            : #else
     238                 :          0 :         auto msg = "Honey database checking not implemented";
     239 [ #  # ][ #  # ]:          0 :         throw Xapian::UnimplementedError(msg);
                 [ #  # ]
     240                 :            : #endif
     241                 :            :       }
     242                 :            : 
     243                 :            :       case BACKEND_OLD:
     244                 :          0 :         break;
     245                 :            : 
     246                 :            :       default:
     247                 :            :         Assert(false);
     248                 :          0 :         break;
     249                 :            :     }
     250                 :            : 
     251                 :            :     // Chert, flint and brass all used the extension ".DB", so check which
     252                 :            :     // to give an appropriate error.
     253                 :            :     struct stat sb;
     254 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iamchert").c_str(), &sb) == 0) {
     255                 :            :         // Chert is no longer supported as of Xapian 1.5.0.
     256 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Chert database support was removed in Xapian 1.5.0");
                 [ #  # ]
     257                 :            :     }
     258 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iamflint").c_str(), &sb) == 0) {
     259                 :            :         // Flint is no longer supported as of Xapian 1.3.0.
     260 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Flint database support was removed in Xapian 1.3.0");
                 [ #  # ]
     261                 :            :     }
     262 [ #  # ][ #  # ]:          0 :     if (stat((dir + "/iambrass").c_str(), &sb) == 0) {
     263                 :            :         // Brass was renamed to glass as of Xapian 1.3.2.
     264 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Brass database support was removed in Xapian 1.3.2");
                 [ #  # ]
     265                 :            :     }
     266                 :            :     // Unaccompanied .DB file.
     267 [ #  # ][ #  # ]:          3 :     throw Xapian::FeatureUnavailableError("Flint, chert and brass database support have all been removed");
                 [ #  # ]
     268                 :            : }
     269                 :            : 
     270                 :            : /** Check a single file DB from an fd.
     271                 :            :  *
     272                 :            :  *  Closes the fd.
     273                 :            :  */
     274                 :            : static size_t
     275                 :          5 : check_db_fd(int fd, int opts, std::ostream* out, int backend)
     276                 :            : {
     277         [ +  + ]:          5 :     if (backend == BACKEND_UNKNOWN) {
     278                 :            :         // FIXME: Actually probe.
     279                 :          1 :         backend = BACKEND_GLASS;
     280                 :            :     }
     281                 :            : 
     282                 :          5 :     size_t errors = 0;
     283      [ +  -  - ]:          5 :     switch (backend) {
     284                 :            :       case BACKEND_GLASS: {
     285                 :            :         // Check a single-file glass database.
     286                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
     287                 :            :         // GlassVersion's destructor will close fd.
     288         [ +  - ]:          5 :         GlassVersion version_file(fd);
     289         [ +  - ]:          5 :         version_file.read();
     290                 :            : 
     291                 :          5 :         Xapian::docid doccount = version_file.get_doccount();
     292                 :          5 :         Xapian::docid db_last_docid = version_file.get_last_docid();
     293         [ -  + ]:          5 :         if (db_last_docid < doccount) {
     294         [ #  # ]:          0 :             if (out)
     295 [ #  # ][ #  # ]:          0 :                 *out << "last_docid = " << db_last_docid << " < doccount = "
                 [ #  # ]
     296 [ #  # ][ #  # ]:          0 :                      << doccount << endl;
     297                 :          0 :             ++errors;
     298                 :            :         }
     299                 :         10 :         vector<Xapian::termcount> doclens;
     300         [ +  - ]:          5 :         reserve_doclens(doclens, db_last_docid, out);
     301                 :            : 
     302                 :            :         // Check all the tables.
     303         [ +  + ]:         35 :         for (auto t : glass_tables) {
     304                 :            :             errors += check_glass_table(t.name, fd, version_file.get_offset(),
     305                 :            :                                         version_file, opts, doclens,
     306         [ +  - ]:         30 :                                         out);
     307                 :            :         }
     308                 :         10 :         break;
     309                 :            : #else
     310                 :            :         (void)opts;
     311                 :            :         (void)out;
     312                 :            :         ::close(fd);
     313                 :            :         throw Xapian::FeatureUnavailableError("Glass database support isn't enabled");
     314                 :            : #endif
     315                 :            :       }
     316                 :            :       case BACKEND_HONEY:
     317                 :            : #ifdef XAPIAN_HAS_HONEY_BACKEND
     318                 :            :         (void)opts;
     319                 :            :         (void)out;
     320                 :          0 :         ::close(fd);
     321 [ #  # ][ #  # ]:          0 :         throw Xapian::UnimplementedError("Honey database checking not implemented");
                 [ #  # ]
     322                 :            : #else
     323                 :            :         (void)opts;
     324                 :            :         (void)out;
     325                 :            :         ::close(fd);
     326                 :            :         throw Xapian::FeatureUnavailableError("Honey database support isn't enabled");
     327                 :            : #endif
     328                 :            :       default:
     329                 :            :         Assert(false);
     330                 :            :     }
     331                 :          5 :     return errors;
     332                 :            : }
     333                 :            : 
     334                 :            : namespace Xapian {
     335                 :            : 
     336                 :            : static size_t
     337                 :         10 : check_stub(const string& stub_path, int opts, std::ostream* out)
     338                 :            : {
     339                 :         10 :     size_t errors = 0;
     340                 :            :     read_stub_file(stub_path,
     341                 :          4 :                    [&errors, opts, out](const string& path) {
     342                 :          4 :                        errors += Database::check(path, opts, out);
     343                 :          3 :                    },
     344                 :          2 :                    [&errors, opts, out](const string& path) {
     345                 :            :                        // FIXME: Doesn't check the database type is glass.
     346                 :          2 :                        errors += Database::check(path, opts, out);
     347                 :          2 :                    },
     348                 :          0 :                    [&errors, opts, out](const string& path) {
     349                 :            :                        // FIXME: Doesn't check the database type is honey.
     350                 :          0 :                        errors += Database::check(path, opts, out);
     351                 :          0 :                    },
     352                 :          4 :                    [](const string&, const string&) {
     353                 :          4 :                        auto msg = "Remote database checking not implemented";
     354 [ +  - ][ +  - ]:          4 :                        throw Xapian::UnimplementedError(msg);
                 [ +  - ]
     355                 :            :                    },
     356                 :          0 :                    [](const string&, unsigned) {
     357                 :          0 :                        auto msg = "Remote database checking not implemented";
     358 [ #  # ][ #  # ]:          0 :                        throw Xapian::UnimplementedError(msg);
                 [ #  # ]
     359                 :            :                    },
     360                 :          1 :                    []() {
     361                 :          1 :                        auto msg = "InMemory database checking not implemented";
     362 [ +  - ][ +  - ]:          1 :                        throw Xapian::UnimplementedError(msg);
                 [ +  - ]
     363         [ +  + ]:         10 :                    });
     364                 :          4 :     return errors;
     365                 :            : }
     366                 :            : 
     367                 :            : size_t
     368                 :         29 : Database::check_(const string * path_ptr, int fd, int opts, std::ostream *out)
     369                 :            : {
     370         [ +  + ]:         29 :     if (!out) {
     371                 :            :         // If we have nowhere to write output, then disable all the options
     372                 :            :         // which only affect what we output.
     373                 :         17 :         opts &= Xapian::DBCHECK_FIX;
     374                 :            :     }
     375                 :            : 
     376         [ +  + ]:         29 :     if (path_ptr == NULL) {
     377         [ +  - ]:          1 :         return check_db_fd(fd, opts, out, BACKEND_UNKNOWN);
     378                 :            :     }
     379                 :            : 
     380                 :         28 :     const string & path = *path_ptr;
     381                 :            :     struct stat sb;
     382         [ +  + ]:         28 :     if (stat(path.c_str(), &sb) == 0) {
     383         [ +  + ]:         25 :         if (S_ISDIR(sb.st_mode)) {
     384         [ +  + ]:         10 :             return check_db_dir(path, opts, out);
     385                 :            :         }
     386                 :            : 
     387         [ +  - ]:         15 :         if (S_ISREG(sb.st_mode)) {
     388         [ +  - ]:         15 :             int backend = test_if_single_file_db(sb, path, &fd);
     389         [ +  + ]:         15 :             if (backend != BACKEND_UNKNOWN) {
     390         [ +  - ]:          4 :                 return check_db_fd(fd, opts, out, backend);
     391                 :            :             }
     392                 :            :             // Could be a single table or a stub database file.  Look at the
     393                 :            :             // extension to determine the type.
     394         [ -  + ]:         11 :             if (endswith(path, ".DB")) {
     395                 :          0 :                 backend = BACKEND_OLD;
     396         [ +  + ]:         11 :             } else if (endswith(path, "." GLASS_TABLE_EXTENSION)) {
     397                 :          1 :                 backend = BACKEND_GLASS;
     398                 :            :             } else {
     399         [ +  + ]:         10 :                 return check_stub(path, opts, out);
     400                 :            :             }
     401                 :            : 
     402         [ +  - ]:          1 :             return check_db_table(path, opts, out, backend);
     403                 :            :         }
     404                 :            : 
     405 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseOpeningError("Not a regular file or directory");
                 [ #  # ]
     406                 :            :     }
     407                 :            : 
     408                 :            :     // The filename passed doesn't exist - see if it's the basename of the
     409                 :            :     // table (perhaps with "." after it), so the user can do xapian-check on
     410                 :            :     // "foo/termlist" or "foo/termlist." (which you would get from filename
     411                 :            :     // completion with older backends).
     412         [ +  - ]:          3 :     string filename = path;
     413         [ +  + ]:          3 :     if (endswith(filename, '.')) {
     414         [ +  - ]:          1 :         filename.resize(filename.size() - 1);
     415                 :            :     }
     416                 :            : 
     417                 :          3 :     int backend = BACKEND_UNKNOWN;
     418 [ +  - ][ -  + ]:          3 :     if (stat((filename + ".DB").c_str(), &sb) == 0) {
     419                 :            :         // Could be chert, flint or brass - we check which below.
     420                 :          0 :         backend = BACKEND_OLD;
     421 [ +  - ][ +  + ]:          3 :     } else if (stat((filename + "." GLASS_TABLE_EXTENSION).c_str(), &sb) == 0) {
     422                 :          2 :         backend = BACKEND_GLASS;
     423                 :            :     } else {
     424                 :          1 :         auto msg = "Couldn't find Xapian database or table to check";
     425 [ +  - ][ +  - ]:          1 :         throw Xapian::DatabaseOpeningError(msg, ENOENT);
     426                 :            :     }
     427                 :            : 
     428         [ +  - ]:         22 :     return check_db_table(path, opts, out, backend);
     429                 :            : }
     430                 :            : 
     431                 :            : }

Generated by: LCOV version 1.11