LCOV - code coverage report
Current view: top level - backends/honey - honey_compact.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 4ba52dacf4fb Lines: 816 1408 58.0 %
Date: 2019-05-20 14:58:19 Functions: 38 80 47.5 %
Branches: 882 4343 20.3 %

           Branch data     Line data    Source code
       1                 :            : /** @file honey_compact.cc
       2                 :            :  * @brief Compact a honey database, or merge and compact several.
       3                 :            :  */
       4                 :            : /* Copyright (C) 2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2018 Olly Betts
       5                 :            :  *
       6                 :            :  * This program is free software; you can redistribute it and/or
       7                 :            :  * modify it under the terms of the GNU General Public License as
       8                 :            :  * published by the Free Software Foundation; either version 2 of the
       9                 :            :  * License, or (at your option) any later version.
      10                 :            :  *
      11                 :            :  * This program is distributed in the hope that it will be useful,
      12                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      13                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14                 :            :  * GNU General Public License for more details.
      15                 :            :  *
      16                 :            :  * You should have received a copy of the GNU General Public License
      17                 :            :  * along with this program; if not, write to the Free Software
      18                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
      19                 :            :  * USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : 
      24                 :            : #include "xapian/compactor.h"
      25                 :            : #include "xapian/constants.h"
      26                 :            : #include "xapian/error.h"
      27                 :            : #include "xapian/types.h"
      28                 :            : 
      29                 :            : #include <algorithm>
      30                 :            : #include <memory>
      31                 :            : #include <queue>
      32                 :            : #include <type_traits>
      33                 :            : 
      34                 :            : #include <cerrno>
      35                 :            : #include <cstdio>
      36                 :            : 
      37                 :            : #include "backends/flint_lock.h"
      38                 :            : #include "compression_stream.h"
      39                 :            : #include "honey_cursor.h"
      40                 :            : #include "honey_database.h"
      41                 :            : #include "honey_defs.h"
      42                 :            : #include "honey_postlist_encodings.h"
      43                 :            : #include "honey_table.h"
      44                 :            : #include "honey_values.h"
      45                 :            : #include "honey_version.h"
      46                 :            : #include "filetests.h"
      47                 :            : #include "internaltypes.h"
      48                 :            : #include "pack.h"
      49                 :            : #include "backends/valuestats.h"
      50                 :            : #include "wordaccess.h"
      51                 :            : 
      52                 :            : #include "../byte_length_strings.h"
      53                 :            : #include "../prefix_compressed_strings.h"
      54                 :            : 
      55                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      56                 :            : # include "../glass/glass_database.h"
      57                 :            : # include "../glass/glass_table.h"
      58                 :            : # include "../glass/glass_values.h"
      59                 :            : #endif
      60                 :            : 
      61                 :            : using namespace std;
      62                 :            : using Honey::encode_valuestats;
      63                 :            : 
      64                 :            : [[noreturn]]
      65                 :            : static void
      66                 :          0 : throw_database_corrupt(const char* item, const char* pos)
      67                 :            : {
      68         [ #  # ]:          0 :     string message;
      69         [ #  # ]:          0 :     if (pos != NULL) {
      70         [ #  # ]:          0 :         message = "Value overflow unpacking termlist: ";
      71                 :            :     } else {
      72         [ #  # ]:          0 :         message = "Out of data unpacking termlist: ";
      73                 :            :     }
      74         [ #  # ]:          0 :     message += item;
      75 [ #  # ][ #  # ]:          0 :     throw Xapian::DatabaseCorruptError(message);
      76                 :            : }
      77                 :            : 
      78                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
      79                 :            : namespace GlassCompact {
      80                 :            : 
      81                 :            : static inline bool
      82                 :     161231 : is_user_metadata_key(const string & key)
      83                 :            : {
      84 [ +  + ][ +  + ]:     161231 :     return key.size() > 1 && key[0] == '\0' && key[1] == '\xc0';
                 [ -  + ]
      85                 :            : }
      86                 :            : 
      87                 :            : static inline bool
      88                 :     161231 : is_valuestats_key(const string & key)
      89                 :            : {
      90 [ +  + ][ +  + ]:     161231 :     return key.size() > 1 && key[0] == '\0' && key[1] == '\xd0';
                 [ +  + ]
      91                 :            : }
      92                 :            : 
      93                 :            : static inline bool
      94                 :     157286 : is_valuechunk_key(const string & key)
      95                 :            : {
      96 [ +  + ][ +  + ]:     157286 :     return key.size() > 1 && key[0] == '\0' && key[1] == '\xd8';
                 [ +  + ]
      97                 :            : }
      98                 :            : 
      99                 :            : static inline bool
     100                 :     148929 : is_doclenchunk_key(const string & key)
     101                 :            : {
     102 [ +  + ][ +  + ]:     148929 :     return key.size() > 1 && key[0] == '\0' && key[1] == '\xe0';
                 [ +  - ]
     103                 :            : }
     104                 :            : 
     105                 :            : }
     106                 :            : 
     107                 :            : static inline bool
     108                 :      29368 : termlist_key_is_values_used(const string& key)
     109                 :            : {
     110                 :      29368 :     const char* p = key.data();
     111                 :      29368 :     const char* e = p + key.size();
     112                 :            :     Xapian::docid did;
     113         [ +  - ]:      29368 :     if (unpack_uint_preserving_sort(&p, e, &did)) {
     114         [ +  + ]:      29368 :         if (p == e)
     115                 :      14684 :             return false;
     116 [ +  - ][ +  - ]:      14684 :         if (e - p == 1 && *p == '\0')
     117                 :      14684 :             return true;
     118                 :            :     }
     119 [ #  # ][ #  # ]:      29368 :     throw Xapian::DatabaseCorruptError("termlist key format");
                 [ #  # ]
     120                 :            : }
     121                 :            : #endif
     122                 :            : 
     123                 :            : // Put all the helpers in a namespace to avoid symbols colliding with those of
     124                 :            : // the same name in other flint-derived backends.
     125                 :            : namespace HoneyCompact {
     126                 :            : 
     127                 :            : /// Return a Honey::KEY_* constant, or a different value for an invalid key.
     128                 :            : static inline int
     129                 :      15712 : key_type(const string& key)
     130                 :            : {
     131         [ +  + ]:      15712 :     if (key[0] != '\0')
     132                 :       1532 :         return Honey::KEY_POSTING_CHUNK;
     133                 :            : 
     134         [ -  + ]:      14180 :     if (key.size() <= 1)
     135                 :          0 :         return -1;
     136                 :            : 
     137                 :      14180 :     unsigned char ch = key[1];
     138 [ +  - ][ +  + ]:      14180 :     if (ch >= Honey::KEY_VALUE_STATS && ch <= Honey::KEY_VALUE_STATS_HI)
     139                 :       4573 :         return Honey::KEY_VALUE_STATS;
     140 [ +  - ][ +  + ]:       9607 :     if (ch >= Honey::KEY_VALUE_CHUNK && ch <= Honey::KEY_VALUE_CHUNK_HI)
     141                 :       9001 :         return Honey::KEY_VALUE_CHUNK;
     142 [ +  - ][ +  - ]:        606 :     if (ch >= Honey::KEY_DOCLEN_CHUNK && ch <= Honey::KEY_DOCLEN_CHUNK_HI)
     143                 :        606 :         return Honey::KEY_DOCLEN_CHUNK;
     144                 :            : 
     145                 :            :     // Handle Honey::KEY_USER_METADATA, Honey::KEY_POSTING_CHUNK, and currently
     146                 :            :     // unused values.
     147                 :          0 :     return ch;
     148                 :            : }
     149                 :            : 
     150                 :            : template<typename T> class PostlistCursor;
     151                 :            : 
     152                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
     153                 :            : // Convert glass to honey.
     154                 :            : template<>
     155                 :        578 : class PostlistCursor<const GlassTable&> : private GlassCursor {
     156                 :            :     Xapian::docid offset;
     157                 :            : 
     158                 :            :   public:
     159                 :            :     string key, tag;
     160                 :            :     Xapian::docid firstdid;
     161                 :            :     Xapian::docid chunk_lastdid;
     162                 :            :     Xapian::termcount tf, cf;
     163                 :            :     Xapian::termcount first_wdf;
     164                 :            :     Xapian::termcount wdf_max;
     165                 :            :     bool have_wdfs;
     166                 :            : 
     167                 :        289 :     PostlistCursor(const GlassTable *in, Xapian::docid offset_)
     168 [ +  - ][ +  - ]:        289 :         : GlassCursor(in), offset(offset_), firstdid(0)
     169                 :            :     {
     170         [ +  - ]:        289 :         rewind();
     171                 :        289 :     }
     172                 :            : 
     173                 :     161520 :     bool next() {
     174 [ +  - ][ +  + ]:     161520 :         if (!GlassCursor::next()) return false;
     175                 :            :         // We put all chunks into the non-initial chunk form here, then fix up
     176                 :            :         // the first chunk for each term in the merged database as we merge.
     177         [ +  - ]:     161231 :         read_tag();
     178         [ +  - ]:     161231 :         key = current_key;
     179         [ +  - ]:     161231 :         tag = current_tag;
     180                 :     161231 :         tf = cf = 0;
     181         [ -  + ]:     161231 :         if (GlassCompact::is_user_metadata_key(key)) {
     182         [ #  # ]:          0 :             key[1] = Honey::KEY_USER_METADATA;
     183                 :          0 :             return true;
     184                 :            :         }
     185         [ +  + ]:     161231 :         if (GlassCompact::is_valuestats_key(key)) {
     186                 :            :             // Adjust key.
     187                 :       3945 :             const char * p = key.data();
     188                 :       3945 :             const char * end = p + key.length();
     189                 :       3945 :             p += 2;
     190                 :            :             Xapian::valueno slot;
     191         [ -  + ]:       3945 :             if (!unpack_uint_last(&p, end, &slot))
     192 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("bad value stats key");
                 [ #  # ]
     193                 :            :             // FIXME: pack_uint_last() is not a good encoding for keys, as the
     194                 :            :             // encoded values do not in general sort the same way as the
     195                 :            :             // numeric values.  The first 256 values will be in order at least.
     196                 :            :             //
     197                 :            :             // We could just buffer up the stats for all the slots - it's
     198                 :            :             // unlikely there are going to be very many.  Another option is
     199                 :            :             // to use multiple cursors or seek a single cursor around.
     200                 :            :             AssertRel(slot, <=, 256);
     201 [ +  - ][ +  - ]:       3945 :             key = Honey::make_valuestats_key(slot);
     202                 :       3945 :             return true;
     203                 :            :         }
     204         [ +  + ]:     157286 :         if (GlassCompact::is_valuechunk_key(key)) {
     205                 :       8357 :             const char * p = key.data();
     206                 :       8357 :             const char * end = p + key.length();
     207                 :       8357 :             p += 2;
     208                 :            :             Xapian::valueno slot;
     209         [ -  + ]:       8357 :             if (!unpack_uint(&p, end, &slot))
     210 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("bad value key");
                 [ #  # ]
     211                 :            :             // FIXME: pack_uint() is not a good encoding for keys, as the
     212                 :            :             // encoded values do not in general sort the same way as the
     213                 :            :             // numeric values.  The first 128 values will be in order at least.
     214                 :            :             //
     215                 :            :             // Buffering up this data for all slots is potentially prohibitively
     216                 :            :             // costly.  We probably need to use multiple cursors or seek a single
     217                 :            :             // cursor around.
     218                 :            :             AssertRel(slot, <=, 128);
     219                 :            :             Xapian::docid first_did;
     220         [ -  + ]:       8357 :             if (!unpack_uint_preserving_sort(&p, end, &first_did))
     221 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("bad value key");
                 [ #  # ]
     222                 :       8357 :             first_did += offset;
     223                 :            : 
     224         [ +  - ]:       8357 :             Glass::ValueChunkReader reader(tag.data(), tag.size(), first_did);
     225                 :       8357 :             Xapian::docid last_did = first_did;
     226 [ +  - ][ +  + ]:     185431 :             while (reader.next(), !reader.at_end()) {
     227                 :     177074 :                 last_did = reader.get_docid();
     228                 :            :             }
     229                 :            : 
     230 [ +  - ][ +  - ]:       8357 :             key = Honey::make_valuechunk_key(slot, last_did);
     231                 :            : 
     232                 :            :             // Add the docid delta across the chunk to the start of the tag.
     233         [ +  - ]:      16714 :             string newtag;
     234         [ +  - ]:       8357 :             pack_uint(newtag, last_did - first_did);
     235         [ +  - ]:       8357 :             tag.insert(0, newtag);
     236                 :            : 
     237                 :       8357 :             return true;
     238                 :            :         }
     239                 :            : 
     240         [ +  + ]:     148929 :         if (GlassCompact::is_doclenchunk_key(key)) {
     241                 :        285 :             const char * d = key.data();
     242                 :        285 :             const char * e = d + key.size();
     243                 :        285 :             d += 2;
     244                 :            : 
     245         [ +  - ]:        285 :             if (d == e) {
     246                 :            :                 // This is an initial chunk, so adjust tag header.
     247                 :        285 :                 d = tag.data();
     248                 :        285 :                 e = d + tag.size();
     249 [ +  - ][ -  + ]:        855 :                 if (!unpack_uint(&d, e, &tf) ||
     250 [ +  - ][ -  + ]:        570 :                     !unpack_uint(&d, e, &cf) ||
     251                 :        285 :                     !unpack_uint(&d, e, &firstdid)) {
     252 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     253                 :            :                 }
     254                 :        285 :                 ++firstdid;
     255         [ +  - ]:        285 :                 tag.erase(0, d - tag.data());
     256                 :            :             } else {
     257                 :            :                 // Not an initial chunk, just unpack firstdid.
     258 [ #  # ][ #  # ]:          0 :                 if (!unpack_uint_preserving_sort(&d, e, &firstdid) || d != e)
                 [ #  # ]
     259 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     260                 :            :             }
     261                 :        285 :             firstdid += offset;
     262                 :            : 
     263                 :            :             // Set key to placeholder value which just indicates that this is a
     264                 :            :             // doclen chunk.
     265                 :            :             static const char doclen_key_prefix[2] = {
     266                 :            :                 0, char(Honey::KEY_DOCLEN_CHUNK)
     267                 :            :             };
     268         [ +  - ]:        285 :             key.assign(doclen_key_prefix, 2);
     269                 :            : 
     270                 :        285 :             d = tag.data();
     271                 :        285 :             e = d + tag.size();
     272                 :            : 
     273                 :            :             // Convert doclen chunk to honey format.
     274         [ +  - ]:        285 :             string newtag;
     275                 :            : 
     276                 :            :             // Skip the "last chunk" flag and increase_to_last.
     277         [ -  + ]:        285 :             if (d == e)
     278                 :            :                 throw Xapian::DatabaseCorruptError("No last chunk flag in "
     279 [ #  # ][ #  # ]:          0 :                                                    "glass docdata chunk");
                 [ #  # ]
     280                 :        285 :             ++d;
     281                 :            :             Xapian::docid increase_to_last;
     282         [ -  + ]:        285 :             if (!unpack_uint(&d, e, &increase_to_last))
     283                 :            :                 throw Xapian::DatabaseCorruptError("Decoding last docid delta "
     284 [ #  # ][ #  # ]:          0 :                                                    "in glass docdata chunk");
                 [ #  # ]
     285                 :            : 
     286                 :        285 :             Xapian::termcount doclen_max = 0;
     287                 :            :             while (true) {
     288                 :            :                 Xapian::termcount doclen;
     289         [ -  + ]:      14684 :                 if (!unpack_uint(&d, e, &doclen))
     290                 :            :                     throw Xapian::DatabaseCorruptError("Decoding doclen in "
     291 [ #  # ][ #  # ]:          0 :                                                        "glass docdata chunk");
                 [ #  # ]
     292         [ +  + ]:      14684 :                 if (doclen > doclen_max)
     293                 :        641 :                     doclen_max = doclen;
     294                 :            :                 unsigned char buf[4];
     295         [ +  - ]:      14684 :                 unaligned_write4(buf, doclen);
     296         [ +  - ]:      14684 :                 newtag.append(reinterpret_cast<char*>(buf), 4);
     297         [ +  + ]:      14684 :                 if (d == e)
     298                 :        285 :                     break;
     299                 :            :                 Xapian::docid gap_size;
     300         [ -  + ]:      14399 :                 if (!unpack_uint(&d, e, &gap_size))
     301                 :            :                     throw Xapian::DatabaseCorruptError("Decoding docid "
     302                 :            :                                                        "gap_size in glass "
     303 [ #  # ][ #  # ]:          0 :                                                        "docdata chunk");
                 [ #  # ]
     304                 :            :                 // FIXME: Split chunk if the gap_size is at all large.
     305         [ +  - ]:      14399 :                 newtag.append(4 * gap_size, '\xff');
     306                 :            :             }
     307                 :            : 
     308                 :            :             Assert(!startswith(newtag, "\xff\xff\xff\xff"));
     309                 :            :             Assert(!endswith(newtag, "\xff\xff\xff\xff"));
     310                 :            : 
     311                 :            :             AssertEq(newtag.size() % 4, 0);
     312                 :        285 :             chunk_lastdid = firstdid - 1 + newtag.size() / 4;
     313                 :            : 
     314                 :            :             // Only encode document lengths using a whole number of bytes for
     315                 :            :             // now.  We could allow arbitrary bit widths, but it complicates
     316                 :            :             // encoding and decoding so we should consider if the fairly small
     317                 :            :             // additional saving is worth it.
     318         [ -  + ]:        285 :             if (doclen_max >= 0xffff) {
     319         [ #  # ]:          0 :                 if (doclen_max >= 0xffffff) {
     320         [ #  # ]:          0 :                     newtag.insert(0, 1, char(32));
     321         [ #  # ]:          0 :                     swap(tag, newtag);
     322         [ #  # ]:          0 :                 } else if (doclen_max >= 0xffffffff) {
     323                 :            :                     // FIXME: Handle these.
     324                 :            :                     const char* m = "Document length values >= 0xffffffff not "
     325                 :          0 :                                     "currently handled";
     326 [ #  # ][ #  # ]:          0 :                     throw Xapian::FeatureUnavailableError(m);
                 [ #  # ]
     327                 :            :                 } else {
     328         [ #  # ]:          0 :                     tag.assign(1, char(24));
     329         [ #  # ]:          0 :                     for (size_t i = 1; i < newtag.size(); i += 4)
     330         [ #  # ]:          0 :                         tag.append(newtag, i, 3);
     331                 :            :                 }
     332                 :            :             } else {
     333         [ +  + ]:        285 :                 if (doclen_max >= 0xff) {
     334         [ +  - ]:         19 :                     tag.assign(1, char(16));
     335         [ +  + ]:      10773 :                     for (size_t i = 2; i < newtag.size(); i += 4)
     336         [ +  - ]:      10754 :                         tag.append(newtag, i, 2);
     337                 :            :                 } else {
     338         [ +  - ]:        266 :                     tag.assign(1, char(8));
     339         [ +  + ]:       4196 :                     for (size_t i = 3; i < newtag.size(); i += 4)
     340         [ +  - ]:       3930 :                         tag.append(newtag, i, 1);
     341                 :            :                 }
     342                 :            :             }
     343                 :            : 
     344                 :        285 :             return true;
     345                 :            :         }
     346                 :            : 
     347                 :            :         // Adjust key if this is *NOT* an initial chunk.
     348                 :            :         // key is: pack_string_preserving_sort(key, tname)
     349                 :            :         // plus optionally: pack_uint_preserving_sort(key, did)
     350                 :     148644 :         const char * d = key.data();
     351                 :     148644 :         const char * e = d + key.size();
     352         [ +  - ]:     148644 :         string tname;
     353 [ +  - ][ -  + ]:     148644 :         if (!unpack_string_preserving_sort(&d, e, tname))
     354 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     355                 :            : 
     356         [ +  - ]:     148644 :         if (d == e) {
     357                 :            :             // This is an initial chunk for a term, so adjust tag header.
     358                 :     148644 :             d = tag.data();
     359                 :     148644 :             e = d + tag.size();
     360 [ +  - ][ -  + ]:     445932 :             if (!unpack_uint(&d, e, &tf) ||
     361 [ +  - ][ -  + ]:     297288 :                 !unpack_uint(&d, e, &cf) ||
     362                 :     148644 :                 !unpack_uint(&d, e, &firstdid)) {
     363 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     364                 :            :             }
     365                 :     148644 :             ++firstdid;
     366                 :     148644 :             have_wdfs = (cf != 0);
     367         [ +  - ]:     148644 :             tag.erase(0, d - tag.data());
     368                 :     148644 :             wdf_max = 0;
     369                 :            :         } else {
     370                 :            :             // Not an initial chunk, so adjust key.
     371                 :          0 :             size_t tmp = d - key.data();
     372 [ #  # ][ #  # ]:          0 :             if (!unpack_uint_preserving_sort(&d, e, &firstdid) || d != e)
                 [ #  # ]
     373 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     374         [ #  # ]:          0 :             key.erase(tmp - 1);
     375                 :            :         }
     376                 :     148644 :         firstdid += offset;
     377                 :            : 
     378                 :     148644 :         d = tag.data();
     379                 :     148644 :         e = d + tag.size();
     380                 :            : 
     381                 :            :         // Convert posting chunk to honey format, but without any header.
     382         [ +  - ]:     297288 :         string newtag;
     383                 :            : 
     384                 :            :         // Skip the "last chunk" flag; decode increase_to_last.
     385         [ -  + ]:     148644 :         if (d == e)
     386                 :            :             throw Xapian::DatabaseCorruptError("No last chunk flag in glass "
     387 [ #  # ][ #  # ]:          0 :                                                "posting chunk");
                 [ #  # ]
     388                 :     148644 :         ++d;
     389                 :            :         Xapian::docid increase_to_last;
     390         [ -  + ]:     148644 :         if (!unpack_uint(&d, e, &increase_to_last))
     391                 :            :             throw Xapian::DatabaseCorruptError("Decoding last docid delta in "
     392 [ #  # ][ #  # ]:          0 :                                                "glass posting chunk");
                 [ #  # ]
     393                 :     148644 :         chunk_lastdid = firstdid + increase_to_last;
     394         [ -  + ]:     148644 :         if (!unpack_uint(&d, e, &first_wdf))
     395                 :            :             throw Xapian::DatabaseCorruptError("Decoding first wdf in glass "
     396 [ #  # ][ #  # ]:          0 :                                                "posting chunk");
                 [ #  # ]
     397         [ -  + ]:     148644 :         if ((first_wdf != 0) != have_wdfs) {
     398                 :            :             // FIXME: Also need to adjust document length, total document length.
     399                 :            :             // Convert wdf=0 to 1 when the term has non-zero wdf elsewhere.
     400                 :            :             // first_wdf = 1;
     401                 :            :             // ++cf;
     402                 :            :             throw Xapian::DatabaseError("Honey does not support a term having "
     403 [ #  # ][ #  # ]:          0 :                                         "both zero and non-zero wdf");
                 [ #  # ]
     404                 :            :         }
     405                 :     148644 :         wdf_max = max(wdf_max, first_wdf);
     406                 :            : 
     407         [ +  + ]:     839852 :         while (d != e) {
     408                 :            :             Xapian::docid delta;
     409         [ -  + ]:     691208 :             if (!unpack_uint(&d, e, &delta))
     410                 :            :                 throw Xapian::DatabaseCorruptError("Decoding docid delta in "
     411 [ #  # ][ #  # ]:          0 :                                                    "glass posting chunk");
                 [ #  # ]
     412         [ +  - ]:     691208 :             pack_uint(newtag, delta);
     413                 :            :             Xapian::termcount wdf;
     414         [ -  + ]:     691208 :             if (!unpack_uint(&d, e, &wdf))
     415                 :            :                 throw Xapian::DatabaseCorruptError("Decoding wdf in glass "
     416 [ #  # ][ #  # ]:          0 :                                                    "posting chunk");
                 [ #  # ]
     417         [ -  + ]:     691208 :             if ((wdf != 0) != have_wdfs) {
     418                 :            :                 // FIXME: Also need to adjust document length, total document length.
     419                 :            :                 // Convert wdf=0 to 1 when the term has non-zero wdf elsewhere.
     420                 :            :                 // wdf = 1;
     421                 :            :                 // ++cf;
     422                 :            :                 throw Xapian::DatabaseError("Honey does not support a term "
     423                 :            :                                             "having both zero and non-zero "
     424 [ #  # ][ #  # ]:          0 :                                             "wdf");
                 [ #  # ]
     425                 :            :             }
     426         [ +  - ]:     691208 :             if (have_wdfs) {
     427         [ +  - ]:     691208 :                 pack_uint(newtag, wdf);
     428                 :     691208 :                 wdf_max = max(wdf_max, wdf);
     429                 :            :             }
     430                 :            :         }
     431                 :            : 
     432         [ +  - ]:     148644 :         swap(tag, newtag);
     433                 :            : 
     434                 :     161520 :         return true;
     435                 :            :     }
     436                 :            : };
     437                 :            : #endif
     438                 :            : 
     439                 :            : template<>
     440                 :         30 : class PostlistCursor<const HoneyTable&> : private HoneyCursor {
     441                 :            :     Xapian::docid offset;
     442                 :            : 
     443                 :            :   public:
     444                 :            :     string key, tag;
     445                 :            :     Xapian::docid firstdid;
     446                 :            :     Xapian::docid chunk_lastdid;
     447                 :            :     Xapian::termcount tf, cf;
     448                 :            :     Xapian::termcount first_wdf;
     449                 :            :     Xapian::termcount wdf_max;
     450                 :            :     bool have_wdfs;
     451                 :            : 
     452                 :         15 :     PostlistCursor(const HoneyTable *in, Xapian::docid offset_)
     453 [ +  - ][ +  - ]:         15 :         : HoneyCursor(in), offset(offset_), firstdid(0)
     454                 :            :     {
     455         [ +  - ]:         15 :         rewind();
     456                 :         15 :     }
     457                 :            : 
     458                 :       1319 :     bool next() {
     459 [ +  - ][ +  + ]:       1319 :         if (!HoneyCursor::next()) return false;
     460                 :            :         // We put all chunks into the non-initial chunk form here, then fix up
     461                 :            :         // the first chunk for each term in the merged database as we merge.
     462         [ +  - ]:       1304 :         read_tag();
     463         [ +  - ]:       1304 :         key = current_key;
     464         [ +  - ]:       1304 :         tag = current_tag;
     465                 :       1304 :         tf = 0;
     466   [ +  +  +  +  :       1304 :         switch (key_type(key)) {
                      - ]
     467                 :            :             case Honey::KEY_USER_METADATA:
     468                 :            :             case Honey::KEY_VALUE_STATS:
     469                 :        168 :                 return true;
     470                 :            :             case Honey::KEY_VALUE_CHUNK: {
     471                 :        176 :                 const char * p = key.data();
     472                 :        176 :                 const char * end = p + key.length();
     473                 :        176 :                 p += 2;
     474                 :            :                 Xapian::valueno slot;
     475         [ +  - ]:        176 :                 if (p[-1] != char(Honey::KEY_VALUE_CHUNK_HI)) {
     476                 :        176 :                     slot = p[-1] - Honey::KEY_VALUE_CHUNK;
     477                 :            :                 } else {
     478         [ #  # ]:          0 :                     if (!unpack_uint_preserving_sort(&p, end, &slot))
     479 [ #  # ][ #  # ]:          0 :                         throw Xapian::DatabaseCorruptError("bad value key");
                 [ #  # ]
     480                 :            :                 }
     481                 :            :                 Xapian::docid did;
     482         [ -  + ]:        176 :                 if (!unpack_uint_preserving_sort(&p, end, &did))
     483 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("bad value key");
                 [ #  # ]
     484 [ +  - ][ +  - ]:        176 :                 key = Honey::make_valuechunk_key(slot, did + offset);
     485                 :        176 :                 return true;
     486                 :            :             }
     487                 :            :             case Honey::KEY_DOCLEN_CHUNK: {
     488                 :         12 :                 Xapian::docid did = Honey::docid_from_key(key);
     489         [ -  + ]:         12 :                 if (did == 0)
     490 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("Bad doclen key");
                 [ #  # ]
     491                 :         12 :                 chunk_lastdid = did + offset;
     492         [ +  - ]:         12 :                 key.resize(2);
     493                 :         12 :                 return true;
     494                 :            :             }
     495                 :            :             case Honey::KEY_POSTING_CHUNK:
     496                 :        948 :                 break;
     497                 :            :             default:
     498                 :            :                 throw Xapian::DatabaseCorruptError("Bad postlist table key "
     499 [ #  # ][ #  # ]:          0 :                                                    "type");
                 [ #  # ]
     500                 :            :         }
     501                 :            : 
     502                 :            :         // Adjust key if this is *NOT* an initial chunk.
     503                 :            :         // key is: pack_string_preserving_sort(key, term)
     504                 :            :         // plus optionally: pack_uint_preserving_sort(key, did)
     505                 :        948 :         const char * d = key.data();
     506                 :        948 :         const char * e = d + key.size();
     507         [ +  - ]:        948 :         string term;
     508 [ +  - ][ -  + ]:        948 :         if (!unpack_string_preserving_sort(&d, e, term))
     509 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     510                 :            : 
     511         [ +  - ]:        948 :         if (d == e) {
     512                 :            :             // This is an initial chunk for a term, so remove tag header.
     513                 :        948 :             d = tag.data();
     514                 :        948 :             e = d + tag.size();
     515                 :            : 
     516                 :            :             Xapian::docid lastdid;
     517         [ -  + ]:        948 :             if (!decode_initial_chunk_header(&d, e, tf, cf,
     518                 :            :                                              firstdid, lastdid, chunk_lastdid,
     519         [ +  - ]:        948 :                                              first_wdf, wdf_max)) {
     520                 :            :                 throw Xapian::DatabaseCorruptError("Bad postlist initial "
     521 [ #  # ][ #  # ]:          0 :                                                    "chunk header");
                 [ #  # ]
     522                 :            :             }
     523                 :            :             // Ignore lastdid - we'll need to recalculate it (at least when
     524                 :            :             // merging, and for simplicity we always do).
     525                 :            :             (void)lastdid;
     526         [ +  - ]:        948 :             tag.erase(0, d - tag.data());
     527                 :            : 
     528 [ +  - ][ +  + ]:        948 :             have_wdfs = (cf != 0) && (cf != tf - 1 + first_wdf);
     529 [ +  + ][ +  + ]:        948 :             if (have_wdfs && tf > 2) {
     530                 :            :                 Xapian::termcount remaining_cf_for_flat_wdf =
     531                 :         40 :                     (tf - 1) * wdf_max;
     532                 :            :                 // Check this matches and that it isn't a false match due
     533                 :            :                 // to overflow of the multiplication above.
     534 [ -  + ][ #  # ]:         40 :                 if (cf - first_wdf == remaining_cf_for_flat_wdf &&
     535                 :          0 :                     usual(remaining_cf_for_flat_wdf / wdf_max == tf - 1)) {
     536                 :            :                     // The wdf is flat so we don't need to store it.
     537                 :        948 :                     have_wdfs = false;
     538                 :            :                 }
     539                 :            :             }
     540                 :            :         } else {
     541         [ #  # ]:          0 :             if (cf > 0) {
     542                 :            :                 // The cf we report should only be non-zero for initial chunks
     543                 :            :                 // (of which there may be several for the same term from
     544                 :            :                 // different instances of this class when merging databases)
     545                 :            :                 // and gets summed to give the total cf for the term, so we
     546                 :            :                 // need to zero it here, after potentially using it to
     547                 :            :                 // calculate first_wdf.
     548                 :            :                 //
     549                 :            :                 // If cf is zero for a term, first_wdf will be zero for all
     550                 :            :                 // chunks so doesn't need setting specially here.
     551         [ #  # ]:          0 :                 if (!have_wdfs)
     552                 :          0 :                     first_wdf = (cf - first_wdf) / (tf - 1);
     553                 :          0 :                 cf = 0;
     554                 :            :             }
     555                 :            : 
     556                 :            :             // Not an initial chunk, so adjust key and remove tag header.
     557                 :          0 :             size_t tmp = d - key.data();
     558 [ #  # ][ #  # ]:          0 :             if (!unpack_uint_preserving_sort(&d, e, &chunk_lastdid) ||
                 [ #  # ]
     559                 :          0 :                 d != e) {
     560 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError("Bad postlist key");
                 [ #  # ]
     561                 :            :             }
     562                 :            :             // -1 to remove the terminating zero byte too.
     563         [ #  # ]:          0 :             key.erase(tmp - 1);
     564                 :            : 
     565                 :          0 :             d = tag.data();
     566                 :          0 :             e = d + tag.size();
     567                 :            : 
     568         [ #  # ]:          0 :             if (have_wdfs) {
     569         [ #  # ]:          0 :                 if (!decode_delta_chunk_header(&d, e, chunk_lastdid, firstdid,
     570         [ #  # ]:          0 :                                                first_wdf)) {
     571                 :            :                     throw Xapian::DatabaseCorruptError("Bad postlist delta "
     572 [ #  # ][ #  # ]:          0 :                                                        "chunk header");
                 [ #  # ]
     573                 :            :                 }
     574                 :            :             } else {
     575         [ #  # ]:          0 :                 if (!decode_delta_chunk_header_no_wdf(&d, e, chunk_lastdid,
     576         [ #  # ]:          0 :                                                       firstdid)) {
     577                 :            :                     throw Xapian::DatabaseCorruptError("Bad postlist delta "
     578 [ #  # ][ #  # ]:          0 :                                                        "chunk header");
                 [ #  # ]
     579                 :            :                 }
     580                 :            :             }
     581         [ #  # ]:          0 :             tag.erase(0, d - tag.data());
     582                 :            :         }
     583                 :        948 :         firstdid += offset;
     584                 :        948 :         chunk_lastdid += offset;
     585                 :       1319 :         return true;
     586                 :            :     }
     587                 :            : };
     588                 :            : 
     589                 :            : template<>
     590                 :          0 : class PostlistCursor<HoneyTable&> : public PostlistCursor<const HoneyTable&> {
     591                 :            :   public:
     592                 :          0 :     PostlistCursor(HoneyTable *in, Xapian::docid offset_)
     593                 :          0 :         : PostlistCursor<const HoneyTable&>(in, offset_) {}
     594                 :            : };
     595                 :            : 
     596                 :            : template<typename T>
     597                 :            : class PostlistCursorGt {
     598                 :            :   public:
     599                 :            :     /** Return true if and only if a's key is strictly greater than b's key.
     600                 :            :      */
     601                 :       1037 :     bool operator()(const T* a, const T* b) const {
     602 [ #  # ][ +  + ]:       1037 :         if (a->key > b->key) return true;
                 [ #  # ]
     603 [ #  # ][ +  + ]:        815 :         if (a->key != b->key) return false;
                 [ #  # ]
     604                 :        262 :         return (a->firstdid > b->firstdid);
     605                 :            :     }
     606                 :            : };
     607                 :            : 
     608                 :            : // U : vector<HoneyTable*>::const_iterator
     609                 :            : template<typename T, typename U> void
     610                 :        298 : merge_postlists(Xapian::Compactor * compactor,
     611                 :            :                 T * out, vector<Xapian::docid>::const_iterator offset,
     612                 :            :                 U b, U e)
     613                 :            : {
     614                 :            :     typedef decltype(**b) table_type; // E.g. HoneyTable
     615                 :            :     typedef PostlistCursor<table_type> cursor_type;
     616                 :            :     typedef PostlistCursorGt<cursor_type> gt_type;
     617 [ #  # ][ #  # ]:        298 :     priority_queue<cursor_type *, vector<cursor_type *>, gt_type> pq;
         [ #  # ][ +  - ]
                 [ +  - ]
     618 [ #  # ][ #  # ]:        602 :     for ( ; b != e; ++b, ++offset) {
         [ #  # ][ +  + ]
                 [ +  + ]
     619                 :        304 :         auto in = *b;
     620         [ #  # ]:        304 :         auto cursor = new cursor_type(in, *offset);
           [ #  #  #  # ]
           [ #  #  #  # ]
           [ #  #  +  - ]
           [ +  -  +  - ]
                 [ +  - ]
     621 [ #  # ][ #  # ]:        304 :         if (cursor->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  + ]
         [ +  - ][ +  + ]
     622 [ #  # ][ #  # ]:        297 :             pq.push(cursor);
         [ #  # ][ +  - ]
                 [ +  - ]
     623                 :            :         } else {
     624                 :            :             // Skip empty tables.
     625 [ #  # ][ #  # ]:          7 :             delete cursor;
         [ #  # ][ +  - ]
                 [ +  - ]
     626                 :            :         }
     627                 :            :     }
     628                 :            : 
     629 [ #  # ][ #  # ]:        596 :     string last_key;
         [ #  # ][ +  - ]
                 [ +  - ]
     630                 :            :     {
     631                 :            :         // Merge user metadata.
     632                 :        298 :         vector<string> tags;
     633 [ #  # ][ #  # ]:        298 :         while (!pq.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     634                 :        292 :             cursor_type * cur = pq.top();
     635                 :        292 :             const string& key = cur->key;
     636 [ #  # ][ #  # ]:        292 :             if (key_type(key) != Honey::KEY_USER_METADATA) break;
         [ #  # ][ +  - ]
                 [ +  - ]
     637                 :            : 
     638 [ #  # ][ #  # ]:          0 :             if (key != last_key) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     639 [ #  # ][ #  # ]:          0 :                 if (!tags.empty()) {
         [ #  # ][ #  # ]
                 [ #  # ]
     640 [ #  # ][ #  # ]:          0 :                     if (tags.size() > 1 && compactor) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     641                 :            :                         Assert(!last_key.empty());
     642                 :            :                         // FIXME: It would be better to merge all duplicates
     643                 :            :                         // for a key in one call, but currently we don't in
     644                 :            :                         // multipass mode.
     645                 :            :                         const string & resolved_tag =
     646                 :            :                             compactor->resolve_duplicate_metadata(last_key,
     647                 :            :                                                                   tags.size(),
     648 [ #  # ][ #  # ]:          0 :                                                                   &tags[0]);
         [ #  # ][ #  # ]
                 [ #  # ]
     649 [ #  # ][ #  # ]:          0 :                         if (!resolved_tag.empty())
         [ #  # ][ #  # ]
                 [ #  # ]
     650 [ #  # ][ #  # ]:          0 :                             out->add(last_key, resolved_tag);
         [ #  # ][ #  # ]
                 [ #  # ]
     651                 :            :                     } else {
     652                 :            :                         Assert(!last_key.empty());
     653 [ #  # ][ #  # ]:          0 :                         out->add(last_key, tags[0]);
         [ #  # ][ #  # ]
                 [ #  # ]
     654                 :            :                     }
     655 [ #  # ][ #  # ]:          0 :                     tags.resize(0);
         [ #  # ][ #  # ]
                 [ #  # ]
     656                 :            :                 }
     657 [ #  # ][ #  # ]:          0 :                 last_key = key;
         [ #  # ][ #  # ]
                 [ #  # ]
     658                 :            :             }
     659 [ #  # ][ #  # ]:          0 :             tags.push_back(cur->tag);
         [ #  # ][ #  # ]
                 [ #  # ]
     660                 :            : 
     661 [ #  # ][ #  # ]:          0 :             pq.pop();
         [ #  # ][ #  # ]
                 [ #  # ]
     662 [ #  # ][ #  # ]:          0 :             if (cur->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     663 [ #  # ][ #  # ]:          0 :                 pq.push(cur);
         [ #  # ][ #  # ]
                 [ #  # ]
     664                 :            :             } else {
     665 [ #  # ][ #  # ]:          0 :                 delete cur;
         [ #  # ][ #  # ]
                 [ #  # ]
     666                 :            :             }
     667                 :            :         }
     668 [ #  # ][ #  # ]:        298 :         if (!tags.empty()) {
         [ #  # ][ -  + ]
                 [ -  + ]
     669 [ #  # ][ #  # ]:          0 :             if (tags.size() > 1 && compactor) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     670                 :            :                 Assert(!last_key.empty());
     671                 :            :                 const string & resolved_tag =
     672                 :            :                     compactor->resolve_duplicate_metadata(last_key,
     673                 :            :                                                           tags.size(),
     674 [ #  # ][ #  # ]:          0 :                                                           &tags[0]);
         [ #  # ][ #  # ]
                 [ #  # ]
     675 [ #  # ][ #  # ]:          0 :                 if (!resolved_tag.empty())
         [ #  # ][ #  # ]
                 [ #  # ]
     676 [ #  # ][ #  # ]:          0 :                     out->add(last_key, resolved_tag);
         [ #  # ][ #  # ]
                 [ #  # ]
     677                 :            :             } else {
     678                 :            :                 Assert(!last_key.empty());
     679 [ #  # ][ #  # ]:          0 :                 out->add(last_key, tags[0]);
         [ #  # ][ #  # ]
                 [ #  # ]
     680                 :            :             }
     681                 :        298 :         }
     682                 :            :     }
     683                 :            : 
     684                 :            :     {
     685                 :            :         // Merge valuestats.
     686                 :        298 :         Xapian::doccount freq = 0;
     687 [ #  # ][ #  # ]:        596 :         string lbound, ubound;
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     688                 :            : 
     689 [ #  # ][ #  # ]:       4411 :         while (!pq.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     690                 :       4405 :             cursor_type * cur = pq.top();
     691                 :       4405 :             const string& key = cur->key;
     692 [ #  # ][ #  # ]:       4405 :             if (key_type(key) != Honey::KEY_VALUE_STATS) break;
         [ #  # ][ +  + ]
                 [ +  + ]
     693 [ #  # ][ #  # ]:       4113 :             if (key != last_key) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  + ]
         [ +  - ][ +  - ]
     694                 :            :                 // For the first valuestats key, last_key will be the previous
     695                 :            :                 // key we wrote, which we don't want to overwrite.  This is the
     696                 :            :                 // only time that freq will be 0, so check that.
     697 [ #  # ][ #  # ]:       4043 :                 if (freq) {
         [ #  # ][ +  + ]
                 [ +  + ]
     698 [ #  # ][ #  # ]:       3751 :                     out->add(last_key, encode_valuestats(freq, lbound, ubound));
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     699                 :       3751 :                     freq = 0;
     700                 :            :                 }
     701 [ #  # ][ #  # ]:       4043 :                 last_key = key;
         [ #  # ][ +  - ]
                 [ +  - ]
     702                 :            :             }
     703                 :            : 
     704                 :       4113 :             const string & tag = cur->tag;
     705                 :            : 
     706                 :       4113 :             const char * pos = tag.data();
     707                 :       4113 :             const char * end = pos + tag.size();
     708                 :            : 
     709                 :            :             Xapian::doccount f;
     710 [ #  # ][ #  # ]:       8226 :             string l, u;
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     711 [ #  # ][ #  # ]:       4113 :             if (!unpack_uint(&pos, end, &f)) {
         [ #  # ][ -  + ]
                 [ -  + ]
     712 [ #  # ][ #  # ]:          0 :                 if (*pos == 0)
         [ #  # ][ #  # ]
                 [ #  # ]
     713 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("Incomplete stats item "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     714                 :          0 :                                                        "in value table");
     715 [ #  # ][ #  # ]:          0 :                 throw Xapian::RangeError("Frequency statistic in value table "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     716                 :          0 :                                          "is too large");
     717                 :            :             }
     718 [ #  # ][ #  # ]:       4113 :             if (!unpack_string(&pos, end, l)) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ -  + ]
         [ +  - ][ -  + ]
     719 [ #  # ][ #  # ]:          0 :                 if (*pos == 0)
         [ #  # ][ #  # ]
                 [ #  # ]
     720 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("Incomplete stats item "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     721                 :          0 :                                                        "in value table");
     722 [ #  # ][ #  # ]:          0 :                 throw Xapian::RangeError("Lower bound in value table is too "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     723                 :          0 :                                          "large");
     724                 :            :             }
     725                 :       4113 :             size_t len = end - pos;
     726 [ #  # ][ #  # ]:       4113 :             if (len == 0) {
         [ #  # ][ +  + ]
                 [ +  + ]
     727 [ #  # ][ #  # ]:        408 :                 u = l;
         [ #  # ][ +  - ]
                 [ +  - ]
     728                 :            :             } else {
     729 [ #  # ][ #  # ]:       3705 :                 u.assign(pos, len);
         [ #  # ][ +  - ]
                 [ +  - ]
     730                 :            :             }
     731 [ #  # ][ #  # ]:       4113 :             if (freq == 0) {
         [ #  # ][ +  + ]
                 [ +  - ]
     732                 :       4043 :                 freq = f;
     733 [ #  # ][ #  # ]:       4043 :                 lbound = l;
         [ #  # ][ +  - ]
                 [ +  - ]
     734 [ #  # ][ #  # ]:       4043 :                 ubound = u;
         [ #  # ][ +  - ]
                 [ +  - ]
     735                 :            :             } else {
     736                 :         70 :                 freq += f;
     737 [ #  # ][ #  # ]:         70 :                 if (l < lbound) lbound = l;
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ +  - ]
         [ +  + ][ +  - ]
         [ #  # ][ #  # ]
                 [ #  # ]
     738 [ #  # ][ #  # ]:         70 :                 if (u > ubound) ubound = u;
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ +  - ]
         [ +  + ][ +  - ]
         [ #  # ][ #  # ]
                 [ #  # ]
     739                 :            :             }
     740                 :            : 
     741 [ #  # ][ #  # ]:       4113 :             pq.pop();
         [ #  # ][ +  - ]
                 [ +  - ]
     742 [ #  # ][ #  # ]:       4113 :             if (cur->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     743 [ #  # ][ #  # ]:       4113 :                 pq.push(cur);
         [ #  # ][ +  - ]
                 [ +  - ]
     744                 :            :             } else {
     745 [ #  # ][ #  # ]:          0 :                 delete cur;
         [ #  # ][ #  # ]
                 [ #  # ]
     746                 :            :             }
     747                 :            :         }
     748                 :            : 
     749 [ #  # ][ #  # ]:        298 :         if (freq) {
         [ #  # ][ +  + ]
                 [ +  + ]
     750 [ #  # ][ #  # ]:        292 :             out->add(last_key, encode_valuestats(freq, lbound, ubound));
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     751                 :        298 :         }
     752                 :            :     }
     753                 :            : 
     754                 :            :     // Merge valuestream chunks.
     755 [ #  # ][ #  # ]:       8831 :     while (!pq.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     756                 :       8825 :         cursor_type * cur = pq.top();
     757                 :       8825 :         const string & key = cur->key;
     758 [ #  # ][ #  # ]:       8825 :         if (key_type(key) != Honey::KEY_VALUE_CHUNK) break;
         [ #  # ][ +  + ]
                 [ +  + ]
     759 [ #  # ][ #  # ]:       8533 :         out->add(key, cur->tag);
         [ #  # ][ +  - ]
                 [ +  - ]
     760 [ #  # ][ #  # ]:       8533 :         pq.pop();
         [ #  # ][ +  - ]
                 [ +  - ]
     761 [ #  # ][ #  # ]:       8533 :         if (cur->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     762 [ #  # ][ #  # ]:       8533 :             pq.push(cur);
         [ #  # ][ +  - ]
                 [ +  - ]
     763                 :            :         } else {
     764 [ #  # ][ #  # ]:       8533 :             delete cur;
         [ #  # ][ #  # ]
                 [ #  # ]
     765                 :            :         }
     766                 :            :     }
     767                 :            : 
     768                 :            :     // Merge doclen chunks.
     769 [ #  # ][ #  # ]:        595 :     while (!pq.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     770                 :        589 :         cursor_type * cur = pq.top();
     771 [ #  # ][ #  # ]:        589 :         if (key_type(cur->key) != Honey::KEY_DOCLEN_CHUNK) break;
         [ #  # ][ +  + ]
                 [ +  + ]
     772                 :        297 :         string tag = std::move(cur->tag);
     773                 :        297 :         auto chunk_lastdid = cur->chunk_lastdid;
     774   [ #  #  #  #  :        297 :         pq.pop();
          #  #  +  -  +  
                      - ]
     775 [ #  # ][ #  # ]:        297 :         if (cur->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     776 [ #  # ][ #  # ]:        297 :             pq.push(cur);
         [ #  # ][ +  - ]
                 [ +  - ]
     777                 :            :         } else {
     778 [ #  # ][ #  # ]:          0 :             delete cur;
         [ #  # ][ #  # ]
                 [ #  # ]
     779                 :            :         }
     780 [ #  # ][ #  # ]:        297 :         while (!pq.empty()) {
         [ #  # ][ +  - ]
                 [ +  - ]
     781                 :        297 :             cur = pq.top();
     782   [ #  #  #  #  :        297 :             if (key_type(cur->key) != Honey::KEY_DOCLEN_CHUNK)
          #  #  +  +  +  
                      - ]
     783                 :        292 :                 break;
     784 [ #  # ][ #  # ]:          5 :             if (tag[0] != cur->tag[0]) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ +  - ]
         [ +  - ][ -  + ]
         [ #  # ][ #  # ]
                 [ #  # ]
     785                 :            :                 // Different width values in the two tags, so punt for now.
     786                 :            :                 // FIXME: We would ideally optimise the total size here.
     787                 :          0 :                 break;
     788                 :            :             }
     789 [ #  # ][ #  # ]:          5 :             size_t byte_width = tag[0] / 8;
         [ #  # ][ +  - ]
                 [ #  # ]
     790                 :          5 :             auto new_size = tag.size();
     791                 :          5 :             Xapian::docid gap_size = cur->firstdid - chunk_lastdid - 1;
     792                 :          5 :             new_size += gap_size * byte_width;
     793   [ #  #  #  #  :          5 :             if (new_size >= HONEY_DOCLEN_CHUNK_MAX) {
          #  #  +  -  #  
                      # ]
     794                 :            :                 // The gap spans beyond HONEY_DOCLEN_CHUNK_MAX.
     795                 :          5 :                 break;
     796                 :            :             }
     797                 :          0 :             new_size += cur->tag.size() - 1;
     798                 :          0 :             auto full_new_size = new_size;
     799   [ #  #  #  #  :          0 :             if (new_size > HONEY_DOCLEN_CHUNK_MAX) {
          #  #  #  #  #  
                      # ]
     800 [ #  # ][ #  # ]:          0 :                 if (byte_width > 1) {
         [ #  # ][ #  # ]
                 [ #  # ]
     801                 :            :                     // HONEY_DOCLEN_CHUNK_MAX should be one more than a
     802                 :            :                     // multiple of 12 so for widths 1,2,3,4 we can fix the
     803                 :            :                     // initial byte which indicates the width for the chunk
     804                 :            :                     // plus an exact number of entries.
     805                 :          0 :                     auto m = (new_size - HONEY_DOCLEN_CHUNK_MAX) % byte_width;
     806                 :            :                     (void)m;
     807                 :            :                     AssertEq(m, 0);
     808                 :            :                 }
     809                 :          0 :                 new_size = HONEY_DOCLEN_CHUNK_MAX;
     810                 :            :             }
     811 [ #  # ][ #  # ]:          0 :             tag.reserve(new_size);
         [ #  # ][ #  # ]
                 [ #  # ]
     812 [ #  # ][ #  # ]:          0 :             tag.append(byte_width * gap_size, '\xff');
         [ #  # ][ #  # ]
                 [ #  # ]
     813 [ #  # ][ #  # ]:          0 :             if (new_size != full_new_size) {
         [ #  # ][ #  # ]
                 [ #  # ]
     814                 :            :                 // Partial copy.
     815                 :          0 :                 auto copy_size = new_size - tag.size();
     816   [ #  #  #  #  :          0 :                 tag.append(cur->tag, 1, copy_size);
          #  #  #  #  #  
                      # ]
     817 [ #  # ][ #  # ]:          0 :                 cur->tag.erase(1, copy_size);
         [ #  # ][ #  # ]
                 [ #  # ]
     818                 :          0 :                 copy_size /= byte_width;
     819                 :          0 :                 cur->firstdid += copy_size;
     820                 :          0 :                 chunk_lastdid += gap_size;
     821                 :          0 :                 chunk_lastdid += copy_size;
     822                 :          0 :                 break;
     823                 :            :             }
     824                 :            : 
     825 [ #  # ][ #  # ]:          0 :             tag.append(cur->tag, 1, string::npos);
         [ #  # ][ #  # ]
                 [ #  # ]
     826                 :          0 :             chunk_lastdid = cur->chunk_lastdid;
     827                 :            : 
     828 [ #  # ][ #  # ]:          0 :             pq.pop();
         [ #  # ][ #  # ]
                 [ #  # ]
     829 [ #  # ][ #  # ]:          0 :             if (cur->next()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     830 [ #  # ][ #  # ]:          0 :                 pq.push(cur);
         [ #  # ][ #  # ]
                 [ #  # ]
     831                 :            :             } else {
     832 [ #  # ][ #  # ]:          0 :                 delete cur;
         [ #  # ][ #  # ]
                 [ #  # ]
     833                 :            :             }
     834                 :            :         }
     835 [ #  # ][ #  # ]:        297 :         out->add(Honey::make_doclenchunk_key(chunk_lastdid), tag);
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
     836                 :            :     }
     837                 :            : 
     838                 :        298 :     Xapian::termcount tf = 0, cf = 0; // Initialise to avoid warnings.
     839                 :            : 
     840                 :     897572 :     struct HoneyPostListChunk {
     841                 :            :         Xapian::docid first, last;
     842                 :            :         Xapian::termcount first_wdf;
     843                 :            :         Xapian::termcount wdf_max;
     844                 :            :         Xapian::doccount tf;
     845                 :            :         Xapian::termcount cf;
     846                 :            :         bool have_wdfs;
     847                 :            :         string data;
     848                 :            : 
     849                 :     149592 :         HoneyPostListChunk(Xapian::docid first_,
     850                 :            :                            Xapian::docid last_,
     851                 :            :                            Xapian::termcount first_wdf_,
     852                 :            :                            Xapian::termcount wdf_max_,
     853                 :            :                            Xapian::doccount tf_,
     854                 :            :                            Xapian::termcount cf_,
     855                 :            :                            bool have_wdfs_,
     856                 :            :                            string&& data_)
     857                 :            :             : first(first_),
     858                 :            :               last(last_),
     859                 :            :               first_wdf(first_wdf_),
     860                 :            :               wdf_max(wdf_max_),
     861                 :            :               tf(tf_),
     862                 :            :               cf(cf_),
     863                 :            :               have_wdfs(have_wdfs_),
     864                 :     149592 :               data(data_) {}
     865                 :            : 
     866                 :      46009 :         void append_postings_to(string& tag, bool want_wdfs) {
     867 [ #  # ][ #  # ]:      46009 :             if (want_wdfs && !have_wdfs) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  + ][ +  + ]
         [ +  + ][ -  + ]
     868                 :            :                 // Need to add wdfs which were implicit.
     869                 :         16 :                 auto wdf = (cf - first_wdf) / (tf - 1);
     870                 :         16 :                 const char* pos = data.data();
     871                 :         16 :                 const char* pos_end = pos + data.size();
     872 [ #  # ][ #  # ]:         36 :                 while (pos != pos_end) {
         [ #  # ][ +  + ]
                 [ #  # ]
     873                 :            :                     Xapian::docid delta;
     874 [ #  # ][ #  # ]:         20 :                     if (!unpack_uint(&pos, pos_end, &delta))
         [ #  # ][ -  + ]
                 [ #  # ]
     875                 :          0 :                         throw_database_corrupt("Decoding docid delta", pos);
     876 [ #  # ][ #  # ]:         20 :                     pack_uint(tag, delta);
         [ #  # ][ +  - ]
                 [ #  # ]
     877 [ #  # ][ #  # ]:         20 :                     pack_uint(tag, wdf);
         [ #  # ][ +  - ]
                 [ #  # ]
     878                 :         16 :                 }
     879 [ #  # ][ #  # ]:      45993 :             } else if (have_wdfs && !want_wdfs) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  + ][ -  + ]
         [ +  - ][ +  + ]
     880                 :            :                 // Need to remove wdfs which are now implicit.
     881                 :      26506 :                 const char* pos = data.data();
     882                 :      26506 :                 const char* pos_end = pos + data.size();
     883 [ #  # ][ #  # ]:     158546 :                 while (pos != pos_end) {
         [ #  # ][ #  # ]
                 [ +  + ]
     884                 :            :                     Xapian::docid delta;
     885 [ #  # ][ #  # ]:     132040 :                     if (!unpack_uint(&pos, pos_end, &delta))
         [ #  # ][ #  # ]
                 [ -  + ]
     886                 :          0 :                         throw_database_corrupt("Decoding docid delta", pos);
     887 [ #  # ][ #  # ]:     132040 :                     pack_uint(tag, delta);
         [ #  # ][ #  # ]
                 [ +  - ]
     888                 :            :                     Xapian::termcount wdf;
     889 [ #  # ][ #  # ]:     132040 :                     if (!unpack_uint(&pos, pos_end, &wdf))
         [ #  # ][ #  # ]
                 [ -  + ]
     890                 :     132040 :                         throw_database_corrupt("Decoding wdf", pos);
     891                 :            :                     (void)wdf;
     892                 :      26506 :                 }
     893                 :            :             } else {
     894                 :      19487 :                 tag += data;
     895                 :            :             }
     896                 :      46009 :         }
     897                 :            :     };
     898                 :        596 :     vector<HoneyPostListChunk> tags;
     899                 :            :     while (true) {
     900                 :     149890 :         cursor_type * cur = NULL;
     901 [ #  # ][ #  # ]:     149890 :         if (!pq.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     902                 :     149592 :             cur = pq.top();
     903 [ #  # ][ #  # ]:     149592 :             pq.pop();
         [ #  # ][ +  - ]
                 [ +  - ]
     904                 :            :         }
     905                 :     149890 :         if (cur) {
     906                 :            :             AssertEq(key_type(cur->key), Honey::KEY_POSTING_CHUNK);
     907                 :            :         }
     908 [ #  # ][ #  # ]:     149890 :         if (cur == NULL || cur->key != last_key) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  + ][ +  - ]
         [ +  + ][ +  + ]
         [ +  + ][ +  - ]
         [ +  - ][ +  - ]
     909 [ #  # ][ #  # ]:     149703 :             if (!tags.empty()) {
         [ #  # ][ +  + ]
                 [ +  + ]
     910                 :     149405 :                 Xapian::termcount first_wdf = tags[0].first_wdf;
     911                 :     149405 :                 Xapian::docid chunk_lastdid = tags[0].last;
     912                 :     149405 :                 Xapian::docid last_did = tags.back().last;
     913                 :            :                 Xapian::termcount wdf_max =
     914 [ #  # ][ #  # ]:     149405 :                     max_element(tags.begin(), tags.end(),
         [ #  # ][ +  - ]
                 [ +  - ]
     915                 :          0 :                                 [](const HoneyPostListChunk& x,
     916                 :        187 :                                    const HoneyPostListChunk& y) {
     917                 :            :                                     return x.wdf_max < y.wdf_max;
     918                 :     149592 :                                 })->wdf_max;
     919                 :            : 
     920                 :     149405 :                 bool have_wdfs = true;
     921   [ #  #  #  #  :     149405 :                 if (cf == 0) {
          #  #  -  +  -  
                      + ]
     922                 :            :                     // wdf must always be zero.
     923                 :          0 :                     have_wdfs = false;
     924 [ #  # ][ #  # ]:     149405 :                 } else if (tf <= 2) {
         [ #  # ][ +  + ]
                 [ +  + ]
     925                 :            :                     // We only need to store cf and first_wdf to know all wdfs.
     926                 :     103396 :                     have_wdfs = false;
     927 [ #  # ][ #  # ]:      46009 :                 } else if (cf == tf - 1 + first_wdf) {
         [ #  # ][ +  + ]
                 [ +  + ]
     928                 :            :                     // wdf must be 1 for second and subsequent entries.
     929                 :      26540 :                     have_wdfs = false;
     930                 :            :                 } else {
     931                 :            :                     Xapian::termcount remaining_cf_for_flat_wdf =
     932                 :      19469 :                         (tf - 1) * wdf_max;
     933                 :            :                     // Check this matches and that it isn't a false match due
     934                 :            :                     // to overflow of the multiplication above.
     935 [ #  # ][ #  # ]:      19469 :                     if (cf - first_wdf == remaining_cf_for_flat_wdf &&
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ -  + ][ #  # ]
         [ +  + ][ +  - ]
     936                 :         38 :                         usual(remaining_cf_for_flat_wdf / wdf_max == tf - 1)) {
     937                 :            :                         // The wdf is flat so we don't need to store it.
     938                 :         38 :                         have_wdfs = false;
     939                 :            :                     }
     940                 :            :                 }
     941                 :            : 
     942                 :     149405 :                 Xapian::docid splice_last = 0;
     943 [ #  # ][ #  # ]:     175911 :                 if (!have_wdfs && tags[0].have_wdfs && tf > 2 &&
           [ #  #  #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
           [ #  #  #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
           [ #  #  #  # ]
         [ #  # ][ +  + ]
                 [ +  + ]
           [ -  +  #  # ]
         [ -  + ][ +  + ]
                 [ +  - ]
           [ +  +  -  + ]
                 [ -  + ]
     944                 :      26506 :                     tags.size() > 1) {
     945                 :            :                     // Stripping wdfs will approximately halve the size so
     946                 :            :                     // double up the chunks.
     947                 :          0 :                     splice_last = chunk_lastdid;
     948                 :          0 :                     chunk_lastdid = tags[1].last;
     949                 :            :                 }
     950                 :            : 
     951 [ #  # ][ #  # ]:     149405 :                 string first_tag;
         [ #  # ][ +  - ]
                 [ +  - ]
     952 [ #  # ][ #  # ]:     149405 :                 encode_initial_chunk_header(tf, cf, tags[0].first, last_did,
         [ #  # ][ +  - ]
                 [ +  - ]
     953                 :            :                                             chunk_lastdid,
     954                 :     149405 :                                             first_wdf, wdf_max, first_tag);
     955                 :            : 
     956 [ #  # ][ #  # ]:     149405 :                 if (tf > 2) {
         [ #  # ][ +  + ]
                 [ +  + ]
     957 [ #  # ][ #  # ]:      46009 :                     tags[0].append_postings_to(first_tag, have_wdfs);
         [ #  # ][ +  - ]
                 [ +  - ]
     958 [ #  # ][ #  # ]:      46009 :                     if (!have_wdfs && splice_last) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  + ][ -  + ]
         [ +  + ][ -  + ]
     959 [ #  # ][ #  # ]:          0 :                         pack_uint(first_tag, tags[1].first - splice_last);
         [ #  # ][ #  # ]
                 [ #  # ]
     960 [ #  # ][ #  # ]:          0 :                         tags[1].append_postings_to(first_tag, have_wdfs);
         [ #  # ][ #  # ]
                 [ #  # ]
     961                 :            :                     }
     962                 :            :                 }
     963 [ #  # ][ #  # ]:     149405 :                 out->add(last_key, first_tag);
         [ #  # ][ +  - ]
                 [ +  - ]
     964                 :            : 
     965                 :            :                 // If tf == 2, the data could be split over two tags when
     966                 :            :                 // merging, but we only output an initial tag in this case.
     967 [ #  # ][ #  # ]:     149405 :                 if (tf > 2 && tags.size() > 1) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ +  + ]
         [ +  + ][ +  + ]
         [ +  + ][ -  + ]
                 [ -  + ]
     968 [ #  # ][ #  # ]:         89 :                     string term;
         [ #  # ][ +  - ]
                 [ #  # ]
     969                 :         89 :                     const char* p = last_key.data();
     970                 :         89 :                     const char* end = p + last_key.size();
     971 [ #  # ][ #  # ]:         89 :                     if (!unpack_string_preserving_sort(&p, end, term) ||
                 [ #  # ]
           [ #  #  #  # ]
         [ #  # ][ #  # ]
           [ #  #  #  # ]
         [ #  # ][ #  # ]
           [ #  #  +  - ]
         [ +  - ][ -  + ]
           [ -  +  #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     972                 :            :                         p != end) {
     973 [ #  # ][ #  # ]:          0 :                         throw Xapian::DatabaseCorruptError("Bad postlist "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     974                 :          0 :                                                            "chunk key");
     975                 :            :                     }
     976                 :            : 
     977                 :         89 :                     auto i = tags.begin();
     978   [ #  #  #  #  :         89 :                     if (splice_last) {
          #  #  -  +  #  
                      # ]
     979                 :          0 :                         splice_last = 0;
     980                 :          0 :                         ++i;
     981                 :            :                     }
     982 [ #  # ][ #  # ]:        178 :                     while (++i != tags.end()) {
         [ #  # ][ +  + ]
                 [ #  # ]
     983                 :         89 :                         last_did = i->last;
     984   [ #  #  #  #  :         89 :                         string tag;
          #  #  +  -  #  
                      # ]
     985 [ #  # ][ #  # ]:         89 :                         if (have_wdfs) {
         [ #  # ][ +  + ]
                 [ #  # ]
     986   [ #  #  #  #  :         43 :                             encode_delta_chunk_header(i->first,
          #  #  +  -  #  
                      # ]
     987                 :            :                                                       last_did,
     988                 :         43 :                                                       i->first_wdf,
     989                 :         86 :                                                       tag);
     990 [ #  # ][ #  # ]:         43 :                             tag += i->data;
         [ #  # ][ +  - ]
                 [ #  # ]
     991                 :            :                         } else {
     992 [ #  # ][ #  # ]:         46 :                             if (i->have_wdfs && i + 1 != tags.end()) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ -  + ]
         [ #  # ][ -  + ]
         [ -  + ][ -  + ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
     993                 :          0 :                                 splice_last = last_did;
     994                 :          0 :                                 last_did = (i + 1)->last;
     995                 :            :                             }
     996                 :            : 
     997 [ #  # ][ #  # ]:         46 :                             encode_delta_chunk_header_no_wdf(i->first,
         [ #  # ][ +  - ]
                 [ #  # ]
     998                 :            :                                                              last_did,
     999                 :         46 :                                                              tag);
    1000 [ #  # ][ #  # ]:         46 :                             tag += i->data;
         [ #  # ][ +  - ]
                 [ #  # ]
    1001 [ #  # ][ #  # ]:         46 :                             if (splice_last) {
         [ #  # ][ -  + ]
                 [ #  # ]
    1002                 :          0 :                                 ++i;
    1003   [ #  #  #  #  :          0 :                                 pack_uint(tag, i->first - splice_last);
          #  #  #  #  #  
                      # ]
    1004                 :          0 :                                 splice_last = 0;
    1005 [ #  # ][ #  # ]:          0 :                                 tag += i->data;
         [ #  # ][ #  # ]
                 [ #  # ]
    1006                 :            :                             }
    1007                 :            :                         }
    1008                 :            : 
    1009 [ #  # ][ #  # ]:         89 :                         out->add(pack_honey_postlist_key(term, last_did), tag);
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ #  # ][ #  # ]
    1010                 :         89 :                     }
    1011                 :     149405 :                 }
    1012                 :            :             }
    1013                 :     149703 :             tags.clear();
    1014 [ #  # ][ #  # ]:     149703 :             if (cur == NULL) break;
         [ #  # ][ +  + ]
                 [ +  + ]
    1015                 :     149405 :             tf = cf = 0;
    1016 [ #  # ][ #  # ]:     149405 :             last_key = cur->key;
         [ #  # ][ +  - ]
                 [ +  - ]
    1017                 :            :         }
    1018                 :            : 
    1019 [ #  # ][ #  # ]:     149592 :         if (tf && cur->tf && (cf == 0) != (cur->cf == 0)) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ +  + ]
         [ +  - ][ -  + ]
         [ -  + ][ #  # ]
                 [ #  # ]
    1020                 :            :             // FIXME: Also need to adjust document length, total document
    1021                 :            :             // length.
    1022                 :            :             // Convert wdf=0 to 1 when the term has non-zero wdf elsewhere.
    1023 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("Honey does not support a term having "
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
    1024                 :          0 :                                         "both zero and non-zero wdf");
    1025                 :            :         }
    1026                 :     149592 :         tf += cur->tf;
    1027                 :     149592 :         cf += cur->cf;
    1028                 :            : 
    1029 [ #  # ][ #  # ]:     149592 :         tags.push_back(HoneyPostListChunk(cur->firstdid,
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ +  - ][ +  - ]
         [ +  - ][ +  - ]
    1030                 :            :                                           cur->chunk_lastdid,
    1031                 :            :                                           cur->first_wdf,
    1032                 :            :                                           cur->wdf_max,
    1033                 :            :                                           cur->tf,
    1034                 :            :                                           cur->cf,
    1035                 :            :                                           cur->have_wdfs,
    1036                 :     149592 :                                           std::move(cur->tag)));
    1037         [ #  # ]:     149592 :         if (cur->next()) {
           [ #  #  #  # ]
           [ #  #  #  # ]
           [ #  #  +  - ]
           [ +  +  +  - ]
                 [ +  + ]
    1038 [ #  # ][ #  # ]:     149295 :             pq.push(cur);
         [ #  # ][ +  - ]
                 [ +  - ]
    1039                 :            :         } else {
    1040 [ #  # ][ #  # ]:     149592 :             delete cur;
         [ #  # ][ +  - ]
                 [ +  - ]
    1041                 :            :         }
    1042                 :        298 :     }
    1043                 :        298 : }
    1044                 :            : 
    1045                 :            : template<typename T> struct MergeCursor;
    1046                 :            : 
    1047                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    1048                 :            : template<>
    1049                 :          0 : struct MergeCursor<const GlassTable&> : public GlassCursor {
    1050                 :          0 :     explicit MergeCursor(const GlassTable *in) : GlassCursor(in) {
    1051         [ #  # ]:          0 :         rewind();
    1052                 :          0 :     }
    1053                 :            : };
    1054                 :            : #endif
    1055                 :            : 
    1056                 :            : template<>
    1057                 :          0 : struct MergeCursor<const HoneyTable&> : public HoneyCursor {
    1058                 :          0 :     explicit MergeCursor(const HoneyTable *in) : HoneyCursor(in) {
    1059         [ #  # ]:          0 :         rewind();
    1060                 :          0 :     }
    1061                 :            : };
    1062                 :            : 
    1063                 :            : template<typename T>
    1064                 :            : struct CursorGt {
    1065                 :            :     /// Return true if and only if a's key is strictly greater than b's key.
    1066                 :          0 :     bool operator()(const T* a, const T* b) const {
    1067 [ #  # ][ #  # ]:          0 :         if (b->after_end()) return false;
    1068 [ #  # ][ #  # ]:          0 :         if (a->after_end()) return true;
    1069                 :          0 :         return (a->current_key > b->current_key);
    1070                 :            :     }
    1071                 :            : };
    1072                 :            : 
    1073                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    1074                 :            : // Convert glass to honey.
    1075                 :            : static void
    1076                 :          0 : merge_spellings(HoneyTable* out,
    1077                 :            :                 vector<const GlassTable*>::const_iterator b,
    1078                 :            :                 vector<const GlassTable*>::const_iterator e)
    1079                 :            : {
    1080                 :            :     typedef MergeCursor<const GlassTable&> cursor_type;
    1081                 :            :     typedef CursorGt<cursor_type> gt_type;
    1082         [ #  # ]:          0 :     priority_queue<cursor_type *, vector<cursor_type *>, gt_type> pq;
    1083         [ #  # ]:          0 :     for ( ; b != e; ++b) {
    1084                 :          0 :         auto in = *b;
    1085 [ #  # ][ #  # ]:          0 :         auto cursor = new cursor_type(in);
    1086 [ #  # ][ #  # ]:          0 :         if (cursor->next()) {
    1087         [ #  # ]:          0 :             pq.push(cursor);
    1088                 :            :         } else {
    1089                 :            :             // Skip empty tables.
    1090         [ #  # ]:          0 :             delete cursor;
    1091                 :            :         }
    1092                 :            :     }
    1093                 :            : 
    1094         [ #  # ]:          0 :     while (!pq.empty()) {
    1095                 :          0 :         cursor_type * cur = pq.top();
    1096         [ #  # ]:          0 :         pq.pop();
    1097                 :            : 
    1098                 :            :         // Glass vs honey spelling key prefix map:
    1099                 :            :         //
    1100                 :            :         //  Type        Glass       Honey
    1101                 :            :         //
    1102                 :            :         //  bookend     Bbd         \x00bd
    1103                 :            :         //  head        Hhe         \x01he
    1104                 :            :         //  middle      Mddl        \x02ddl
    1105                 :            :         //  tail        Til         \x03il
    1106                 :            :         //  word        Wword       word
    1107                 :            :         //
    1108                 :            :         // In honey, if the word's first byte is <= '\x04', we add a prefix
    1109                 :            :         // of '\x04' (which is unlikely in real world use but ensures that
    1110                 :            :         // we can handle arbitrary strings).
    1111                 :            :         //
    1112                 :            :         // The prefixes for honey are chosen such that we save a byte for the
    1113                 :            :         // key of all real-world words, and so that more first bytes are used
    1114                 :            :         // so that a top-level array index is more effective, especially for
    1115                 :            :         // random-access lookup of word entries (which we do to check the
    1116                 :            :         // frequency of potential spelling candidates).
    1117                 :            :         //
    1118                 :            :         // The other prefixes are chosen such that we can do look up in key
    1119                 :            :         // order at search time, which is more efficient for a cursor which can
    1120                 :            :         // only efficiently move forwards.
    1121                 :            :         //
    1122                 :            :         // Note that the key ordering is the same for glass and honey, which
    1123                 :            :         // makes translating during compaction simpler.
    1124         [ #  # ]:          0 :         string key = cur->current_key;
    1125 [ #  # ][ #  #  :          0 :         switch (key[0]) {
             #  #  #  # ]
    1126                 :            :             case 'B':
    1127         [ #  # ]:          0 :                 key[0] = Honey::KEY_PREFIX_BOOKEND;
    1128                 :          0 :                 break;
    1129                 :            :             case 'H':
    1130         [ #  # ]:          0 :                 key[0] = Honey::KEY_PREFIX_HEAD;
    1131                 :          0 :                 break;
    1132                 :            :             case 'M':
    1133         [ #  # ]:          0 :                 key[0] = Honey::KEY_PREFIX_MIDDLE;
    1134                 :          0 :                 break;
    1135                 :            :             case 'T':
    1136         [ #  # ]:          0 :                 key[0] = Honey::KEY_PREFIX_TAIL;
    1137                 :          0 :                 break;
    1138                 :            :             case 'W':
    1139 [ #  # ][ #  # ]:          0 :                 if (static_cast<unsigned char>(key[1]) > Honey::KEY_PREFIX_WORD)
    1140         [ #  # ]:          0 :                     key.erase(0, 1);
    1141                 :            :                 else
    1142         [ #  # ]:          0 :                     key[0] = Honey::KEY_PREFIX_WORD;
    1143                 :          0 :                 break;
    1144                 :            :             default: {
    1145         [ #  # ]:          0 :                 string m = "Bad spelling key prefix: ";
    1146 [ #  # ][ #  # ]:          0 :                 m += static_cast<unsigned char>(key[0]);
    1147 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCorruptError(m);
    1148                 :            :             }
    1149                 :            :         }
    1150                 :            : 
    1151 [ #  # ][ #  # ]:          0 :         if (pq.empty() || pq.top()->current_key > key) {
         [ #  # ][ #  # ]
    1152                 :            :             // No merging to do for this key so just copy the tag value,
    1153                 :            :             // adjusting if necessary.  If we don't need to adjust it, just
    1154                 :            :             // copy the compressed value.
    1155                 :            :             bool compressed;
    1156         [ #  # ]:          0 :             switch (key[0]) {
              [ #  #  # ]
    1157                 :            :                 case Honey::KEY_PREFIX_HEAD: {
    1158                 :            :                     // Honey omits the first two bytes from the first entry
    1159                 :            :                     // since we know what those most be from the key
    1160                 :            :                     // (subsequent entries won't include those anyway, since
    1161                 :            :                     // they'll be part of the reused prefix).
    1162         [ #  # ]:          0 :                     compressed = cur->read_tag(false);
    1163         [ #  # ]:          0 :                     unsigned char len = cur->current_tag[0] ^ MAGIC_XOR_VALUE;
    1164         [ #  # ]:          0 :                     cur->current_tag[0] = (len - 2) ^ MAGIC_XOR_VALUE;
    1165                 :            :                     AssertEq(cur->current_tag[1], key[1]);
    1166                 :            :                     AssertEq(cur->current_tag[2], key[2]);
    1167         [ #  # ]:          0 :                     cur->current_tag.erase(1, 2);
    1168                 :          0 :                     break;
    1169                 :            :                 }
    1170                 :            :                 case Honey::KEY_PREFIX_TAIL:
    1171                 :            :                 case Honey::KEY_PREFIX_BOOKEND: {
    1172                 :            :                     // Need to repack because honey omits the last 2 or 1 bytes
    1173                 :            :                     // from each entry (2 for tail, 1 for bookend) since we
    1174                 :            :                     // know what those must be from the key.
    1175         [ #  # ]:          0 :                     compressed = cur->read_tag(false);
    1176         [ #  # ]:          0 :                     PrefixCompressedStringItor spell_in(cur->current_tag);
    1177         [ #  # ]:          0 :                     string new_tag;
    1178         [ #  # ]:          0 :                     PrefixCompressedStringWriter spell_out(new_tag, key);
    1179         [ #  # ]:          0 :                     while (!spell_in.at_end()) {
    1180         [ #  # ]:          0 :                         spell_out.append(*spell_in);
    1181         [ #  # ]:          0 :                         ++spell_in;
    1182                 :            :                     }
    1183         [ #  # ]:          0 :                     cur->current_tag = std::move(new_tag);
    1184                 :          0 :                     break;
    1185                 :            :                 }
    1186                 :            :                 default:
    1187         [ #  # ]:          0 :                     compressed = cur->read_tag(true);
    1188                 :          0 :                     break;
    1189                 :            :             }
    1190         [ #  # ]:          0 :             out->add(key, cur->current_tag, compressed);
    1191 [ #  # ][ #  # ]:          0 :             if (cur->next()) {
    1192         [ #  # ]:          0 :                 pq.push(cur);
    1193                 :            :             } else {
    1194         [ #  # ]:          0 :                 delete cur;
    1195                 :            :             }
    1196                 :          0 :             continue;
    1197                 :            :         }
    1198                 :            : 
    1199                 :            :         // Merge tag values with the same key:
    1200         [ #  # ]:          0 :         string tag;
    1201 [ #  # ][ #  # ]:          0 :         if (static_cast<unsigned char>(key[0]) < Honey::KEY_PREFIX_WORD) {
    1202                 :            :             // We just want the union of words, so copy over the first instance
    1203                 :            :             // and skip any identical ones.
    1204                 :            :             priority_queue<PrefixCompressedStringItor *,
    1205                 :            :                            vector<PrefixCompressedStringItor *>,
    1206         [ #  # ]:          0 :                            PrefixCompressedStringItorGt> pqtag;
    1207                 :            :             // Stick all the cursor_type pointers in a vector because their
    1208                 :            :             // current_tag members must remain valid while we're merging their
    1209                 :            :             // tags, but we need to call next() on them all afterwards.
    1210                 :          0 :             vector<cursor_type *> vec;
    1211         [ #  # ]:          0 :             vec.reserve(pq.size());
    1212                 :            : 
    1213                 :            :             while (true) {
    1214         [ #  # ]:          0 :                 cur->read_tag();
    1215 [ #  # ][ #  # ]:          0 :                 pqtag.push(new PrefixCompressedStringItor(cur->current_tag));
                 [ #  # ]
    1216         [ #  # ]:          0 :                 vec.push_back(cur);
    1217 [ #  # ][ #  # ]:          0 :                 if (pq.empty() || pq.top()->current_key != key) break;
         [ #  # ][ #  # ]
    1218                 :          0 :                 cur = pq.top();
    1219         [ #  # ]:          0 :                 pq.pop();
    1220                 :            :             }
    1221                 :            : 
    1222         [ #  # ]:          0 :             PrefixCompressedStringWriter wr(tag, key);
    1223         [ #  # ]:          0 :             string lastword;
    1224         [ #  # ]:          0 :             while (!pqtag.empty()) {
    1225                 :          0 :                 PrefixCompressedStringItor * it = pqtag.top();
    1226         [ #  # ]:          0 :                 pqtag.pop();
    1227         [ #  # ]:          0 :                 string word = **it;
    1228 [ #  # ][ #  # ]:          0 :                 if (word != lastword) {
    1229         [ #  # ]:          0 :                     lastword = word;
    1230         [ #  # ]:          0 :                     wr.append(lastword);
    1231                 :            :                 }
    1232         [ #  # ]:          0 :                 ++*it;
    1233         [ #  # ]:          0 :                 if (!it->at_end()) {
    1234         [ #  # ]:          0 :                     pqtag.push(it);
    1235                 :            :                 } else {
    1236         [ #  # ]:          0 :                     delete it;
    1237                 :            :                 }
    1238                 :          0 :             }
    1239                 :            : 
    1240         [ #  # ]:          0 :             for (auto i : vec) {
    1241                 :          0 :                 cur = i;
    1242 [ #  # ][ #  # ]:          0 :                 if (cur->next()) {
    1243         [ #  # ]:          0 :                     pq.push(cur);
    1244                 :            :                 } else {
    1245         [ #  # ]:          0 :                     delete cur;
    1246                 :            :                 }
    1247                 :          0 :             }
    1248                 :            :         } else {
    1249                 :            :             // We want to sum the frequencies from tags for the same key.
    1250                 :          0 :             Xapian::termcount tot_freq = 0;
    1251                 :            :             while (true) {
    1252         [ #  # ]:          0 :                 cur->read_tag();
    1253                 :            :                 Xapian::termcount freq;
    1254                 :          0 :                 const char * p = cur->current_tag.data();
    1255                 :          0 :                 const char * end = p + cur->current_tag.size();
    1256 [ #  # ][ #  # ]:          0 :                 if (!unpack_uint_last(&p, end, &freq) || freq == 0) {
                 [ #  # ]
    1257                 :          0 :                     throw_database_corrupt("Bad spelling word freq", p);
    1258                 :            :                 }
    1259                 :          0 :                 tot_freq += freq;
    1260 [ #  # ][ #  # ]:          0 :                 if (cur->next()) {
    1261         [ #  # ]:          0 :                     pq.push(cur);
    1262                 :            :                 } else {
    1263         [ #  # ]:          0 :                     delete cur;
    1264                 :            :                 }
    1265 [ #  # ][ #  # ]:          0 :                 if (pq.empty() || pq.top()->current_key != key) break;
         [ #  # ][ #  # ]
    1266                 :          0 :                 cur = pq.top();
    1267         [ #  # ]:          0 :                 pq.pop();
    1268                 :            :             }
    1269         [ #  # ]:          0 :             tag.resize(0);
    1270         [ #  # ]:          0 :             pack_uint_last(tag, tot_freq);
    1271                 :            :         }
    1272         [ #  # ]:          0 :         out->add(key, tag);
    1273         [ #  # ]:          0 :     }
    1274                 :          0 : }
    1275                 :            : #endif
    1276                 :            : 
    1277                 :            : static void
    1278                 :          0 : merge_spellings(HoneyTable* out,
    1279                 :            :                 vector<const HoneyTable*>::const_iterator b,
    1280                 :            :                 vector<const HoneyTable*>::const_iterator e)
    1281                 :            : {
    1282                 :            :     typedef MergeCursor<const HoneyTable&> cursor_type;
    1283                 :            :     typedef CursorGt<cursor_type> gt_type;
    1284         [ #  # ]:          0 :     priority_queue<cursor_type *, vector<cursor_type *>, gt_type> pq;
    1285         [ #  # ]:          0 :     for ( ; b != e; ++b) {
    1286                 :          0 :         auto in = *b;
    1287 [ #  # ][ #  # ]:          0 :         auto cursor = new cursor_type(in);
    1288 [ #  # ][ #  # ]:          0 :         if (cursor->next()) {
    1289         [ #  # ]:          0 :             pq.push(cursor);
    1290                 :            :         } else {
    1291                 :            :             // Skip empty tables.
    1292         [ #  # ]:          0 :             delete cursor;
    1293                 :            :         }
    1294                 :            :     }
    1295                 :            : 
    1296         [ #  # ]:          0 :     while (!pq.empty()) {
    1297                 :          0 :         cursor_type * cur = pq.top();
    1298         [ #  # ]:          0 :         pq.pop();
    1299                 :            : 
    1300         [ #  # ]:          0 :         string key = cur->current_key;
    1301 [ #  # ][ #  # ]:          0 :         if (pq.empty() || pq.top()->current_key > key) {
         [ #  # ][ #  # ]
    1302                 :            :             // No need to merge the tags, just copy the (possibly compressed)
    1303                 :            :             // tag value.
    1304         [ #  # ]:          0 :             bool compressed = cur->read_tag(true);
    1305         [ #  # ]:          0 :             out->add(key, cur->current_tag, compressed);
    1306 [ #  # ][ #  # ]:          0 :             if (cur->next()) {
    1307         [ #  # ]:          0 :                 pq.push(cur);
    1308                 :            :             } else {
    1309         [ #  # ]:          0 :                 delete cur;
    1310                 :            :             }
    1311                 :          0 :             continue;
    1312                 :            :         }
    1313                 :            : 
    1314                 :            :         // Merge tag values with the same key:
    1315 [ #  # ][ #  # ]:          0 :         string tag;
    1316 [ #  # ][ #  # ]:          0 :         if (static_cast<unsigned char>(key[0]) < Honey::KEY_PREFIX_WORD) {
    1317                 :            :             // We just want the union of words, so copy over the first instance
    1318                 :            :             // and skip any identical ones.
    1319                 :            :             priority_queue<PrefixCompressedStringItor *,
    1320                 :            :                            vector<PrefixCompressedStringItor *>,
    1321         [ #  # ]:          0 :                            PrefixCompressedStringItorGt> pqtag;
    1322                 :            :             // Stick all the cursor_type pointers in a vector because their
    1323                 :            :             // current_tag members must remain valid while we're merging their
    1324                 :            :             // tags, but we need to call next() on them all afterwards.
    1325                 :          0 :             vector<cursor_type *> vec;
    1326         [ #  # ]:          0 :             vec.reserve(pq.size());
    1327                 :            : 
    1328                 :            :             while (true) {
    1329         [ #  # ]:          0 :                 cur->read_tag();
    1330                 :            :                 pqtag.push(new PrefixCompressedStringItor(cur->current_tag,
    1331 [ #  # ][ #  # ]:          0 :                                                           key));
                 [ #  # ]
    1332         [ #  # ]:          0 :                 vec.push_back(cur);
    1333 [ #  # ][ #  # ]:          0 :                 if (pq.empty() || pq.top()->current_key != key) break;
         [ #  # ][ #  # ]
    1334                 :          0 :                 cur = pq.top();
    1335         [ #  # ]:          0 :                 pq.pop();
    1336                 :            :             }
    1337                 :            : 
    1338         [ #  # ]:          0 :             PrefixCompressedStringWriter wr(tag, key);
    1339         [ #  # ]:          0 :             string lastword;
    1340         [ #  # ]:          0 :             while (!pqtag.empty()) {
    1341                 :          0 :                 PrefixCompressedStringItor * it = pqtag.top();
    1342         [ #  # ]:          0 :                 pqtag.pop();
    1343         [ #  # ]:          0 :                 string word = **it;
    1344 [ #  # ][ #  # ]:          0 :                 if (word != lastword) {
    1345         [ #  # ]:          0 :                     lastword = word;
    1346         [ #  # ]:          0 :                     wr.append(lastword);
    1347                 :            :                 }
    1348         [ #  # ]:          0 :                 ++*it;
    1349         [ #  # ]:          0 :                 if (!it->at_end()) {
    1350         [ #  # ]:          0 :                     pqtag.push(it);
    1351                 :            :                 } else {
    1352         [ #  # ]:          0 :                     delete it;
    1353                 :            :                 }
    1354                 :          0 :             }
    1355                 :            : 
    1356         [ #  # ]:          0 :             for (auto i : vec) {
    1357                 :          0 :                 cur = i;
    1358 [ #  # ][ #  # ]:          0 :                 if (cur->next()) {
    1359         [ #  # ]:          0 :                     pq.push(cur);
    1360                 :            :                 } else {
    1361         [ #  # ]:          0 :                     delete cur;
    1362                 :            :                 }
    1363                 :          0 :             }
    1364                 :            :         } else {
    1365                 :            :             // We want to sum the frequencies from tags for the same key.
    1366                 :          0 :             Xapian::termcount tot_freq = 0;
    1367                 :            :             while (true) {
    1368         [ #  # ]:          0 :                 cur->read_tag();
    1369                 :            :                 Xapian::termcount freq;
    1370                 :          0 :                 const char * p = cur->current_tag.data();
    1371                 :          0 :                 const char * end = p + cur->current_tag.size();
    1372 [ #  # ][ #  # ]:          0 :                 if (!unpack_uint_last(&p, end, &freq) || freq == 0) {
                 [ #  # ]
    1373                 :          0 :                     throw_database_corrupt("Bad spelling word freq", p);
    1374                 :            :                 }
    1375                 :          0 :                 tot_freq += freq;
    1376 [ #  # ][ #  # ]:          0 :                 if (cur->next()) {
    1377         [ #  # ]:          0 :                     pq.push(cur);
    1378                 :            :                 } else {
    1379         [ #  # ]:          0 :                     delete cur;
    1380                 :            :                 }
    1381 [ #  # ][ #  # ]:          0 :                 if (pq.empty() || pq.top()->current_key != key) break;
         [ #  # ][ #  # ]
    1382                 :          0 :                 cur = pq.top();
    1383         [ #  # ]:          0 :                 pq.pop();
    1384                 :            :             }
    1385         [ #  # ]:          0 :             tag.resize(0);
    1386         [ #  # ]:          0 :             pack_uint_last(tag, tot_freq);
    1387                 :            :         }
    1388         [ #  # ]:          0 :         out->add(key, tag);
    1389                 :          0 :     }
    1390                 :          0 : }
    1391                 :            : 
    1392                 :            : // U : vector<HoneyTable*>::const_iterator
    1393                 :            : template<typename T, typename U> void
    1394                 :          0 : merge_synonyms(T* out, U b, U e)
    1395                 :            : {
    1396                 :            :     typedef decltype(**b) table_type; // E.g. HoneyTable
    1397                 :            :     typedef MergeCursor<table_type> cursor_type;
    1398                 :            :     typedef CursorGt<cursor_type> gt_type;
    1399 [ #  # ][ #  # ]:          0 :     priority_queue<cursor_type *, vector<cursor_type *>, gt_type> pq;
    1400 [ #  # ][ #  # ]:          0 :     for ( ; b != e; ++b) {
    1401                 :          0 :         auto in = *b;
    1402         [ #  # ]:          0 :         auto cursor = new cursor_type(in);
           [ #  #  #  # ]
                 [ #  # ]
    1403 [ #  # ][ #  # ]:          0 :         if (cursor->next()) {
         [ #  # ][ #  # ]
    1404 [ #  # ][ #  # ]:          0 :             pq.push(cursor);
    1405                 :            :         } else {
    1406                 :            :             // Skip empty tables.
    1407 [ #  # ][ #  # ]:          0 :             delete cursor;
    1408                 :            :         }
    1409                 :            :     }
    1410                 :            : 
    1411 [ #  # ][ #  # ]:          0 :     while (!pq.empty()) {
    1412                 :          0 :         cursor_type * cur = pq.top();
    1413   [ #  #  #  # ]:          0 :         pq.pop();
    1414                 :            : 
    1415 [ #  # ][ #  # ]:          0 :         string key = cur->current_key;
    1416 [ #  # ][ #  # ]:          0 :         if (pq.empty() || pq.top()->current_key > key) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
    1417                 :            :             // No need to merge the tags, just copy the (possibly compressed)
    1418                 :            :             // tag value.
    1419 [ #  # ][ #  # ]:          0 :             bool compressed = cur->read_tag(true);
    1420 [ #  # ][ #  # ]:          0 :             out->add(key, cur->current_tag, compressed);
    1421 [ #  # ][ #  # ]:          0 :             if (cur->next()) {
         [ #  # ][ #  # ]
    1422 [ #  # ][ #  # ]:          0 :                 pq.push(cur);
    1423                 :            :             } else {
    1424 [ #  # ][ #  # ]:          0 :                 delete cur;
    1425                 :            :             }
    1426                 :          0 :             continue;
    1427                 :            :         }
    1428                 :            : 
    1429                 :            :         // Merge tag values with the same key:
    1430 [ #  # ][ #  # ]:          0 :         string tag;
         [ #  # ][ #  # ]
    1431                 :            : 
    1432                 :            :         // We just want the union of words, so copy over the first instance
    1433                 :            :         // and skip any identical ones.
    1434                 :            :         priority_queue<ByteLengthPrefixedStringItor *,
    1435                 :            :                        vector<ByteLengthPrefixedStringItor *>,
    1436 [ #  # ][ #  # ]:          0 :                        ByteLengthPrefixedStringItorGt> pqtag;
    1437                 :          0 :         vector<cursor_type *> vec;
    1438                 :            : 
    1439                 :            :         while (true) {
    1440 [ #  # ][ #  # ]:          0 :             cur->read_tag();
    1441 [ #  # ][ #  # ]:          0 :             pqtag.push(new ByteLengthPrefixedStringItor(cur->current_tag));
         [ #  # ][ #  # ]
    1442 [ #  # ][ #  # ]:          0 :             vec.push_back(cur);
    1443 [ #  # ][ #  # ]:          0 :             if (pq.empty() || pq.top()->current_key != key) break;
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
    1444                 :          0 :             cur = pq.top();
    1445 [ #  # ][ #  # ]:          0 :             pq.pop();
    1446                 :            :         }
    1447                 :            : 
    1448 [ #  # ][ #  # ]:          0 :         string lastword;
    1449 [ #  # ][ #  # ]:          0 :         while (!pqtag.empty()) {
    1450                 :          0 :             ByteLengthPrefixedStringItor * it = pqtag.top();
    1451   [ #  #  #  # ]:          0 :             pqtag.pop();
    1452 [ #  # ][ #  # ]:          0 :             if (**it != lastword) {
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
    1453 [ #  # ][ #  # ]:          0 :                 lastword = **it;
         [ #  # ][ #  # ]
    1454   [ #  #  #  # ]:          0 :                 tag += uint8_t(lastword.size() ^ MAGIC_XOR_VALUE);
    1455 [ #  # ][ #  # ]:          0 :                 tag += lastword;
    1456                 :            :             }
    1457 [ #  # ][ #  # ]:          0 :             ++*it;
    1458 [ #  # ][ #  # ]:          0 :             if (!it->at_end()) {
    1459 [ #  # ][ #  # ]:          0 :                 pqtag.push(it);
    1460                 :            :             } else {
    1461                 :          0 :                 delete it;
    1462                 :            :             }
    1463                 :            :         }
    1464                 :            : 
    1465 [ #  # ][ #  # ]:          0 :         for (auto i : vec) {
    1466                 :          0 :             cur = i;
    1467         [ #  # ]:          0 :             if (cur->next()) {
           [ #  #  #  # ]
                 [ #  # ]
    1468 [ #  # ][ #  # ]:          0 :                 pq.push(cur);
    1469                 :            :             } else {
    1470 [ #  # ][ #  # ]:          0 :                 delete cur;
    1471                 :            :             }
    1472                 :            :         }
    1473                 :            : 
    1474 [ #  # ][ #  # ]:          0 :         out->add(key, tag);
    1475                 :          0 :     }
    1476                 :          0 : }
    1477                 :            : 
    1478                 :            : template<typename T, typename U> void
    1479                 :          0 : multimerge_postlists(Xapian::Compactor * compactor,
    1480                 :            :                      T* out, const char * tmpdir,
    1481                 :            :                      const vector<U*>& in,
    1482                 :            :                      vector<Xapian::docid> off)
    1483                 :            : {
    1484 [ #  # ][ #  # ]:          0 :     if (in.size() <= 3) {
    1485 [ #  # ][ #  # ]:          0 :         merge_postlists(compactor, out, off.begin(), in.begin(), in.end());
    1486                 :          0 :         return;
    1487                 :            :     }
    1488                 :          0 :     unsigned int c = 0;
    1489                 :          0 :     vector<HoneyTable *> tmp;
    1490   [ #  #  #  # ]:          0 :     tmp.reserve(in.size() / 2);
    1491                 :            :     {
    1492                 :          0 :         vector<Xapian::docid> newoff;
    1493   [ #  #  #  # ]:          0 :         newoff.resize(in.size() / 2);
    1494 [ #  # ][ #  # ]:          0 :         for (unsigned int i = 0, j; i < in.size(); i = j) {
    1495                 :          0 :             j = i + 2;
    1496 [ #  # ][ #  # ]:          0 :             if (j == in.size() - 1) ++j;
    1497                 :            : 
    1498 [ #  # ][ #  # ]:          0 :             string dest = tmpdir;
    1499                 :            :             char buf[64];
    1500                 :          0 :             sprintf(buf, "/tmp%u_%u.", c, i / 2);
    1501   [ #  #  #  # ]:          0 :             dest += buf;
    1502                 :            : 
    1503 [ #  # ][ #  # ]:          0 :             HoneyTable * tmptab = new HoneyTable("postlist", dest, false);
         [ #  # ][ #  # ]
    1504                 :            : 
    1505                 :            :             // Don't compress entries in temporary tables, even if the final
    1506                 :            :             // table would do so.  Any already compressed entries will get
    1507                 :            :             // copied in compressed form.
    1508 [ #  # ][ #  # ]:          0 :             Honey::RootInfo root_info;
    1509 [ #  # ][ #  # ]:          0 :             root_info.init(0);
    1510                 :          0 :             const int flags = Xapian::DB_DANGEROUS|Xapian::DB_NO_SYNC;
    1511 [ #  # ][ #  # ]:          0 :             tmptab->create_and_open(flags, root_info);
    1512                 :            : 
    1513   [ #  #  #  # ]:          0 :             merge_postlists(compactor, tmptab, off.begin() + i,
    1514                 :          0 :                             in.begin() + i, in.begin() + j);
    1515 [ #  # ][ #  # ]:          0 :             tmp.push_back(tmptab);
    1516 [ #  # ][ #  # ]:          0 :             tmptab->flush_db();
    1517 [ #  # ][ #  # ]:          0 :             tmptab->commit(1, &root_info);
    1518                 :            :         }
    1519                 :          0 :         swap(off, newoff);
    1520                 :          0 :         ++c;
    1521                 :            :     }
    1522                 :            : 
    1523 [ #  # ][ #  # ]:          0 :     while (tmp.size() > 3) {
    1524                 :          0 :         vector<HoneyTable *> tmpout;
    1525   [ #  #  #  # ]:          0 :         tmpout.reserve(tmp.size() / 2);
    1526                 :          0 :         vector<Xapian::docid> newoff;
    1527   [ #  #  #  # ]:          0 :         newoff.resize(tmp.size() / 2);
    1528 [ #  # ][ #  # ]:          0 :         for (unsigned int i = 0, j; i < tmp.size(); i = j) {
    1529                 :          0 :             j = i + 2;
    1530 [ #  # ][ #  # ]:          0 :             if (j == tmp.size() - 1) ++j;
    1531                 :            : 
    1532 [ #  # ][ #  # ]:          0 :             string dest = tmpdir;
    1533                 :            :             char buf[64];
    1534                 :          0 :             sprintf(buf, "/tmp%u_%u.", c, i / 2);
    1535   [ #  #  #  # ]:          0 :             dest += buf;
    1536                 :            : 
    1537 [ #  # ][ #  # ]:          0 :             HoneyTable * tmptab = new HoneyTable("postlist", dest, false);
         [ #  # ][ #  # ]
    1538                 :            : 
    1539                 :            :             // Don't compress entries in temporary tables, even if the final
    1540                 :            :             // table would do so.  Any already compressed entries will get
    1541                 :            :             // copied in compressed form.
    1542 [ #  # ][ #  # ]:          0 :             Honey::RootInfo root_info;
    1543 [ #  # ][ #  # ]:          0 :             root_info.init(0);
    1544                 :          0 :             const int flags = Xapian::DB_DANGEROUS|Xapian::DB_NO_SYNC;
    1545 [ #  # ][ #  # ]:          0 :             tmptab->create_and_open(flags, root_info);
    1546                 :            : 
    1547   [ #  #  #  # ]:          0 :             merge_postlists(compactor, tmptab, off.begin() + i,
    1548                 :          0 :                             tmp.begin() + i, tmp.begin() + j);
    1549 [ #  # ][ #  # ]:          0 :             if (c > 0) {
    1550 [ #  # ][ #  # ]:          0 :                 for (unsigned int k = i; k < j; ++k) {
    1551                 :            :                     // FIXME: unlink(tmp[k]->get_path().c_str());
    1552 [ #  # ][ #  # ]:          0 :                     delete tmp[k];
    1553                 :          0 :                     tmp[k] = NULL;
    1554                 :            :                 }
    1555                 :            :             }
    1556 [ #  # ][ #  # ]:          0 :             tmpout.push_back(tmptab);
    1557 [ #  # ][ #  # ]:          0 :             tmptab->flush_db();
    1558 [ #  # ][ #  # ]:          0 :             tmptab->commit(1, &root_info);
    1559                 :            :         }
    1560                 :          0 :         swap(tmp, tmpout);
    1561                 :          0 :         swap(off, newoff);
    1562                 :          0 :         ++c;
    1563                 :            :     }
    1564 [ #  # ][ #  # ]:          0 :     merge_postlists(compactor, out, off.begin(), tmp.begin(), tmp.end());
    1565 [ #  # ][ #  # ]:          0 :     if (c > 0) {
    1566 [ #  # ][ #  # ]:          0 :         for (size_t k = 0; k < tmp.size(); ++k) {
    1567                 :            :             // FIXME: unlink(tmp[k]->get_path().c_str());
    1568 [ #  # ][ #  # ]:          0 :             delete tmp[k];
    1569                 :          0 :             tmp[k] = NULL;
    1570                 :            :         }
    1571                 :          0 :     }
    1572                 :            : }
    1573                 :            : 
    1574                 :            : template<typename T> class PositionCursor;
    1575                 :            : 
    1576                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    1577                 :            : template<>
    1578                 :        570 : class PositionCursor<const GlassTable&> : private GlassCursor {
    1579                 :            :     Xapian::docid offset;
    1580                 :            : 
    1581                 :            :   public:
    1582                 :            :     string key;
    1583                 :            :     Xapian::docid firstdid;
    1584                 :            : 
    1585                 :        285 :     PositionCursor(const GlassTable *in, Xapian::docid offset_)
    1586         [ +  - ]:        285 :         : GlassCursor(in), offset(offset_), firstdid(0) {
    1587         [ +  - ]:        285 :         rewind();
    1588                 :        285 :     }
    1589                 :            : 
    1590                 :     840137 :     bool next() {
    1591 [ +  - ][ +  + ]:     840137 :         if (!GlassCursor::next()) return false;
    1592         [ +  - ]:     839852 :         read_tag();
    1593                 :     839852 :         const char * d = current_key.data();
    1594                 :     839852 :         const char * e = d + current_key.size();
    1595         [ +  - ]:     839852 :         string term;
    1596                 :            :         Xapian::docid did;
    1597   [ +  -  +  - ]:    2519556 :         if (!unpack_string_preserving_sort(&d, e, term) ||
                 [ -  + ]
    1598 [ +  - ][ -  + ]:    1679704 :             !unpack_uint_preserving_sort(&d, e, &did) ||
    1599                 :     839852 :             d != e) {
    1600 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Bad position key");
                 [ #  # ]
    1601                 :            :         }
    1602                 :            : 
    1603         [ +  - ]:     839852 :         key.resize(0);
    1604         [ +  - ]:     839852 :         pack_string_preserving_sort(key, term);
    1605         [ +  - ]:     839852 :         pack_uint_preserving_sort(key, did + offset);
    1606                 :     840137 :         return true;
    1607                 :            :     }
    1608                 :            : 
    1609                 :     839852 :     const string & get_tag() const {
    1610                 :     839852 :         return current_tag;
    1611                 :            :     }
    1612                 :            : };
    1613                 :            : #endif
    1614                 :            : 
    1615                 :            : template<>
    1616                 :         24 : class PositionCursor<const HoneyTable&> : private HoneyCursor {
    1617                 :            :     Xapian::docid offset;
    1618                 :            : 
    1619                 :            :   public:
    1620                 :            :     string key;
    1621                 :            :     Xapian::docid firstdid;
    1622                 :            : 
    1623                 :         12 :     PositionCursor(const HoneyTable *in, Xapian::docid offset_)
    1624         [ +  - ]:         12 :         : HoneyCursor(in), offset(offset_), firstdid(0) {
    1625         [ +  - ]:         12 :         rewind();
    1626                 :         12 :     }
    1627                 :            : 
    1628                 :       1360 :     bool next() {
    1629 [ +  - ][ +  + ]:       1360 :         if (!HoneyCursor::next()) return false;
    1630         [ +  - ]:       1348 :         read_tag();
    1631                 :       1348 :         const char * d = current_key.data();
    1632                 :       1348 :         const char * e = d + current_key.size();
    1633         [ +  - ]:       1348 :         string term;
    1634                 :            :         Xapian::docid did;
    1635   [ +  -  +  - ]:       4044 :         if (!unpack_string_preserving_sort(&d, e, term) ||
                 [ -  + ]
    1636 [ +  - ][ -  + ]:       2696 :             !unpack_uint_preserving_sort(&d, e, &did) ||
    1637                 :       1348 :             d != e) {
    1638 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Bad position key");
                 [ #  # ]
    1639                 :            :         }
    1640                 :            : 
    1641         [ +  - ]:       1348 :         key.resize(0);
    1642         [ +  - ]:       1348 :         pack_string_preserving_sort(key, term);
    1643         [ +  - ]:       1348 :         pack_uint_preserving_sort(key, did + offset);
    1644                 :       1360 :         return true;
    1645                 :            :     }
    1646                 :            : 
    1647                 :       1348 :     const string & get_tag() const {
    1648                 :       1348 :         return current_tag;
    1649                 :            :     }
    1650                 :            : };
    1651                 :            : 
    1652                 :            : template<typename T>
    1653                 :            : class PositionCursorGt {
    1654                 :            :   public:
    1655                 :            :     /** Return true if and only if a's key is strictly greater than b's key.
    1656                 :            :      */
    1657                 :       1041 :     bool operator()(const T* a, const T* b) const {
    1658                 :       1041 :         return a->key > b->key;
    1659                 :            :     }
    1660                 :            : };
    1661                 :            : 
    1662                 :            : template<typename T, typename U> void
    1663                 :        292 : merge_positions(T* out, const vector<U*> & inputs,
    1664                 :            :                 const vector<Xapian::docid> & offset)
    1665                 :            : {
    1666                 :            :     typedef decltype(*inputs[0]) table_type; // E.g. HoneyTable
    1667                 :            :     typedef PositionCursor<table_type> cursor_type;
    1668                 :            :     typedef PositionCursorGt<cursor_type> gt_type;
    1669 [ +  - ][ +  - ]:        292 :     priority_queue<cursor_type *, vector<cursor_type *>, gt_type> pq;
    1670 [ +  + ][ +  + ]:        589 :     for (size_t i = 0; i < inputs.size(); ++i) {
    1671                 :        297 :         auto in = inputs[i];
    1672         [ +  - ]:        297 :         auto cursor = new cursor_type(in, offset[i]);
           [ +  -  +  - ]
                 [ +  - ]
    1673 [ +  - ][ +  - ]:        297 :         if (cursor->next()) {
         [ +  - ][ +  - ]
    1674 [ +  - ][ +  - ]:        297 :             pq.push(cursor);
    1675                 :            :         } else {
    1676                 :            :             // Skip empty tables.
    1677 [ #  # ][ #  # ]:          0 :             delete cursor;
    1678                 :            :         }
    1679                 :            :     }
    1680                 :            : 
    1681 [ +  + ][ +  + ]:     841492 :     while (!pq.empty()) {
    1682                 :     841200 :         cursor_type * cur = pq.top();
    1683   [ +  -  +  - ]:     841200 :         pq.pop();
    1684 [ +  - ][ +  - ]:     841200 :         out->add(cur->key, cur->get_tag());
    1685 [ +  - ][ +  + ]:     841200 :         if (cur->next()) {
         [ +  - ][ +  + ]
    1686 [ +  - ][ +  - ]:     840903 :             pq.push(cur);
    1687                 :            :         } else {
    1688 [ +  - ][ +  - ]:     841200 :             delete cur;
    1689                 :            :         }
    1690                 :        292 :     }
    1691                 :        292 : }
    1692                 :            : 
    1693                 :            : template<typename T, typename U> void
    1694                 :         16 : merge_docid_keyed(T *out, const vector<U*> & inputs,
    1695                 :            :                   const vector<Xapian::docid> & offset,
    1696                 :            :                   int = 0)
    1697                 :            : {
    1698         [ +  + ]:         43 :     for (size_t i = 0; i < inputs.size(); ++i) {
    1699                 :         27 :         Xapian::docid off = offset[i];
    1700                 :            : 
    1701                 :         27 :         auto in = inputs[i];
    1702         [ +  - ]:         27 :         HoneyCursor cur(in);
    1703         [ +  - ]:         27 :         cur.rewind();
    1704                 :            : 
    1705         [ +  - ]:         54 :         string key;
    1706 [ +  - ][ +  + ]:        139 :         while (cur.next()) {
    1707                 :            :             // Adjust the key if this isn't the first database.
    1708         [ +  + ]:        112 :             if (off) {
    1709                 :            :                 Xapian::docid did;
    1710                 :         28 :                 const char * d = cur.current_key.data();
    1711                 :         28 :                 const char * e = d + cur.current_key.size();
    1712         [ -  + ]:         28 :                 if (!unpack_uint_preserving_sort(&d, e, &did)) {
    1713         [ #  # ]:          0 :                     string msg = "Bad key in ";
    1714         [ #  # ]:          0 :                     msg += inputs[i]->get_path();
    1715 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError(msg);
    1716                 :            :                 }
    1717                 :         28 :                 did += off;
    1718         [ +  - ]:         28 :                 key.resize(0);
    1719         [ +  - ]:         28 :                 pack_uint_preserving_sort(key, did);
    1720         [ -  + ]:         28 :                 if (d != e) {
    1721                 :            :                     // Copy over anything extra in the key (e.g. the zero byte
    1722                 :            :                     // at the end of "used value slots" in the termlist table).
    1723         [ #  # ]:         28 :                     key.append(d, e - d);
    1724                 :            :                 }
    1725                 :            :             } else {
    1726         [ +  - ]:         84 :                 key = cur.current_key;
    1727                 :            :             }
    1728         [ +  - ]:        112 :             bool compressed = cur.read_tag(true);
    1729         [ +  - ]:        112 :             out->add(key, cur.current_tag, compressed);
    1730                 :            :         }
    1731                 :            :     }
    1732                 :         16 : }
    1733                 :            : 
    1734                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    1735                 :            : template<typename T> void
    1736                 :        574 : merge_docid_keyed(T *out, const vector<const GlassTable*> & inputs,
    1737                 :            :                   const vector<Xapian::docid> & offset,
    1738                 :            :                   Xapian::termcount & ut_lb, Xapian::termcount & ut_ub,
    1739                 :            :                   int table_type = 0)
    1740                 :            : {
    1741         [ +  + ]:       1148 :     for (size_t i = 0; i < inputs.size(); ++i) {
    1742                 :        574 :         Xapian::docid off = offset[i];
    1743                 :            : 
    1744                 :        574 :         auto in = inputs[i];
    1745         [ +  + ]:        574 :         if (in->empty()) continue;
    1746                 :            : 
    1747         [ +  - ]:        570 :         GlassCursor cur(in);
    1748         [ +  - ]:        570 :         cur.rewind();
    1749                 :            : 
    1750         [ +  - ]:       1140 :         string key;
    1751 [ +  - ][ +  + ]:      28588 :         while (cur.next()) {
    1752                 :            : next_without_next:
    1753                 :            :             // Adjust the key if this isn't the first database.
    1754         [ -  + ]:      28018 :             if (off) {
    1755                 :            :                 Xapian::docid did;
    1756                 :          0 :                 const char * d = cur.current_key.data();
    1757                 :          0 :                 const char * e = d + cur.current_key.size();
    1758         [ #  # ]:          0 :                 if (!unpack_uint_preserving_sort(&d, e, &did)) {
    1759         [ #  # ]:          0 :                     string msg = "Bad key in ";
    1760 [ #  # ][ #  # ]:          0 :                     msg += inputs[i]->get_path();
    1761 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError(msg);
    1762                 :            :                 }
    1763                 :          0 :                 did += off;
    1764         [ #  # ]:          0 :                 key.resize(0);
    1765         [ #  # ]:          0 :                 pack_uint_preserving_sort(key, did);
    1766         [ #  # ]:          0 :                 if (d != e) {
    1767                 :            :                     // Copy over anything extra in the key (e.g. the zero byte
    1768                 :            :                     // at the end of "used value slots" in the termlist table).
    1769         [ #  # ]:          0 :                     key.append(d, e - d);
    1770                 :            :                 }
    1771                 :            :             } else {
    1772         [ +  - ]:      28018 :                 key = cur.current_key;
    1773                 :            :             }
    1774         [ +  + ]:      28018 :             if (table_type == Honey::TERMLIST) {
    1775 [ +  - ][ -  + ]:      14684 :                 if (termlist_key_is_values_used(key)) {
    1776 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseCorruptError("value slots used "
                 [ #  # ]
    1777                 :            :                                                        "entry without a "
    1778                 :            :                                                        "corresponding "
    1779                 :          0 :                                                        "termlist entry");
    1780                 :            :                 }
    1781         [ +  - ]:      14684 :                 cur.read_tag();
    1782         [ +  - ]:      14684 :                 string tag = cur.current_tag;
    1783                 :            : 
    1784         [ +  - ]:      14684 :                 bool next_result = cur.next();
    1785                 :      14684 :                 bool next_already_done = true;
    1786                 :      14684 :                 unsigned bitmap_slots_used = 0;
    1787         [ +  - ]:      29368 :                 string encoded_slots_used;
              [ +  -  - ]
    1788 [ +  - ][ +  - ]:      29368 :                 if (next_result &&
                 [ +  - ]
    1789         [ +  - ]:      14684 :                     termlist_key_is_values_used(cur.current_key)) {
    1790                 :      14684 :                     next_already_done = false;
    1791         [ +  - ]:      14684 :                     cur.read_tag();
    1792                 :      14684 :                     const string& valtag = cur.current_tag;
    1793                 :            : 
    1794                 :      14684 :                     const char* p = valtag.data();
    1795                 :      14684 :                     const char* end = p + valtag.size();
    1796                 :            : 
    1797                 :      14684 :                     Xapian::VecCOW<Xapian::termpos> slots;
    1798                 :            : 
    1799                 :            :                     Xapian::valueno first_slot;
    1800         [ -  + ]:      14684 :                     if (!unpack_uint(&p, end, &first_slot)) {
    1801                 :          0 :                         throw_database_corrupt("Value slot encoding corrupt",
    1802                 :            :                                                p);
    1803                 :            :                     }
    1804         [ +  - ]:      14684 :                     slots.push_back(first_slot);
    1805                 :      14684 :                     Xapian::valueno last_slot = first_slot;
    1806         [ +  + ]:     185431 :                     while (p != end) {
    1807                 :            :                         Xapian::valueno slot;
    1808         [ -  + ]:     170747 :                         if (!unpack_uint(&p, end, &slot)) {
    1809                 :          0 :                             throw_database_corrupt("Value slot encoding "
    1810                 :            :                                                    "corrupt", p);
    1811                 :            :                         }
    1812                 :     170747 :                         slot += last_slot + 1;
    1813                 :     170747 :                         last_slot = slot;
    1814                 :            : 
    1815         [ +  - ]:     170747 :                         slots.push_back(slot);
    1816                 :            :                     }
    1817                 :            : 
    1818 [ +  - ][ -  + ]:      14684 :                     if (slots.back() <= 6) {
    1819                 :            :                         // Encode as a bitmap if only slots in the range 0-6
    1820                 :            :                         // are used.
    1821 [ #  # ][ #  # ]:          0 :                         for (auto slot : slots) {
                 [ #  # ]
    1822                 :          0 :                             bitmap_slots_used |= 1 << slot;
    1823                 :            :                         }
    1824                 :            :                     } else {
    1825         [ +  - ]:      14684 :                         string enc;
    1826         [ +  - ]:      14684 :                         pack_uint(enc, last_slot);
    1827         [ +  - ]:      14684 :                         if (slots.size() > 1) {
    1828         [ +  - ]:      14684 :                             BitWriter slots_used(enc);
    1829         [ +  - ]:      14684 :                             slots_used.encode(first_slot, last_slot);
    1830         [ +  - ]:      14684 :                             slots_used.encode(slots.size() - 2,
    1831                 :      29368 :                                               last_slot - first_slot);
    1832         [ +  - ]:      14684 :                             slots_used.encode_interpolative(slots, 0,
    1833                 :      14684 :                                                             slots.size() - 1);
    1834 [ +  - ][ +  - ]:      14684 :                             encoded_slots_used = slots_used.freeze();
    1835                 :            :                         } else {
    1836         [ #  # ]:          0 :                             encoded_slots_used = std::move(enc);
    1837                 :      14684 :                         }
    1838                 :      14684 :                     }
    1839                 :            :                 }
    1840                 :            : 
    1841                 :      14684 :                 const char* pos = tag.data();
    1842                 :      14684 :                 const char* end = pos + tag.size();
    1843                 :            : 
    1844         [ +  - ]:      29368 :                 string newtag;
              [ +  -  - ]
    1845         [ -  + ]:      14684 :                 if (encoded_slots_used.empty()) {
    1846         [ #  # ]:          0 :                     newtag += char(bitmap_slots_used);
    1847                 :            :                 } else {
    1848                 :      14684 :                     auto size = encoded_slots_used.size();
    1849         [ +  - ]:      14684 :                     if (size < 0x80) {
    1850         [ +  - ]:      14684 :                         newtag += char(0x80 | size);
    1851                 :            :                     } else {
    1852         [ #  # ]:          0 :                         newtag += '\x80';
    1853         [ #  # ]:          0 :                         pack_uint(newtag, size);
    1854                 :            :                     }
    1855         [ +  - ]:      14684 :                     newtag += encoded_slots_used;
    1856                 :            :                 }
    1857                 :            : 
    1858         [ +  + ]:      14684 :                 if (pos != end) {
    1859                 :            :                     Xapian::termcount doclen;
    1860         [ -  + ]:      13296 :                     if (!unpack_uint(&pos, end, &doclen)) {
    1861                 :          0 :                         throw_database_corrupt("doclen", pos);
    1862                 :            :                     }
    1863                 :            :                     Xapian::termcount termlist_size;
    1864         [ -  + ]:      13296 :                     if (!unpack_uint(&pos, end, &termlist_size)) {
    1865                 :          0 :                         throw_database_corrupt("termlist length", pos);
    1866                 :            :                     }
    1867                 :            : 
    1868                 :      13296 :                     auto uniq_terms = min(termlist_size, doclen);
    1869 [ +  - ][ +  + ]:      13296 :                     if (uniq_terms &&
                 [ +  + ]
    1870                 :            :                         (ut_lb == 0 || uniq_terms < ut_lb)) {
    1871                 :        718 :                         ut_lb = uniq_terms;
    1872                 :            :                     }
    1873         [ +  + ]:      13296 :                     if (uniq_terms > ut_ub)
    1874                 :        630 :                         ut_ub = uniq_terms;
    1875                 :            : 
    1876         [ +  - ]:      13296 :                     pack_uint(newtag, termlist_size - 1);
    1877         [ +  - ]:      13296 :                     pack_uint(newtag, doclen);
    1878                 :            : 
    1879         [ +  - ]:      13296 :                     string current_term;
    1880         [ +  + ]:     853148 :                     while (pos != end) {
    1881                 :     839852 :                         Xapian::termcount current_wdf = 0;
    1882                 :            : 
    1883         [ +  + ]:     839852 :                         if (!current_term.empty()) {
    1884                 :     826556 :                             size_t reuse = static_cast<unsigned char>(*pos++);
    1885         [ +  - ]:     826556 :                             newtag += char(reuse);
    1886                 :            : 
    1887         [ +  - ]:     826556 :                             if (reuse > current_term.size()) {
    1888                 :     826556 :                                 current_wdf = reuse / (current_term.size() + 1);
    1889                 :     826556 :                                 reuse = reuse % (current_term.size() + 1);
    1890                 :            :                             }
    1891         [ +  - ]:     826556 :                             current_term.resize(reuse);
    1892                 :            : 
    1893         [ -  + ]:     826556 :                             if (pos == end)
    1894                 :     826556 :                                 throw_database_corrupt("term", NULL);
    1895                 :            :                         }
    1896                 :            : 
    1897                 :     839852 :                         size_t append = static_cast<unsigned char>(*pos++);
    1898         [ -  + ]:     839852 :                         if (size_t(end - pos) < append)
    1899                 :          0 :                             throw_database_corrupt("term", NULL);
    1900                 :            : 
    1901         [ +  - ]:     839852 :                         current_term.append(pos, append);
    1902                 :     839852 :                         pos += append;
    1903                 :            : 
    1904         [ +  + ]:     839852 :                         if (current_wdf) {
    1905                 :     826556 :                             --current_wdf;
    1906                 :            :                         } else {
    1907         [ -  + ]:      13296 :                             if (!unpack_uint(&pos, end, &current_wdf)) {
    1908                 :          0 :                                 throw_database_corrupt("wdf", pos);
    1909                 :            :                             }
    1910         [ +  - ]:      13296 :                             pack_uint(newtag, current_wdf);
    1911                 :            :                         }
    1912                 :            : 
    1913         [ +  - ]:     839852 :                         newtag += char(append);
    1914         [ +  - ]:     839852 :                         newtag.append(current_term.end() - append,
    1915 [ +  - ][ +  - ]:    1679704 :                                       current_term.end());
    1916                 :      13296 :                     }
    1917                 :            :                 }
    1918         [ +  - ]:      14684 :                 if (!newtag.empty())
    1919         [ +  - ]:      14684 :                     out->add(key, newtag);
    1920         [ -  + ]:      14684 :                 if (!next_result) break;
    1921         [ -  + ]:      29368 :                 if (next_already_done) goto next_without_next;
              [ +  -  - ]
    1922                 :            :             } else {
    1923         [ +  - ]:      13334 :                 bool compressed = cur.read_tag(true);
    1924         [ +  - ]:      13334 :                 out->add(key, cur.current_tag, compressed);
    1925                 :            :             }
    1926                 :            :         }
    1927                 :            :     }
    1928                 :        574 : }
    1929                 :            : #endif
    1930                 :            : 
    1931                 :            : }
    1932                 :            : 
    1933                 :            : using namespace HoneyCompact;
    1934                 :            : 
    1935                 :            : void
    1936                 :        298 : HoneyDatabase::compact(Xapian::Compactor* compactor,
    1937                 :            :                        const char* destdir,
    1938                 :            :                        int fd,
    1939                 :            :                        int source_backend,
    1940                 :            :                        const vector<const Xapian::Database::Internal*>& sources,
    1941                 :            :                        const vector<Xapian::docid>& offset,
    1942                 :            :                        Xapian::Compactor::compaction_level compaction,
    1943                 :            :                        unsigned flags,
    1944                 :            :                        Xapian::docid last_docid)
    1945                 :            : {
    1946                 :            :     // Currently unused for honey.
    1947                 :            :     (void)compaction;
    1948                 :            : 
    1949                 :            :     struct table_list {
    1950                 :            :         // The "base name" of the table.
    1951                 :            :         char name[9];
    1952                 :            :         // The type.
    1953                 :            :         Honey::table_type type;
    1954                 :            :         // Create tables after position lazily.
    1955                 :            :         bool lazy;
    1956                 :            :     };
    1957                 :            : 
    1958                 :            :     static const table_list tables[] = {
    1959                 :            :         // name         type                    lazy
    1960                 :            :         { "postlist", Honey::POSTLIST,        false },
    1961                 :            :         { "docdata",  Honey::DOCDATA,         true },
    1962                 :            :         { "termlist", Honey::TERMLIST,        false },
    1963                 :            :         { "position", Honey::POSITION,        true },
    1964                 :            :         { "spelling", Honey::SPELLING,        true },
    1965                 :            :         { "synonym",  Honey::SYNONYM,         true }
    1966                 :            :     };
    1967                 :        298 :     const table_list * tables_end = tables +
    1968                 :        596 :         (sizeof(tables) / sizeof(tables[0]));
    1969                 :            : 
    1970                 :        298 :     const int FLAGS = Xapian::DB_DANGEROUS;
    1971                 :            : 
    1972                 :        298 :     bool single_file = (flags & Xapian::DBCOMPACT_SINGLE_FILE);
    1973                 :        298 :     bool multipass = (flags & Xapian::DBCOMPACT_MULTIPASS);
    1974         [ +  + ]:        298 :     if (single_file) {
    1975                 :            :         // FIXME: Support this combination - we need to put temporary files
    1976                 :            :         // somewhere.
    1977                 :          2 :         multipass = false;
    1978                 :            :     }
    1979                 :            : 
    1980         [ +  + ]:        298 :     if (single_file) {
    1981         [ +  + ]:          4 :         for (size_t i = 0; i != sources.size(); ++i) {
    1982                 :            :             bool has_uncommitted_changes;
    1983         [ -  + ]:          2 :             if (source_backend == Xapian::DB_BACKEND_GLASS) {
    1984                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    1985                 :          0 :                 auto db = static_cast<const GlassDatabase*>(sources[i]);
    1986         [ #  # ]:          0 :                 has_uncommitted_changes = db->has_uncommitted_changes();
    1987                 :            : #else
    1988                 :            :                 throw Xapian::FeatureUnavailableError("Glass backend disabled");
    1989                 :            : #endif
    1990                 :            :             } else {
    1991                 :          2 :                 auto db = static_cast<const HoneyDatabase*>(sources[i]);
    1992                 :          2 :                 has_uncommitted_changes = db->has_uncommitted_changes();
    1993                 :            :             }
    1994         [ -  + ]:          2 :             if (has_uncommitted_changes) {
    1995                 :            :                 const char * m =
    1996                 :            :                     "Can't compact from a WritableDatabase with uncommitted "
    1997                 :            :                     "changes - either call commit() first, or create a new "
    1998                 :          0 :                     "Database object from the filename on disk";
    1999 [ #  # ][ #  # ]:          0 :                 throw Xapian::InvalidOperationError(m);
                 [ #  # ]
    2000                 :            :             }
    2001                 :            :         }
    2002                 :            :     }
    2003                 :            : 
    2004 [ +  + ][ +  - ]:        298 :     FlintLock lock(destdir ? destdir : "");
                 [ +  - ]
    2005         [ +  + ]:        298 :     if (!single_file) {
    2006         [ +  - ]:        296 :         string explanation;
    2007         [ +  - ]:        296 :         FlintLock::reason why = lock.lock(true, false, explanation);
    2008         [ -  + ]:        296 :         if (why != FlintLock::SUCCESS) {
    2009         [ #  # ]:          0 :             lock.throw_databaselockerror(why, destdir, explanation);
    2010                 :        296 :         }
    2011                 :            :     }
    2012                 :            : 
    2013         [ +  - ]:        298 :     unique_ptr<HoneyVersion> version_file_out;
    2014         [ +  + ]:        298 :     if (single_file) {
    2015         [ -  + ]:          2 :         if (destdir) {
    2016         [ #  # ]:          0 :             fd = open(destdir, O_RDWR|O_CREAT|O_TRUNC|O_BINARY|O_CLOEXEC, 0666);
    2017         [ #  # ]:          0 :             if (fd < 0) {
    2018 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseCreateError("open() failed", errno);
    2019                 :            :             }
    2020                 :            :         }
    2021 [ +  - ][ +  - ]:          2 :         version_file_out.reset(new HoneyVersion(fd));
    2022                 :            :     } else {
    2023                 :        296 :         fd = -1;
    2024 [ +  - ][ +  - ]:        296 :         version_file_out.reset(new HoneyVersion(destdir));
                 [ +  - ]
    2025                 :            :     }
    2026                 :            : 
    2027                 :            :     // Set to true if stat() failed (which can happen if the files are > 2GB
    2028                 :            :     // and off_t is 32 bit) or one of the totals overflowed.
    2029                 :        298 :     bool bad_totals = false;
    2030                 :        298 :     off_t in_total = 0, out_total = 0;
    2031                 :            : 
    2032         [ +  - ]:        298 :     version_file_out->create();
    2033         [ +  + ]:        602 :     for (size_t i = 0; i != sources.size(); ++i) {
    2034                 :        304 :         bool source_single_file = false;
    2035         [ +  + ]:        304 :         if (source_backend == Xapian::DB_BACKEND_GLASS) {
    2036                 :            : #ifdef XAPIAN_HAS_GLASS_BACKEND
    2037                 :        289 :             auto db = static_cast<const GlassDatabase*>(sources[i]);
    2038                 :        289 :             auto& v_in = db->version_file;
    2039                 :        289 :             auto& v_out = version_file_out;
    2040                 :            :             // Glass backend doesn't track unique term bounds, hence setting
    2041                 :            :             // them to 0. We calculate the unique term bounds as we convert the
    2042                 :            :             // termlist table and fill in the correct values before we write out
    2043                 :            :             // version_file_out.
    2044                 :            :             v_out->merge_stats(v_in.get_doccount(),
    2045                 :            :                                v_in.get_doclength_lower_bound(),
    2046                 :            :                                v_in.get_doclength_upper_bound(),
    2047                 :            :                                v_in.get_wdf_upper_bound(),
    2048                 :            :                                v_in.get_total_doclen(),
    2049                 :            :                                v_in.get_spelling_wordfreq_upper_bound(),
    2050                 :            :                                0,
    2051         [ +  - ]:        289 :                                0);
    2052                 :        289 :             source_single_file = db->single_file();
    2053                 :            : #else
    2054                 :            :             Assert(false);
    2055                 :            : #endif
    2056                 :            :         } else {
    2057                 :         15 :             auto db = static_cast<const HoneyDatabase*>(sources[i]);
    2058         [ +  - ]:         15 :             version_file_out->merge_stats(db->version_file);
    2059                 :         15 :             source_single_file = db->single_file();
    2060                 :            :         }
    2061         [ -  + ]:        304 :         if (source_single_file) {
    2062                 :            :             // Add single file input DB sizes to in_total here.  For other
    2063                 :            :             // databases, we sum per-table below.
    2064         [ #  # ]:          0 :             string path;
    2065         [ #  # ]:          0 :             sources[i]->get_backend_info(&path);
    2066                 :          0 :             off_t db_size = file_size(path);
    2067         [ #  # ]:          0 :             if (errno == 0) {
    2068                 :            :                 // FIXME: check overflow and set bad_totals
    2069                 :          0 :                 in_total += db_size;
    2070                 :            :             } else {
    2071                 :          0 :                 bad_totals = true;
    2072                 :          0 :             }
    2073                 :            :         }
    2074                 :            :     }
    2075                 :            : 
    2076 [ +  - ][ +  - ]:        596 :     string fl_serialised;
    2077                 :            : #if 0
    2078                 :            :     if (single_file) {
    2079                 :            :         HoneyFreeList fl;
    2080                 :            :         fl.set_first_unused_block(1); // FIXME: Assumption?
    2081                 :            :         fl.pack(fl_serialised);
    2082                 :            :     }
    2083                 :            : #endif
    2084                 :            : 
    2085                 :            :     // FIXME: sort out indentation.
    2086         [ +  + ]:        298 : if (source_backend == Xapian::DB_BACKEND_GLASS) {
    2087                 :            : #ifndef XAPIAN_HAS_GLASS_BACKEND
    2088                 :            :     throw Xapian::FeatureUnavailableError("Glass backend disabled");
    2089                 :            : #else
    2090                 :        289 :     vector<HoneyTable *> tabs;
    2091         [ +  - ]:        289 :     tabs.reserve(tables_end - tables);
    2092                 :        289 :     off_t prev_size = 0;
    2093         [ +  + ]:       2023 :     for (const table_list * t = tables; t < tables_end; ++t) {
    2094                 :            :         // The postlist table requires an N-way merge, adjusting the
    2095                 :            :         // headers of various blocks.  The spelling and synonym tables also
    2096                 :            :         // need special handling.  The other tables have keys sorted in
    2097                 :            :         // docid order, so we can merge them by simply copying all the keys
    2098                 :            :         // from each source table in turn.
    2099         [ -  + ]:       1734 :         if (compactor)
    2100 [ #  # ][ #  # ]:          0 :             compactor->set_status(t->name, string());
                 [ #  # ]
    2101                 :            : 
    2102         [ +  - ]:       1734 :         string dest;
    2103         [ +  - ]:       1734 :         if (!single_file) {
    2104         [ +  - ]:       1734 :             dest = destdir;
    2105         [ +  - ]:       1734 :             dest += '/';
    2106         [ +  - ]:       1734 :             dest += t->name;
    2107         [ +  - ]:       1734 :             dest += '.';
    2108                 :            :         }
    2109                 :            : 
    2110                 :       1734 :         bool output_will_exist = !t->lazy;
    2111                 :            : 
    2112                 :            :         // Sometimes stat can fail for benign reasons (e.g. >= 2GB file
    2113                 :            :         // on certain systems).
    2114                 :       1734 :         bool bad_stat = false;
    2115                 :            : 
    2116                 :            :         // We can't currently report input sizes if there's a single file DB
    2117                 :            :         // amongst the inputs.
    2118                 :       1734 :         bool single_file_in = false;
    2119                 :            : 
    2120                 :       1734 :         off_t in_size = 0;
    2121                 :            : 
    2122      [ +  -  + ]:       3468 :         vector<const GlassTable*> inputs;
    2123         [ +  - ]:       1734 :         inputs.reserve(sources.size());
    2124                 :       1734 :         size_t inputs_present = 0;
    2125         [ +  + ]:       3468 :         for (auto src : sources) {
    2126                 :       1734 :             auto db = static_cast<const GlassDatabase*>(src);
    2127                 :            :             const GlassTable * table;
    2128   [ +  +  +  +  :       1734 :             switch (t->type) {
                +  +  - ]
    2129                 :            :                 case Honey::POSTLIST:
    2130                 :        289 :                     table = &(db->postlist_table);
    2131                 :        289 :                     break;
    2132                 :            :                 case Honey::DOCDATA:
    2133                 :        289 :                     table = &(db->docdata_table);
    2134                 :        289 :                     break;
    2135                 :            :                 case Honey::TERMLIST:
    2136                 :        289 :                     table = &(db->termlist_table);
    2137                 :        289 :                     break;
    2138                 :            :                 case Honey::POSITION:
    2139                 :        289 :                     table = &(db->position_table);
    2140                 :        289 :                     break;
    2141                 :            :                 case Honey::SPELLING:
    2142                 :        289 :                     table = &(db->spelling_table);
    2143                 :        289 :                     break;
    2144                 :            :                 case Honey::SYNONYM:
    2145                 :        289 :                     table = &(db->synonym_table);
    2146                 :        289 :                     break;
    2147                 :            :                 default:
    2148                 :            :                     Assert(false);
    2149                 :          0 :                     return;
    2150                 :            :             }
    2151                 :            : 
    2152         [ -  + ]:       1734 :             if (db->single_file()) {
    2153 [ #  # ][ #  # ]:          0 :                 if (t->lazy && !GlassCursor(table).next()) {
         [ #  # ][ #  # ]
                 [ #  # ]
           [ #  #  #  # ]
    2154                 :            :                     // Lazy table with no entries, so essentially doesn't
    2155                 :            :                     // exist.
    2156                 :            :                 } else {
    2157                 :            :                     // FIXME: Find actual size somehow?
    2158                 :            :                     // in_size += table->size() / 1024;
    2159                 :          0 :                     single_file_in = true;
    2160                 :          0 :                     output_will_exist = true;
    2161                 :          0 :                     ++inputs_present;
    2162                 :            :                 }
    2163                 :            :             } else {
    2164         [ +  - ]:       1734 :                 off_t db_size = file_size(table->get_path());
    2165         [ +  + ]:       1734 :                 if (errno == 0) {
    2166                 :            :                     // FIXME: check overflow and set bad_totals
    2167                 :       1148 :                     in_total += db_size;
    2168                 :       1148 :                     in_size += db_size / 1024;
    2169                 :       1148 :                     output_will_exist = true;
    2170                 :       1148 :                     ++inputs_present;
    2171         [ -  + ]:        586 :                 } else if (errno != ENOENT) {
    2172                 :            :                     // We get ENOENT for an optional table.
    2173                 :          0 :                     bad_totals = bad_stat = true;
    2174                 :          0 :                     output_will_exist = true;
    2175                 :          0 :                     ++inputs_present;
    2176                 :            :                 }
    2177                 :            :             }
    2178         [ +  - ]:       1734 :             inputs.push_back(table);
    2179                 :            :         }
    2180                 :            : 
    2181                 :            :         // If any inputs lack a termlist table, suppress it in the output.
    2182 [ +  + ][ -  + ]:       1734 :         if (t->type == Honey::TERMLIST && inputs_present != sources.size()) {
                 [ -  + ]
    2183         [ #  # ]:          0 :             if (inputs_present != 0) {
    2184         [ #  # ]:          0 :                 if (compactor) {
    2185         [ #  # ]:          0 :                     string m = str(inputs_present);
    2186         [ #  # ]:          0 :                     m += " of ";
    2187 [ #  # ][ #  # ]:          0 :                     m += str(sources.size());
    2188         [ #  # ]:          0 :                     m += " inputs present, so suppressing output";
    2189 [ #  # ][ #  # ]:          0 :                     compactor->set_status(t->name, m);
    2190                 :            :                 }
    2191                 :          0 :                 continue;
    2192                 :            :             }
    2193                 :          0 :             output_will_exist = false;
    2194                 :            :         }
    2195                 :            : 
    2196         [ +  + ]:       1734 :         if (!output_will_exist) {
    2197         [ -  + ]:        586 :             if (compactor)
    2198 [ #  # ][ #  # ]:          0 :                 compactor->set_status(t->name, "doesn't exist");
                 [ #  # ]
    2199                 :        586 :             continue;
    2200                 :            :         }
    2201                 :            : 
    2202                 :            :         HoneyTable * out;
    2203                 :       1148 :         off_t table_start_offset = -1;
    2204         [ -  + ]:       1148 :         if (single_file) {
    2205         [ #  # ]:          0 :             if (t == tables) {
    2206                 :            :                 // Start first table HONEY_VERSION_MAX_SIZE bytes in to allow
    2207                 :            :                 // space for version file.  It's tricky to exactly know the
    2208                 :            :                 // size of the version file beforehand.
    2209                 :          0 :                 table_start_offset = HONEY_VERSION_MAX_SIZE;
    2210         [ #  # ]:          0 :                 if (lseek(fd, table_start_offset, SEEK_SET) < 0)
    2211 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseError("lseek() failed", errno);
    2212                 :            :             } else {
    2213                 :          0 :                 table_start_offset = lseek(fd, 0, SEEK_CUR);
    2214                 :            :             }
    2215                 :          0 :             out = new HoneyTable(t->name, fd, version_file_out->get_offset(),
    2216 [ #  # ][ #  # ]:          0 :                                  false, false);
    2217                 :            :         } else {
    2218 [ +  - ][ +  - ]:       1148 :             out = new HoneyTable(t->name, dest, false, t->lazy);
    2219                 :            :         }
    2220         [ +  - ]:       1148 :         tabs.push_back(out);
    2221                 :       1148 :         Honey::RootInfo * root_info = version_file_out->root_to_set(t->type);
    2222         [ -  + ]:       1148 :         if (single_file) {
    2223         [ #  # ]:          0 :             root_info->set_free_list(fl_serialised);
    2224                 :          0 :             root_info->set_offset(table_start_offset);
    2225                 :            :             out->open(FLAGS,
    2226                 :          0 :                       version_file_out->get_root(t->type),
    2227         [ #  # ]:          0 :                       version_file_out->get_revision());
    2228                 :            :         } else {
    2229         [ +  - ]:       1148 :             out->create_and_open(FLAGS, *root_info);
    2230                 :            :         }
    2231                 :            : 
    2232   [ +  -  -  +  :       1148 :         switch (t->type) {
                      + ]
    2233                 :            :             case Honey::POSTLIST: {
    2234 [ -  + ][ #  # ]:        289 :                 if (multipass && inputs.size() > 3) {
                 [ -  + ]
    2235                 :            :                     multimerge_postlists(compactor, out, destdir,
    2236 [ #  # ][ #  # ]:          0 :                                          inputs, offset);
    2237                 :            :                 } else {
    2238                 :            :                     merge_postlists(compactor, out, offset.begin(),
    2239         [ +  - ]:        289 :                                     inputs.begin(), inputs.end());
    2240                 :            :                 }
    2241                 :        289 :                 break;
    2242                 :            :             }
    2243                 :            :             case Honey::SPELLING:
    2244         [ #  # ]:          0 :                 merge_spellings(out, inputs.cbegin(), inputs.cend());
    2245                 :          0 :                 break;
    2246                 :            :             case Honey::SYNONYM:
    2247         [ #  # ]:          0 :                 merge_synonyms(out, inputs.begin(), inputs.end());
    2248                 :          0 :                 break;
    2249                 :            :             case Honey::POSITION:
    2250         [ +  - ]:        285 :                 merge_positions(out, inputs, offset);
    2251                 :        285 :                 break;
    2252                 :            :             default: {
    2253                 :            :                 // DocData, Termlist
    2254                 :        574 :                 auto & v_out = version_file_out;
    2255                 :        574 :                 auto ut_lb = v_out->get_unique_terms_lower_bound();
    2256                 :        574 :                 auto ut_ub = v_out->get_unique_terms_upper_bound();
    2257         [ +  - ]:        574 :                 merge_docid_keyed(out, inputs, offset, ut_lb, ut_ub, t->type);
    2258                 :        574 :                 version_file_out->set_unique_terms_lower_bound(ut_lb);
    2259                 :        574 :                 version_file_out->set_unique_terms_upper_bound(ut_ub);
    2260                 :        574 :                 break;
    2261                 :            :             }
    2262                 :            :         }
    2263                 :            : 
    2264                 :            :         // Commit as revision 1.
    2265         [ +  - ]:       1148 :         out->flush_db();
    2266         [ +  - ]:       1148 :         out->commit(1, root_info);
    2267         [ +  - ]:       1148 :         out->sync();
    2268 [ -  + ][ #  # ]:       1148 :         if (single_file) fl_serialised = root_info->get_free_list();
    2269                 :            : 
    2270                 :       1148 :         off_t out_size = 0;
    2271 [ +  - ][ +  - ]:       1148 :         if (!bad_stat && !single_file_in) {
    2272                 :            :             off_t db_size;
    2273         [ -  + ]:       1148 :             if (single_file) {
    2274                 :          0 :                 db_size = file_size(fd);
    2275                 :            :             } else {
    2276         [ +  - ]:       1148 :                 db_size = file_size(dest + HONEY_TABLE_EXTENSION);
    2277                 :            :             }
    2278         [ +  - ]:       1148 :             if (errno == 0) {
    2279         [ -  + ]:       1148 :                 if (single_file) {
    2280                 :          0 :                     off_t old_prev_size = prev_size;
    2281                 :          0 :                     prev_size = db_size;
    2282                 :          0 :                     db_size -= old_prev_size;
    2283                 :            :                 }
    2284                 :            :                 // FIXME: check overflow and set bad_totals
    2285                 :       1148 :                 out_total += db_size;
    2286                 :       1148 :                 out_size = db_size / 1024;
    2287         [ #  # ]:          0 :             } else if (errno != ENOENT) {
    2288                 :       1148 :                 bad_totals = bad_stat = true;
    2289                 :            :             }
    2290                 :            :         }
    2291         [ -  + ]:       1148 :         if (bad_stat) {
    2292         [ #  # ]:          0 :             if (compactor)
    2293                 :            :                 compactor->set_status(t->name,
    2294 [ #  # ][ #  # ]:          0 :                                       "Done (couldn't stat all the DB files)");
                 [ #  # ]
    2295         [ -  + ]:       1148 :         } else if (single_file_in) {
    2296         [ #  # ]:          0 :             if (compactor)
    2297                 :            :                 compactor->set_status(t->name,
    2298                 :            :                                       "Done (table sizes unknown for single "
    2299 [ #  # ][ #  # ]:          0 :                                       "file DB input)");
                 [ #  # ]
    2300                 :            :         } else {
    2301         [ +  - ]:       1148 :             string status;
    2302         [ +  + ]:       1148 :             if (out_size == in_size) {
    2303         [ +  - ]:          8 :                 status = "Size unchanged (";
    2304                 :            :             } else {
    2305                 :            :                 off_t delta;
    2306         [ +  - ]:       1140 :                 if (out_size < in_size) {
    2307                 :       1140 :                     delta = in_size - out_size;
    2308         [ +  - ]:       1140 :                     status = "Reduced by ";
    2309                 :            :                 } else {
    2310                 :          0 :                     delta = out_size - in_size;
    2311         [ #  # ]:          0 :                     status = "INCREASED by ";
    2312                 :            :                 }
    2313         [ +  - ]:       1140 :                 if (in_size) {
    2314 [ +  - ][ +  - ]:       1140 :                     status += str(100 * delta / in_size);
    2315         [ +  - ]:       1140 :                     status += "% ";
    2316                 :            :                 }
    2317 [ +  - ][ +  - ]:       1140 :                 status += str(delta);
    2318         [ +  - ]:       1140 :                 status += "K (";
    2319 [ +  - ][ +  - ]:       1140 :                 status += str(in_size);
    2320         [ +  - ]:       1140 :                 status += "K -> ";
    2321                 :            :             }
    2322 [ +  - ][ +  - ]:       1148 :             status += str(out_size);
    2323         [ +  - ]:       1148 :             status += "K)";
    2324         [ -  + ]:       1148 :             if (compactor)
    2325 [ #  # ][ #  # ]:       1734 :                 compactor->set_status(t->name, status);
              [ +  -  + ]
    2326                 :            :         }
    2327                 :       1734 :     }
    2328                 :            : 
    2329                 :            :     // If compacting to a single file output and all the tables are empty, pad
    2330                 :            :     // the output so that it isn't mistaken for a stub database when we try to
    2331                 :            :     // open it.  For this it needs to at least HONEY_MIN_DB_SIZE in size.
    2332 [ -  + ][ #  # ]:        289 :     if (single_file && prev_size < HONEY_MIN_DB_SIZE) {
    2333                 :          0 :         out_total = HONEY_MIN_DB_SIZE;
    2334                 :            : #ifdef HAVE_FTRUNCATE
    2335         [ #  # ]:          0 :         if (ftruncate(fd, HONEY_MIN_DB_SIZE) < 0) {
    2336                 :            :             throw Xapian::DatabaseError("Failed to set size of output "
    2337 [ #  # ][ #  # ]:          0 :                                         "database", errno);
    2338                 :            :         }
    2339                 :            : #else
    2340                 :            :         const off_t off = HONEY_MIN_DB_SIZE - 1;
    2341                 :            :         if (lseek(fd, off, SEEK_SET) != off || write(fd, "", 1) != 1) {
    2342                 :            :             throw Xapian::DatabaseError("Failed to set size of output "
    2343                 :            :                                         "database", errno);
    2344                 :            :         }
    2345                 :            : #endif
    2346                 :            :     }
    2347                 :            : 
    2348         [ -  + ]:        289 :     if (single_file) {
    2349         [ #  # ]:          0 :         if (lseek(fd, version_file_out->get_offset(), SEEK_SET) == -1) {
    2350 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("lseek() failed", errno);
    2351                 :            :         }
    2352                 :            :     }
    2353                 :        289 :     version_file_out->set_last_docid(last_docid);
    2354         [ +  - ]:        289 :     string tmpfile = version_file_out->write(1, FLAGS);
    2355         [ -  + ]:        289 :     if (single_file) {
    2356                 :          0 :         off_t version_file_size = lseek(fd, 0, SEEK_CUR);
    2357         [ #  # ]:          0 :         if (version_file_size < 0) {
    2358 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("lseek() failed", errno);
    2359                 :            :         }
    2360         [ #  # ]:          0 :         if (version_file_size > HONEY_VERSION_MAX_SIZE) {
    2361                 :            :             throw Xapian::DatabaseError("Didn't allow enough space for "
    2362 [ #  # ][ #  # ]:          0 :                                         "version file data");
                 [ #  # ]
    2363                 :            :         }
    2364                 :            :     }
    2365         [ +  + ]:       1437 :     for (unsigned j = 0; j != tabs.size(); ++j) {
    2366         [ +  - ]:       1148 :         tabs[j]->sync();
    2367                 :            :     }
    2368                 :            :     // Commit with revision 1.
    2369         [ +  - ]:        289 :     version_file_out->sync(tmpfile, 1, FLAGS);
    2370         [ +  + ]:       1437 :     for (unsigned j = 0; j != tabs.size(); ++j) {
    2371         [ +  - ]:       1148 :         delete tabs[j];
    2372         [ +  - ]:        289 :     }
    2373                 :            : #endif
    2374                 :            : } else {
    2375                 :          9 :     vector<HoneyTable *> tabs;
    2376         [ +  - ]:          9 :     tabs.reserve(tables_end - tables);
    2377                 :          9 :     off_t prev_size = HONEY_MIN_DB_SIZE;
    2378         [ +  + ]:         63 :     for (const table_list * t = tables; t < tables_end; ++t) {
    2379                 :            :         // The postlist table requires an N-way merge, adjusting the
    2380                 :            :         // headers of various blocks.  The spelling and synonym tables also
    2381                 :            :         // need special handling.  The other tables have keys sorted in
    2382                 :            :         // docid order, so we can merge them by simply copying all the keys
    2383                 :            :         // from each source table in turn.
    2384         [ -  + ]:         54 :         if (compactor)
    2385 [ #  # ][ #  # ]:          0 :             compactor->set_status(t->name, string());
                 [ #  # ]
    2386                 :            : 
    2387         [ +  - ]:         54 :         string dest;
    2388         [ +  + ]:         54 :         if (!single_file) {
    2389         [ +  - ]:         42 :             dest = destdir;
    2390         [ +  - ]:         42 :             dest += '/';
    2391         [ +  - ]:         42 :             dest += t->name;
    2392         [ +  - ]:         42 :             dest += '.';
    2393                 :            :         }
    2394                 :            : 
    2395                 :         54 :         bool output_will_exist = !t->lazy;
    2396                 :            : 
    2397                 :            :         // Sometimes stat can fail for benign reasons (e.g. >= 2GB file
    2398                 :            :         // on certain systems).
    2399                 :         54 :         bool bad_stat = false;
    2400                 :            : 
    2401                 :            :         // We can't currently report input sizes if there's a single file DB
    2402                 :            :         // amongst the inputs.
    2403                 :         54 :         bool single_file_in = false;
    2404                 :            : 
    2405                 :         54 :         off_t in_size = 0;
    2406                 :            : 
    2407      [ +  -  + ]:        108 :         vector<const HoneyTable*> inputs;
    2408         [ +  - ]:         54 :         inputs.reserve(sources.size());
    2409                 :         54 :         size_t inputs_present = 0;
    2410         [ +  + ]:        144 :         for (auto src : sources) {
    2411                 :         90 :             auto db = static_cast<const HoneyDatabase*>(src);
    2412                 :            :             const HoneyTable * table;
    2413   [ +  +  +  +  :         90 :             switch (t->type) {
                +  +  - ]
    2414                 :            :                 case Honey::POSTLIST:
    2415                 :         15 :                     table = &(db->postlist_table);
    2416                 :         15 :                     break;
    2417                 :            :                 case Honey::DOCDATA:
    2418                 :         15 :                     table = &(db->docdata_table);
    2419                 :         15 :                     break;
    2420                 :            :                 case Honey::TERMLIST:
    2421                 :         15 :                     table = &(db->termlist_table);
    2422                 :         15 :                     break;
    2423                 :            :                 case Honey::POSITION:
    2424                 :         15 :                     table = &(db->position_table);
    2425                 :         15 :                     break;
    2426                 :            :                 case Honey::SPELLING:
    2427                 :         15 :                     table = &(db->spelling_table);
    2428                 :         15 :                     break;
    2429                 :            :                 case Honey::SYNONYM:
    2430                 :         15 :                     table = &(db->synonym_table);
    2431                 :         15 :                     break;
    2432                 :            :                 default:
    2433                 :            :                     Assert(false);
    2434                 :          0 :                     return;
    2435                 :            :             }
    2436                 :            : 
    2437         [ -  + ]:         90 :             if (db->single_file()) {
    2438 [ #  # ][ #  # ]:          0 :                 if (t->lazy && !HoneyCursor(table).next()) {
         [ #  # ][ #  # ]
                 [ #  # ]
           [ #  #  #  # ]
    2439                 :            :                     // Lazy table with no entries, so essentially doesn't
    2440                 :            :                     // exist.
    2441                 :            :                 } else {
    2442                 :            :                     // FIXME: Find actual size somehow?
    2443                 :            :                     // in_size += table->size() / 1024;
    2444                 :          0 :                     single_file_in = true;
    2445                 :          0 :                     output_will_exist = true;
    2446                 :          0 :                     ++inputs_present;
    2447                 :            :                 }
    2448                 :            :             } else {
    2449                 :         90 :                 off_t db_size = file_size(table->get_path());
    2450         [ +  + ]:         90 :                 if (errno == 0) {
    2451                 :            :                     // FIXME: check overflow and set bad_totals
    2452                 :         54 :                     in_total += db_size;
    2453                 :         54 :                     in_size += db_size / 1024;
    2454                 :         54 :                     output_will_exist = true;
    2455                 :         54 :                     ++inputs_present;
    2456         [ -  + ]:         36 :                 } else if (errno != ENOENT) {
    2457                 :            :                     // We get ENOENT for an optional table.
    2458                 :          0 :                     bad_totals = bad_stat = true;
    2459                 :          0 :                     output_will_exist = true;
    2460                 :          0 :                     ++inputs_present;
    2461                 :            :                 }
    2462                 :            :             }
    2463         [ +  - ]:         90 :             inputs.push_back(table);
    2464                 :            :         }
    2465                 :            : 
    2466                 :            :         // If any inputs lack a termlist table, suppress it in the output.
    2467 [ +  + ][ -  + ]:         54 :         if (t->type == Honey::TERMLIST && inputs_present != sources.size()) {
                 [ -  + ]
    2468         [ #  # ]:          0 :             if (inputs_present != 0) {
    2469         [ #  # ]:          0 :                 if (compactor) {
    2470         [ #  # ]:          0 :                     string m = str(inputs_present);
    2471         [ #  # ]:          0 :                     m += " of ";
    2472 [ #  # ][ #  # ]:          0 :                     m += str(sources.size());
    2473         [ #  # ]:          0 :                     m += " inputs present, so suppressing output";
    2474 [ #  # ][ #  # ]:          0 :                     compactor->set_status(t->name, m);
    2475                 :            :                 }
    2476                 :          0 :                 continue;
    2477                 :            :             }
    2478                 :          0 :             output_will_exist = false;
    2479                 :            :         }
    2480                 :            : 
    2481         [ +  + ]:         54 :         if (!output_will_exist) {
    2482         [ -  + ]:         22 :             if (compactor)
    2483 [ #  # ][ #  # ]:          0 :                 compactor->set_status(t->name, "doesn't exist");
                 [ #  # ]
    2484                 :         22 :             continue;
    2485                 :            :         }
    2486                 :            : 
    2487                 :            :         HoneyTable * out;
    2488                 :         32 :         off_t table_start_offset = -1;
    2489         [ +  + ]:         32 :         if (single_file) {
    2490         [ +  + ]:          8 :             if (t == tables) {
    2491                 :            :                 // Start first table HONEY_VERSION_MAX_SIZE bytes in to allow
    2492                 :            :                 // space for version file.  It's tricky to exactly know the
    2493                 :            :                 // size of the version file beforehand.
    2494                 :          2 :                 table_start_offset = HONEY_VERSION_MAX_SIZE;
    2495         [ -  + ]:          2 :                 if (lseek(fd, table_start_offset, SEEK_SET) < 0)
    2496 [ #  # ][ #  # ]:          0 :                     throw Xapian::DatabaseError("lseek() failed", errno);
    2497                 :            :             } else {
    2498                 :          6 :                 table_start_offset = lseek(fd, 0, SEEK_CUR);
    2499                 :            :             }
    2500                 :          8 :             out = new HoneyTable(t->name, fd, version_file_out->get_offset(),
    2501 [ +  - ][ +  - ]:          8 :                                  false, false);
    2502                 :            :         } else {
    2503 [ +  - ][ +  - ]:         24 :             out = new HoneyTable(t->name, dest, false, t->lazy);
    2504                 :            :         }
    2505         [ +  - ]:         32 :         tabs.push_back(out);
    2506                 :         32 :         Honey::RootInfo * root_info = version_file_out->root_to_set(t->type);
    2507         [ +  + ]:         32 :         if (single_file) {
    2508         [ +  - ]:          8 :             root_info->set_free_list(fl_serialised);
    2509                 :          8 :             root_info->set_offset(table_start_offset);
    2510                 :            :             out->open(FLAGS,
    2511                 :          8 :                       version_file_out->get_root(t->type),
    2512         [ +  - ]:         16 :                       version_file_out->get_revision());
    2513                 :            :         } else {
    2514         [ +  - ]:         24 :             out->create_and_open(FLAGS, *root_info);
    2515                 :            :         }
    2516                 :            : 
    2517   [ +  -  -  +  :         32 :         switch (t->type) {
                      + ]
    2518                 :            :             case Honey::POSTLIST: {
    2519 [ -  + ][ #  # ]:          9 :                 if (multipass && inputs.size() > 3) {
                 [ -  + ]
    2520                 :            :                     multimerge_postlists(compactor, out, destdir,
    2521 [ #  # ][ #  # ]:          0 :                                          inputs, offset);
    2522                 :            :                 } else {
    2523                 :            :                     merge_postlists(compactor, out, offset.begin(),
    2524         [ +  - ]:          9 :                                     inputs.begin(), inputs.end());
    2525                 :            :                 }
    2526                 :          9 :                 break;
    2527                 :            :             }
    2528                 :            :             case Honey::SPELLING:
    2529         [ #  # ]:          0 :                 merge_spellings(out, inputs.begin(), inputs.end());
    2530                 :          0 :                 break;
    2531                 :            :             case Honey::SYNONYM:
    2532         [ #  # ]:          0 :                 merge_synonyms(out, inputs.begin(), inputs.end());
    2533                 :          0 :                 break;
    2534                 :            :             case Honey::POSITION:
    2535         [ +  - ]:          7 :                 merge_positions(out, inputs, offset);
    2536                 :          7 :                 break;
    2537                 :            :             default:
    2538                 :            :                 // DocData, Termlist
    2539         [ +  - ]:         16 :                 merge_docid_keyed(out, inputs, offset);
    2540                 :         16 :                 break;
    2541                 :            :         }
    2542                 :            : 
    2543                 :            :         // Commit as revision 1.
    2544         [ +  - ]:         32 :         out->flush_db();
    2545         [ +  - ]:         32 :         out->commit(1, root_info);
    2546         [ +  - ]:         32 :         out->sync();
    2547 [ +  + ][ +  - ]:         32 :         if (single_file) fl_serialised = root_info->get_free_list();
    2548                 :            : 
    2549                 :         32 :         off_t out_size = 0;
    2550 [ +  - ][ +  - ]:         32 :         if (!bad_stat && !single_file_in) {
    2551                 :            :             off_t db_size;
    2552         [ +  + ]:         32 :             if (single_file) {
    2553                 :          8 :                 db_size = file_size(fd);
    2554                 :            :             } else {
    2555         [ +  - ]:         24 :                 db_size = file_size(dest + HONEY_TABLE_EXTENSION);
    2556                 :            :             }
    2557         [ +  - ]:         32 :             if (errno == 0) {
    2558         [ +  + ]:         32 :                 if (single_file) {
    2559                 :          8 :                     off_t old_prev_size = prev_size;
    2560                 :          8 :                     prev_size = db_size;
    2561                 :          8 :                     db_size -= old_prev_size;
    2562                 :            :                 }
    2563                 :            :                 // FIXME: check overflow and set bad_totals
    2564                 :         32 :                 out_total += db_size;
    2565                 :         32 :                 out_size = db_size / 1024;
    2566         [ #  # ]:          0 :             } else if (errno != ENOENT) {
    2567                 :         32 :                 bad_totals = bad_stat = true;
    2568                 :            :             }
    2569                 :            :         }
    2570         [ -  + ]:         32 :         if (bad_stat) {
    2571         [ #  # ]:          0 :             if (compactor)
    2572                 :            :                 compactor->set_status(t->name,
    2573 [ #  # ][ #  # ]:          0 :                                       "Done (couldn't stat all the DB files)");
                 [ #  # ]
    2574         [ -  + ]:         32 :         } else if (single_file_in) {
    2575         [ #  # ]:          0 :             if (compactor)
    2576                 :            :                 compactor->set_status(t->name,
    2577                 :            :                                       "Done (table sizes unknown for single "
    2578 [ #  # ][ #  # ]:          0 :                                       "file DB input)");
                 [ #  # ]
    2579                 :            :         } else {
    2580         [ +  - ]:         32 :             string status;
    2581         [ +  + ]:         32 :             if (out_size == in_size) {
    2582         [ +  - ]:         23 :                 status = "Size unchanged (";
    2583                 :            :             } else {
    2584                 :            :                 off_t delta;
    2585         [ +  + ]:          9 :                 if (out_size < in_size) {
    2586                 :          7 :                     delta = in_size - out_size;
    2587         [ +  - ]:          7 :                     status = "Reduced by ";
    2588                 :            :                 } else {
    2589                 :          2 :                     delta = out_size - in_size;
    2590         [ +  - ]:          2 :                     status = "INCREASED by ";
    2591                 :            :                 }
    2592         [ +  + ]:          9 :                 if (in_size) {
    2593 [ +  - ][ +  - ]:          7 :                     status += str(100 * delta / in_size);
    2594         [ +  - ]:          7 :                     status += "% ";
    2595                 :            :                 }
    2596 [ +  - ][ +  - ]:          9 :                 status += str(delta);
    2597         [ +  - ]:          9 :                 status += "K (";
    2598 [ +  - ][ +  - ]:          9 :                 status += str(in_size);
    2599         [ +  - ]:          9 :                 status += "K -> ";
    2600                 :            :             }
    2601 [ +  - ][ +  - ]:         32 :             status += str(out_size);
    2602         [ +  - ]:         32 :             status += "K)";
    2603         [ -  + ]:         32 :             if (compactor)
    2604 [ #  # ][ #  # ]:         54 :                 compactor->set_status(t->name, status);
              [ +  -  + ]
    2605                 :            :         }
    2606                 :         54 :     }
    2607                 :            : 
    2608                 :            :     // If compacting to a single file output and all the tables are empty, pad
    2609                 :            :     // the output so that it isn't mistaken for a stub database when we try to
    2610                 :            :     // open it.  For this it needs to at least HONEY_MIN_DB_SIZE in size.
    2611 [ +  + ][ -  + ]:          9 :     if (single_file && prev_size < HONEY_MIN_DB_SIZE) {
    2612                 :          0 :         out_total = HONEY_MIN_DB_SIZE;
    2613                 :            : #ifdef HAVE_FTRUNCATE
    2614         [ #  # ]:          0 :         if (ftruncate(fd, HONEY_MIN_DB_SIZE) < 0) {
    2615                 :            :             throw Xapian::DatabaseError("Failed to set size of output "
    2616 [ #  # ][ #  # ]:          0 :                                         "database", errno);
    2617                 :            :         }
    2618                 :            : #else
    2619                 :            :         const off_t off = HONEY_MIN_DB_SIZE - 1;
    2620                 :            :         if (lseek(fd, off, SEEK_SET) != off || write(fd, "", 1) != 1) {
    2621                 :            :             throw Xapian::DatabaseError("Failed to set size of output "
    2622                 :            :                                         "database", errno);
    2623                 :            :         }
    2624                 :            : #endif
    2625                 :            :     }
    2626                 :            : 
    2627         [ +  + ]:          9 :     if (single_file) {
    2628         [ -  + ]:          2 :         if (lseek(fd, version_file_out->get_offset(), SEEK_SET) == -1) {
    2629 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("lseek() failed", errno);
    2630                 :            :         }
    2631                 :            :     }
    2632                 :          9 :     version_file_out->set_last_docid(last_docid);
    2633         [ +  - ]:          9 :     string tmpfile = version_file_out->write(1, FLAGS);
    2634         [ +  + ]:         41 :     for (unsigned j = 0; j != tabs.size(); ++j) {
    2635         [ +  - ]:         32 :         tabs[j]->sync();
    2636                 :            :     }
    2637                 :            :     // Commit with revision 1.
    2638         [ +  - ]:          9 :     version_file_out->sync(tmpfile, 1, FLAGS);
    2639         [ +  + ]:         41 :     for (unsigned j = 0; j != tabs.size(); ++j) {
    2640         [ +  - ]:         32 :         delete tabs[j];
    2641         [ +  - ]:          9 :     }
    2642                 :            : }
    2643                 :            : 
    2644 [ +  + ][ +  - ]:        298 :     if (!single_file) lock.release();
    2645                 :            : 
    2646 [ +  - ][ -  + ]:        298 :     if (!bad_totals && compactor) {
    2647         [ #  # ]:          0 :         string status;
    2648                 :          0 :         in_total /= 1024;
    2649                 :          0 :         out_total /= 1024;
    2650         [ #  # ]:          0 :         if (out_total == in_total) {
    2651         [ #  # ]:          0 :             status = "Size unchanged (";
    2652                 :            :         } else {
    2653                 :            :             off_t delta;
    2654         [ #  # ]:          0 :             if (out_total < in_total) {
    2655                 :          0 :                 delta = in_total - out_total;
    2656         [ #  # ]:          0 :                 status = "Reduced by ";
    2657                 :            :             } else {
    2658                 :          0 :                 delta = out_total - in_total;
    2659         [ #  # ]:          0 :                 status = "INCREASED by ";
    2660                 :            :             }
    2661         [ #  # ]:          0 :             if (in_total) {
    2662 [ #  # ][ #  # ]:          0 :                 status += str(100 * delta / in_total);
    2663         [ #  # ]:          0 :                 status += "% ";
    2664                 :            :             }
    2665 [ #  # ][ #  # ]:          0 :             status += str(delta);
    2666         [ #  # ]:          0 :             status += "K (";
    2667 [ #  # ][ #  # ]:          0 :             status += str(in_total);
    2668         [ #  # ]:          0 :             status += "K -> ";
    2669                 :            :         }
    2670 [ #  # ][ #  # ]:          0 :         status += str(out_total);
    2671         [ #  # ]:          0 :         status += "K)";
    2672 [ #  # ][ #  # ]:        298 :         compactor->set_status("Total", status);
                 [ +  - ]
    2673                 :        298 :     }
    2674                 :            : }

Generated by: LCOV version 1.11