LCOV - code coverage report
Current view: top level - backends/glass - glass_values.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 954b5873a738 Lines: 263 279 94.3 %
Date: 2019-06-30 05:20:33 Functions: 22 22 100.0 %
Branches: 270 506 53.4 %

           Branch data     Line data    Source code
       1                 :            : /** @file glass_values.cc
       2                 :            :  * @brief GlassValueManager class
       3                 :            :  */
       4                 :            : /* Copyright (C) 2008,2009,2010,2011,2012,2016,2017 Olly Betts
       5                 :            :  * Copyright (C) 2008,2009 Lemur Consulting Ltd
       6                 :            :  *
       7                 :            :  * This program is free software; you can redistribute it and/or modify
       8                 :            :  * it under the terms of the GNU General Public License as published by
       9                 :            :  * the Free Software Foundation; either version 2 of the License, or
      10                 :            :  * (at your option) any later version.
      11                 :            :  *
      12                 :            :  * This program is distributed in the hope that it will be useful,
      13                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      14                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15                 :            :  * GNU General Public License for more details.
      16                 :            :  *
      17                 :            :  * You should have received a copy of the GNU General Public License
      18                 :            :  * along with this program; if not, write to the Free Software
      19                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : 
      24                 :            : #include "glass_values.h"
      25                 :            : 
      26                 :            : #include "glass_cursor.h"
      27                 :            : #include "glass_postlist.h"
      28                 :            : #include "glass_termlist.h"
      29                 :            : #include "debuglog.h"
      30                 :            : #include "backends/documentinternal.h"
      31                 :            : #include "pack.h"
      32                 :            : 
      33                 :            : #include "xapian/error.h"
      34                 :            : #include "xapian/valueiterator.h"
      35                 :            : 
      36                 :            : #include <algorithm>
      37                 :            : #include <memory>
      38                 :            : 
      39                 :            : using namespace Glass;
      40                 :            : using namespace std;
      41                 :            : 
      42                 :            : // FIXME:
      43                 :            : //  * put the "used slots" entry in the same termlist tag as the terms?
      44                 :            : //  * multi-values?
      45                 :            : //  * values named instead of numbered?
      46                 :            : 
      47                 :            : /** Generate a key for the "used slots" data. */
      48                 :            : static inline string
      49                 :     348671 : make_slot_key(Xapian::docid did)
      50                 :            : {
      51                 :            :     LOGCALL_STATIC(DB, string, "make_slot_key", did);
      52                 :            :     // Add an extra character so that it can't clash with a termlist entry key
      53                 :            :     // and will sort just after the corresponding termlist entry key.
      54                 :            :     // FIXME: should we store this in the *same entry* as the list of terms?
      55                 :     348671 :     string key;
      56         [ +  - ]:     348671 :     pack_uint_preserving_sort(key, did);
      57         [ +  - ]:     348671 :     key += '\0';
      58                 :     348671 :     RETURN(key);
      59                 :            : }
      60                 :            : 
      61                 :            : /** Generate a key for a value statistics item. */
      62                 :            : static inline string
      63                 :     451585 : make_valuestats_key(Xapian::valueno slot)
      64                 :            : {
      65                 :            :     LOGCALL_STATIC(DB, string, "make_valuestats_key", slot);
      66         [ +  - ]:     451585 :     string key("\0\xd0", 2);
      67         [ +  - ]:     451585 :     pack_uint_last(key, slot);
      68                 :     451585 :     RETURN(key);
      69                 :            : }
      70                 :            : 
      71                 :            : void
      72                 :    1964077 : ValueChunkReader::assign(const char * p_, size_t len, Xapian::docid did_)
      73                 :            : {
      74                 :    1964077 :     p = p_;
      75                 :    1964077 :     end = p_ + len;
      76                 :    1964077 :     did = did_;
      77         [ -  + ]:    1964077 :     if (!unpack_string(&p, end, value))
      78 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Failed to unpack first value");
                 [ #  # ]
      79                 :    1964077 : }
      80                 :            : 
      81                 :            : void
      82                 :     708587 : ValueChunkReader::next()
      83                 :            : {
      84         [ +  + ]:     708587 :     if (p == end) {
      85                 :      19531 :         p = NULL;
      86                 :     708587 :         return;
      87                 :            :     }
      88                 :            : 
      89                 :            :     Xapian::docid delta;
      90         [ -  + ]:     689056 :     if (!unpack_uint(&p, end, &delta))
      91 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Failed to unpack streamed value docid");
                 [ #  # ]
      92                 :     689056 :     did += delta + 1;
      93 [ +  - ][ -  + ]:     689056 :     if (!unpack_string(&p, end, value))
      94 [ #  # ][ #  # ]:     689056 :         throw Xapian::DatabaseCorruptError("Failed to unpack streamed value");
                 [ #  # ]
      95                 :            : }
      96                 :            : 
      97                 :            : void
      98                 :    2073824 : ValueChunkReader::skip_to(Xapian::docid target)
      99                 :            : {
     100 [ +  - ][ +  + ]:    2073824 :     if (p == NULL || target <= did)
     101                 :    2073824 :         return;
     102                 :            : 
     103                 :            :     size_t value_len;
     104         [ +  + ]:  364123130 :     while (p != end) {
     105                 :            :         // Get the next docid
     106                 :            :         Xapian::docid delta;
     107         [ -  + ]:  364103822 :         if (rare(!unpack_uint(&p, end, &delta)))
     108 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed value docid");
                 [ #  # ]
     109                 :  364103822 :         did += delta + 1;
     110                 :            : 
     111                 :            :         // Get the length of the string
     112         [ -  + ]:  364103822 :         if (rare(!unpack_uint(&p, end, &value_len))) {
     113 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed value length");
                 [ #  # ]
     114                 :            :         }
     115                 :            : 
     116                 :            :         // Check that it's not too long
     117         [ -  + ]:  364103822 :         if (rare(value_len > size_t(end - p))) {
     118 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed value");
                 [ #  # ]
     119                 :            :         }
     120                 :            : 
     121                 :            :         // Assign the value and return only if we've reached the target
     122         [ +  + ]:  364103822 :         if (did >= target) {
     123         [ +  - ]:    1970822 :             value.assign(p, value_len);
     124                 :    1970822 :             p += value_len;
     125                 :    1970822 :             return;
     126                 :            :         }
     127                 :  362133000 :         p += value_len;
     128                 :            :     }
     129                 :      19308 :     p = NULL;
     130                 :            : }
     131                 :            : 
     132                 :            : void
     133                 :     546762 : GlassValueManager::add_value(Xapian::docid did, Xapian::valueno slot,
     134                 :            :                              const string & val)
     135                 :            : {
     136         [ +  - ]:     546762 :     auto i = changes.find(slot);
     137         [ +  + ]:     546762 :     if (i == changes.end()) {
     138 [ +  - ][ +  - ]:     225312 :         i = changes.insert(make_pair(slot, map<Xapian::docid, string>())).first;
                 [ +  - ]
     139                 :            :     }
     140 [ +  - ][ +  - ]:     546762 :     i->second[did] = val;
     141                 :     546762 : }
     142                 :            : 
     143                 :            : void
     144                 :      10879 : GlassValueManager::remove_value(Xapian::docid did, Xapian::valueno slot)
     145                 :            : {
     146         [ +  - ]:      10879 :     auto i = changes.find(slot);
     147         [ +  + ]:      10879 :     if (i == changes.end()) {
     148 [ +  - ][ +  - ]:        191 :         i = changes.insert(make_pair(slot, map<Xapian::docid, string>())).first;
                 [ +  - ]
     149                 :            :     }
     150 [ +  - ][ +  - ]:      10879 :     i->second[did] = string();
                 [ +  - ]
     151                 :      10879 : }
     152                 :            : 
     153                 :            : Xapian::docid
     154                 :    1911257 : GlassValueManager::get_chunk_containing_did(Xapian::valueno slot,
     155                 :            :                                             Xapian::docid did,
     156                 :            :                                             string &chunk) const
     157                 :            : {
     158                 :            :     LOGCALL(DB, Xapian::docid, "GlassValueManager::get_chunk_containing_did", slot | did | chunk);
     159         [ +  + ]:    1911257 :     if (!cursor.get())
     160                 :        149 :         cursor.reset(postlist_table->cursor_get());
     161         [ -  + ]:    1911257 :     if (!cursor.get()) RETURN(0);
     162                 :            : 
     163         [ +  - ]:    1911257 :     bool exact = cursor->find_entry(make_valuechunk_key(slot, did));
     164         [ +  + ]:    1911257 :     if (!exact) {
     165                 :            :         // If we didn't find a chunk starting with docid did, then we need
     166                 :            :         // to check if the chunk contains did.
     167                 :    1843517 :         const char * p = cursor->current_key.data();
     168                 :    1843517 :         const char * end = p + cursor->current_key.size();
     169                 :            : 
     170                 :            :         // Check that it is a value stream chunk.
     171 [ +  + ][ +  - ]:    1843550 :         if (end - p < 2 || *p++ != '\0' || *p++ != '\xd8') RETURN(0);
         [ -  + ][ +  + ]
     172                 :            : 
     173                 :            :         // Check that it's for the right value slot.
     174                 :            :         Xapian::valueno v;
     175         [ -  + ]:    1841511 :         if (!unpack_uint(&p, end, &v)) {
     176 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Bad value key");
                 [ #  # ]
     177                 :            :         }
     178         [ +  + ]:    1841511 :         if (v != slot) RETURN(0);
     179                 :            : 
     180                 :            :         // And get the first docid for the chunk so we can return it.
     181 [ +  - ][ -  + ]:    1841478 :         if (!unpack_uint_preserving_sort(&p, end, &did) || p != end) {
                 [ -  + ]
     182 [ #  # ][ #  # ]:    1841478 :             throw Xapian::DatabaseCorruptError("Bad value key");
                 [ #  # ]
     183                 :            :         }
     184                 :            :     }
     185                 :            : 
     186                 :    1909218 :     cursor->read_tag();
     187                 :    1909218 :     swap(chunk, cursor->current_tag);
     188                 :            : 
     189                 :    1911257 :     RETURN(did);
     190                 :            : }
     191                 :            : 
     192                 :            : static const size_t CHUNK_SIZE_THRESHOLD = 2000;
     193                 :            : 
     194                 :            : namespace Glass {
     195                 :            : 
     196                 :            : class ValueUpdater {
     197                 :            :     GlassPostListTable * table;
     198                 :            : 
     199                 :            :     Xapian::valueno slot;
     200                 :            : 
     201                 :            :     string ctag;
     202                 :            : 
     203                 :            :     ValueChunkReader reader;
     204                 :            : 
     205                 :            :     string tag;
     206                 :            : 
     207                 :            :     Xapian::docid prev_did;
     208                 :            : 
     209                 :            :     Xapian::docid first_did;
     210                 :            : 
     211                 :            :     Xapian::docid new_first_did;
     212                 :            : 
     213                 :            :     Xapian::docid last_allowed_did;
     214                 :            : 
     215                 :     586188 :     void append_to_stream(Xapian::docid did, const string & value) {
     216                 :            :         Assert(did);
     217         [ +  + ]:     586188 :         if (tag.empty()) {
     218                 :     231318 :             new_first_did = did;
     219                 :            :         } else {
     220                 :            :             AssertRel(did,>,prev_did);
     221                 :     354870 :             pack_uint(tag, did - prev_did - 1);
     222                 :            :         }
     223                 :     586188 :         prev_did = did;
     224                 :     586188 :         pack_string(tag, value);
     225         [ +  + ]:     586188 :         if (tag.size() >= CHUNK_SIZE_THRESHOLD) write_tag();
     226                 :     586188 :     }
     227                 :            : 
     228                 :     231435 :     void write_tag() {
     229                 :            :         // If the first docid has changed, delete the old entry.
     230 [ +  + ][ +  + ]:     231435 :         if (first_did && new_first_did != first_did) {
     231         [ +  - ]:        182 :             table->del(make_valuechunk_key(slot, first_did));
     232                 :            :         }
     233         [ +  + ]:     231435 :         if (!tag.empty()) {
     234 [ +  - ][ +  - ]:     231318 :             table->add(make_valuechunk_key(slot, new_first_did), tag);
     235                 :            :         }
     236                 :     231435 :         first_did = 0;
     237                 :     231435 :         tag.resize(0);
     238                 :     231435 :     }
     239                 :            : 
     240                 :            :   public:
     241                 :     225502 :     ValueUpdater(GlassPostListTable * table_, Xapian::valueno slot_)
     242 [ +  - ][ +  - ]:     225502 :         : table(table_), slot(slot_), first_did(0), last_allowed_did(0) { }
     243                 :            : 
     244                 :     451004 :     ~ValueUpdater() {
     245         [ +  + ]:     234501 :         while (!reader.at_end()) {
     246                 :            :             // FIXME: use skip_to and some splicing magic instead?
     247                 :       8999 :             append_to_stream(reader.get_docid(), reader.get_value());
     248                 :       8999 :             reader.next();
     249                 :            :         }
     250                 :     225502 :         write_tag();
     251                 :     225502 :     }
     252                 :            : 
     253                 :     547489 :     void update(Xapian::docid did, const string & value) {
     254 [ +  + ][ +  + ]:     547489 :         if (last_allowed_did && did > last_allowed_did) {
     255                 :            :             // The next change needs to go in a later existing chunk than the
     256                 :            :             // one we're currently updating, so we copy over the rest of the
     257                 :            :             // entries from the current chunk, write out the updated chunk and
     258                 :            :             // drop through to the case below will read in that later chunk.
     259                 :            :             // FIXME: use some string splicing magic instead of this loop.
     260         [ +  + ]:       3324 :             while (!reader.at_end()) {
     261                 :            :                 // last_allowed_did should be an upper bound for this chunk.
     262                 :            :                 AssertRel(reader.get_docid(),<=,last_allowed_did);
     263                 :       3265 :                 append_to_stream(reader.get_docid(), reader.get_value());
     264                 :       3265 :                 reader.next();
     265                 :            :             }
     266                 :         59 :             write_tag();
     267                 :         59 :             last_allowed_did = 0;
     268                 :            :         }
     269         [ +  + ]:     547489 :         if (last_allowed_did == 0) {
     270                 :     225561 :             last_allowed_did = GLASS_MAX_DOCID;
     271                 :            :             Assert(tag.empty());
     272                 :     225561 :             new_first_did = 0;
     273         [ +  - ]:     225561 :             unique_ptr<GlassCursor> cursor(table->cursor_get());
     274 [ +  - ][ +  - ]:     225561 :             if (cursor->find_entry(make_valuechunk_key(slot, did))) {
                 [ +  + ]
     275                 :            :                 // We found an exact match, so the first docid is the one
     276                 :            :                 // we looked for.
     277                 :        206 :                 first_did = did;
     278                 :            :             } else {
     279                 :            :                 Assert(!cursor->after_end());
     280                 :            :                 // Otherwise we need to unpack it from the key we found.
     281                 :            :                 // We may have found a non-value-chunk entry in which case
     282                 :            :                 // docid_from_key() returns 0.
     283         [ +  - ]:     225355 :                 first_did = docid_from_key(slot, cursor->current_key);
     284                 :            :             }
     285                 :            : 
     286                 :            :             // If there are no further chunks, then the last docid that can go
     287                 :            :             // in this chunk is the highest valid docid.  If there are further
     288                 :            :             // chunks then it's one less than the first docid of the next
     289                 :            :             // chunk.
     290         [ +  + ]:     225561 :             if (first_did) {
     291                 :            :                 // We found a value chunk.
     292         [ +  - ]:        272 :                 cursor->read_tag();
     293                 :            :                 // FIXME:swap(cursor->current_tag, ctag);
     294         [ +  - ]:        272 :                 ctag = cursor->current_tag;
     295         [ +  - ]:        272 :                 reader.assign(ctag.data(), ctag.size(), first_did);
     296                 :            :             }
     297 [ +  - ][ +  + ]:     225561 :             if (cursor->next()) {
     298                 :     225534 :                 const string & key = cursor->current_key;
     299         [ +  - ]:     225534 :                 Xapian::docid next_first_did = docid_from_key(slot, key);
     300         [ +  + ]:     225534 :                 if (next_first_did) last_allowed_did = next_first_did - 1;
     301                 :            :                 Assert(last_allowed_did);
     302                 :            :                 AssertRel(last_allowed_did,>=,first_did);
     303                 :     225561 :             }
     304                 :            :         }
     305                 :            : 
     306                 :            :         // Copy over entries until we get to the one we want to
     307                 :            :         // add/modify/delete.
     308                 :            :         // FIXME: use skip_to and some splicing magic instead?
     309 [ +  + ][ +  + ]:     578348 :         while (!reader.at_end() && reader.get_docid() < did) {
                 [ +  + ]
     310                 :      30859 :             append_to_stream(reader.get_docid(), reader.get_value());
     311                 :      30859 :             reader.next();
     312                 :            :         }
     313 [ +  + ][ +  - ]:     547489 :         if (!reader.at_end() && reader.get_docid() == did) reader.next();
                 [ +  + ]
     314         [ +  + ]:     547489 :         if (!value.empty()) {
     315                 :            :             // Add/update entry for did.
     316                 :     543065 :             append_to_stream(did, value);
     317                 :            :         }
     318                 :     547489 :     }
     319                 :            : };
     320                 :            : 
     321                 :            : }
     322                 :            : 
     323                 :            : void
     324                 :      18614 : GlassValueManager::merge_changes()
     325                 :            : {
     326         [ +  + ]:      18614 :     if (termlist_table->is_open()) {
     327 [ +  - ][ +  + ]:      83140 :         for (auto i : slots) {
     328         [ +  - ]:     129058 :             string key = make_slot_key(i.first);
     329                 :      64529 :             const string& enc = i.second;
     330         [ +  + ]:      64529 :             if (!enc.empty()) {
     331 [ +  - ][ +  - ]:      60483 :                 termlist_table->add(key, enc);
     332                 :            :             } else {
     333         [ +  - ]:       4046 :                 termlist_table->del(key);
     334                 :            :             }
     335                 :      64529 :         }
     336                 :      18611 :         slots.clear();
     337                 :            :     }
     338                 :            : 
     339 [ +  - ][ +  + ]:     244116 :     for (auto i : changes) {
     340                 :     225502 :         Xapian::valueno slot = i.first;
     341         [ +  - ]:     451004 :         Glass::ValueUpdater updater(postlist_table, slot);
     342                 :     225502 :         const map<Xapian::docid, string>& slot_changes = i.second;
     343 [ +  - ][ +  + ]:     772991 :         for (auto j : slot_changes) {
     344         [ +  - ]:     547489 :             updater.update(j.first, j.second);
     345                 :     547489 :         }
     346                 :     225502 :     }
     347                 :      18614 :     changes.clear();
     348                 :      18614 : }
     349                 :            : 
     350                 :            : void
     351                 :     132291 : GlassValueManager::add_document(Xapian::docid did, const Xapian::Document &doc,
     352                 :            :                                 map<Xapian::valueno, ValueStats> & value_stats)
     353                 :            : {
     354                 :            :     // FIXME: Use BitWriter and interpolative coding?  Or is it not worthwhile
     355                 :            :     // for this?
     356         [ +  - ]:     132291 :     string slots_used;
     357                 :     132291 :     Xapian::valueno prev_slot = static_cast<Xapian::valueno>(-1);
     358         [ +  - ]:     264582 :     Xapian::ValueIterator it = doc.values_begin();
     359         [ +  + ]:     679053 :     while (it != doc.values_end()) {
     360         [ +  - ]:     546762 :         Xapian::valueno slot = it.get_valueno();
     361         [ +  - ]:     546762 :         string value = *it;
     362                 :            :         Assert(!value.empty());
     363                 :            : 
     364                 :            :         // Update the statistics.
     365                 :     546762 :         std::pair<map<Xapian::valueno, ValueStats>::iterator, bool> i;
     366 [ +  - ][ +  - ]:     546762 :         i = value_stats.insert(make_pair(slot, ValueStats()));
     367                 :     546762 :         ValueStats & stats = i.first->second;
     368         [ +  + ]:     546762 :         if (i.second) {
     369                 :            :             // There were no statistics stored already, so read them.
     370         [ +  - ]:     225312 :             get_value_stats(slot, stats);
     371                 :            :         }
     372                 :            : 
     373                 :            :         // Now, modify the stored statistics.
     374         [ +  + ]:     546762 :         if ((stats.freq)++ == 0) {
     375                 :            :             // If the value count was previously zero, set the upper and lower
     376                 :            :             // bounds to the newly added value.
     377         [ +  - ]:     225311 :             stats.lower_bound = value;
     378         [ +  - ]:     225311 :             stats.upper_bound = value;
     379                 :            :         } else {
     380                 :            :             // Otherwise, simply make sure they reflect the new value.
     381 [ +  - ][ +  + ]:     321451 :             if (value < stats.lower_bound) {
     382         [ +  - ]:       5011 :                 stats.lower_bound = value;
     383 [ +  - ][ +  + ]:     316440 :             } else if (value > stats.upper_bound) {
     384         [ +  - ]:       5843 :                 stats.upper_bound = value;
     385                 :            :             }
     386                 :            :         }
     387                 :            : 
     388         [ +  - ]:     546762 :         add_value(did, slot, value);
     389         [ +  + ]:     546762 :         if (termlist_table->is_open()) {
     390         [ +  - ]:     546761 :             pack_uint(slots_used, slot - prev_slot - 1);
     391                 :     546761 :             prev_slot = slot;
     392                 :            :         }
     393         [ +  - ]:     546762 :         ++it;
     394                 :     546762 :     }
     395 [ +  + ][ +  - ]:     132291 :     if (slots_used.empty() && slots.find(did) == slots.end()) {
         [ +  + ][ +  + ]
         [ +  + ][ +  +  
             #  #  #  # ]
     396                 :            :         // Adding a new document with no values which we didn't just remove.
     397                 :            :     } else {
     398 [ +  - ][ +  - ]:      68994 :         swap(slots[did], slots_used);
     399                 :     132291 :     }
     400                 :     132291 : }
     401                 :            : 
     402                 :            : void
     403                 :      25200 : GlassValueManager::delete_document(Xapian::docid did,
     404                 :            :                                    map<Xapian::valueno, ValueStats> & value_stats)
     405                 :            : {
     406                 :            :     Assert(termlist_table->is_open());
     407         [ +  - ]:      25200 :     map<Xapian::docid, string>::iterator it = slots.find(did);
     408         [ +  - ]:      25200 :     string s;
     409         [ +  + ]:      25200 :     if (it != slots.end()) {
     410         [ +  - ]:       4509 :         swap(s, it->second);
     411                 :            :     } else {
     412                 :            :         // Get from table, making a swift exit if this document has no values.
     413 [ +  - ][ +  - ]:      45891 :         if (!termlist_table->get_exact_entry(make_slot_key(did), s)) return;
                 [ +  + ]
     414 [ +  - ][ +  - ]:       6904 :         slots.insert(make_pair(did, string()));
                 [ +  - ]
     415                 :            :     }
     416                 :      11413 :     const char * p = s.data();
     417                 :      11413 :     const char * end = p + s.size();
     418                 :      11413 :     Xapian::valueno prev_slot = static_cast<Xapian::valueno>(-1);
     419 [ +  + ][ +  + ]:      36079 :     while (p != end) {
     420                 :            :         Xapian::valueno slot;
     421         [ -  + ]:      10879 :         if (!unpack_uint(&p, end, &slot)) {
     422 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Value slot encoding corrupt");
                 [ #  # ]
     423                 :            :         }
     424                 :      10879 :         slot += prev_slot + 1;
     425                 :      10879 :         prev_slot = slot;
     426                 :            : 
     427                 :      10879 :         std::pair<map<Xapian::valueno, ValueStats>::iterator, bool> i;
     428 [ +  - ][ +  - ]:      10879 :         i = value_stats.insert(make_pair(slot, ValueStats()));
     429                 :      10879 :         ValueStats & stats = i.first->second;
     430         [ +  + ]:      10879 :         if (i.second) {
     431                 :            :             // There were no statistics stored already, so read them.
     432         [ +  - ]:        191 :             get_value_stats(slot, stats);
     433                 :            :         }
     434                 :            : 
     435                 :            :         // Now, modify the stored statistics.
     436                 :            :         AssertRelParanoid(stats.freq, >, 0);
     437         [ +  + ]:      10879 :         if (--(stats.freq) == 0) {
     438         [ +  - ]:        102 :             stats.lower_bound.resize(0);
     439         [ +  - ]:        102 :             stats.upper_bound.resize(0);
     440                 :            :         }
     441                 :            : 
     442         [ +  - ]:      10879 :         remove_value(did, slot);
     443                 :      25200 :     }
     444                 :            : }
     445                 :            : 
     446                 :            : void
     447                 :      12278 : GlassValueManager::replace_document(Xapian::docid did,
     448                 :            :                                     const Xapian::Document &doc,
     449                 :            :                                     map<Xapian::valueno, ValueStats> & value_stats)
     450                 :            : {
     451         [ +  + ]:      12278 :     if (doc.get_docid() == did) {
     452                 :            :         // If we're replacing a document with itself, but the optimisation for
     453                 :            :         // this higher up hasn't kicked in (e.g. because we've added/replaced
     454                 :            :         // a document since this one was read) and the values haven't changed,
     455                 :            :         // then the call to delete_document() below will remove the values
     456                 :            :         // before the subsequent add_document() can read them.
     457                 :            :         //
     458                 :            :         // The simplest way to handle this is to force the document to read its
     459                 :            :         // values, which we only need to do this is the docid matches.  Note
     460                 :            :         // that this check can give false positives as we don't also check the
     461                 :            :         // database, so for example replacing document 4 in one database with
     462                 :            :         // document 4 from another will unnecessarily trigger this, but forcing
     463                 :            :         // the values to be read is fairly harmless, and this is unlikely to be
     464                 :            :         // a common case.
     465                 :         70 :         doc.internal->ensure_values_fetched();
     466                 :            :     }
     467                 :      12278 :     delete_document(did, value_stats);
     468                 :      12278 :     add_document(did, doc, value_stats);
     469                 :      12278 : }
     470                 :            : 
     471                 :            : string
     472                 :    1923604 : GlassValueManager::get_value(Xapian::docid did, Xapian::valueno slot) const
     473                 :            : {
     474         [ +  - ]:    1923604 :     auto i = changes.find(slot);
     475         [ +  + ]:    1923604 :     if (i != changes.end()) {
     476         [ +  - ]:      24401 :         auto j = i->second.find(did);
     477 [ +  + ][ +  - ]:      24401 :         if (j != i->second.end()) return j->second;
     478                 :            :     }
     479                 :            : 
     480                 :            :     // Read it from the table.
     481         [ +  - ]:    1911257 :     string chunk;
     482                 :            :     Xapian::docid first_did;
     483         [ +  - ]:    1911257 :     first_did = get_chunk_containing_did(slot, did, chunk);
     484 [ +  + ][ +  - ]:    1911257 :     if (first_did == 0) return string();
     485                 :            : 
     486         [ +  - ]:    3818436 :     ValueChunkReader reader(chunk.data(), chunk.size(), first_did);
     487         [ +  - ]:    1909218 :     reader.skip_to(did);
     488 [ +  + ][ +  + ]:    1909218 :     if (reader.at_end() || reader.get_docid() != did) return string();
         [ +  + ][ +  - ]
     489         [ +  - ]:    3822614 :     return reader.get_value();
     490                 :            : }
     491                 :            : 
     492                 :            : void
     493                 :     274363 : GlassValueManager::get_all_values(map<Xapian::valueno, string> & values,
     494                 :            :                                   Xapian::docid did) const
     495                 :            : {
     496                 :            :     Assert(values.empty());
     497         [ -  + ]:     274363 :     if (!termlist_table->is_open()) {
     498                 :            :         // Either the database has been closed, or else there's no termlist
     499                 :            :         // table.  Check if the postlist table is open to determine which is
     500                 :            :         // the case.
     501         [ #  # ]:          0 :         if (!postlist_table->is_open())
     502                 :          0 :             GlassTable::throw_database_closed();
     503 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Database has no termlist");
                 [ #  # ]
     504                 :            :     }
     505         [ +  - ]:     274363 :     map<Xapian::docid, string>::const_iterator i = slots.find(did);
     506         [ +  - ]:     274363 :     string s;
     507         [ +  + ]:     274363 :     if (i != slots.end()) {
     508         [ +  - ]:      10912 :         s = i->second;
     509                 :            :     } else {
     510                 :            :         // Get from table.
     511 [ +  - ][ +  - ]:     537814 :         if (!termlist_table->get_exact_entry(make_slot_key(did), s)) return;
                 [ +  + ]
     512                 :            :     }
     513                 :     248170 :     const char * p = s.data();
     514                 :     248170 :     const char * end = p + s.size();
     515                 :     248170 :     Xapian::valueno prev_slot = static_cast<Xapian::valueno>(-1);
     516 [ +  + ][ +  + ]:    1995669 :     while (p != end) {
     517                 :            :         Xapian::valueno slot;
     518         [ -  + ]:    1721306 :         if (!unpack_uint(&p, end, &slot)) {
     519 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Value slot encoding corrupt");
                 [ #  # ]
     520                 :            :         }
     521                 :    1721306 :         slot += prev_slot + 1;
     522                 :    1721306 :         prev_slot = slot;
     523 [ +  - ][ +  - ]:    1721306 :         values.insert(make_pair(slot, get_value(did, slot)));
                 [ +  - ]
     524                 :     274363 :     }
     525                 :            : }
     526                 :            : 
     527                 :            : void
     528                 :        580 : GlassValueManager::get_value_stats(Xapian::valueno slot) const
     529                 :            : {
     530                 :            :     LOGCALL_VOID(DB, "GlassValueManager::get_value_stats", slot);
     531                 :            :     // Invalidate the cache first in case an exception is thrown.
     532                 :        580 :     mru_slot = Xapian::BAD_VALUENO;
     533                 :        580 :     get_value_stats(slot, mru_valstats);
     534                 :        580 :     mru_slot = slot;
     535                 :        580 : }
     536                 :            : 
     537                 :            : void
     538                 :     226083 : GlassValueManager::get_value_stats(Xapian::valueno slot, ValueStats & stats) const
     539                 :            : {
     540                 :            :     LOGCALL_VOID(DB, "GlassValueManager::get_value_stats", slot | Literal("[stats]"));
     541                 :            : 
     542         [ +  - ]:     226083 :     string tag;
     543 [ +  - ][ +  - ]:     226083 :     if (postlist_table->get_exact_entry(make_valuestats_key(slot), tag)) {
                 [ +  + ]
     544                 :        744 :         const char * pos = tag.data();
     545                 :        744 :         const char * end = pos + tag.size();
     546                 :            : 
     547         [ -  + ]:        744 :         if (!unpack_uint(&pos, end, &(stats.freq))) {
     548 [ #  # ][ #  # ]:          0 :             if (pos == 0) throw Xapian::DatabaseCorruptError("Incomplete stats item in value table");
         [ #  # ][ #  # ]
     549 [ #  # ][ #  # ]:          0 :             throw Xapian::RangeError("Frequency statistic in value table is too large");
                 [ #  # ]
     550                 :            :         }
     551 [ +  - ][ -  + ]:        744 :         if (!unpack_string(&pos, end, stats.lower_bound)) {
     552 [ #  # ][ #  # ]:          0 :             if (pos == 0) throw Xapian::DatabaseCorruptError("Incomplete stats item in value table");
         [ #  # ][ #  # ]
     553 [ #  # ][ #  # ]:          0 :             throw Xapian::RangeError("Lower bound in value table is too large");
                 [ #  # ]
     554                 :            :         }
     555 [ -  + ][ #  # ]:        744 :         if (stats.lower_bound.empty() && stats.freq != 0) {
                 [ -  + ]
     556                 :            :             Assert(false);
     557                 :            :             // This shouldn't happen, but let's code defensively and set the
     558                 :            :             // smallest valid lower bound (a single zero byte) if it somehow
     559                 :            :             // does (it's been reported by a notmuch user).
     560         [ #  # ]:          0 :             stats.lower_bound.assign(1, '\0');
     561                 :            :         }
     562                 :        744 :         size_t len = end - pos;
     563         [ +  + ]:        744 :         if (len == 0) {
     564         [ +  - ]:        235 :             stats.upper_bound = stats.lower_bound;
     565                 :            :         } else {
     566         [ +  - ]:        744 :             stats.upper_bound.assign(pos, len);
     567                 :            :         }
     568                 :            :     } else {
     569         [ +  - ]:     225339 :         stats.clear();
     570                 :     226083 :     }
     571                 :     226083 : }
     572                 :            : 
     573                 :            : void
     574                 :      13252 : GlassValueManager::set_value_stats(map<Xapian::valueno, ValueStats> & value_stats)
     575                 :            : {
     576                 :            :     LOGCALL_VOID(DB, "GlassValueManager::set_value_stats", value_stats);
     577                 :      13252 :     map<Xapian::valueno, ValueStats>::const_iterator i;
     578         [ +  + ]:     238754 :     for (i = value_stats.begin(); i != value_stats.end(); ++i) {
     579         [ +  - ]:     225502 :         string key = make_valuestats_key(i->first);
     580                 :     225502 :         const ValueStats & stats = i->second;
     581         [ +  + ]:     225502 :         if (stats.freq != 0) {
     582         [ +  - ]:     225425 :             string new_value;
     583         [ +  - ]:     225425 :             pack_uint(new_value, stats.freq);
     584         [ +  - ]:     225425 :             pack_string(new_value, stats.lower_bound);
     585                 :            :             // We don't store or count empty values, so neither of the bounds
     586                 :            :             // can be empty.  So we can safely store an empty upper bound when
     587                 :            :             // the bounds are equal.
     588 [ +  - ][ +  + ]:     225425 :             if (stats.lower_bound != stats.upper_bound)
     589         [ +  - ]:       4294 :                 new_value += stats.upper_bound;
     590 [ +  - ][ +  - ]:     225425 :             postlist_table->add(key, new_value);
     591                 :            :         } else {
     592         [ +  - ]:         77 :             postlist_table->del(key);
     593                 :            :         }
     594                 :     225502 :     }
     595                 :      13252 :     value_stats.clear();
     596                 :      13252 :     mru_slot = Xapian::BAD_VALUENO;
     597                 :      13252 : }

Generated by: LCOV version 1.11