LCOV - code coverage report
Current view: top level - backends/honey - honey_values.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 4ba52dacf4fb Lines: 91 303 30.0 %
Date: 2019-05-20 14:58:19 Functions: 8 20 40.0 %
Branches: 75 516 14.5 %

           Branch data     Line data    Source code
       1                 :            : /** @file honey_values.cc
       2                 :            :  * @brief HoneyValueManager class
       3                 :            :  */
       4                 :            : /* Copyright (C) 2008,2009,2010,2011,2012,2016,2017,2018 Olly Betts
       5                 :            :  * Copyright (C) 2008,2009 Lemur Consulting Ltd
       6                 :            :  *
       7                 :            :  * This program is free software; you can redistribute it and/or modify
       8                 :            :  * it under the terms of the GNU General Public License as published by
       9                 :            :  * the Free Software Foundation; either version 2 of the License, or
      10                 :            :  * (at your option) any later version.
      11                 :            :  *
      12                 :            :  * This program is distributed in the hope that it will be useful,
      13                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      14                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15                 :            :  * GNU General Public License for more details.
      16                 :            :  *
      17                 :            :  * You should have received a copy of the GNU General Public License
      18                 :            :  * along with this program; if not, write to the Free Software
      19                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : 
      24                 :            : #include "honey_values.h"
      25                 :            : 
      26                 :            : #include "honey_cursor.h"
      27                 :            : #include "honey_postlist.h"
      28                 :            : #include "honey_postlisttable.h"
      29                 :            : #include "honey_termlist.h"
      30                 :            : #include "honey_termlisttable.h"
      31                 :            : 
      32                 :            : #include "bitstream.h"
      33                 :            : #include "debuglog.h"
      34                 :            : #include "backends/documentinternal.h"
      35                 :            : #include "pack.h"
      36                 :            : 
      37                 :            : #include "xapian/error.h"
      38                 :            : #include "xapian/valueiterator.h"
      39                 :            : 
      40                 :            : #include <algorithm>
      41                 :            : #include <memory>
      42                 :            : 
      43                 :            : using namespace Honey;
      44                 :            : using namespace std;
      45                 :            : 
      46                 :            : // FIXME:
      47                 :            : //  * multi-values?
      48                 :            : //  * values named instead of numbered?
      49                 :            : 
      50                 :            : void
      51                 :      40332 : ValueChunkReader::assign(const char * p_, size_t len, Xapian::docid last_did)
      52                 :            : {
      53                 :      40332 :     p = p_;
      54                 :      40332 :     end = p_ + len;
      55         [ -  + ]:      40332 :     if (!unpack_uint(&p, end, &did))
      56 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Failed to unpack docid delta");
                 [ #  # ]
      57                 :      40332 :     did = last_did - did;
      58         [ -  + ]:      40332 :     if (!unpack_string(&p, end, value))
      59 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Failed to unpack first value");
                 [ #  # ]
      60                 :      40332 : }
      61                 :            : 
      62                 :            : void
      63                 :       1873 : ValueChunkReader::next()
      64                 :            : {
      65         [ +  + ]:       1873 :     if (p == end) {
      66                 :        131 :         p = NULL;
      67                 :       1873 :         return;
      68                 :            :     }
      69                 :            : 
      70                 :            :     Xapian::docid delta;
      71         [ -  + ]:       1742 :     if (!unpack_uint(&p, end, &delta)) {
      72                 :            :         throw Xapian::DatabaseCorruptError("Failed to unpack streamed value "
      73 [ #  # ][ #  # ]:          0 :                                            "docid");
                 [ #  # ]
      74                 :            :     }
      75                 :       1742 :     did += delta + 1;
      76 [ +  - ][ -  + ]:       1742 :     if (!unpack_string(&p, end, value))
      77 [ #  # ][ #  # ]:       1742 :         throw Xapian::DatabaseCorruptError("Failed to unpack streamed value");
                 [ #  # ]
      78                 :            : }
      79                 :            : 
      80                 :            : void
      81                 :      76229 : ValueChunkReader::skip_to(Xapian::docid target)
      82                 :            : {
      83 [ +  - ][ +  + ]:      76229 :     if (p == NULL || target <= did)
      84                 :      76229 :         return;
      85                 :            : 
      86                 :            :     size_t value_len;
      87         [ +  + ]:    6810828 :     while (p != end) {
      88                 :            :         // Get the next docid
      89                 :            :         Xapian::docid delta;
      90         [ -  + ]:    6806794 :         if (rare(!unpack_uint(&p, end, &delta))) {
      91                 :            :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed "
      92 [ #  # ][ #  # ]:          0 :                                                "value docid");
                 [ #  # ]
      93                 :            :         }
      94                 :    6806794 :         did += delta + 1;
      95                 :            : 
      96                 :            :         // Get the length of the string
      97         [ -  + ]:    6806794 :         if (rare(!unpack_uint(&p, end, &value_len))) {
      98                 :            :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed "
      99 [ #  # ][ #  # ]:          0 :                                                "value length");
                 [ #  # ]
     100                 :            :         }
     101                 :            : 
     102                 :            :         // Check that it's not too long
     103         [ -  + ]:    6806794 :         if (rare(value_len > size_t(end - p))) {
     104                 :            :             throw Xapian::DatabaseCorruptError("Failed to unpack streamed "
     105 [ #  # ][ #  # ]:          0 :                                                "value");
                 [ #  # ]
     106                 :            :         }
     107                 :            : 
     108                 :            :         // Assign the value and return only if we've reached the target
     109         [ +  + ]:    6806794 :         if (did >= target) {
     110         [ +  - ]:      65727 :             value.assign(p, value_len);
     111                 :      65727 :             p += value_len;
     112                 :      65727 :             return;
     113                 :            :         }
     114                 :    6741067 :         p += value_len;
     115                 :            :     }
     116                 :       4034 :     p = NULL;
     117                 :            : }
     118                 :            : 
     119                 :            : void
     120                 :          0 : HoneyValueManager::add_value(Xapian::docid did, Xapian::valueno slot,
     121                 :            :                              const string & val)
     122                 :            : {
     123         [ #  # ]:          0 :     auto i = changes.find(slot);
     124         [ #  # ]:          0 :     if (i == changes.end()) {
     125 [ #  # ][ #  # ]:          0 :         i = changes.insert(make_pair(slot, map<Xapian::docid, string>())).first;
     126                 :            :     }
     127 [ #  # ][ #  # ]:          0 :     i->second[did] = val;
     128                 :          0 : }
     129                 :            : 
     130                 :            : void
     131                 :          0 : HoneyValueManager::remove_value(Xapian::docid did, Xapian::valueno slot)
     132                 :            : {
     133         [ #  # ]:          0 :     auto i = changes.find(slot);
     134         [ #  # ]:          0 :     if (i == changes.end()) {
     135 [ #  # ][ #  # ]:          0 :         i = changes.insert(make_pair(slot, map<Xapian::docid, string>())).first;
     136                 :            :     }
     137 [ #  # ][ #  # ]:          0 :     i->second[did] = string();
                 [ #  # ]
     138                 :          0 : }
     139                 :            : 
     140                 :            : Xapian::docid
     141                 :      33495 : HoneyValueManager::get_chunk_containing_did(Xapian::valueno slot,
     142                 :            :                                             Xapian::docid did,
     143                 :            :                                             string &chunk) const
     144                 :            : {
     145                 :            :     LOGCALL(DB, Xapian::docid, "HoneyValueManager::get_chunk_containing_did", slot | did | chunk);
     146         [ +  + ]:      33495 :     if (!cursor.get())
     147                 :         21 :         cursor.reset(postlist_table.cursor_get());
     148         [ -  + ]:      33495 :     if (!cursor.get()) RETURN(0);
     149                 :            : 
     150         [ +  + ]:      33495 :     bool exact = cursor->find_entry_ge(make_valuechunk_key(slot, did));
     151         [ +  + ]:      33494 :     if (!exact) {
     152                 :            :         // The chunk doesn't end with docid did, so we need to get
     153                 :            :         // the last docid in the chunk from the key to return.
     154                 :      32292 :         did = docid_from_key(slot, cursor->current_key);
     155                 :            :     }
     156                 :            : 
     157                 :      33494 :     cursor->read_tag();
     158                 :      33494 :     chunk = cursor->current_tag;
     159                 :            :     // FIXME: fails, not sure why: swap(chunk, cursor->current_tag);
     160                 :            : 
     161                 :      33494 :     RETURN(did);
     162                 :            : }
     163                 :            : 
     164                 :            : static const size_t CHUNK_SIZE_THRESHOLD = 2000;
     165                 :            : 
     166                 :            : namespace Honey {
     167                 :            : 
     168                 :            : class ValueUpdater {
     169                 :            :     HoneyPostListTable& table;
     170                 :            : 
     171                 :            :     Xapian::valueno slot;
     172                 :            : 
     173                 :            :     string ctag;
     174                 :            : 
     175                 :            :     ValueChunkReader reader;
     176                 :            : 
     177                 :            :     string tag;
     178                 :            : 
     179                 :            :     Xapian::docid prev_did;
     180                 :            : 
     181                 :            :     Xapian::docid last_did;
     182                 :            : 
     183                 :            :     Xapian::docid new_last_did;
     184                 :            : 
     185                 :            :     Xapian::docid last_allowed_did;
     186                 :            : 
     187                 :          0 :     void append_to_stream(Xapian::docid did, const string & value) {
     188                 :            :         Assert(did);
     189         [ #  # ]:          0 :         if (!tag.empty()) {
     190                 :            :             AssertRel(did,>,prev_did);
     191                 :          0 :             pack_uint(tag, did - prev_did - 1);
     192                 :            :         }
     193                 :          0 :         prev_did = did;
     194                 :          0 :         new_last_did = did;
     195                 :          0 :         pack_string(tag, value);
     196         [ #  # ]:          0 :         if (tag.size() >= CHUNK_SIZE_THRESHOLD) write_tag();
     197                 :          0 :     }
     198                 :            : 
     199                 :          0 :     void write_tag() {
     200                 :            :         // If the last docid has changed, delete the old entry.
     201 [ #  # ][ #  # ]:          0 :         if (last_did && new_last_did != last_did) {
     202                 :          0 :             table.del(make_valuechunk_key(slot, last_did));
     203                 :            :         }
     204         [ #  # ]:          0 :         if (!tag.empty()) {
     205         [ #  # ]:          0 :             table.add(make_valuechunk_key(slot, new_last_did), tag);
     206                 :            :         }
     207                 :          0 :         last_did = 0;
     208                 :          0 :         tag.resize(0);
     209                 :          0 :     }
     210                 :            : 
     211                 :            :   public:
     212                 :          0 :     ValueUpdater(HoneyPostListTable& table_, Xapian::valueno slot_)
     213 [ #  # ][ #  # ]:          0 :         : table(table_), slot(slot_), last_did(0), last_allowed_did(0) { }
     214                 :            : 
     215                 :          0 :     ~ValueUpdater() {
     216         [ #  # ]:          0 :         while (!reader.at_end()) {
     217                 :            :             // FIXME: use skip_to and some splicing magic instead?
     218                 :          0 :             append_to_stream(reader.get_docid(), reader.get_value());
     219                 :          0 :             reader.next();
     220                 :            :         }
     221                 :          0 :         write_tag();
     222                 :          0 :     }
     223                 :            : 
     224                 :          0 :     void update(Xapian::docid did, const string & value) {
     225 [ #  # ][ #  # ]:          0 :         if (last_allowed_did && did > last_allowed_did) {
     226                 :            :             // The next change needs to go in a later existing chunk than the
     227                 :            :             // one we're currently updating, so we copy over the rest of the
     228                 :            :             // entries from the current chunk, write out the updated chunk and
     229                 :            :             // drop through to the case below will read in that later chunk.
     230                 :            :             // FIXME: use some string splicing magic instead of this loop.
     231         [ #  # ]:          0 :             while (!reader.at_end()) {
     232                 :            :                 // last_allowed_did should be an upper bound for this chunk.
     233                 :            :                 AssertRel(reader.get_docid(),<=,last_allowed_did);
     234                 :          0 :                 append_to_stream(reader.get_docid(), reader.get_value());
     235                 :          0 :                 reader.next();
     236                 :            :             }
     237                 :          0 :             write_tag();
     238                 :          0 :             last_allowed_did = 0;
     239                 :            :         }
     240         [ #  # ]:          0 :         if (last_allowed_did == 0) {
     241                 :          0 :             last_allowed_did = HONEY_MAX_DOCID;
     242                 :            :             Assert(tag.empty());
     243                 :          0 :             new_last_did = 0;
     244         [ #  # ]:          0 :             unique_ptr<HoneyCursor> cursor(table.cursor_get());
     245 [ #  # ][ #  # ]:          0 :             if (cursor->find_entry_ge(make_valuechunk_key(slot, did))) {
                 [ #  # ]
     246                 :            :                 // We found an exact match, so the last docid is the one
     247                 :            :                 // we looked for.
     248                 :          0 :                 last_did = did;
     249                 :            :             } else {
     250                 :            :                 Assert(!cursor->after_end());
     251                 :            :                 // Otherwise we need to unpack it from the key we found.
     252                 :            :                 // We may have found a non-value-chunk entry in which case
     253                 :            :                 // docid_from_key() returns 0.
     254         [ #  # ]:          0 :                 last_did = docid_from_key(slot, cursor->current_key);
     255                 :            :             }
     256                 :            : 
     257                 :            :             // If there are no further chunks, then the last docid that can go
     258                 :            :             // in this chunk is the highest valid docid.  If there are further
     259                 :            :             // chunks then it's one less than the first docid of the next
     260                 :            :             // chunk.
     261         [ #  # ]:          0 :             if (last_did) {
     262                 :            :                 // We found a value chunk.
     263         [ #  # ]:          0 :                 cursor->read_tag();
     264                 :            :                 // FIXME:swap(cursor->current_tag, ctag);
     265         [ #  # ]:          0 :                 ctag = cursor->current_tag;
     266         [ #  # ]:          0 :                 reader.assign(ctag.data(), ctag.size(), last_did);
     267                 :            :             }
     268 [ #  # ][ #  # ]:          0 :             if (cursor->next()) {
     269                 :          0 :                 const string & key = cursor->current_key;
     270         [ #  # ]:          0 :                 Xapian::docid next_last_did = docid_from_key(slot, key);
     271         [ #  # ]:          0 :                 if (next_last_did) {
     272         [ #  # ]:          0 :                     cursor->read_tag();
     273                 :            :                     Xapian::docid delta;
     274                 :          0 :                     const char* p = cursor->current_tag.data();
     275                 :          0 :                     const char* e = p + cursor->current_tag.size();
     276         [ #  # ]:          0 :                     if (!unpack_uint(&p, e, &delta)) {
     277                 :            :                         throw Xapian::DatabaseCorruptError("Failed to unpack "
     278 [ #  # ][ #  # ]:          0 :                                                            "docid delta");
                 [ #  # ]
     279                 :            :                     }
     280                 :          0 :                     Xapian::docid next_first_did = next_last_did - delta;
     281                 :          0 :                     last_allowed_did = next_first_did - 1;
     282                 :            :                 }
     283                 :            :                 Assert(last_allowed_did);
     284                 :            :                 AssertRel(last_allowed_did,>=,last_did);
     285                 :          0 :             }
     286                 :            :         }
     287                 :            : 
     288                 :            :         // Copy over entries until we get to the one we want to
     289                 :            :         // add/modify/delete.
     290                 :            :         // FIXME: use skip_to and some splicing magic instead?
     291 [ #  # ][ #  # ]:          0 :         while (!reader.at_end() && reader.get_docid() < did) {
                 [ #  # ]
     292                 :          0 :             append_to_stream(reader.get_docid(), reader.get_value());
     293                 :          0 :             reader.next();
     294                 :            :         }
     295 [ #  # ][ #  # ]:          0 :         if (!reader.at_end() && reader.get_docid() == did) reader.next();
                 [ #  # ]
     296         [ #  # ]:          0 :         if (!value.empty()) {
     297                 :            :             // Add/update entry for did.
     298                 :          0 :             append_to_stream(did, value);
     299                 :            :         }
     300                 :          0 :     }
     301                 :            : };
     302                 :            : 
     303                 :            : }
     304                 :            : 
     305                 :            : void
     306                 :          0 : HoneyValueManager::merge_changes()
     307                 :            : {
     308         [ #  # ]:          0 :     for (auto&& i : changes) {
     309                 :          0 :         Xapian::valueno slot = i.first;
     310         [ #  # ]:          0 :         Honey::ValueUpdater updater(postlist_table, slot);
     311         [ #  # ]:          0 :         for (auto&& j : i.second) {
     312         [ #  # ]:          0 :             updater.update(j.first, j.second);
     313                 :            :         }
     314                 :          0 :     }
     315                 :          0 :     changes.clear();
     316                 :          0 : }
     317                 :            : 
     318                 :            : string
     319                 :          0 : HoneyValueManager::add_document(Xapian::docid did, const Xapian::Document &doc,
     320                 :            :                                 map<Xapian::valueno, ValueStats> &val_stats)
     321                 :            : {
     322         [ #  # ]:          0 :     Xapian::ValueIterator it = doc.values_begin();
     323         [ #  # ]:          0 :     if (it == doc.values_end()) {
     324                 :            :         // No document values.
     325         [ #  # ]:          0 :         auto i = slots.find(did);
     326         [ #  # ]:          0 :         if (i != slots.end()) {
     327                 :            :             // Document's values already added or modified in this batch.
     328 [ #  # ][ #  # ]:          0 :             i->second = string();
     329                 :            :         }
     330         [ #  # ]:          0 :         return string();
     331                 :            :     }
     332                 :            : 
     333         [ #  # ]:          0 :     Xapian::valueno count = doc.internal->values_count();
     334         [ #  # ]:          0 :     Xapian::VecCOW<Xapian::termpos> slotvec(count);
     335                 :            : 
     336         [ #  # ]:          0 :     Xapian::valueno first_slot = it.get_valueno();
     337                 :          0 :     Xapian::valueno last_slot = first_slot;
     338         [ #  # ]:          0 :     while (it != doc.values_end()) {
     339         [ #  # ]:          0 :         Xapian::valueno slot = it.get_valueno();
     340         [ #  # ]:          0 :         slotvec.push_back(slot);
     341         [ #  # ]:          0 :         const string& value = *it;
     342                 :            : 
     343                 :            :         // Update the statistics.
     344 [ #  # ][ #  # ]:          0 :         auto i = val_stats.insert(make_pair(slot, ValueStats()));
     345                 :          0 :         ValueStats& stats = i.first->second;
     346         [ #  # ]:          0 :         if (i.second) {
     347                 :            :             // There were no statistics stored already, so read them.
     348         [ #  # ]:          0 :             get_value_stats(slot, stats);
     349                 :            :         }
     350                 :            : 
     351                 :            :         // Now, modify the stored statistics.
     352         [ #  # ]:          0 :         if ((stats.freq)++ == 0) {
     353                 :            :             // If the value count was previously zero, set the upper and lower
     354                 :            :             // bounds to the newly added value.
     355         [ #  # ]:          0 :             stats.lower_bound = value;
     356         [ #  # ]:          0 :             stats.upper_bound = value;
     357                 :            :         } else {
     358                 :            :             // Otherwise, simply make sure they reflect the new value.
     359                 :            :             //
     360                 :            :             // Check the upper bound first, as for some common uses of value
     361                 :            :             // slots (dates) the values will tend to get larger not smaller
     362                 :            :             // over time.
     363         [ #  # ]:          0 :             int cmp = value.compare(stats.upper_bound);
     364         [ #  # ]:          0 :             if (cmp >= 0) {
     365 [ #  # ][ #  # ]:          0 :                 if (cmp > 0) stats.upper_bound = value;
     366 [ #  # ][ #  # ]:          0 :             } else if (value < stats.lower_bound) {
     367         [ #  # ]:          0 :                 stats.lower_bound = value;
     368                 :            :             }
     369                 :            :         }
     370                 :            : 
     371         [ #  # ]:          0 :         add_value(did, slot, value);
     372                 :          0 :         last_slot = slot;
     373         [ #  # ]:          0 :         ++it;
     374                 :          0 :     }
     375                 :            : 
     376         [ #  # ]:          0 :     if (!termlist_table.is_open()) {
     377         [ #  # ]:          0 :         return string();
     378                 :            :     }
     379                 :            : 
     380         [ #  # ]:          0 :     string enc;
     381         [ #  # ]:          0 :     pack_uint(enc, last_slot);
     382         [ #  # ]:          0 :     BitWriter slots_used(enc);
     383         [ #  # ]:          0 :     if (count > 1) {
     384         [ #  # ]:          0 :         slots_used.encode(first_slot, last_slot);
     385         [ #  # ]:          0 :         slots_used.encode(count - 2, last_slot - first_slot);
     386         [ #  # ]:          0 :         slots_used.encode_interpolative(slotvec, 0, slotvec.size() - 1);
     387                 :            :     }
     388                 :            : 
     389 [ #  # ][ #  # ]:          0 :     return slots_used.freeze();
     390                 :            : }
     391                 :            : 
     392                 :            : void
     393                 :          0 : HoneyValueManager::delete_document(Xapian::docid did,
     394                 :            :                                    map<Xapian::valueno, ValueStats>& val_stats)
     395                 :            : {
     396                 :            :     Assert(termlist_table.is_open());
     397         [ #  # ]:          0 :     map<Xapian::docid, string>::iterator it = slots.find(did);
     398         [ #  # ]:          0 :     string s;
     399         [ #  # ]:          0 :     if (it != slots.end()) {
     400         [ #  # ]:          0 :         swap(s, it->second);
     401                 :            :     } else {
     402                 :            :         // Get from table, making a swift exit if this document has no terms or
     403                 :            :         // values.
     404 [ #  # ][ #  # ]:          0 :         if (!termlist_table.get_exact_entry(termlist_table.make_key(did), s))
                 [ #  # ]
     405                 :          0 :             return;
     406 [ #  # ][ #  # ]:          0 :         slots.insert(make_pair(did, string()));
     407                 :            :     }
     408                 :            : 
     409                 :          0 :     const char* p = s.data();
     410                 :          0 :     const char* end = p + s.size();
     411                 :            :     size_t slot_enc_size;
     412         [ #  # ]:          0 :     if (!unpack_uint(&p, end, &slot_enc_size)) {
     413 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Termlist encoding corrupt");
                 [ #  # ]
     414                 :            :     }
     415         [ #  # ]:          0 :     if (slot_enc_size == 0)
     416                 :          0 :         return;
     417                 :            : 
     418                 :          0 :     end = p + slot_enc_size;
     419                 :            :     Xapian::valueno last_slot;
     420         [ #  # ]:          0 :     if (!unpack_uint(&p, end, &last_slot)) {
     421 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Slots used data corrupt");
                 [ #  # ]
     422                 :            :     }
     423                 :            : 
     424         [ #  # ]:          0 :     if (p != end) {
     425                 :          0 :         BitReader rd(p, end);
     426         [ #  # ]:          0 :         Xapian::valueno first_slot = rd.decode(last_slot);
     427         [ #  # ]:          0 :         Xapian::valueno slot_count = rd.decode(last_slot - first_slot) + 2;
     428         [ #  # ]:          0 :         rd.decode_interpolative(0, slot_count - 1, first_slot, last_slot);
     429                 :            : 
     430                 :          0 :         Xapian::valueno slot = first_slot;
     431         [ #  # ]:          0 :         while (slot != last_slot) {
     432 [ #  # ][ #  # ]:          0 :             auto i = val_stats.insert(make_pair(slot, ValueStats()));
     433                 :          0 :             ValueStats & stats = i.first->second;
     434         [ #  # ]:          0 :             if (i.second) {
     435                 :            :                 // There were no statistics stored already, so read them.
     436         [ #  # ]:          0 :                 get_value_stats(slot, stats);
     437                 :            :             }
     438                 :            : 
     439                 :            :             // Now, modify the stored statistics.
     440                 :            :             AssertRelParanoid(stats.freq, >, 0);
     441         [ #  # ]:          0 :             if (--(stats.freq) == 0) {
     442         [ #  # ]:          0 :                 stats.lower_bound.resize(0);
     443         [ #  # ]:          0 :                 stats.upper_bound.resize(0);
     444                 :            :             }
     445                 :            : 
     446         [ #  # ]:          0 :             remove_value(did, slot);
     447                 :            : 
     448         [ #  # ]:          0 :             slot = rd.decode_interpolative_next();
     449                 :          0 :         }
     450                 :            :     }
     451                 :            : 
     452                 :          0 :     Xapian::valueno slot = last_slot;
     453                 :            :     {
     454                 :            :         {
     455                 :            :             // FIXME: share code with above
     456 [ #  # ][ #  # ]:          0 :             auto i = val_stats.insert(make_pair(slot, ValueStats()));
     457                 :          0 :             ValueStats & stats = i.first->second;
     458         [ #  # ]:          0 :             if (i.second) {
     459                 :            :                 // There were no statistics stored already, so read them.
     460         [ #  # ]:          0 :                 get_value_stats(slot, stats);
     461                 :            :             }
     462                 :            : 
     463                 :            :             // Now, modify the stored statistics.
     464                 :            :             AssertRelParanoid(stats.freq, >, 0);
     465         [ #  # ]:          0 :             if (--(stats.freq) == 0) {
     466         [ #  # ]:          0 :                 stats.lower_bound.resize(0);
     467         [ #  # ]:          0 :                 stats.upper_bound.resize(0);
     468                 :            :             }
     469                 :            : 
     470 [ #  # ][ #  # ]:          0 :             remove_value(did, slot);
     471                 :            :         }
     472                 :          0 :     }
     473                 :            : }
     474                 :            : 
     475                 :            : string
     476                 :          0 : HoneyValueManager::replace_document(Xapian::docid did,
     477                 :            :                                     const Xapian::Document &doc,
     478                 :            :                                     map<Xapian::valueno, ValueStats>& val_stats)
     479                 :            : {
     480         [ #  # ]:          0 :     if (doc.get_docid() == did) {
     481                 :            :         // If we're replacing a document with itself, but the optimisation for
     482                 :            :         // this higher up hasn't kicked in (e.g. because we've added/replaced
     483                 :            :         // a document since this one was read) and the values haven't changed,
     484                 :            :         // then the call to delete_document() below will remove the values
     485                 :            :         // before the subsequent add_document() can read them.
     486                 :            :         //
     487                 :            :         // The simplest way to handle this is to force the document to read its
     488                 :            :         // values, which we only need to do this is the docid matches.  Note
     489                 :            :         // that this check can give false positives as we don't also check the
     490                 :            :         // database, so for example replacing document 4 in one database with
     491                 :            :         // document 4 from another will unnecessarily trigger this, but forcing
     492                 :            :         // the values to be read is fairly harmless, and this is unlikely to be
     493                 :            :         // a common case.  (Currently we will end up fetching the values
     494                 :            :         // anyway, but there's scope to change that in the future).
     495                 :          0 :         doc.internal->ensure_values_fetched();
     496                 :            :     }
     497                 :          0 :     delete_document(did, val_stats);
     498                 :          0 :     return add_document(did, doc, val_stats);
     499                 :            : }
     500                 :            : 
     501                 :            : string
     502                 :      33495 : HoneyValueManager::get_value(Xapian::docid did, Xapian::valueno slot) const
     503                 :            : {
     504         [ +  - ]:      33495 :     auto i = changes.find(slot);
     505         [ -  + ]:      33495 :     if (i != changes.end()) {
     506         [ #  # ]:          0 :         auto j = i->second.find(did);
     507 [ #  # ][ #  # ]:          0 :         if (j != i->second.end()) return j->second;
     508                 :            :     }
     509                 :            : 
     510                 :            :     // Read it from the table.
     511         [ +  - ]:      33495 :     string chunk;
     512                 :            :     Xapian::docid last_did;
     513         [ +  + ]:      33495 :     last_did = get_chunk_containing_did(slot, did, chunk);
     514 [ +  + ][ +  - ]:      33494 :     if (last_did == 0) return string();
     515                 :            : 
     516         [ +  - ]:      66982 :     ValueChunkReader reader(chunk.data(), chunk.size(), last_did);
     517         [ +  - ]:      33491 :     reader.skip_to(did);
     518 [ +  - ][ +  + ]:      33491 :     if (reader.at_end() || reader.get_docid() != did) return string();
         [ +  + ][ +  - ]
     519         [ +  - ]:      63804 :     return reader.get_value();
     520                 :            : }
     521                 :            : 
     522                 :            : void
     523                 :         50 : HoneyValueManager::get_all_values(map<Xapian::valueno, string> & values,
     524                 :            :                                   Xapian::docid did) const
     525                 :            : {
     526                 :            :     Assert(values.empty());
     527         [ -  + ]:         50 :     if (!termlist_table.is_open()) {
     528                 :            :         // Either the database has been closed, or else there's no termlist
     529                 :            :         // table.  Check if the postlist table is open to determine which is
     530                 :            :         // the case.
     531         [ #  # ]:          0 :         if (!postlist_table.is_open())
     532         [ #  # ]:          0 :             HoneyTable::throw_database_closed();
     533 [ #  # ][ #  # ]:          0 :         throw Xapian::FeatureUnavailableError("Database has no termlist");
                 [ #  # ]
     534                 :            :     }
     535                 :            : 
     536         [ +  - ]:         50 :     string s;
     537 [ +  - ][ +  - ]:         50 :     if (!termlist_table.get_exact_entry(termlist_table.make_key(did), s))
                 [ -  + ]
     538                 :          0 :         return;
     539                 :            : 
     540                 :         50 :     const char* p = s.data();
     541                 :         50 :     const char* end = p + s.size();
     542                 :         50 :     size_t slot_enc_size = *p++;
     543                 :            : 
     544         [ -  + ]:         50 :     if ((slot_enc_size & 0x80) == 0) {
     545                 :            :         // If the top bit is clear we have a 7-bit bitmap of slots used.
     546                 :          0 :         Xapian::valueno slot = 0;
     547         [ #  # ]:          0 :         while (slot_enc_size) {
     548         [ #  # ]:          0 :             if (slot_enc_size & 1) {
     549 [ #  # ][ #  # ]:          0 :                 values.insert(make_pair(slot, get_value(did, slot)));
     550                 :            :             }
     551                 :          0 :             ++slot;
     552                 :          0 :             slot_enc_size >>= 1;
     553                 :            :         }
     554                 :          0 :         return;
     555                 :            :     }
     556                 :            : 
     557                 :         50 :     slot_enc_size &= 0x7f;
     558         [ -  + ]:         50 :     if (slot_enc_size == 0) {
     559         [ #  # ]:          0 :         if (!unpack_uint(&p, end, &slot_enc_size)) {
     560 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseCorruptError("Termlist encoding corrupt");
                 [ #  # ]
     561                 :            :         }
     562                 :            :     }
     563                 :            : 
     564                 :         50 :     end = p + slot_enc_size;
     565                 :            :     Xapian::valueno last_slot;
     566         [ -  + ]:         50 :     if (!unpack_uint(&p, end, &last_slot)) {
     567 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseCorruptError("Slots used data corrupt");
                 [ #  # ]
     568                 :            :     }
     569                 :            : 
     570         [ +  - ]:         50 :     if (p != end) {
     571                 :         50 :         BitReader rd(p, end);
     572         [ +  - ]:         50 :         Xapian::valueno first_slot = rd.decode(last_slot);
     573         [ +  - ]:         50 :         Xapian::valueno slot_count = rd.decode(last_slot - first_slot) + 2;
     574         [ +  - ]:         50 :         rd.decode_interpolative(0, slot_count - 1, first_slot, last_slot);
     575                 :            : 
     576                 :         50 :         Xapian::valueno slot = first_slot;
     577         [ +  + ]:        700 :         while (slot != last_slot) {
     578 [ +  - ][ +  - ]:        650 :             values.insert(make_pair(slot, get_value(did, slot)));
     579         [ +  - ]:        650 :             slot = rd.decode_interpolative_next();
     580                 :         50 :         }
     581                 :            :     }
     582 [ +  - ][ +  - ]:         50 :     values.insert(make_pair(last_slot, get_value(did, last_slot)));
                 [ +  - ]
     583                 :            : }
     584                 :            : 
     585                 :            : void
     586                 :         46 : HoneyValueManager::get_value_stats(Xapian::valueno slot) const
     587                 :            : {
     588                 :            :     LOGCALL_VOID(DB, "HoneyValueManager::get_value_stats", slot);
     589                 :            :     // Invalidate the cache first in case an exception is thrown.
     590                 :         46 :     mru_slot = Xapian::BAD_VALUENO;
     591                 :         46 :     get_value_stats(slot, mru_valstats);
     592                 :         46 :     mru_slot = slot;
     593                 :         46 : }
     594                 :            : 
     595                 :            : void
     596                 :         46 : HoneyValueManager::get_value_stats(Xapian::valueno slot,
     597                 :            :                                    ValueStats& stats) const
     598                 :            : {
     599                 :            :     LOGCALL_VOID(DB, "HoneyValueManager::get_value_stats", slot | Literal("[stats]"));
     600                 :            : 
     601         [ +  - ]:         46 :     string tag;
     602 [ +  - ][ +  - ]:         46 :     if (postlist_table.get_exact_entry(Honey::make_valuestats_key(slot), tag)) {
                 [ +  + ]
     603                 :         43 :         const char * pos = tag.data();
     604                 :         43 :         const char * end = pos + tag.size();
     605                 :            : 
     606         [ -  + ]:         43 :         if (!unpack_uint(&pos, end, &(stats.freq))) {
     607         [ #  # ]:          0 :             if (pos == 0) {
     608                 :            :                 throw Xapian::DatabaseCorruptError("Incomplete stats item in "
     609 [ #  # ][ #  # ]:          0 :                                                    "value table");
                 [ #  # ]
     610                 :            :             }
     611                 :            :             throw Xapian::RangeError("Frequency statistic in value table is "
     612 [ #  # ][ #  # ]:          0 :                                      "too large");
                 [ #  # ]
     613                 :            :         }
     614 [ +  - ][ -  + ]:         43 :         if (!unpack_string(&pos, end, stats.lower_bound)) {
     615         [ #  # ]:          0 :             if (pos == 0) {
     616                 :            :                 throw Xapian::DatabaseCorruptError("Incomplete stats item in "
     617 [ #  # ][ #  # ]:          0 :                                                    "value table");
                 [ #  # ]
     618                 :            :             }
     619                 :            :             throw Xapian::RangeError("Lower bound in value table is too "
     620 [ #  # ][ #  # ]:          0 :                                      "large");
                 [ #  # ]
     621                 :            :         }
     622                 :         43 :         size_t len = end - pos;
     623         [ +  + ]:         43 :         if (len == 0) {
     624         [ +  - ]:          1 :             stats.upper_bound = stats.lower_bound;
     625                 :            :         } else {
     626         [ +  - ]:         43 :             stats.upper_bound.assign(pos, len);
     627                 :            :         }
     628                 :            :     } else {
     629         [ +  - ]:          3 :         stats.clear();
     630                 :         46 :     }
     631                 :         46 : }
     632                 :            : 
     633                 :            : void
     634                 :          0 : HoneyValueManager::set_value_stats(map<Xapian::valueno, ValueStats>& val_stats)
     635                 :            : {
     636                 :            :     LOGCALL_VOID(DB, "HoneyValueManager::set_value_stats", val_stats);
     637                 :          0 :     map<Xapian::valueno, ValueStats>::const_iterator i;
     638         [ #  # ]:          0 :     for (i = val_stats.begin(); i != val_stats.end(); ++i) {
     639         [ #  # ]:          0 :         string key = Honey::make_valuestats_key(i->first);
     640                 :          0 :         const ValueStats & stats = i->second;
     641         [ #  # ]:          0 :         if (stats.freq != 0) {
     642                 :            :             postlist_table.add(key, encode_valuestats(stats.freq,
     643                 :            :                                                       stats.lower_bound,
     644 [ #  # ][ #  # ]:          0 :                                                       stats.upper_bound));
     645                 :            :         } else {
     646                 :          0 :             postlist_table.del(key);
     647                 :            :         }
     648                 :          0 :     }
     649                 :          0 :     val_stats.clear();
     650                 :          0 :     mru_slot = Xapian::BAD_VALUENO;
     651                 :          0 : }

Generated by: LCOV version 1.11