LCOV - code coverage report
Current view: top level - common - io_utils.cc (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core 954b5873a738 Lines: 63 116 54.3 %
Date: 2019-06-30 05:20:33 Functions: 9 11 81.8 %
Branches: 30 120 25.0 %

           Branch data     Line data    Source code
       1                 :            : /** @file io_utils.cc
       2                 :            :  * @brief Wrappers for low-level POSIX I/O routines.
       3                 :            :  */
       4                 :            : /* Copyright (C) 2004,2006,2007,2008,2009,2011,2012,2014,2015,2016,2018 Olly Betts
       5                 :            :  * Copyright (C) 2010 Richard Boulton
       6                 :            :  *
       7                 :            :  * This program is free software; you can redistribute it and/or modify
       8                 :            :  * it under the terms of the GNU General Public License as published by
       9                 :            :  * the Free Software Foundation; either version 2 of the License, or
      10                 :            :  * (at your option) any later version.
      11                 :            :  *
      12                 :            :  * This program is distributed in the hope that it will be useful,
      13                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      14                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15                 :            :  * GNU General Public License for more details.
      16                 :            :  *
      17                 :            :  * You should have received a copy of the GNU General Public License
      18                 :            :  * along with this program; if not, write to the Free Software
      19                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
      20                 :            :  */
      21                 :            : 
      22                 :            : #include <config.h>
      23                 :            : 
      24                 :            : #include "io_utils.h"
      25                 :            : #include "posixy_wrapper.h"
      26                 :            : 
      27                 :            : #include "safeunistd.h"
      28                 :            : 
      29                 :            : #include <cerrno>
      30                 :            : #include <cstring>
      31                 :            : #include <string>
      32                 :            : 
      33                 :            : #include <xapian/error.h>
      34                 :            : 
      35                 :            : #include "omassert.h"
      36                 :            : #include "str.h"
      37                 :            : 
      38                 :            : // Trying to include the correct headers with the correct defines set to
      39                 :            : // get pread() and pwrite() prototyped on every platform without breaking any
      40                 :            : // other platform is a real can of worms.  So instead we probe for what
      41                 :            : // prototypes (if any) are required in configure and put them into
      42                 :            : // PREAD_PROTOTYPE and PWRITE_PROTOTYPE.
      43                 :            : #if defined HAVE_PREAD && defined PREAD_PROTOTYPE
      44                 :            : PREAD_PROTOTYPE
      45                 :            : #endif
      46                 :            : #if defined HAVE_PWRITE && defined PWRITE_PROTOTYPE
      47                 :            : PWRITE_PROTOTYPE
      48                 :            : #endif
      49                 :            : 
      50                 :            : bool
      51                 :       4014 : io_unlink(const std::string & filename)
      52                 :            : {
      53         [ +  + ]:       4014 :     if (posixy_unlink(filename.c_str()) == 0) {
      54                 :        684 :         return true;
      55                 :            :     }
      56         [ -  + ]:       3330 :     if (errno != ENOENT) {
      57 [ #  # ][ #  # ]:          0 :         throw Xapian::DatabaseError(filename + ": delete failed", errno);
      58                 :            :     }
      59                 :       4014 :     return false;
      60                 :            : }
      61                 :            : 
      62                 :            : // The smallest fd we want to use for a writable handle.
      63                 :            : const int MIN_WRITE_FD = 3;
      64                 :            : 
      65                 :            : int
      66                 :       7592 : io_open_block_wr(const char * fname, bool anew)
      67                 :            : {
      68                 :            :     // Use auto because on AIX O_CLOEXEC may be a 64-bit integer constant.
      69                 :       7592 :     auto flags = O_RDWR | O_BINARY | O_CLOEXEC;
      70         [ +  + ]:       7592 :     if (anew) flags |= O_CREAT | O_TRUNC;
      71                 :       7592 :     int fd = ::open(fname, flags, 0666);
      72 [ +  + ][ +  + ]:       7592 :     if (fd >= MIN_WRITE_FD || fd < 0) return fd;
      73                 :            : 
      74                 :            :     // We want to avoid using fd < MIN_WRITE_FD, in case some other code in
      75                 :            :     // the same process tries to write to stdout or stderr, which would end up
      76                 :            :     // corrupting our database.
      77                 :          2 :     int badfd = fd;
      78                 :            : #ifdef F_DUPFD_CLOEXEC
      79                 :            :     // dup to the first unused fd >= MIN_WRITE_FD.
      80                 :          2 :     fd = fcntl(badfd, F_DUPFD_CLOEXEC, MIN_WRITE_FD);
      81                 :            :     // F_DUPFD_CLOEXEC may not be supported.
      82 [ -  + ][ #  # ]:          2 :     if (fd < 0 && errno == EINVAL)
      83                 :            : #endif
      84                 :            : #ifdef F_DUPFD
      85                 :            :     {
      86                 :          0 :         fd = fcntl(badfd, F_DUPFD, MIN_WRITE_FD);
      87                 :            : # ifdef FD_CLOEXEC
      88         [ #  # ]:          0 :         if (fd >= 0)
      89                 :          0 :             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
      90                 :            : # endif
      91                 :            :     }
      92                 :          2 :     int save_errno = errno;
      93                 :          2 :     (void)close(badfd);
      94                 :          2 :     errno = save_errno;
      95                 :            : #else
      96                 :            :     {
      97                 :            :         char toclose[MIN_WRITE_FD];
      98                 :            :         memset(toclose, 0, sizeof(toclose));
      99                 :            :         fd = badfd;
     100                 :            :         do {
     101                 :            :             toclose[fd] = 1;
     102                 :            :             fd = dup(fd);
     103                 :            :         } while (fd >= 0 && fd < MIN_WRITE_FD);
     104                 :            :         int save_errno = errno;
     105                 :            :         for (badfd = 0; badfd != MIN_WRITE_FD; ++badfd)
     106                 :            :             if (toclose[badfd])
     107                 :            :                 close(badfd);
     108                 :            :         if (fd < 0) {
     109                 :            :             errno = save_errno;
     110                 :            :         } else {
     111                 :            : # ifdef FD_CLOEXEC
     112                 :            :             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
     113                 :            : # endif
     114                 :            :         }
     115                 :            :     }
     116                 :            : #endif
     117                 :            :     Assert(fd >= MIN_WRITE_FD || fd < 0);
     118                 :          2 :     return fd;
     119                 :            : }
     120                 :            : 
     121                 :            : size_t
     122                 :       3953 : io_read(int fd, char * p, size_t n, size_t min)
     123                 :            : {
     124                 :       3953 :     size_t total = 0;
     125         [ +  + ]:       7905 :     while (n) {
     126                 :       7265 :         ssize_t c = read(fd, p, n);
     127         [ +  + ]:       7265 :         if (c <= 0) {
     128         [ +  - ]:       3313 :             if (c == 0) {
     129         [ +  + ]:       3313 :                 if (total >= min) break;
     130 [ +  - ][ +  - ]:          1 :                 throw Xapian::DatabaseCorruptError("Couldn't read enough (EOF)");
                 [ +  - ]
     131                 :            :             }
     132         [ #  # ]:          0 :             if (errno == EINTR) continue;
     133 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("Error reading from file", errno);
     134                 :            :         }
     135                 :       3952 :         p += c;
     136                 :       3952 :         total += c;
     137                 :       3952 :         n -= c;
     138                 :            :     }
     139                 :       3952 :     return total;
     140                 :            : }
     141                 :            : 
     142                 :            : /** Write n bytes from block pointed to by p to file descriptor fd. */
     143                 :            : void
     144                 :      12819 : io_write(int fd, const char * p, size_t n)
     145                 :            : {
     146         [ +  + ]:      25638 :     while (n) {
     147                 :      12819 :         ssize_t c = write(fd, p, n);
     148         [ -  + ]:      12819 :         if (c < 0) {
     149         [ #  # ]:          0 :             if (errno == EINTR) continue;
     150 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("Error writing to file", errno);
     151                 :            :         }
     152                 :      12819 :         p += c;
     153                 :      12819 :         n -= c;
     154                 :            :     }
     155                 :      12819 : }
     156                 :            : 
     157                 :            : size_t
     158                 :    3549491 : io_pread(int fd, char * p, size_t n, off_t o, size_t min)
     159                 :            : {
     160                 :    3549491 :     size_t total = 0;
     161                 :            : #ifdef HAVE_PREAD
     162                 :            :     while (true) {
     163                 :    3549491 :         ssize_t c = pread(fd, p, n, o);
     164                 :            :         // We should get a full read most of the time, so streamline that case.
     165         [ +  + ]:    3549491 :         if (usual(c == ssize_t(n)))
     166                 :    3224694 :             return total + n;
     167                 :            :         // -1 is error, 0 is EOF
     168         [ -  + ]:     324797 :         if (c <= 0) {
     169         [ #  # ]:          0 :             if (c == 0) {
     170         [ #  # ]:          0 :                 if (min == 0)
     171                 :          0 :                     return total;
     172 [ #  # ][ #  # ]:          0 :                 throw Xapian::DatabaseError("EOF reading database");
                 [ #  # ]
     173                 :            :             }
     174                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     175                 :            :             // In this case we should retry the read.
     176         [ #  # ]:          0 :             if (errno == EINTR) continue;
     177 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("Error reading database", errno);
     178                 :            :         }
     179                 :     324797 :         total += c;
     180         [ +  - ]:     324797 :         if (total >= min)
     181                 :     324797 :             return total;
     182                 :          0 :         p += c;
     183                 :          0 :         n -= c;
     184                 :          0 :         o += c;
     185                 :    3549491 :     }
     186                 :            : #else
     187                 :            :     if (rare(lseek(fd, o, SEEK_SET) < 0))
     188                 :            :         throw Xapian::DatabaseError("Error seeking database", errno);
     189                 :            :     while (true) {
     190                 :            :         ssize_t c = read(fd, p, n);
     191                 :            :         // We should get a full read most of the time, so streamline that case.
     192                 :            :         if (usual(c == ssize_t(n)))
     193                 :            :             return total + n;
     194                 :            :         if (c <= 0) {
     195                 :            :             if (c == 0) {
     196                 :            :                 if (min == 0)
     197                 :            :                     return total;
     198                 :            :                 throw Xapian::DatabaseError("EOF reading database");
     199                 :            :             }
     200                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     201                 :            :             // In this case we should retry the read.
     202                 :            :             if (errno == EINTR) continue;
     203                 :            :             throw Xapian::DatabaseError("Error reading database", errno);
     204                 :            :         }
     205                 :            :         total += c;
     206                 :            :         if (total >= min)
     207                 :            :             return total;
     208                 :            :         p += c;
     209                 :            :         n -= c;
     210                 :            :     }
     211                 :            : #endif
     212                 :            : }
     213                 :            : 
     214                 :            : void
     215                 :          0 : io_pwrite(int fd, const char * p, size_t n, off_t o)
     216                 :            : {
     217                 :            : #ifdef HAVE_PWRITE
     218         [ #  # ]:          0 :     while (n) {
     219                 :          0 :         ssize_t c = pwrite(fd, p, n, o);
     220                 :            :         // We should get a full write most of the time, so streamline that
     221                 :            :         // case.
     222         [ #  # ]:          0 :         if (usual(c == ssize_t(n)))
     223                 :          0 :             return;
     224         [ #  # ]:          0 :         if (c < 0) {
     225         [ #  # ]:          0 :             if (errno == EINTR) continue;
     226 [ #  # ][ #  # ]:          0 :             throw Xapian::DatabaseError("Error writing to file", errno);
     227                 :            :         }
     228                 :          0 :         p += c;
     229                 :          0 :         n -= c;
     230                 :          0 :         o += c;
     231                 :            :     }
     232                 :            : #else
     233                 :            :     if (rare(lseek(fd, o, SEEK_SET) < 0))
     234                 :            :         throw Xapian::DatabaseError("Error seeking database", errno);
     235                 :            :     io_write(fd, p, n);
     236                 :            : #endif
     237                 :            : }
     238                 :            : 
     239                 :            : [[noreturn]]
     240                 :            : static void
     241                 :          0 : throw_block_error(const char * s, off_t b, int e = 0)
     242                 :            : {
     243         [ #  # ]:          0 :     std::string m = s;
     244 [ #  # ][ #  # ]:          0 :     m += str(b);
     245         [ #  # ]:          0 :     throw Xapian::DatabaseError(m, e);
     246                 :            : }
     247                 :            : 
     248                 :            : #ifdef HAVE_POSIX_FADVISE
     249                 :            : bool
     250                 :       1256 : io_readahead_block(int fd, size_t n, off_t b, off_t o)
     251                 :            : {
     252                 :       1256 :     o += b * n;
     253                 :            :     // Assume that any failure is likely to also happen for another call with
     254                 :            :     // the same fd.
     255                 :       1256 :     return posix_fadvise(fd, o, n, POSIX_FADV_WILLNEED) == 0;
     256                 :            : }
     257                 :            : #endif
     258                 :            : 
     259                 :            : void
     260                 :    5109861 : io_read_block(int fd, char * p, size_t n, off_t b, off_t o)
     261                 :            : {
     262                 :    5109861 :     o += b * n;
     263                 :            :     // Prefer pread if available since it's typically implemented as a
     264                 :            :     // separate syscall, and that eliminates the overhead of an extra syscall
     265                 :            :     // per block read.
     266                 :            : #ifdef HAVE_PREAD
     267                 :            :     while (true) {
     268                 :    5109861 :         ssize_t c = pread(fd, p, n, o);
     269                 :            :         // We should get a full read most of the time, so streamline that case.
     270         [ +  - ]:    5109861 :         if (usual(c == ssize_t(n)))
     271                 :    5109861 :             return;
     272                 :            :         // -1 is error, 0 is EOF
     273         [ #  # ]:          0 :         if (c <= 0) {
     274         [ #  # ]:          0 :             if (c == 0)
     275                 :          0 :                 throw_block_error("EOF reading block ", b);
     276                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     277                 :            :             // In this case we should retry the read.
     278         [ #  # ]:          0 :             if (errno == EINTR) continue;
     279                 :          0 :             throw_block_error("Error reading block ", b, errno);
     280                 :            :         }
     281                 :          0 :         p += c;
     282                 :          0 :         n -= c;
     283                 :          0 :         o += c;
     284                 :          0 :     }
     285                 :            : #else
     286                 :            :     if (rare(lseek(fd, o, SEEK_SET) < 0))
     287                 :            :         throw_block_error("Error seeking to block ", b, errno);
     288                 :            :     while (true) {
     289                 :            :         ssize_t c = read(fd, p, n);
     290                 :            :         // We should get a full read most of the time, so streamline that case.
     291                 :            :         if (usual(c == ssize_t(n)))
     292                 :            :             return;
     293                 :            :         if (c <= 0) {
     294                 :            :             if (c == 0)
     295                 :            :                 throw_block_error("EOF reading block ", b);
     296                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     297                 :            :             // In this case we should retry the read.
     298                 :            :             if (errno == EINTR) continue;
     299                 :            :             throw_block_error("Error reading block ", b, errno);
     300                 :            :         }
     301                 :            :         p += c;
     302                 :            :         n -= c;
     303                 :            :     }
     304                 :            : #endif
     305                 :            : }
     306                 :            : 
     307                 :            : void
     308                 :     632224 : io_write_block(int fd, const char * p, size_t n, off_t b, off_t o)
     309                 :            : {
     310                 :     632224 :     o += b * n;
     311                 :            :     // Prefer pwrite if available since it's typically implemented as a
     312                 :            :     // separate syscall, and that eliminates the overhead of an extra syscall
     313                 :            :     // per block write.
     314                 :            : #ifdef HAVE_PWRITE
     315                 :            :     while (true) {
     316                 :     632224 :         ssize_t c = pwrite(fd, p, n, o);
     317                 :            :         // We should get a full write most of the time, so streamline that case.
     318         [ +  - ]:     632224 :         if (usual(c == ssize_t(n)))
     319                 :     632224 :             return;
     320         [ #  # ]:          0 :         if (c < 0) {
     321                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     322                 :            :             // In this case we should retry the write.
     323         [ #  # ]:          0 :             if (errno == EINTR) continue;
     324                 :          0 :             throw_block_error("Error writing block ", b, errno);
     325                 :            :         }
     326                 :          0 :         p += c;
     327                 :          0 :         n -= c;
     328                 :          0 :         o += c;
     329                 :          0 :     }
     330                 :            : #else
     331                 :            :     if (rare(lseek(fd, o, SEEK_SET) < 0))
     332                 :            :         throw_block_error("Error seeking to block ", b, errno);
     333                 :            :     while (true) {
     334                 :            :         ssize_t c = write(fd, p, n);
     335                 :            :         // We should get a full write most of the time, so streamline that case.
     336                 :            :         if (usual(c == ssize_t(n)))
     337                 :            :             return;
     338                 :            :         if (c < 0) {
     339                 :            :             // We get EINTR if the syscall was interrupted by a signal.
     340                 :            :             // In this case we should retry the write.
     341                 :            :             if (errno == EINTR) continue;
     342                 :            :             throw_block_error("Error writing block ", b, errno);
     343                 :            :         }
     344                 :            :         p += c;
     345                 :            :         n -= c;
     346                 :            :     }
     347                 :            : #endif
     348                 :            : }
     349                 :            : 
     350                 :            : bool
     351                 :      10156 : io_tmp_rename(const std::string & tmp_file, const std::string & real_file)
     352                 :            : {
     353                 :            : #ifdef EXDEV
     354                 :            :     // We retry on EXDEV a few times as some older Linux kernels are buggy and
     355                 :            :     // fail with EXDEV when the two files are on the same device (as they
     356                 :            :     // always ought to be when this function is used).  Don't retry forever in
     357                 :            :     // case someone calls this with files on different devices.
     358                 :            :     //
     359                 :            :     // We're not sure exactly which kernels are buggy in this way, but there's
     360                 :            :     // discussion here: https://www.spinics.net/lists/linux-nfs/msg17306.html
     361                 :            :     //
     362                 :            :     // Reported at: https://trac.xapian.org/ticket/698
     363                 :      10156 :     int retries = 5;
     364                 :            : retry:
     365                 :            : #endif
     366         [ -  + ]:      10156 :     if (posixy_rename(tmp_file.c_str(), real_file.c_str()) < 0) {
     367                 :            : #ifdef EXDEV
     368 [ #  # ][ #  # ]:          0 :         if (errno == EXDEV && --retries > 0) goto retry;
                 [ #  # ]
     369                 :            : #endif
     370                 :            :         // With NFS, rename() failing may just mean that the server crashed
     371                 :            :         // after successfully renaming, but before reporting this, and then
     372                 :            :         // the retried operation fails.  So we need to check if the source
     373                 :            :         // file still exists, which we do by calling unlink(), since we want
     374                 :            :         // to remove the temporary file anyway.
     375                 :          0 :         int saved_errno = errno;
     376 [ #  # ][ #  # ]:          0 :         if (unlink(tmp_file.c_str()) == 0 || errno != ENOENT) {
                 [ #  # ]
     377                 :          0 :             errno = saved_errno;
     378                 :          0 :             return false;
     379                 :            :         }
     380                 :            :     }
     381                 :      10156 :     return true;
     382                 :            : }

Generated by: LCOV version 1.11