LCOV - code coverage report
Current view: top level - tests/harness - testsuite.h (source / functions) Hit Total Coverage
Test: Test Coverage for xapian-core fcfb185a9dd5 Lines: 18 18 100.0 %
Date: 2019-04-18 16:33:14 Functions: 4 4 100.0 %
Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : /** @file testsuite.h
       2                 :            :  * @brief a generic test suite engine
       3                 :            :  */
       4                 :            : /* Copyright 1999,2000,2001 BrightStation PLC
       5                 :            :  * Copyright 2002,2003,2005,2006,2007,2008,2009,2013,2015,2016,2018 Olly Betts
       6                 :            :  * Copyright 2007 Richard Boulton
       7                 :            :  *
       8                 :            :  * This program is free software; you can redistribute it and/or
       9                 :            :  * modify it under the terms of the GNU General Public License as
      10                 :            :  * published by the Free Software Foundation; either version 2 of the
      11                 :            :  * License, or (at your option) any later version.
      12                 :            :  *
      13                 :            :  * This program is distributed in the hope that it will be useful,
      14                 :            :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      15                 :            :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      16                 :            :  * GNU General Public License for more details.
      17                 :            :  *
      18                 :            :  * You should have received a copy of the GNU General Public License
      19                 :            :  * along with this program; if not, write to the Free Software
      20                 :            :  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
      21                 :            :  * USA
      22                 :            :  */
      23                 :            : 
      24                 :            : #ifndef XAPIAN_INCLUDED_TESTSUITE_H
      25                 :            : #define XAPIAN_INCLUDED_TESTSUITE_H
      26                 :            : 
      27                 :            : #ifndef XAPIAN_UNITTEST
      28                 :            : # include "output.h"
      29                 :            : # define UNITTEST_CHECK_EXCEPTION
      30                 :            : #endif
      31                 :            : 
      32                 :            : #include "stringutils.h" // For STRINGIZE().
      33                 :            : 
      34                 :            : #include <iomanip>
      35                 :            : #include <map>
      36                 :            : #include <sstream>
      37                 :            : #include <string>
      38                 :            : #include <vector>
      39                 :            : 
      40                 :            : #include <cfloat> // For DBL_DIG.
      41                 :            : 
      42                 :            : /** Class which is thrown when a test case fails.
      43                 :            :  */
      44                 :            : class TestFail { };
      45                 :            : 
      46                 :            : /** Class which is thrown when a test case is to be skipped.
      47                 :            :  *
      48                 :            :  *  This happens when something can't be tested for some reason, but
      49                 :            :  *  that reason isn't grounds for causing the test to fail.
      50                 :            :  */
      51                 :            : class TestSkip { };
      52                 :            : 
      53                 :            : /// Helper macro.
      54                 :            : #define THROW_TEST_(EXCEPTION, MSG) \
      55                 :            :     do { \
      56                 :            :         if (verbose) { \
      57                 :            :             tout << __FILE__ ":" STRINGIZE(__LINE__) ": " << MSG << std::endl; \
      58                 :            :         } \
      59                 :            :         throw EXCEPTION(); \
      60                 :            :     } while (0)
      61                 :            : 
      62                 :            : /** Fail the current testcase with message MSG.
      63                 :            :  *
      64                 :            :  *  MSG is written to an std::ostream and so can contain <<.
      65                 :            :  */
      66                 :            : #define FAIL_TEST(MSG) THROW_TEST_(TestFail, MSG)
      67                 :            : 
      68                 :            : /** Skip the current testcase with message MSG.
      69                 :            :  *
      70                 :            :  *  MSG is written to an std::ostream and so can contain <<.
      71                 :            :  */
      72                 :            : #define SKIP_TEST(MSG) THROW_TEST_(TestSkip, MSG)
      73                 :            : 
      74                 :            : /// Type for a test function.
      75                 :            : typedef bool (*test_func)();
      76                 :            : 
      77                 :            : /// Structure holding a description of a test.
      78                 :            : struct test_desc {
      79                 :            :     /// The name of the test.
      80                 :            :     const char *name;
      81                 :            : 
      82                 :            :     /// The function to run to perform the test.
      83                 :            :     test_func run;
      84                 :            : };
      85                 :            : 
      86                 :            : /// The global verbose flag.
      87                 :            : //
      88                 :            : //  If verbose is non-zero, then the test harness will display diagnostic output
      89                 :            : //  for tests which fail or skip.  If it is > 1, then the diagnostic output will
      90                 :            : //  also be displayed for tests which pass.  Individual tests may use this flag
      91                 :            : //  to avoid needless generation of diagnostic output in cases when it's
      92                 :            : //  expensive.
      93                 :            : extern int verbose;
      94                 :            : 
      95                 :            : /** Set to a string explanation for testcases expected to fail. */
      96                 :            : extern const char* expected_failure;
      97                 :            : 
      98                 :            : /// The exception type we were expecting in TEST_EXCEPTION.
      99                 :            : //  Used to detect if such an exception was mishandled by the
     100                 :            : //  compiler/runtime.
     101                 :            : extern const char * expected_exception;
     102                 :            : 
     103                 :            : /** The output stream.  Data written to this stream will only appear
     104                 :            :  *  when a test fails.
     105                 :            :  */
     106                 :            : extern std::ostringstream tout;
     107                 :            : 
     108                 :            : /// The test driver.  This class takes care of running the tests.
     109                 :        354 : class test_driver {
     110                 :            :         /// Write out anything in tout and clear it.
     111                 :            :         void write_and_clear_tout();
     112                 :            : 
     113                 :            :     public:
     114                 :            :         /** A structure used to report the summary of tests passed
     115                 :            :          *  and failed.
     116                 :            :          */
     117                 :            :         struct result {
     118                 :            :             /// The number of tests which succeeded.
     119                 :            :             unsigned int succeeded = 0;
     120                 :            : 
     121                 :            :             /// The number of tests which failed.
     122                 :            :             unsigned int failed = 0;
     123                 :            : 
     124                 :            :             /// The number of tests which were skipped.
     125                 :            :             unsigned int skipped = 0;
     126                 :            : 
     127                 :            :             /** Number of tests with result XFAIL.
     128                 :            :              *
     129                 :            :              *  I.e. tests which were expected to fail and did.
     130                 :            :              */
     131                 :            :             unsigned int xfailed = 0;
     132                 :            : 
     133                 :            :             /** Number of tests with result XFAIL.
     134                 :            :              *
     135                 :            :              *  I.e. tests which were expected to fail but passed.
     136                 :            :              */
     137                 :            :             unsigned int xpassed = 0;
     138                 :            : 
     139                 :        185 :             result & operator+=(const result & o) {
     140                 :        185 :                 succeeded += o.succeeded;
     141                 :        185 :                 failed += o.failed;
     142                 :        185 :                 skipped += o.skipped;
     143                 :        185 :                 xfailed += o.xfailed;
     144                 :        185 :                 xpassed += o.xpassed;
     145                 :        185 :                 return *this;
     146                 :            :             }
     147                 :            : 
     148                 :          8 :             void reset() {
     149                 :          8 :                 succeeded = 0;
     150                 :          8 :                 failed = 0;
     151                 :          8 :                 skipped = 0;
     152                 :          8 :                 xfailed = 0;
     153                 :          8 :                 xpassed = 0;
     154                 :          8 :             }
     155                 :            :         };
     156                 :            : 
     157                 :            :         /** Add a test-specific command line option.
     158                 :            :          *
     159                 :            :          *  The recognised option will be described as:
     160                 :            :          *
     161                 :            :          *   -<s> <l>
     162                 :            :          *
     163                 :            :          *  And any value set will be put into arg.
     164                 :            :          */
     165                 :            :         static void add_command_line_option(const std::string &l, char s,
     166                 :            :                                             std::string * arg);
     167                 :            : 
     168                 :            :         /** Parse the command line arguments.
     169                 :            :          *
     170                 :            :          *  @param  argc        The argument count passed into ::main()
     171                 :            :          *  @param  argv        The argument list passed into ::main()
     172                 :            :          */
     173                 :            :         static void parse_command_line(int argc, char **argv);
     174                 :            : 
     175                 :            :         [[noreturn]]
     176                 :            :         static void usage();
     177                 :            : 
     178                 :            :         static int run(const test_desc *tests);
     179                 :            : 
     180                 :            :         /** The constructor, which sets up the test driver.
     181                 :            :          *
     182                 :            :          *  @param tests The zero-terminated array of tests to run.
     183                 :            :          */
     184                 :            :         test_driver(const test_desc *tests_);
     185                 :            : 
     186                 :            :         /** Run all the tests supplied and return the results
     187                 :            :          */
     188                 :            :         result run_tests();
     189                 :            : 
     190                 :            :         /** Run the tests in the list and return the results
     191                 :            :          */
     192                 :            :         result run_tests(std::vector<std::string>::const_iterator b,
     193                 :            :                          std::vector<std::string>::const_iterator e);
     194                 :            : 
     195                 :            :         /** Read srcdir from environment and if not present, make a valiant
     196                 :            :          *  attempt to guess a value
     197                 :            :          */
     198                 :            :         static std::string get_srcdir();
     199                 :            : 
     200                 :            :         // Running subtotal for current backend.
     201                 :            :         static result subtotal;
     202                 :            : 
     203                 :            :         // Running total for the whole test run.
     204                 :            :         static result total;
     205                 :            : 
     206                 :            :         /// Print summary of tests passed, failed, and skipped.
     207                 :            :         static void report(const test_driver::result &r, const std::string &desc);
     208                 :            : 
     209                 :            :     private:
     210                 :            :         /** Prevent copying */
     211                 :            :         test_driver(const test_driver &);
     212                 :            :         test_driver & operator=(const test_driver &);
     213                 :            : 
     214                 :            :         enum test_result {
     215                 :            :             XPASS = 3, XFAIL = 2, PASS = 1, FAIL = 0, SKIP = -1
     216                 :            :         };
     217                 :            : 
     218                 :            :         static std::map<int, std::string *> short_opts;
     219                 :            : 
     220                 :            :         static std::string opt_help;
     221                 :            : 
     222                 :            :         static std::vector<std::string> test_names;
     223                 :            : 
     224                 :            :         /** Runs the test function and returns its result.  It will
     225                 :            :          *  also trap exceptions and some memory leaks and force a
     226                 :            :          *  failure in those cases.
     227                 :            :          *
     228                 :            :          *  @param test A description of the test to run.
     229                 :            :          */
     230                 :            :         test_result runtest(const test_desc *test);
     231                 :            : 
     232                 :            :         /** The implementation used by run_tests.
     233                 :            :          *  it runs test(s) (with runtest()), prints out messages for
     234                 :            :          *  the user, and tracks the successes and failures.
     235                 :            :          *
     236                 :            :          *  @param b, e  If b != e, a vector of the test(s) to run.
     237                 :            :          *               If b == e, all tests will be run.
     238                 :            :          */
     239                 :            :         result do_run_tests(std::vector<std::string>::const_iterator b,
     240                 :            :                             std::vector<std::string>::const_iterator e);
     241                 :            : 
     242                 :            :         // abort tests at the first failure
     243                 :            :         static bool abort_on_error;
     244                 :            : 
     245                 :            :         // the default stream to output to
     246                 :            :         std::ostream out;
     247                 :            : 
     248                 :            :         // the list of tests to run.
     249                 :            :         const test_desc *tests;
     250                 :            : 
     251                 :            :         // how many test runs we've done - no summary if just one run
     252                 :            :         static int runs;
     253                 :            : 
     254                 :            :         // program name
     255                 :            :         static std::string argv0;
     256                 :            : 
     257                 :            :         // strings to use for colouring - empty if output isn't a tty
     258                 :            :         static std::string col_red, col_green, col_yellow, col_reset;
     259                 :            : 
     260                 :            :         // use \r to not advance a line when a test passes (this only
     261                 :            :         // really makes sense if the output is a tty)
     262                 :            :         static bool use_cr;
     263                 :            : };
     264                 :            : 
     265                 :            : /** Test a condition, and display the test with an extra explanation if
     266                 :            :  *  the condition fails.
     267                 :            :  *  NB: wrapped in do { ... } while (0) so a trailing ';' works correctly.
     268                 :            :  */
     269                 :            : #define TEST_AND_EXPLAIN(a, b) do {\
     270                 :            :         bool test_and_explain_fail_ = !(a);\
     271                 :            :         UNITTEST_CHECK_EXCEPTION\
     272                 :            :         if (test_and_explain_fail_)\
     273                 :            :             FAIL_TEST(STRINGIZE(a) << std::endl << b << std::endl);\
     274                 :            :     } while (0)
     275                 :            : 
     276                 :            : /// Test a condition, without an additional explanation for failure.
     277                 :            : #define TEST(a) TEST_AND_EXPLAIN(a, "")
     278                 :            : 
     279                 :            : /// Test for equality of two things.
     280                 :            : #define TEST_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
     281                 :            :         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be equal:" \
     282                 :            :         " were " << (a) << " and " << (b))
     283                 :            : 
     284                 :            : /** Test for equality of two strings.
     285                 :            :  *
     286                 :            :  *  If they aren't equal, show each on a separate line so the difference can
     287                 :            :  *  be seen clearly.
     288                 :            :  */
     289                 :            : #define TEST_STRINGS_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
     290                 :            :         "Expected " STRINGIZE(a) " and " STRINGIZE(b) " to be equal, were:\n\"" \
     291                 :            :         << (a) << "\"\n\"" << (b) << '"')
     292                 :            : 
     293                 :            : /// Helper function for TEST_EQUAL_DOUBLE macro.
     294                 :            : extern bool TEST_EQUAL_DOUBLE_(double a, double b);
     295                 :            : 
     296                 :            : /// Test two doubles for near equality.
     297                 :            : #define TEST_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(TEST_EQUAL_DOUBLE_((a), (b)), \
     298                 :            :         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be (nearly) equal:" \
     299                 :            :         " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
     300                 :            : 
     301                 :            : /// Test two doubles for non-near-equality.
     302                 :            : #define TEST_NOT_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(!TEST_EQUAL_DOUBLE_((a), (b)), \
     303                 :            :         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be (nearly) equal:" \
     304                 :            :         " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
     305                 :            : 
     306                 :            : /// Test for non-equality of two things.
     307                 :            : #define TEST_NOT_EQUAL(a, b) TEST_AND_EXPLAIN(((a) != (b)), \
     308                 :            :         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be equal:" \
     309                 :            :         " were " << (a) << " and " << (b))
     310                 :            : 
     311                 :            : #define DEFINE_TESTCASE(S,COND) bool test_##S()
     312                 :            : 
     313                 :            : // Newer test macros:
     314                 :            : #include "testmacros.h"
     315                 :            : 
     316                 :            : /** Mark a testcase as expected to fail.
     317                 :            :  *
     318                 :            :  *  @param msg  An static string explaining why the testcase is expected to
     319                 :            :  *              fail.  Must not be NULL.
     320                 :            :  *
     321                 :            :  *  This is intended to be used temporarily to mark tests for known bugs before
     322                 :            :  *  the bugs are fixed.  If the test fails, the result will be shown as "XFAIL"
     323                 :            :  *  and this won't cause the test run to fail.  However, if a test marked in
     324                 :            :  *  this way actually passed, the result will be shown as "XPASS" and the test
     325                 :            :  *  run *will* fail.  (So XFAIL is explicitly not suitable for marking "flaky"
     326                 :            :  *  testcases - please fix flaky testcases rather than trying to find a way to
     327                 :            :  *  mark them as flaky!)
     328                 :            :  *
     329                 :            :  *  This macro should be used inside the testcase code.  It can be used inside
     330                 :            :  *  a conditional if the testcase is only expected to fail in certain situations
     331                 :            :  *  (for example, only for some backends) - it only has an effect if it is
     332                 :            :  *  actually executed.
     333                 :            :  */
     334                 :          6 : inline void XFAIL(const char* msg) {
     335                 :          6 :     expected_failure = msg;
     336                 :          6 : }
     337                 :            : 
     338                 :            : #endif // XAPIAN_INCLUDED_TESTSUITE_H

Generated by: LCOV version 1.11