Branch data Line data Source code
1 : : /** @file testsuite.h
2 : : * @brief a generic test suite engine
3 : : */
4 : : /* Copyright 1999,2000,2001 BrightStation PLC
5 : : * Copyright 2002,2003,2005,2006,2007,2008,2009,2013,2015,2016,2018 Olly Betts
6 : : * Copyright 2007 Richard Boulton
7 : : *
8 : : * This program is free software; you can redistribute it and/or
9 : : * modify it under the terms of the GNU General Public License as
10 : : * published by the Free Software Foundation; either version 2 of the
11 : : * License, or (at your option) any later version.
12 : : *
13 : : * This program is distributed in the hope that it will be useful,
14 : : * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 : : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 : : * GNU General Public License for more details.
17 : : *
18 : : * You should have received a copy of the GNU General Public License
19 : : * along with this program; if not, write to the Free Software
20 : : * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
21 : : * USA
22 : : */
23 : :
24 : : #ifndef XAPIAN_INCLUDED_TESTSUITE_H
25 : : #define XAPIAN_INCLUDED_TESTSUITE_H
26 : :
27 : : #ifndef XAPIAN_UNITTEST
28 : : # include "output.h"
29 : : # define UNITTEST_CHECK_EXCEPTION
30 : : #endif
31 : :
32 : : #include "stringutils.h" // For STRINGIZE().
33 : :
34 : : #include <iomanip>
35 : : #include <map>
36 : : #include <sstream>
37 : : #include <string>
38 : : #include <vector>
39 : :
40 : : #include <cfloat> // For DBL_DIG.
41 : :
42 : : /** Class which is thrown when a test case fails.
43 : : */
44 : : class TestFail { };
45 : :
46 : : /** Class which is thrown when a test case is to be skipped.
47 : : *
48 : : * This happens when something can't be tested for some reason, but
49 : : * that reason isn't grounds for causing the test to fail.
50 : : */
51 : : class TestSkip { };
52 : :
53 : : /// Helper macro.
54 : : #define THROW_TEST_(EXCEPTION, MSG) \
55 : : do { \
56 : : if (verbose) { \
57 : : tout << __FILE__ ":" STRINGIZE(__LINE__) ": " << MSG << std::endl; \
58 : : } \
59 : : throw EXCEPTION(); \
60 : : } while (0)
61 : :
62 : : /** Fail the current testcase with message MSG.
63 : : *
64 : : * MSG is written to an std::ostream and so can contain <<.
65 : : */
66 : : #define FAIL_TEST(MSG) THROW_TEST_(TestFail, MSG)
67 : :
68 : : /** Skip the current testcase with message MSG.
69 : : *
70 : : * MSG is written to an std::ostream and so can contain <<.
71 : : */
72 : : #define SKIP_TEST(MSG) THROW_TEST_(TestSkip, MSG)
73 : :
74 : : /// Type for a test function.
75 : : typedef bool (*test_func)();
76 : :
77 : : /// Structure holding a description of a test.
78 : : struct test_desc {
79 : : /// The name of the test.
80 : : const char *name;
81 : :
82 : : /// The function to run to perform the test.
83 : : test_func run;
84 : : };
85 : :
86 : : /// The global verbose flag.
87 : : //
88 : : // If verbose is non-zero, then the test harness will display diagnostic output
89 : : // for tests which fail or skip. If it is > 1, then the diagnostic output will
90 : : // also be displayed for tests which pass. Individual tests may use this flag
91 : : // to avoid needless generation of diagnostic output in cases when it's
92 : : // expensive.
93 : : extern int verbose;
94 : :
95 : : /** Set to a string explanation for testcases expected to fail. */
96 : : extern const char* expected_failure;
97 : :
98 : : /// The exception type we were expecting in TEST_EXCEPTION.
99 : : // Used to detect if such an exception was mishandled by the
100 : : // compiler/runtime.
101 : : extern const char * expected_exception;
102 : :
103 : : /** The output stream. Data written to this stream will only appear
104 : : * when a test fails.
105 : : */
106 : : extern std::ostringstream tout;
107 : :
108 : : /// The test driver. This class takes care of running the tests.
109 : 354 : class test_driver {
110 : : /// Write out anything in tout and clear it.
111 : : void write_and_clear_tout();
112 : :
113 : : public:
114 : : /** A structure used to report the summary of tests passed
115 : : * and failed.
116 : : */
117 : : struct result {
118 : : /// The number of tests which succeeded.
119 : : unsigned int succeeded = 0;
120 : :
121 : : /// The number of tests which failed.
122 : : unsigned int failed = 0;
123 : :
124 : : /// The number of tests which were skipped.
125 : : unsigned int skipped = 0;
126 : :
127 : : /** Number of tests with result XFAIL.
128 : : *
129 : : * I.e. tests which were expected to fail and did.
130 : : */
131 : : unsigned int xfailed = 0;
132 : :
133 : : /** Number of tests with result XFAIL.
134 : : *
135 : : * I.e. tests which were expected to fail but passed.
136 : : */
137 : : unsigned int xpassed = 0;
138 : :
139 : 185 : result & operator+=(const result & o) {
140 : 185 : succeeded += o.succeeded;
141 : 185 : failed += o.failed;
142 : 185 : skipped += o.skipped;
143 : 185 : xfailed += o.xfailed;
144 : 185 : xpassed += o.xpassed;
145 : 185 : return *this;
146 : : }
147 : :
148 : 8 : void reset() {
149 : 8 : succeeded = 0;
150 : 8 : failed = 0;
151 : 8 : skipped = 0;
152 : 8 : xfailed = 0;
153 : 8 : xpassed = 0;
154 : 8 : }
155 : : };
156 : :
157 : : /** Add a test-specific command line option.
158 : : *
159 : : * The recognised option will be described as:
160 : : *
161 : : * -<s> <l>
162 : : *
163 : : * And any value set will be put into arg.
164 : : */
165 : : static void add_command_line_option(const std::string &l, char s,
166 : : std::string * arg);
167 : :
168 : : /** Parse the command line arguments.
169 : : *
170 : : * @param argc The argument count passed into ::main()
171 : : * @param argv The argument list passed into ::main()
172 : : */
173 : : static void parse_command_line(int argc, char **argv);
174 : :
175 : : [[noreturn]]
176 : : static void usage();
177 : :
178 : : static int run(const test_desc *tests);
179 : :
180 : : /** The constructor, which sets up the test driver.
181 : : *
182 : : * @param tests The zero-terminated array of tests to run.
183 : : */
184 : : test_driver(const test_desc *tests_);
185 : :
186 : : /** Run all the tests supplied and return the results
187 : : */
188 : : result run_tests();
189 : :
190 : : /** Run the tests in the list and return the results
191 : : */
192 : : result run_tests(std::vector<std::string>::const_iterator b,
193 : : std::vector<std::string>::const_iterator e);
194 : :
195 : : /** Read srcdir from environment and if not present, make a valiant
196 : : * attempt to guess a value
197 : : */
198 : : static std::string get_srcdir();
199 : :
200 : : // Running subtotal for current backend.
201 : : static result subtotal;
202 : :
203 : : // Running total for the whole test run.
204 : : static result total;
205 : :
206 : : /// Print summary of tests passed, failed, and skipped.
207 : : static void report(const test_driver::result &r, const std::string &desc);
208 : :
209 : : private:
210 : : /** Prevent copying */
211 : : test_driver(const test_driver &);
212 : : test_driver & operator=(const test_driver &);
213 : :
214 : : enum test_result {
215 : : XPASS = 3, XFAIL = 2, PASS = 1, FAIL = 0, SKIP = -1
216 : : };
217 : :
218 : : static std::map<int, std::string *> short_opts;
219 : :
220 : : static std::string opt_help;
221 : :
222 : : static std::vector<std::string> test_names;
223 : :
224 : : /** Runs the test function and returns its result. It will
225 : : * also trap exceptions and some memory leaks and force a
226 : : * failure in those cases.
227 : : *
228 : : * @param test A description of the test to run.
229 : : */
230 : : test_result runtest(const test_desc *test);
231 : :
232 : : /** The implementation used by run_tests.
233 : : * it runs test(s) (with runtest()), prints out messages for
234 : : * the user, and tracks the successes and failures.
235 : : *
236 : : * @param b, e If b != e, a vector of the test(s) to run.
237 : : * If b == e, all tests will be run.
238 : : */
239 : : result do_run_tests(std::vector<std::string>::const_iterator b,
240 : : std::vector<std::string>::const_iterator e);
241 : :
242 : : // abort tests at the first failure
243 : : static bool abort_on_error;
244 : :
245 : : // the default stream to output to
246 : : std::ostream out;
247 : :
248 : : // the list of tests to run.
249 : : const test_desc *tests;
250 : :
251 : : // how many test runs we've done - no summary if just one run
252 : : static int runs;
253 : :
254 : : // program name
255 : : static std::string argv0;
256 : :
257 : : // strings to use for colouring - empty if output isn't a tty
258 : : static std::string col_red, col_green, col_yellow, col_reset;
259 : :
260 : : // use \r to not advance a line when a test passes (this only
261 : : // really makes sense if the output is a tty)
262 : : static bool use_cr;
263 : : };
264 : :
265 : : /** Test a condition, and display the test with an extra explanation if
266 : : * the condition fails.
267 : : * NB: wrapped in do { ... } while (0) so a trailing ';' works correctly.
268 : : */
269 : : #define TEST_AND_EXPLAIN(a, b) do {\
270 : : bool test_and_explain_fail_ = !(a);\
271 : : UNITTEST_CHECK_EXCEPTION\
272 : : if (test_and_explain_fail_)\
273 : : FAIL_TEST(STRINGIZE(a) << std::endl << b << std::endl);\
274 : : } while (0)
275 : :
276 : : /// Test a condition, without an additional explanation for failure.
277 : : #define TEST(a) TEST_AND_EXPLAIN(a, "")
278 : :
279 : : /// Test for equality of two things.
280 : : #define TEST_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
281 : : "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be equal:" \
282 : : " were " << (a) << " and " << (b))
283 : :
284 : : /** Test for equality of two strings.
285 : : *
286 : : * If they aren't equal, show each on a separate line so the difference can
287 : : * be seen clearly.
288 : : */
289 : : #define TEST_STRINGS_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
290 : : "Expected " STRINGIZE(a) " and " STRINGIZE(b) " to be equal, were:\n\"" \
291 : : << (a) << "\"\n\"" << (b) << '"')
292 : :
293 : : /// Helper function for TEST_EQUAL_DOUBLE macro.
294 : : extern bool TEST_EQUAL_DOUBLE_(double a, double b);
295 : :
296 : : /// Test two doubles for near equality.
297 : : #define TEST_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(TEST_EQUAL_DOUBLE_((a), (b)), \
298 : : "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be (nearly) equal:" \
299 : : " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
300 : :
301 : : /// Test two doubles for non-near-equality.
302 : : #define TEST_NOT_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(!TEST_EQUAL_DOUBLE_((a), (b)), \
303 : : "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be (nearly) equal:" \
304 : : " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
305 : :
306 : : /// Test for non-equality of two things.
307 : : #define TEST_NOT_EQUAL(a, b) TEST_AND_EXPLAIN(((a) != (b)), \
308 : : "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be equal:" \
309 : : " were " << (a) << " and " << (b))
310 : :
311 : : #define DEFINE_TESTCASE(S,COND) bool test_##S()
312 : :
313 : : // Newer test macros:
314 : : #include "testmacros.h"
315 : :
316 : : /** Mark a testcase as expected to fail.
317 : : *
318 : : * @param msg An static string explaining why the testcase is expected to
319 : : * fail. Must not be NULL.
320 : : *
321 : : * This is intended to be used temporarily to mark tests for known bugs before
322 : : * the bugs are fixed. If the test fails, the result will be shown as "XFAIL"
323 : : * and this won't cause the test run to fail. However, if a test marked in
324 : : * this way actually passed, the result will be shown as "XPASS" and the test
325 : : * run *will* fail. (So XFAIL is explicitly not suitable for marking "flaky"
326 : : * testcases - please fix flaky testcases rather than trying to find a way to
327 : : * mark them as flaky!)
328 : : *
329 : : * This macro should be used inside the testcase code. It can be used inside
330 : : * a conditional if the testcase is only expected to fail in certain situations
331 : : * (for example, only for some backends) - it only has an effect if it is
332 : : * actually executed.
333 : : */
334 : 6 : inline void XFAIL(const char* msg) {
335 : 6 : expected_failure = msg;
336 : 6 : }
337 : :
338 : : #endif // XAPIAN_INCLUDED_TESTSUITE_H
|
