libwreport  3.40
utils/tests.h
1 #ifndef WREPORT_TESTS_H
2 #define WREPORT_TESTS_H
3 
13 #include <cstdint>
14 #include <exception>
15 #include <filesystem>
16 #include <functional>
17 #include <sstream>
18 #include <string>
19 #include <vector>
20 
21 namespace wreport {
22 namespace tests {
23 struct LocationInfo;
24 }
25 } // namespace wreport
26 
27 /*
28  * These global arguments will be shadowed by local variables in functions that
29  * implement tests.
30  *
31  * They are here to act as default root nodes to fulfill method signatures when
32  * tests are called from outside other tests.
33  */
34 extern const wreport::tests::LocationInfo wreport_test_location_info;
35 
36 namespace wreport {
37 namespace tests {
38 
56 struct LocationInfo : public std::stringstream
57 {
58  LocationInfo() {}
59 
64  std::ostream& operator()();
65 };
66 
69 {
70  const char* file;
71  int line;
72  const char* call;
73  std::string local_info;
74 
75  TestStackFrame(const char* file_, int line_, const char* call_)
76  : file(file_), line(line_), call(call_)
77  {
78  }
79 
80  TestStackFrame(const char* file_, int line_, const char* call_,
81  const LocationInfo& local_info_)
82  : file(file_), line(line_), call(call_), local_info(local_info_.str())
83  {
84  }
85 
86  std::string format() const;
87 
88  void format(std::ostream& out) const;
89 };
90 
91 struct TestStack : public std::vector<TestStackFrame>
92 {
93  using vector::vector;
94 
96  std::string backtrace() const;
97 
99  void backtrace(std::ostream& out) const;
100 };
101 
106 struct TestFailed : public std::exception
107 {
108  std::string message;
109  TestStack stack;
110 
111  explicit TestFailed(const std::exception& e);
112 
113  template <typename... Args>
114  TestFailed(const std::exception& e, Args&&... args) : TestFailed(e)
115  {
116  add_stack_info(std::forward<Args>(args)...);
117  }
118 
119  explicit TestFailed(const std::string& message_) : message(message_) {}
120 
121  template <typename... Args>
122  TestFailed(const std::string& message_, Args&&... args)
123  : TestFailed(message_)
124  {
125  add_stack_info(std::forward<Args>(args)...);
126  }
127 
128  const char* what() const noexcept override { return message.c_str(); }
129 
130  template <typename... Args> void add_stack_info(Args&&... args)
131  {
132  stack.emplace_back(std::forward<Args>(args)...);
133  }
134 };
135 
139 struct TestSkipped : public std::exception
140 {
141  std::string reason;
142 
143  TestSkipped();
144  explicit TestSkipped(const std::string& reason);
145 };
146 
151 #define WREPORT_TEST_INFO(name) \
152  wreport::tests::LocationInfo wreport_test_location_info; \
153  wreport::tests::LocationInfo& name = wreport_test_location_info
154 
162 template <typename A> void assert_true(const A& actual)
164 {
165  if (actual)
166  return;
167  std::stringstream ss;
168  ss << "actual value " << actual << " is not true";
169  throw TestFailed(ss.str());
170 }
171 
172 [[noreturn]] void assert_true(std::nullptr_t actual);
173 
175 template <typename A> void assert_false(const A& actual)
176 {
177  if (!actual)
178  return;
179  std::stringstream ss;
180  ss << "actual value " << actual << " is not false";
181  throw TestFailed(ss.str());
182 }
183 
184 void assert_false(std::nullptr_t actual);
185 
186 template <typename LIST>
187 static inline void _format_list(std::ostream& o, const LIST& list)
188 {
189  bool first = true;
190  o << "[";
191  for (const auto& v : list)
192  {
193  if (first)
194  first = false;
195  else
196  o << ", ";
197  o << v;
198  }
199  o << "]";
200 }
201 
202 template <typename T>
203 void assert_equal(const std::vector<T>& actual, const std::vector<T>& expected)
204 {
205  if (actual == expected)
206  return;
207  std::stringstream ss;
208  ss << "value ";
209  _format_list(ss, actual);
210  ss << " is different than the expected ";
211  _format_list(ss, expected);
212  throw TestFailed(ss.str());
213 }
214 
215 template <typename T>
216 void assert_equal(const std::vector<T>& actual,
217  const std::initializer_list<T>& expected)
218 {
219  if (actual == expected)
220  return;
221  std::stringstream ss;
222  ss << "value ";
223  _format_list(ss, actual);
224  ss << " is different than the expected ";
225  _format_list(ss, expected);
226  throw TestFailed(ss.str());
227 }
228 
233 template <typename A, typename E>
234 void assert_equal(const A& actual, const E& expected)
235 {
236  if (actual == expected)
237  return;
238  std::stringstream ss;
239  ss << "value '" << actual << "' is different than the expected '"
240  << expected << "'";
241  throw TestFailed(ss.str());
242 }
243 
248 template <typename A, typename E>
249 void assert_not_equal(const A& actual, const E& expected)
250 {
251  if (actual != expected)
252  return;
253  std::stringstream ss;
254  ss << "value '" << actual << "' is not different than the expected '"
255  << expected << "'";
256  throw TestFailed(ss.str());
257 }
258 
260 template <typename A, typename E>
261 void assert_less(const A& actual, const E& expected)
262 {
263  if (actual < expected)
264  return;
265  std::stringstream ss;
266  ss << "value '" << actual << "' is not less than the expected '" << expected
267  << "'";
268  throw TestFailed(ss.str());
269 }
270 
272 template <typename A, typename E>
273 void assert_less_equal(const A& actual, const E& expected)
274 {
275  if (actual <= expected)
276  return;
277  std::stringstream ss;
278  ss << "value '" << actual
279  << "' is not less than or equals to the expected '" << expected << "'";
280  throw TestFailed(ss.str());
281 }
282 
284 template <typename A, typename E>
285 void assert_greater(const A& actual, const E& expected)
286 {
287  if (actual > expected)
288  return;
289  std::stringstream ss;
290  ss << "value '" << actual << "' is not greater than the expected '"
291  << expected << "'";
292  throw TestFailed(ss.str());
293 }
294 
296 template <typename A, typename E>
297 void assert_greater_equal(const A& actual, const E& expected)
298 {
299  if (actual >= expected)
300  return;
301  std::stringstream ss;
302  ss << "value '" << actual
303  << "' is not greater than or equals to the expected '" << expected
304  << "'";
305  throw TestFailed(ss.str());
306 }
307 
309 void assert_startswith(const std::string& actual, const std::string& expected);
310 
312 void assert_endswith(const std::string& actual, const std::string& expected);
313 
315 void assert_contains(const std::string& actual, const std::string& expected);
316 
318 void assert_not_contains(const std::string& actual,
319  const std::string& expected);
320 
327 void assert_re_matches(const std::string& actual, const std::string& expected);
328 
335 void assert_not_re_matches(const std::string& actual,
336  const std::string& expected);
337 
338 template <class A> struct Actual
339 {
340  A _actual;
341  Actual(const A& actual) : _actual(actual) {}
342  Actual(const Actual&) = default;
343  Actual(Actual&&) = default;
344  ~Actual() = default;
345  Actual& operator=(const Actual&) = delete;
346  Actual& operator=(Actual&&) = delete;
347 
348  void istrue() const { assert_true(_actual); }
349  void isfalse() const { assert_false(_actual); }
350  template <typename E> void operator==(const E& expected) const
351  {
352  assert_equal(_actual, expected);
353  }
354  template <typename E> void operator!=(const E& expected) const
355  {
356  assert_not_equal(_actual, expected);
357  }
358  template <typename E> void operator<(const E& expected) const
359  {
360  return assert_less(_actual, expected);
361  }
362  template <typename E> void operator<=(const E& expected) const
363  {
364  return assert_less_equal(_actual, expected);
365  }
366  template <typename E> void operator>(const E& expected) const
367  {
368  return assert_greater(_actual, expected);
369  }
370  template <typename E> void operator>=(const E& expected) const
371  {
372  return assert_greater_equal(_actual, expected);
373  }
374 };
375 
377 {
378  const char* _actual;
379  ActualCString(const char* s) : _actual(s) {}
380 
381  void istrue() const { return assert_true(_actual); }
382  void isfalse() const { return assert_false(_actual); }
383  void operator==(const char* expected) const;
384  void operator==(const std::string& expected) const;
385  void operator!=(const char* expected) const;
386  void operator!=(const std::string& expected) const;
387  void operator<(const std::string& expected) const;
388  void operator<=(const std::string& expected) const;
389  void operator>(const std::string& expected) const;
390  void operator>=(const std::string& expected) const;
391  void startswith(const std::string& expected) const;
392  void endswith(const std::string& expected) const;
393  void contains(const std::string& expected) const;
394  void not_contains(const std::string& expected) const;
395  void matches(const std::string& re) const;
396  void not_matches(const std::string& re) const;
397 };
398 
399 struct ActualStdString : public Actual<std::string>
400 {
401  explicit ActualStdString(const std::string& s) : Actual<std::string>(s) {}
402 
404  void operator==(const std::vector<uint8_t>& expected) const;
406  void operator!=(const std::vector<uint8_t>& expected) const;
407  void startswith(const std::string& expected) const;
408  void endswith(const std::string& expected) const;
409  void contains(const std::string& expected) const;
410  void not_contains(const std::string& expected) const;
411  void matches(const std::string& re) const;
412  void not_matches(const std::string& re) const;
413 };
414 
415 struct ActualPath : public Actual<std::filesystem::path>
416 {
417  explicit ActualPath(const std::filesystem::path& p)
419  {
420  }
421 
424 
425  // Check if the normalized paths match
426  void is(const std::filesystem::path& expected) const;
427  [[deprecated("Use path_startswith")]] void
428  startswith(const std::string& data) const;
429 
430  void path_startswith(const std::filesystem::path& expected) const;
431  void path_endswith(const std::filesystem::path& expected) const;
432  void path_contains(const std::filesystem::path& expected) const;
433  void path_not_contains(const std::filesystem::path& expected) const;
434 
435  void exists() const;
436  void not_exists() const;
437  void empty() const;
438  void not_empty() const;
439 
440  void contents_startwith(const std::string& data) const;
441  void contents_equal(const std::string& data) const;
442  void contents_equal(const std::vector<uint8_t>& data) const;
443  void contents_equal(const std::initializer_list<std::string>& lines) const;
444  void contents_match(const std::string& data_re) const;
445  void
446  contents_match(const std::initializer_list<std::string>& lines_re) const;
447 };
448 
449 struct ActualDouble : public Actual<double>
450 {
451  using Actual::Actual;
452 
453  void almost_equal(double expected, unsigned places) const;
454  void not_almost_equal(double expected, unsigned places) const;
455 };
456 
457 template <typename A> inline Actual<A> actual(const A& actual)
458 {
459  return Actual<A>(actual);
460 }
461 inline ActualCString actual(const char* actual)
462 {
463  return ActualCString(actual);
464 }
465 inline ActualCString actual(char* actual) { return ActualCString(actual); }
466 inline ActualStdString actual(const std::string& actual)
467 {
468  return ActualStdString(actual);
469 }
470 inline ActualStdString actual(const std::vector<uint8_t>& actual)
471 {
472  return ActualStdString(std::string(actual.begin(), actual.end()));
473 }
474 inline ActualPath actual(const std::filesystem::path& actual)
475 {
476  return ActualPath(actual);
477 }
478 inline ActualDouble actual(double actual) { return ActualDouble(actual); }
479 
480 struct ActualFunction : public Actual<std::function<void()>>
481 {
482  using Actual::Actual;
483 
484  void throws(const std::string& what_match) const;
485 };
486 
487 inline ActualFunction actual_function(std::function<void()> actual)
488 {
489  return ActualFunction(actual);
490 }
491 
492 inline ActualPath actual_path(const char* pathname)
493 {
494  return ActualPath(pathname);
495 }
496 inline ActualPath actual_path(const std::string& pathname)
497 {
498  return ActualPath(pathname);
499 }
500 inline ActualPath actual_file(const char* pathname)
501 {
502  return ActualPath(pathname);
503 }
504 inline ActualPath actual_file(const std::string& pathname)
505 {
506  return ActualPath(pathname);
507 }
508 
516 #define wassert(...) \
517  do \
518  { \
519  try \
520  { \
521  __VA_ARGS__; \
522  } \
523  catch (wreport::tests::TestFailed & e1) \
524  { \
525  e1.add_stack_info(__FILE__, __LINE__, #__VA_ARGS__, \
526  wreport_test_location_info); \
527  throw; \
528  } \
529  catch (std::exception & e2) \
530  { \
531  throw wreport::tests::TestFailed(e2, __FILE__, __LINE__, \
532  #__VA_ARGS__, \
533  wreport_test_location_info); \
534  } \
535  } while (0)
536 
538 #define wassert_true(...) wassert(actual(__VA_ARGS__).istrue())
539 
541 #define wassert_false(...) wassert(actual(__VA_ARGS__).isfalse())
542 
548 #define wassert_throws(exc, ...) \
549  [&]() { \
550  try \
551  { \
552  __VA_ARGS__; \
553  wfail_test(#__VA_ARGS__ " did not throw " #exc); \
554  } \
555  catch (TestFailed & e1) \
556  { \
557  throw; \
558  } \
559  catch (exc & e2) \
560  { \
561  return e2; \
562  } \
563  catch (std::exception & e3) \
564  { \
565  std::string msg(#__VA_ARGS__ " did not throw " #exc \
566  " but threw "); \
567  msg += typeid(e3).name(); \
568  msg += " instead"; \
569  wfail_test(msg); \
570  } \
571  }()
572 
580 #define wcallchecked(func) \
581  [&]() { \
582  try \
583  { \
584  return func; \
585  } \
586  catch (wreport::tests::TestFailed & e) \
587  { \
588  e.add_stack_info(__FILE__, __LINE__, #func, \
589  wreport_test_location_info); \
590  throw; \
591  } \
592  catch (std::exception & e) \
593  { \
594  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #func, \
595  wreport_test_location_info); \
596  } \
597  }()
598 
602 #define wfail_test(msg) wassert(throw wreport::tests::TestFailed((msg)))
603 
604 struct TestCase;
605 struct TestController;
606 struct TestRegistry;
607 struct TestCaseResult;
608 struct TestMethod;
609 struct TestMethodResult;
610 
615 {
617  std::string name;
618 
620  std::string doc;
621 
627  std::function<void()> test_function;
628 
629  TestMethod(const std::string& name_) : name(name_), test_function() {}
630 
631  TestMethod(const std::string& name_, std::function<void()> test_function_)
632  : name(name_), test_function(test_function_)
633  {
634  }
635 };
636 
641 struct TestCase
642 {
644  std::string name;
645 
647  std::vector<TestMethod> methods;
648 
650  bool tests_registered = false;
651 
652  TestCase(const std::string& name);
653  virtual ~TestCase() {}
654 
658  void register_tests_once();
659 
667  virtual void register_tests() = 0;
668 
672  virtual void setup() {}
673 
677  virtual void teardown() {}
678 
682  virtual void test_setup() {}
683 
687  virtual void test_teardown() {}
688 
693 
698 
706  virtual TestCaseResult run_tests(TestController& controller);
707 
720  virtual TestMethodResult run_test(TestController& controller,
721  TestMethod& method);
722 
727  TestMethod& add_method(const std::string& name_)
728  {
729  methods.emplace_back(name_);
730  return methods.back();
731  }
732 
736  template <typename... Args>
737  TestMethod& add_method(const std::string& name_,
738  std::function<void()> test_function)
739  {
740  methods.emplace_back(name_, test_function);
741  return methods.back();
742  }
743 
747  template <typename... Args>
748  TestMethod& add_method(const std::string& name_, const std::string& doc,
749  std::function<void()> test_function)
750  {
751  methods.emplace_back(name_, test_function);
752  methods.back().doc = doc;
753  return methods.back();
754  }
755 };
756 
767 struct Fixture
768 {
769  // Called before each test
770  void test_setup() {}
771 
772  // Called after each test
773  void test_teardown() {}
774 };
775 
776 template <typename Fixture, typename... Args>
777 static inline Fixture* fixture_factory(Args... args)
778 {
779  return new Fixture(args...);
780 }
781 
785 template <typename FIXTURE> class FixtureTestCase : public TestCase
786 {
787 public:
788  typedef FIXTURE Fixture;
789 
790  Fixture* fixture = nullptr;
791  std::function<Fixture*()> make_fixture;
792 
793  template <typename... Args>
794  FixtureTestCase(const std::string& name_, Args... args) : TestCase(name_)
795  {
796  make_fixture = std::bind(fixture_factory<FIXTURE, Args...>, args...);
797  }
798  FixtureTestCase(const FixtureTestCase&) = delete;
799  FixtureTestCase(FixtureTestCase&&) = delete;
800  FixtureTestCase& operator=(const FixtureTestCase&) = delete;
801  FixtureTestCase& operator=(FixtureTestCase&) = delete;
802 
803  void setup() override
804  {
805  TestCase::setup();
806  fixture = make_fixture();
807  }
808 
809  void teardown() override
810  {
811  delete fixture;
812  fixture = nullptr;
814  }
815 
816  void method_setup(TestMethodResult& mr) override
817  {
819  if (fixture)
820  fixture->test_setup();
821  }
822 
824  {
825  if (fixture)
826  fixture->test_teardown();
828  }
829 
834  template <typename... Args>
835  TestMethod& add_method(const std::string& name_,
836  std::function<void(FIXTURE&)> test_function)
837  {
838  return TestCase::add_method(name_, [=]() { test_function(*fixture); });
839  }
840 
845  template <typename... Args>
846  TestMethod& add_method(const std::string& name_, const std::string& doc,
847  std::function<void(FIXTURE&)> test_function)
848  {
849  return TestCase::add_method(name_, doc,
850  [=]() { test_function(*fixture); });
851  }
852 };
853 
854 } // namespace tests
855 } // namespace wreport
856 #endif
virtual TestCaseResult run_tests(TestController &controller)
Call setup(), run all the tests that have been registered, then call teardown().
Result of running a whole test case.
Definition: testrunner.h:92
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: utils/tests.h:641
std::string name
Name of the test case.
Definition: utils/tests.h:644
Information about one stack frame in the test execution stack.
Definition: utils/tests.h:68
Add information to the test backtrace for the tests run in the current scope.
Definition: utils/tests.h:56
Exception thrown when a test or a test case needs to be skipped.
Definition: utils/tests.h:139
virtual void method_setup(TestMethodResult &)
Set up before the test method is run.
Definition: utils/tests.h:692
void register_tests_once()
Idempotent wrapper for register_tests()
virtual void test_teardown()
Clean up after each test method is run.
Definition: utils/tests.h:687
bool tests_registered
Set to true the first time register_tests_once is run.
Definition: utils/tests.h:650
Abstract interface for the objects that supervise test execution.
Definition: testrunner.h:157
Definition: utils/tests.h:338
virtual void test_setup()
Set up before each test method is run.
Definition: utils/tests.h:682
virtual TestMethodResult run_test(TestController &controller, TestMethod &method)
Run a test method.
void teardown() override
Clean up after the test case is run.
Definition: utils/tests.h:809
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: utils/tests.h:106
Definition: utils/tests.h:415
void method_teardown(TestMethodResult &mr) override
Clean up after the test method is run.
Definition: utils/tests.h:823
std::vector< TestMethod > methods
All registered test methods.
Definition: utils/tests.h:647
std::string backtrace() const
Return the formatted backtrace for this location.
TestMethod & add_method(const std::string &name_, const std::string &doc, std::function< void()> test_function)
Register a new test method, including documentation.
Definition: utils/tests.h:748
Definition: utils/tests.h:91
virtual void setup()
Set up the test case before it is run.
Definition: utils/tests.h:672
Definition: utils/tests.h:376
std::string doc
Documentation attached to this test method.
Definition: utils/tests.h:620
Definition: utils/tests.h:480
Definition: utils/tests.h:449
Result of running a test method.
Definition: testrunner.h:25
TestMethod & add_method(const std::string &name_, const std::string &doc, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument, including documentation...
Definition: utils/tests.h:846
void setup() override
Set up the test case before it is run.
Definition: utils/tests.h:803
std::function< void()> test_function
Main body of the test method.
Definition: utils/tests.h:627
void method_setup(TestMethodResult &mr) override
Set up before the test method is run.
Definition: utils/tests.h:816
Definition: utils/tests.h:399
virtual void register_tests()=0
This will be called before running the test case, to populate it with its test methods.
TestMethod & add_method(const std::string &name_)
Register a new test method, with the actual test function to be added later.
Definition: utils/tests.h:727
String functions.
Definition: benchmark.h:13
Test case that includes a fixture.
Definition: utils/tests.h:785
TestMethod & add_method(const std::string &name_, std::function< void()> test_function)
Register a new test method.
Definition: utils/tests.h:737
Test method information.
Definition: utils/tests.h:614
virtual void method_teardown(TestMethodResult &)
Clean up after the test method is run.
Definition: utils/tests.h:697
virtual void teardown()
Clean up after the test case is run.
Definition: utils/tests.h:677
TestMethod & add_method(const std::string &name_, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument.
Definition: utils/tests.h:835
std::string name
Name of the test method.
Definition: utils/tests.h:617
Base class for test fixtures.
Definition: utils/tests.h:767
std::ostream & operator()()
Clear the current information and return the output stream to which new information can be sent...