21753596263

Committed 06 Feb 2026 02:13PM UTC coverage: 90.063% (-0.01%) from 90.073%

Build # 21753596263

Build Type

Pull #5289

github

Committed by

web-flow

Commit Message

Merge 587099284 into 8ea0ca252

Pull Request Pull Request #5289: Further misc header reductions, forward declarations, etc

Run Details

102237 of 113517 relevant lines covered (90.06%)

11402137.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.76

/src/tests/runner/test_runner.cpp

/*
* (C) 2017 Jack Lloyd
* (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include "../tests.h"

#include "test_runner.h"
#include "test_stdout_reporter.h"
#include "test_xml_reporter.h"

#include <botan/hex.h>
#include <botan/version.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_CPUID)
   #include <botan/internal/cpuid.h>
#endif

#if defined(BOTAN_HAS_THREAD_UTILS)
   #include <botan/internal/rwlock.h>
   #include <botan/internal/thread_pool.h>
#endif

#include <shared_mutex>

namespace Botan_Tests {

Test_Runner::Test_Runner(std::ostream& out) : m_output(out) {}

Test_Runner::~Test_Runner() = default;

bool Test_Runner::run(const Test_Options& options) {
   if(!options.no_stdout()) {
      m_reporters.emplace_back(std::make_unique<StdoutReporter>(options, output()));
   }
   if(!options.xml_results_dir().empty()) {
#if defined(BOTAN_TARGET_OS_HAS_FILESYSTEM)
      m_reporters.emplace_back(std::make_unique<XmlReporter>(options, options.xml_results_dir()));
#else
      output() << "Generating test report files is not supported on this platform\n";
#endif
   }

   auto req = Botan_Tests::Test::filter_registered_tests(options.requested_tests(), options.skip_tests());

   // TODO: Test runner should not be aware of certain test's environmental requirements.
   if(options.pkcs11_lib().empty()) {
      // do not run pkcs11 tests by default unless pkcs11-lib set
      for(auto iter = req.begin(); iter != req.end();) {
         if((*iter).find("pkcs11") != std::string::npos) {
            iter = req.erase(iter);
         } else {
            ++iter;
         }
      }
   }

   if(req.empty()) {
      throw Test_Error("No tests to run");
   }

   std::vector<uint8_t> seed = Botan::hex_decode(options.drbg_seed());
   if(seed.empty()) {
      const uint64_t ts = Botan_Tests::Test::timestamp();
      seed.resize(8);
      Botan::store_be(ts, seed.data());
   }

   for(auto& reporter : m_reporters) {
#if defined(BOTAN_HAS_CPUID)
      const std::string cpuid = Botan::CPUID::to_string();
      if(!cpuid.empty()) {
         reporter->set_property("CPU flags", cpuid);
      }
#endif

      if(!options.pkcs11_lib().empty()) {
         reporter->set_property("pkcs11 library", options.pkcs11_lib());
      }

      if(!options.provider().empty()) {
         reporter->set_property("provider", options.provider());
      }

      reporter->set_property("drbg_seed", Botan::hex_encode(seed));
   }

   Botan_Tests::Test::set_test_options(options);

   for(size_t i = 0; i != options.test_runs(); ++i) {
      Botan_Tests::Test::set_test_rng_seed(seed, i);

      for(const auto& reporter : m_reporters) {
         reporter->next_test_run();
      }

      const bool passed =
         (options.test_threads() == 1) ? run_tests(req) : run_tests_multithreaded(req, options.test_threads());

      for(const auto& reporter : m_reporters) {
         reporter->render();
      }

      if(!passed) {
         return false;
      }
   }

   return true;
}

namespace {

std::vector<Test::Result> run_a_test(const std::string& test_name) {
   std::vector<Test::Result> results;

   try {
#if defined(BOTAN_HAS_CPUID) && defined(BOTAN_HAS_SIMD_4X32)
      if(test_name == "simd_4x32" && !Botan::CPUID::has(Botan::CPUID::Feature::SIMD_4X32)) {
         results.push_back(Test::Result::Note(test_name, "SIMD 4x32 not available on this platform"));
         return results;
      }
#endif

      if(std::unique_ptr<Test> test = Test::get_test(test_name)) {
         std::vector<Test::Result> test_results = test->run();
         for(auto& result : test_results) {
            if(!result.code_location() && test->registration_location()) {
               // If a test result has no specific code location associated to it,
               // we fall back to the test case's registration location.
               result.set_code_location(test->registration_location().value());
            }
         }
         results.insert(results.end(), test_results.begin(), test_results.end());
      } else {
         results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));
      }
   } catch(std::exception& e) {
      results.push_back(Test::Result::Failure(test_name, e.what()));
   } catch(...) {
      results.push_back(Test::Result::Failure(test_name, "unknown exception"));
   }

   return results;
}

bool all_passed(const std::vector<Test::Result>& results) {
   return std::all_of(results.begin(), results.end(), [](const auto& r) { return r.tests_failed() == 0; });
}

}  // namespace

bool Test_Runner::run_tests_multithreaded(const std::vector<std::string>& tests_to_run, size_t test_threads) {
   // If 0 then we let thread pool select the count
   BOTAN_ASSERT_NOMSG(test_threads != 1);

#if !defined(BOTAN_HAS_THREAD_UTILS)
   output() << "Running tests in multiple threads not enabled in this build\n";
   return run_tests(tests_to_run);

#else
   Botan::Thread_Pool pool(test_threads);
   Botan::RWLock rwlock;

   std::vector<std::future<std::vector<Test::Result>>> fut_results;

   auto run_test_exclusive = [&](const std::string& test_name) {
      const std::unique_lock lk(rwlock);
      return run_a_test(test_name);
   };

   auto run_test_shared = [&](const std::string& test_name) {
      const std::shared_lock lk(rwlock);
      return run_a_test(test_name);
   };

   for(const auto& test_name : tests_to_run) {
      if(Test::test_needs_serialization(test_name)) {
         fut_results.push_back(pool.run(run_test_exclusive, test_name));
      } else {
         fut_results.push_back(pool.run(run_test_shared, test_name));
      }
   }

   bool passed = true;
   for(size_t i = 0; i != fut_results.size(); ++i) {
      for(auto& reporter : m_reporters) {
         reporter->waiting_for_next_results(tests_to_run[i]);
      }
      const auto results = fut_results[i].get();
      for(auto& reporter : m_reporters) {
         reporter->record(tests_to_run[i], results);
      }
      passed &= all_passed(results);
   }

   pool.shutdown();

   return passed;
#endif
}

bool Test_Runner::run_tests(const std::vector<std::string>& tests_to_run) {
   bool passed = true;
   for(const auto& test_name : tests_to_run) {
      for(auto& reporter : m_reporters) {
         reporter->waiting_for_next_results(test_name);
      }
      const auto results = run_a_test(test_name);

      for(auto& reporter : m_reporters) {
         reporter->record(test_name, results);
      }
      passed &= all_passed(results);
   }

   return passed;
}

}  // namespace Botan_Tests

1	/*
2	* (C) 2017 Jack Lloyd
3	* (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include "../tests.h"
9
10	#include "test_runner.h"
11	#include "test_stdout_reporter.h"
12	#include "test_xml_reporter.h"
13
14	#include <botan/hex.h>
15	#include <botan/version.h>
16	#include <botan/internal/loadstor.h>
17
18	#if defined(BOTAN_HAS_CPUID)
19	#include <botan/internal/cpuid.h>
20	#endif
21
22	#if defined(BOTAN_HAS_THREAD_UTILS)
23	#include <botan/internal/rwlock.h>
24	#include <botan/internal/thread_pool.h>
25	#endif
26
27	#include <shared_mutex>
28
29	namespace Botan_Tests {
30
31	Test_Runner::Test_Runner(std::ostream& out) : m_output(out) {}	1✔
32
33	Test_Runner::~Test_Runner() = default;	1✔
34
35	bool Test_Runner::run(const Test_Options& options) {	1✔
36	if(!options.no_stdout()) {	1✔
37	m_reporters.emplace_back(std::make_unique<StdoutReporter>(options, output()));	2✔
38	}
39	if(!options.xml_results_dir().empty()) {	1✔
40	#if defined(BOTAN_TARGET_OS_HAS_FILESYSTEM)
41	m_reporters.emplace_back(std::make_unique<XmlReporter>(options, options.xml_results_dir()));	2✔
42	#else
43	output() << "Generating test report files is not supported on this platform\n";
44	#endif
45	}
46
47	auto req = Botan_Tests::Test::filter_registered_tests(options.requested_tests(), options.skip_tests());	1✔
48
49	// TODO: Test runner should not be aware of certain test's environmental requirements.
50	if(options.pkcs11_lib().empty()) {	1✔
51	// do not run pkcs11 tests by default unless pkcs11-lib set
52	for(auto iter = req.begin(); iter != req.end();) {	×
53	if((*iter).find("pkcs11") != std::string::npos) {	×
54	iter = req.erase(iter);	×
55	} else {
56	++iter;	×
57	}
58	}
59	}
60
61	if(req.empty()) {	1✔
62	throw Test_Error("No tests to run");	×
63	}
64
65	std::vector<uint8_t> seed = Botan::hex_decode(options.drbg_seed());	1✔
66	if(seed.empty()) {	1✔
67	const uint64_t ts = Botan_Tests::Test::timestamp();	1✔
68	seed.resize(8);	1✔
69	Botan::store_be(ts, seed.data());	1✔
70	}
71
72	for(auto& reporter : m_reporters) {	3✔
73	#if defined(BOTAN_HAS_CPUID)
74	const std::string cpuid = Botan::CPUID::to_string();	2✔
75	if(!cpuid.empty()) {	2✔
76	reporter->set_property("CPU flags", cpuid);	4✔
77	}
78	#endif
79
80	if(!options.pkcs11_lib().empty()) {	2✔
81	reporter->set_property("pkcs11 library", options.pkcs11_lib());	4✔
82	}
83
84	if(!options.provider().empty()) {	2✔
85	reporter->set_property("provider", options.provider());	×
86	}
87
88	reporter->set_property("drbg_seed", Botan::hex_encode(seed));	4✔
89	}	2✔
90
91	Botan_Tests::Test::set_test_options(options);	1✔
92
93	for(size_t i = 0; i != options.test_runs(); ++i) {	2✔
94	Botan_Tests::Test::set_test_rng_seed(seed, i);	1✔
95
96	for(const auto& reporter : m_reporters) {	3✔
97	reporter->next_test_run();	2✔
98	}
99
100	const bool passed =	1✔
101	(options.test_threads() == 1) ? run_tests(req) : run_tests_multithreaded(req, options.test_threads());	1✔
102
103	for(const auto& reporter : m_reporters) {	3✔
104	reporter->render();	2✔
105	}
106
107	if(!passed) {	1✔
108	return false;
109	}
110	}
111
112	return true;
113	}	1✔
114
115	namespace {
116
117	std::vector<Test::Result> run_a_test(const std::string& test_name) {	419✔
118	std::vector<Test::Result> results;	419✔
119
120	try {	419✔
121	#if defined(BOTAN_HAS_CPUID) && defined(BOTAN_HAS_SIMD_4X32)
122	if(test_name == "simd_4x32" && !Botan::CPUID::has(Botan::CPUID::Feature::SIMD_4X32)) {	419✔
123	results.push_back(Test::Result::Note(test_name, "SIMD 4x32 not available on this platform"));	×
124	return results;	×
125	}
126	#endif
127
128	if(std::unique_ptr<Test> test = Test::get_test(test_name)) {	419✔
129	std::vector<Test::Result> test_results = test->run();	419✔
130	for(auto& result : test_results) {	51,111✔
131	if(!result.code_location() && test->registration_location()) {	50,692✔
132	// If a test result has no specific code location associated to it,
133	// we fall back to the test case's registration location.
134	result.set_code_location(test->registration_location().value());	50,692✔
135	}
136	}
137	results.insert(results.end(), test_results.begin(), test_results.end());	419✔
138	} else {	419✔
139	results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));	×
140	}	419✔
141	} catch(std::exception& e) {	×
142	results.push_back(Test::Result::Failure(test_name, e.what()));	×
143	} catch(...) {	×
144	results.push_back(Test::Result::Failure(test_name, "unknown exception"));	×
145	}	×
146
147	return results;
148	}	×
149
150	bool all_passed(const std::vector<Test::Result>& results) {	419✔
151	return std::all_of(results.begin(), results.end(), [](const auto& r) { return r.tests_failed() == 0; });	13,081✔
152	}
153
154	} // namespace
155
156	bool Test_Runner::run_tests_multithreaded(const std::vector<std::string>& tests_to_run, size_t test_threads) {	1✔
157	// If 0 then we let thread pool select the count
158	BOTAN_ASSERT_NOMSG(test_threads != 1);	1✔
159
160	#if !defined(BOTAN_HAS_THREAD_UTILS)
161	output() << "Running tests in multiple threads not enabled in this build\n";
162	return run_tests(tests_to_run);
163
164	#else
165	Botan::Thread_Pool pool(test_threads);	1✔
166	Botan::RWLock rwlock;	1✔
167
168	std::vector<std::future<std::vector<Test::Result>>> fut_results;	1✔
169
170	auto run_test_exclusive = [&](const std::string& test_name) {	22✔
171	const std::unique_lock lk(rwlock);	21✔
172	return run_a_test(test_name);	21✔
173	};	21✔
174
175	auto run_test_shared = [&](const std::string& test_name) {	399✔
176	const std::shared_lock lk(rwlock);	398✔
177	return run_a_test(test_name);	398✔
178	};	398✔
179
180	for(const auto& test_name : tests_to_run) {	420✔
181	if(Test::test_needs_serialization(test_name)) {	419✔
182	fut_results.push_back(pool.run(run_test_exclusive, test_name));	42✔
183	} else {
184	fut_results.push_back(pool.run(run_test_shared, test_name));	796✔
185	}
186	}
187
188	bool passed = true;
189	for(size_t i = 0; i != fut_results.size(); ++i) {	420✔
190	for(auto& reporter : m_reporters) {	1,257✔
191	reporter->waiting_for_next_results(tests_to_run[i]);	838✔
192	}
193	const auto results = fut_results[i].get();	419✔
194	for(auto& reporter : m_reporters) {	1,257✔
195	reporter->record(tests_to_run[i], results);	838✔
196	}
197	passed &= all_passed(results);	419✔
198	}	419✔
199
200	pool.shutdown();	1✔
201
202	return passed;	1✔
203	#endif
204	}	2✔
205
206	bool Test_Runner::run_tests(const std::vector<std::string>& tests_to_run) {	×
207	bool passed = true;	×
208	for(const auto& test_name : tests_to_run) {	×
209	for(auto& reporter : m_reporters) {	×
210	reporter->waiting_for_next_results(test_name);	×
211	}
212	const auto results = run_a_test(test_name);	×
213
214	for(auto& reporter : m_reporters) {	×
215	reporter->record(test_name, results);	×
216	}
217	passed &= all_passed(results);	×
218	}	×
219
220	return passed;	×
221	}
222
223	} // namespace Botan_Tests

randombit / botan / 21753596263

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous