5079590438

Committed 25 May 2023 12:28PM UTC coverage: 92.228% (+0.5%) from 91.723%

Build # 5079590438

Build Type

Pull #3502

github

Pull Request Pull Request #3502: Apply clang-format to the codebase

Run Details

75589 of 81959 relevant lines covered (92.23%)

12139530.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.81

/src/tests/runner/test_runner.cpp

/*
* (C) 2017 Jack Lloyd
* (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include "../tests.h"

#include "test_runner.h"
#include "test_stdout_reporter.h"
#include "test_xml_reporter.h"

#include <botan/version.h>
#include <botan/internal/cpuid.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_THREAD_UTILS)
   #include <botan/internal/rwlock.h>
   #include <botan/internal/thread_pool.h>
#endif

#include <shared_mutex>

namespace Botan_Tests {

Test_Runner::Test_Runner(std::ostream& out) : m_output(out) {}

Test_Runner::~Test_Runner() = default;

namespace {

/*
* This is a fast, simple, deterministic PRNG that's used for running
* the tests. It is not intended to be cryptographically secure.
*/
class Testsuite_RNG final : public Botan::RandomNumberGenerator {
   public:
      std::string name() const override { return "Testsuite_RNG"; }

      void clear() override { m_x = 0; }

      bool accepts_input() const override { return true; }

      bool is_seeded() const override { return true; }

      void fill_bytes_with_input(std::span<uint8_t> output, std::span<const uint8_t> input) override {
         for(const auto byte : input) {
            mix(byte);
         }

         for(auto& byte : output) {
            byte = mix();
         }
      }

      Testsuite_RNG(const std::vector<uint8_t>& seed, uint64_t test_counter) {
         m_x = ~test_counter;
         add_entropy(seed.data(), seed.size());
      }

   private:
      uint8_t mix(uint8_t input = 0) {
         m_x ^= input;
         m_x *= 0xF2E16957;
         m_x += 0xE50B590F;
         return static_cast<uint8_t>(m_x >> 27);
      }

      uint64_t m_x;
};

}

bool Test_Runner::run(const Test_Options& opts) {
   m_reporters.emplace_back(std::make_unique<StdoutReporter>(opts, output()));
   if(!opts.xml_results_dir().empty()) {
#if defined(BOTAN_TARGET_OS_HAS_FILESYSTEM)
      m_reporters.emplace_back(std::make_unique<XmlReporter>(opts, opts.xml_results_dir()));
#else
      output() << "Generating test report files is not supported on this platform\n";
#endif
   }

   auto req = Botan_Tests::Test::filter_registered_tests(opts.requested_tests(), opts.skip_tests());

   // TODO: Test runner should not be aware of certain test's environmental requirements.
   if(opts.pkcs11_lib().empty()) {
      // do not run pkcs11 tests by default unless pkcs11-lib set
      for(auto iter = req.begin(); iter != req.end();) {
         if((*iter).find("pkcs11") != std::string::npos) {
            iter = req.erase(iter);
         } else {
            ++iter;
         }
      }
   }

   if(req.empty()) {
      throw Test_Error("No tests to run");
   }

   std::vector<uint8_t> seed = Botan::hex_decode(opts.drbg_seed());
   if(seed.empty()) {
      const uint64_t ts = Botan_Tests::Test::timestamp();
      seed.resize(8);
      Botan::store_be(ts, seed.data());
   }

   for(auto& reporter : m_reporters) {
      const std::string cpuid = Botan::CPUID::to_string();
      if(!cpuid.empty())
         reporter->set_property("CPU flags", cpuid);

      if(!opts.pkcs11_lib().empty())
         reporter->set_property("pkcs11 library", opts.pkcs11_lib());

      if(!opts.provider().empty())
         reporter->set_property("provider", opts.provider());

      reporter->set_property("drbg_seed", Botan::hex_encode(seed));
   }

   Botan_Tests::Test::set_test_options(opts);

   for(size_t i = 0; i != opts.test_runs(); ++i) {
      Botan_Tests::Test::set_test_rng(std::make_shared<Testsuite_RNG>(seed, i));

      for(const auto& reporter : m_reporters) {
         reporter->next_test_run();
      }

      const bool passed =
         (opts.test_threads() == 1) ? run_tests(req) : run_tests_multithreaded(req, opts.test_threads());

      for(const auto& reporter : m_reporters) {
         reporter->render();
      }

      if(!passed)
         return false;
   }

   return true;
}

namespace {

std::vector<Test::Result> run_a_test(const std::string& test_name) {
   std::vector<Test::Result> results;

   try {
      if(test_name == "simd_32" && Botan::CPUID::has_simd_32() == false) {
         results.push_back(Test::Result::Note(test_name, "SIMD not available on this platform"));
      } else if(std::unique_ptr<Test> test = Test::get_test(test_name)) {
         std::vector<Test::Result> test_results = test->run();
         for(auto& result : test_results) {
            if(!result.code_location() && test->registration_location()) {
               // If a test result has no specific code location associated to it,
               // we fall back to the test case's registration location.
               result.set_code_location(test->registration_location().value());
            }
         }
         results.insert(results.end(), test_results.begin(), test_results.end());
      } else {
         results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));
      }
   } catch(std::exception& e) { results.push_back(Test::Result::Failure(test_name, e.what())); } catch(...) {
      results.push_back(Test::Result::Failure(test_name, "unknown exception"));
   }

   return results;
}

bool all_passed(const std::vector<Test::Result>& results) {
   return std::all_of(results.begin(), results.end(), [](const auto& r) { return r.tests_failed() == 0; });
}

}

bool Test_Runner::run_tests_multithreaded(const std::vector<std::string>& tests_to_run, size_t test_threads) {
   // If 0 then we let thread pool select the count
   BOTAN_ASSERT_NOMSG(test_threads != 1);

#if !defined(BOTAN_HAS_THREAD_UTILS)
   output() << "Running tests in multiple threads not enabled in this build\n";
   return run_tests(tests_to_run);

#else
   Botan::Thread_Pool pool(test_threads);
   Botan::RWLock rwlock;

   std::vector<std::future<std::vector<Test::Result>>> m_fut_results;

   auto run_test_exclusive = [&](const std::string& test_name) {
      std::unique_lock lk(rwlock);
      return run_a_test(test_name);
   };

   auto run_test_shared = [&](const std::string& test_name) {
      std::shared_lock lk(rwlock);
      return run_a_test(test_name);
   };

   for(const auto& test_name : tests_to_run) {
      if(Test::test_needs_serialization(test_name)) {
         m_fut_results.push_back(pool.run(run_test_exclusive, test_name));
      } else {
         m_fut_results.push_back(pool.run(run_test_shared, test_name));
      }
   }

   bool passed = true;
   for(size_t i = 0; i != m_fut_results.size(); ++i) {
      const auto results = m_fut_results[i].get();
      for(auto& reporter : m_reporters) {
         reporter->record(tests_to_run[i], results);
      }
      passed &= all_passed(results);
   }

   pool.shutdown();

   return passed;
#endif
}

bool Test_Runner::run_tests(const std::vector<std::string>& tests_to_run) {
   bool passed = true;
   for(const auto& test_name : tests_to_run) {
      const auto results = run_a_test(test_name);

      for(auto& reporter : m_reporters) {
         reporter->record(test_name, results);
      }
      passed &= all_passed(results);
   }

   return passed;
}

}

1	/*
2	* (C) 2017 Jack Lloyd
3	* (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include "../tests.h"
9
10	#include "test_runner.h"
11	#include "test_stdout_reporter.h"
12	#include "test_xml_reporter.h"
13
14	#include <botan/version.h>
15	#include <botan/internal/cpuid.h>
16	#include <botan/internal/loadstor.h>
17
18	#if defined(BOTAN_HAS_THREAD_UTILS)
19	#include <botan/internal/rwlock.h>
20	#include <botan/internal/thread_pool.h>
21	#endif
22
23	#include <shared_mutex>
24
25	namespace Botan_Tests {
26
27	Test_Runner::Test_Runner(std::ostream& out) : m_output(out) {}	1✔
28
29	Test_Runner::~Test_Runner() = default;	1✔
30
31	namespace {
32
33	/*
34	* This is a fast, simple, deterministic PRNG that's used for running
35	* the tests. It is not intended to be cryptographically secure.
36	*/
37	class Testsuite_RNG final : public Botan::RandomNumberGenerator {	1✔
38	public:
39	std::string name() const override { return "Testsuite_RNG"; }	×
40
41	void clear() override { m_x = 0; }	×
42
43	bool accepts_input() const override { return true; }	×
44
45	bool is_seeded() const override { return true; }	23,242✔
46
47	void fill_bytes_with_input(std::span<uint8_t> output, std::span<const uint8_t> input) override {	1,605,876✔
48	for(const auto byte : input) {	1,605,884✔
49	mix(byte);	8✔
50	}
51
52	for(auto& byte : output) {	9,821,331✔
53	byte = mix();	8,215,455✔
54	}
55	}	1,605,876✔
56
57	Testsuite_RNG(const std::vector<uint8_t>& seed, uint64_t test_counter) {	1✔
58	m_x = ~test_counter;	1✔
59	add_entropy(seed.data(), seed.size());	1✔
60	}	1✔
61
62	private:
63	uint8_t mix(uint8_t input = 0) {	8,215,463✔
64	m_x ^= input;	8,215,463✔
65	m_x *= 0xF2E16957;	8,215,463✔
66	m_x += 0xE50B590F;	8,215,463✔
67	return static_cast<uint8_t>(m_x >> 27);	8,215,463✔
68	}
69
70	uint64_t m_x;
71	};
72
73	}
74
75	bool Test_Runner::run(const Test_Options& opts) {	1✔
76	m_reporters.emplace_back(std::make_unique<StdoutReporter>(opts, output()));	1✔
77	if(!opts.xml_results_dir().empty()) {	1✔
78	#if defined(BOTAN_TARGET_OS_HAS_FILESYSTEM)
79	m_reporters.emplace_back(std::make_unique<XmlReporter>(opts, opts.xml_results_dir()));	2✔
80	#else
81	output() << "Generating test report files is not supported on this platform\n";
82	#endif
83	}
84
85	auto req = Botan_Tests::Test::filter_registered_tests(opts.requested_tests(), opts.skip_tests());	1✔
86
87	// TODO: Test runner should not be aware of certain test's environmental requirements.
88	if(opts.pkcs11_lib().empty()) {	1✔
89	// do not run pkcs11 tests by default unless pkcs11-lib set
90	for(auto iter = req.begin(); iter != req.end();) {	×
91	if((*iter).find("pkcs11") != std::string::npos) {	×
92	iter = req.erase(iter);	×
93	} else {
94	++iter;	×
95	}
96	}
97	}
98
99	if(req.empty()) {	1✔
100	throw Test_Error("No tests to run");	×
101	}
102
103	std::vector<uint8_t> seed = Botan::hex_decode(opts.drbg_seed());	1✔
104	if(seed.empty()) {	1✔
105	const uint64_t ts = Botan_Tests::Test::timestamp();	1✔
106	seed.resize(8);	1✔
107	Botan::store_be(ts, seed.data());	1✔
108	}
109
110	for(auto& reporter : m_reporters) {	3✔
111	const std::string cpuid = Botan::CPUID::to_string();	2✔
112	if(!cpuid.empty())	2✔
113	reporter->set_property("CPU flags", cpuid);	4✔
114
115	if(!opts.pkcs11_lib().empty())	2✔
116	reporter->set_property("pkcs11 library", opts.pkcs11_lib());	4✔
117
118	if(!opts.provider().empty())	2✔
119	reporter->set_property("provider", opts.provider());	×
120
121	reporter->set_property("drbg_seed", Botan::hex_encode(seed));	6✔
122	}	2✔
123
124	Botan_Tests::Test::set_test_options(opts);	1✔
125
126	for(size_t i = 0; i != opts.test_runs(); ++i) {	2✔
127	Botan_Tests::Test::set_test_rng(std::make_shared<Testsuite_RNG>(seed, i));	2✔
128
129	for(const auto& reporter : m_reporters) {	3✔
130	reporter->next_test_run();	2✔
131	}
132
133	const bool passed =	1✔
134	(opts.test_threads() == 1) ? run_tests(req) : run_tests_multithreaded(req, opts.test_threads());	1✔
135
136	for(const auto& reporter : m_reporters) {	3✔
137	reporter->render();	2✔
138	}
139
140	if(!passed)	1✔
141	return false;	×
142	}
143
144	return true;	1✔
145	}	1✔
146
147	namespace {
148
149	std::vector<Test::Result> run_a_test(const std::string& test_name) {	311✔
150	std::vector<Test::Result> results;	311✔
151
152	try {	311✔
153	if(test_name == "simd_32" && Botan::CPUID::has_simd_32() == false) {	311✔
154	results.push_back(Test::Result::Note(test_name, "SIMD not available on this platform"));	×
155	} else if(std::unique_ptr<Test> test = Test::get_test(test_name)) {	311✔
156	std::vector<Test::Result> test_results = test->run();	311✔
157	for(auto& result : test_results) {	45,469✔
158	if(!result.code_location() && test->registration_location()) {	45,158✔
159	// If a test result has no specific code location associated to it,
160	// we fall back to the test case's registration location.
161	result.set_code_location(test->registration_location().value());	45,158✔
162	}
163	}
164	results.insert(results.end(), test_results.begin(), test_results.end());	311✔
165	} else {	311✔
166	results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));	×
167	}	311✔
168	} catch(std::exception& e) { results.push_back(Test::Result::Failure(test_name, e.what())); } catch(...) {	×
169	results.push_back(Test::Result::Failure(test_name, "unknown exception"));	×
170	}	×
171
172	return results;	311✔
173	}	×
174
175	bool all_passed(const std::vector<Test::Result>& results) {	311✔
176	return std::all_of(results.begin(), results.end(), [](const auto& r) { return r.tests_failed() == 0; });	11,594✔
177	}
178
179	}
180
181	bool Test_Runner::run_tests_multithreaded(const std::vector<std::string>& tests_to_run, size_t test_threads) {	1✔
182	// If 0 then we let thread pool select the count
183	BOTAN_ASSERT_NOMSG(test_threads != 1);	1✔
184
185	#if !defined(BOTAN_HAS_THREAD_UTILS)
186	output() << "Running tests in multiple threads not enabled in this build\n";
187	return run_tests(tests_to_run);
188
189	#else
190	Botan::Thread_Pool pool(test_threads);	1✔
191	Botan::RWLock rwlock;	1✔
192
193	std::vector<std::future<std::vector<Test::Result>>> m_fut_results;	1✔
194
195	auto run_test_exclusive = [&](const std::string& test_name) {	18✔
196	std::unique_lock lk(rwlock);	17✔
197	return run_a_test(test_name);	17✔
198	};	17✔
199
200	auto run_test_shared = [&](const std::string& test_name) {	295✔
201	std::shared_lock lk(rwlock);	294✔
202	return run_a_test(test_name);	294✔
203	};	294✔
204
205	for(const auto& test_name : tests_to_run) {	312✔
206	if(Test::test_needs_serialization(test_name)) {	311✔
207	m_fut_results.push_back(pool.run(run_test_exclusive, test_name));	34✔
208	} else {
209	m_fut_results.push_back(pool.run(run_test_shared, test_name));	588✔
210	}
211	}
212
213	bool passed = true;
214	for(size_t i = 0; i != m_fut_results.size(); ++i) {	312✔
215	const auto results = m_fut_results[i].get();	311✔
216	for(auto& reporter : m_reporters) {	933✔
217	reporter->record(tests_to_run[i], results);	622✔
218	}
219	passed &= all_passed(results);	311✔
220	}	311✔
221
222	pool.shutdown();	1✔
223
224	return passed;	1✔
225	#endif
226	}	2✔
227
228	bool Test_Runner::run_tests(const std::vector<std::string>& tests_to_run) {	×
229	bool passed = true;	×
230	for(const auto& test_name : tests_to_run) {	×
231	const auto results = run_a_test(test_name);	×
232
233	for(auto& reporter : m_reporters) {	×
234	reporter->record(test_name, results);	×
235	}
236	passed &= all_passed(results);	×
237	}	×
238
239	return passed;	×
240	}
241
242	}

randombit / botan / 5079590438

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous