#117

Committed 25 May 2025 03:01PM UTC coverage: 66.932% (-0.07%) from 67.006%

Build # #117

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

iwyu and reduce build-time inter-dependencies (#144)

Run Details

26 of 145 new or added lines in 20 files covered. (17.93%)

89 existing lines in 5 files now uncovered.

9738 of 14549 relevant lines covered (66.93%)

3041.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/benchmarking.cpp

// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: 2024 Mikkel Schubert <mikkelsch@gmail.com>
#include "benchmarking.hpp" // for benchmark_toggles, benchmarker
#include "debug.hpp"        // for AR_REQUIRE
#include "logging.hpp"      // for log
#include "mathutils.hpp"    // for arithmetic_mean, standard_deviation
#include "strutils.hpp"     // for to_lower
#include <algorithm>        // for find, accumulate
#include <array>            // for array
#include <cmath>            // for round
#include <cstdint>          // for uint64_t
#include <iomanip>          // for fixed, setsetprecision
#include <iostream>         // for cout
#include <numeric>          // for accumulate
#include <sstream>          // for ostringstream
#include <string>           // for string, to_string
#include <utility>          // for move
#include <vector>           // for vector

namespace adapterremoval {

namespace {

//! Number of loops to perform prior to benchmarking as burn-in
const size_t BENCHMARK_BURN_IN = 1;
//! Benchmarks must be repeated at least this number of times
const size_t BENCHMARK_MIN_LOOPS = 10;
//! Benchmarks must be repeated at most this number of times
const size_t BENCHMARK_MAX_LOOPS = 1000;
//! Benchmarks must run for at this this number of nano-seconds
const double BENCHMARK_MIN_TIME_NS = 5'000'000'000;
//! Benchmark loops shorter than this number of nano-seconds cannot be measured
const double BENCHMARK_CUTOFF_TIME_NS = 10'000;
//! Number of NS between terminal updates
const size_t BENCHMARK_UPDATE_INTERVAL = 50'000'000;

} // namespace

benchmark_toggles::benchmark_toggles(std::vector<std::string> keys)
  : m_toggles(std::move(keys))
  , m_enabled(m_toggles.size(), false)
  , m_defaults()
{
}

bool
benchmark_toggles::update_toggles(const std::vector<std::string>& keys)
{
  bool any_errors = false;
  m_defaults = keys.empty();
  for (const auto& key : keys) {
    const auto it =
      std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));

    if (it == m_toggles.end()) {
      log::error() << "Unknown benchmarking toggle '" << key << "'";
      any_errors = true;
    } else {
      m_enabled.at(it - m_toggles.begin()) = true;
    }
  }

  if (any_errors) {
    auto msg = log::error();
    msg << "Valid toggles are " << join_text(m_toggles, ", ", ", and ");
  }

  return !any_errors;
}

bool
benchmark_toggles::is_set(const std::string& key) const
{
  const auto it = std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));

  AR_REQUIRE(it != m_toggles.end());
  return m_enabled.at(it - m_toggles.begin());
}

benchmarker::benchmarker(std::string desc, std::vector<std::string> toggles)
  : m_description(std::move(desc))
  , m_toggles(std::move(toggles))
{
}

void
benchmarker::run_if_toggled(const benchmark_toggles& toggles)
{
  const auto s = enabled(toggles);
  if (s != strategy::skip) {
    run(s);
  }
}

/** Called before `setup` to perform any per batch setup */
void
benchmarker::setup()
{
}

strategy
benchmarker::enabled(const benchmark_toggles& toggles) const
{
  if (toggles.defaults()) {
    return strategy::benchmark;
  }

  for (const auto& toggle : m_toggles) {
    if (toggles.is_set(toggle)) {
      return strategy::benchmark;
    }
  }

  return m_required ? strategy::passthrough : strategy::skip;
}

void
benchmarker::run(const strategy s)
{
  static bool header = false;
  if (!header) {
    std::cout << "             Benchmark |       Min |      Mean |       Max | "
                 "SD (%) | Loops | Outliers"
              << std::endl;
    header = true;
  }

  if (s != strategy::passthrough) {
    log::cerr() << "\r\033[K" << std::setw(22) << m_description << " burn-in";

    for (size_t i = 1; i <= BENCHMARK_BURN_IN; ++i) {
      setup();
      execute();

      log::cerr() << ".";
    }
  } else {
    log::cerr() << "\r\033[K" << std::setw(22) << m_description << " (setup)";
  }

  size_t loops = 0;
  uint64_t next_update = 0;
  do {
    uint64_t elapsed =
      std::accumulate(m_durations.begin(), m_durations.end(), uint64_t());

    do {
      setup();
      const auto start = clock::now();
      execute();
      const auto duration = (clock::now() - start).count();

      loops++;
      elapsed += duration;
      m_durations.push_back(duration);

      if (elapsed >= next_update) {
        log::cerr() << "\r\033[K" << summarize(loops);
        next_update += BENCHMARK_UPDATE_INTERVAL;
      }
    } while (s == strategy::benchmark &&
             m_durations.size() < BENCHMARK_MAX_LOOPS &&
             (m_durations.size() < BENCHMARK_MIN_LOOPS ||
              (elapsed < BENCHMARK_MIN_TIME_NS &&
               elapsed / m_durations.size() >= BENCHMARK_CUTOFF_TIME_NS)));
  } while (s == strategy::benchmark && (loops < 2 * m_durations.size()) &&
           grubbs_test_prune(m_durations));

  log::cerr() << "\r\033[K";

  if (s != strategy::passthrough) {
    std::cout << summarize(loops) << std::endl;
  }
}

std::string
benchmarker::summarize(size_t loops) const
{
  const std::array<size_t, 7> COLUMN_WIDTHS{ 22, 9, 9, 9, 6, 5, 8 };
  std::array<std::string, COLUMN_WIDTHS.size()> values{
    m_description,
    "",
    "",
    "",
    "",
    std::to_string(m_durations.size()),
    std::to_string(loops - m_durations.size()),
  };

  if (!m_durations.empty()) {
    const auto min_max =
      std::minmax_element(m_durations.begin(), m_durations.end());
    const auto mean = arithmetic_mean(m_durations);

    values.at(1) = format_thousand_sep(*min_max.first / 1e3);
    values.at(2) = format_thousand_sep(std::round(mean / 1e3));
    values.at(3) = format_thousand_sep(*min_max.second / 1e3);

    if (m_durations.size() > 1) {
      const auto sd = standard_deviation(m_durations);
      values.at(4) = format_fraction(1e9 * sd, 1e7 * mean);
    }
  }

  std::ostringstream ss;
  for (size_t i = 0; i < values.size(); ++i) {
    if (i) {
      ss << " | ";
    }

    ss << std::setw(COLUMN_WIDTHS.at(i)) << values.at(i);
  }

  return ss.str();
}

} // namespace adapterremoval

1	// SPDX-License-Identifier: GPL-3.0-or-later
2	// SPDX-FileCopyrightText: 2024 Mikkel Schubert <mikkelsch@gmail.com>
3	#include "benchmarking.hpp" // for benchmark_toggles, benchmarker
4	#include "debug.hpp" // for AR_REQUIRE
5	#include "logging.hpp" // for log
6	#include "mathutils.hpp" // for arithmetic_mean, standard_deviation
7	#include "strutils.hpp" // for to_lower
8	#include <algorithm> // for find, accumulate
9	#include <array> // for array
10	#include <cmath> // for round
11	#include <cstdint> // for uint64_t
12	#include <iomanip> // for fixed, setsetprecision
13	#include <iostream> // for cout
14	#include <numeric> // for accumulate
15	#include <sstream> // for ostringstream
16	#include <string> // for string, to_string
17	#include <utility> // for move
18	#include <vector> // for vector
19
20	namespace adapterremoval {
21
22	namespace {
23
24	//! Number of loops to perform prior to benchmarking as burn-in
25	const size_t BENCHMARK_BURN_IN = 1;
26	//! Benchmarks must be repeated at least this number of times
27	const size_t BENCHMARK_MIN_LOOPS = 10;
28	//! Benchmarks must be repeated at most this number of times
29	const size_t BENCHMARK_MAX_LOOPS = 1000;
30	//! Benchmarks must run for at this this number of nano-seconds
31	const double BENCHMARK_MIN_TIME_NS = 5'000'000'000;
32	//! Benchmark loops shorter than this number of nano-seconds cannot be measured
33	const double BENCHMARK_CUTOFF_TIME_NS = 10'000;
34	//! Number of NS between terminal updates
35	const size_t BENCHMARK_UPDATE_INTERVAL = 50'000'000;
36
37	} // namespace
38
NEW 39	benchmark_toggles::benchmark_toggles(std::vector<std::string> keys)	×
40	: m_toggles(std::move(keys))	×
41	, m_enabled(m_toggles.size(), false)	×
42	, m_defaults()	×
43	{
44	}
45
46	bool
NEW 47	benchmark_toggles::update_toggles(const std::vector<std::string>& keys)	×
48	{
49	bool any_errors = false;	×
50	m_defaults = keys.empty();	×
51	for (const auto& key : keys) {	×
52	const auto it =	×
53	std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));	×
54
55	if (it == m_toggles.end()) {	×
56	log::error() << "Unknown benchmarking toggle '" << key << "'";	×
57	any_errors = true;	×
58	} else {
59	m_enabled.at(it - m_toggles.begin()) = true;	×
60	}
61	}
62
63	if (any_errors) {	×
64	auto msg = log::error();	×
65	msg << "Valid toggles are " << join_text(m_toggles, ", ", ", and ");	×
66	}
67
68	return !any_errors;	×
69	}
70
71	bool
72	benchmark_toggles::is_set(const std::string& key) const	×
73	{
74	const auto it = std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));	×
75
76	AR_REQUIRE(it != m_toggles.end());	×
77	return m_enabled.at(it - m_toggles.begin());	×
78	}
79
NEW 80	benchmarker::benchmarker(std::string desc, std::vector<std::string> toggles)	×
81	: m_description(std::move(desc))	×
82	, m_toggles(std::move(toggles))	×
83	{
84	}
85
86	void
87	benchmarker::run_if_toggled(const benchmark_toggles& toggles)	×
88	{
89	const auto s = enabled(toggles);	×
90	if (s != strategy::skip) {	×
91	run(s);	×
92	}
93	}
94
95	/** Called before `setup` to perform any per batch setup */
96	void
NEW 97	benchmarker::setup()	×
98	{
99	}
100
101	strategy
102	benchmarker::enabled(const benchmark_toggles& toggles) const	×
103	{
104	if (toggles.defaults()) {	×
105	return strategy::benchmark;
106	}
107
108	for (const auto& toggle : m_toggles) {	×
109	if (toggles.is_set(toggle)) {	×
110	return strategy::benchmark;	×
111	}
112	}
113
114	return m_required ? strategy::passthrough : strategy::skip;	×
115	}
116
117	void
118	benchmarker::run(const strategy s)	×
119	{
120	static bool header = false;
121	if (!header) {	×
122	std::cout << " Benchmark \| Min \| Mean \| Max \| "	×
123	"SD (%) \| Loops \| Outliers"	×
124	<< std::endl;	×
125	header = true;	×
126	}
127
128	if (s != strategy::passthrough) {	×
129	log::cerr() << "\r\033[K" << std::setw(22) << m_description << " burn-in";	×
130
131	for (size_t i = 1; i <= BENCHMARK_BURN_IN; ++i) {	×
132	setup();	×
133	execute();	×
134
135	log::cerr() << ".";	×
136	}
137	} else {
138	log::cerr() << "\r\033[K" << std::setw(22) << m_description << " (setup)";	×
139	}
140
141	size_t loops = 0;
142	uint64_t next_update = 0;
143	do {	×
144	uint64_t elapsed =	×
145	std::accumulate(m_durations.begin(), m_durations.end(), uint64_t());	×
146
147	do {	×
148	setup();	×
149	const auto start = clock::now();	×
150	execute();	×
151	const auto duration = (clock::now() - start).count();	×
152
153	loops++;	×
154	elapsed += duration;	×
155	m_durations.push_back(duration);	×
156
157	if (elapsed >= next_update) {	×
158	log::cerr() << "\r\033[K" << summarize(loops);	×
159	next_update += BENCHMARK_UPDATE_INTERVAL;	×
160	}
161	} while (s == strategy::benchmark &&	×
162	m_durations.size() < BENCHMARK_MAX_LOOPS &&	×
163	(m_durations.size() < BENCHMARK_MIN_LOOPS \|\|	×
164	(elapsed < BENCHMARK_MIN_TIME_NS &&	×
165	elapsed / m_durations.size() >= BENCHMARK_CUTOFF_TIME_NS)));	×
166	} while (s == strategy::benchmark && (loops < 2 * m_durations.size()) &&	×
167	grubbs_test_prune(m_durations));	×
168
169	log::cerr() << "\r\033[K";	×
170
171	if (s != strategy::passthrough) {	×
172	std::cout << summarize(loops) << std::endl;	×
173	}
174	}
175
176	std::string
177	benchmarker::summarize(size_t loops) const	×
178	{
179	const std::array<size_t, 7> COLUMN_WIDTHS{ 22, 9, 9, 9, 6, 5, 8 };	×
180	std::array<std::string, COLUMN_WIDTHS.size()> values{	×
181	m_description,	×
182	"",
183	"",
184	"",
185	"",
186	std::to_string(m_durations.size()),	×
187	std::to_string(loops - m_durations.size()),	×
188	};
189
190	if (!m_durations.empty()) {	×
191	const auto min_max =	×
192	std::minmax_element(m_durations.begin(), m_durations.end());	×
193	const auto mean = arithmetic_mean(m_durations);	×
194
195	values.at(1) = format_thousand_sep(*min_max.first / 1e3);	×
196	values.at(2) = format_thousand_sep(std::round(mean / 1e3));	×
197	values.at(3) = format_thousand_sep(*min_max.second / 1e3);	×
198
199	if (m_durations.size() > 1) {	×
200	const auto sd = standard_deviation(m_durations);	×
201	values.at(4) = format_fraction(1e9 * sd, 1e7 * mean);	×
202	}
203	}
204
205	std::ostringstream ss;	×
206	for (size_t i = 0; i < values.size(); ++i) {	×
207	if (i) {	×
208	ss << " \| ";	×
209	}
210
211	ss << std::setw(COLUMN_WIDTHS.at(i)) << values.at(i);	×
212	}
213
214	return ss.str();	×
215	}
216
217	} // namespace adapterremoval

MikkelSchubert / adapterremoval / #117

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous