MikkelSchubert / adapterremoval / #45

Committed 20 Sep 2024 06:49PM UTC coverage: 26.244% (-49.2%) from 75.443%

Build # #45

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

attempt to fix coveralls run

Run Details

2458 of 9366 relevant lines covered (26.24%)

4362.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/benchmarking.cpp

/*************************************************************************\
 * AdapterRemoval - cleaning next-generation sequencing reads            *
 *                                                                       *
 * Copyright (C) 2024 by Mikkel Schubert - mikkelsch@gmail.com           *
 *                                                                       *
 * This program is free software: you can redistribute it and/or modify  *
 * it under the terms of the GNU General Public License as published by  *
 * the Free Software Foundation, either version 3 of the License, or     *
 * (at your option) any later version.                                   *
 *                                                                       *
 * This program is distributed in the hope that it will be useful,       *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 * GNU General Public License for more details.                          *
 *                                                                       *
 * You should have received a copy of the GNU General Public License     *
 * along with this program.  If not, see <http://www.gnu.org/licenses/>. *
\*************************************************************************/
#include "benchmarking.hpp" // for benchmark_toggles, benchmarker
#include "debug.hpp"        // for AR_REQUIRE
#include "logging.hpp"      // for log
#include "mathutils.hpp"    // for arithmetic_mean, standard_deviation
#include "simd.hpp"         // for supported
#include "strutils.hpp"     // for to_lower
#include <algorithm>        // for find, accumulate
#include <iomanip>          // for fixed, setsetprecision
#include <numeric>          // for accumulate

namespace adapterremoval {

namespace {

//! Number of loops to perform prior to benchmarking as burn-in
const size_t BENCHMARK_BURN_IN = 1;
//! Benchmarks must be repeated at least this number of times
const size_t BENCHMARK_MIN_LOOPS = 10;
//! Benchmarks must be repeated at most this number of times
const size_t BENCHMARK_MAX_LOOPS = 1000;
//! Benchmarks must run for at this this number of nano-seconds
const double BENCHMARK_MIN_TIME_NS = 10'000'000'000;
//! Benchmark loops shorter than this number of nano-seconds cannot be measured
const double BENCHMARK_CUTOFF_TIME_NS = BENCHMARK_MIN_TIME_NS / 100'000;

} // namespace

benchmark_toggles::benchmark_toggles(string_vec keys)
  : m_toggles(std::move(keys))
  , m_enabled(m_toggles.size(), false)
  , m_defaults()
{
}

bool
benchmark_toggles::update_toggles(const string_vec& keys)
{
  bool any_errors = false;
  m_defaults = keys.empty();
  for (const auto& key : keys) {
    const auto it =
      std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));

    if (it == m_toggles.end()) {
      log::error() << "Unknown benchmarking toggle '" << key << "'";
      any_errors = true;
    } else {
      m_enabled.at(it - m_toggles.begin()) = true;
    }
  }

  if (any_errors) {
    auto msg = log::error();
    msg << "Valid toggles are ";
    for (size_t i = 0; i < m_toggles.size() - 1; ++i) {
      msg << m_toggles.at(i) << ", ";
    }
    msg << " and " << m_toggles.back();
  }

  return !any_errors;
}

bool
benchmark_toggles::is_set(const std::string& key) const
{
  const auto it = std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));

  AR_REQUIRE(it != m_toggles.end());
  return m_enabled.at(it - m_toggles.begin());
}

benchmarker::benchmarker(std::string desc, string_vec toggles)
  : m_description(std::move(desc))
  , m_toggles(std::move(toggles))
{
}

void
benchmarker::run_if_toggled(const benchmark_toggles& toggles)
{
  const auto s = enabled(toggles);
  if (s != strategy::skip) {
    run(s);
  }
}

/** Called before `setup` to perform any per batch setup */
void
benchmarker::setup(){};

strategy
benchmarker::enabled(const benchmark_toggles& toggles) const
{
  if (toggles.defaults()) {
    return strategy::benchmark;
  }

  for (const auto& toggle : m_toggles) {
    if (toggles.is_set(toggle)) {
      return strategy::benchmark;
    }
  }

  return m_required ? strategy::passthrough : strategy::skip;
}

void
benchmarker::run(const strategy s)
{
  log::cerr() << "\r\033[KBenchmarking: " << summarize(0);

  if (s != strategy::passthrough) {
    for (size_t i = 1; i <= BENCHMARK_BURN_IN; ++i) {
      setup();
      execute();

      log::cerr() << std::fixed << std::setprecision(5)
                  << "\rBenchmarking: " << m_description << " burn-in loop "
                  << i << " completed";
    }
  }

  size_t loops = 0;
  do {
    uint64_t elapsed =
      std::accumulate(m_durations.begin(), m_durations.end(), uint64_t());

    do {
      setup();
      const auto start = clock::now();
      execute();
      const auto duration = (clock::now() - start).count();

      loops++;
      elapsed += duration;
      m_durations.push_back(duration);

      log::cerr() << "\r\033[KBenchmarking: " << summarize(loops);
    } while (s == strategy::benchmark &&
             m_durations.size() < BENCHMARK_MAX_LOOPS &&
             (m_durations.size() < BENCHMARK_MIN_LOOPS ||
              (elapsed < BENCHMARK_MIN_TIME_NS &&
               elapsed / m_durations.size() >= BENCHMARK_CUTOFF_TIME_NS)));
  } while (s == strategy::benchmark && (loops < 2 * m_durations.size()) &&
           grubbs_test_prune(m_durations));

  log::cerr() << "\r\033[K";
  log::info() << "  " << summarize(loops);
}

std::string
benchmarker::summarize(size_t loops) const
{
  std::ostringstream ss;
  if (loops) {
    ss << m_durations.size();
    if (loops > m_durations.size()) {
      ss << " + " << loops - m_durations.size();
    }

    ss << (loops != 1 ? " loops of " : " loop of ");
  }

  ss << m_description;

  if (!m_durations.empty()) {
    ss << " in " << std::fixed << std::setprecision(5)
       << arithmetic_mean(m_durations) / 1e9;

    if (m_durations.size() > 1) {
      ss << " +/- " << std::setprecision(6)
         << standard_deviation(m_durations) / 1e9;
    }

    ss << " seconds";
  }

  return ss.str();
}

} // namespace adapterremoval

1	/*************************************************************************\
2	* AdapterRemoval - cleaning next-generation sequencing reads *
3	* *
4	* Copyright (C) 2024 by Mikkel Schubert - mikkelsch@gmail.com *
5	* *
6	* This program is free software: you can redistribute it and/or modify *
7	* it under the terms of the GNU General Public License as published by *
8	* the Free Software Foundation, either version 3 of the License, or *
9	* (at your option) any later version. *
10	* *
11	* This program is distributed in the hope that it will be useful, *
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of *
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
14	* GNU General Public License for more details. *
15	* *
16	* You should have received a copy of the GNU General Public License *
17	* along with this program. If not, see <http://www.gnu.org/licenses/>. *
18	\*************************************************************************/
19	#include "benchmarking.hpp" // for benchmark_toggles, benchmarker
20	#include "debug.hpp" // for AR_REQUIRE
21	#include "logging.hpp" // for log
22	#include "mathutils.hpp" // for arithmetic_mean, standard_deviation
23	#include "simd.hpp" // for supported
24	#include "strutils.hpp" // for to_lower
25	#include <algorithm> // for find, accumulate
26	#include <iomanip> // for fixed, setsetprecision
27	#include <numeric> // for accumulate
28
29	namespace adapterremoval {
30
31	namespace {
32
33	//! Number of loops to perform prior to benchmarking as burn-in
34	const size_t BENCHMARK_BURN_IN = 1;
35	//! Benchmarks must be repeated at least this number of times
36	const size_t BENCHMARK_MIN_LOOPS = 10;
37	//! Benchmarks must be repeated at most this number of times
38	const size_t BENCHMARK_MAX_LOOPS = 1000;
39	//! Benchmarks must run for at this this number of nano-seconds
40	const double BENCHMARK_MIN_TIME_NS = 10'000'000'000;
41	//! Benchmark loops shorter than this number of nano-seconds cannot be measured
42	const double BENCHMARK_CUTOFF_TIME_NS = BENCHMARK_MIN_TIME_NS / 100'000;
43
44	} // namespace
45
46	benchmark_toggles::benchmark_toggles(string_vec keys)	×
47	: m_toggles(std::move(keys))	×
48	, m_enabled(m_toggles.size(), false)	×
49	, m_defaults()	×
50	{
51	}
52
53	bool
54	benchmark_toggles::update_toggles(const string_vec& keys)	×
55	{
56	bool any_errors = false;	×
57	m_defaults = keys.empty();	×
58	for (const auto& key : keys) {	×
59	const auto it =	×
60	std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));	×
61
62	if (it == m_toggles.end()) {	×
63	log::error() << "Unknown benchmarking toggle '" << key << "'";	×
64	any_errors = true;	×
65	} else {
66	m_enabled.at(it - m_toggles.begin()) = true;	×
67	}
68	}
69
70	if (any_errors) {	×
71	auto msg = log::error();	×
72	msg << "Valid toggles are ";	×
73	for (size_t i = 0; i < m_toggles.size() - 1; ++i) {	×
74	msg << m_toggles.at(i) << ", ";	×
75	}
76	msg << " and " << m_toggles.back();	×
77	}
78
79	return !any_errors;	×
80	}
81
82	bool
83	benchmark_toggles::is_set(const std::string& key) const	×
84	{
85	const auto it = std::find(m_toggles.begin(), m_toggles.end(), to_lower(key));	×
86
87	AR_REQUIRE(it != m_toggles.end());	×
88	return m_enabled.at(it - m_toggles.begin());	×
89	}
90
91	benchmarker::benchmarker(std::string desc, string_vec toggles)	×
92	: m_description(std::move(desc))	×
93	, m_toggles(std::move(toggles))	×
94	{
95	}
96
97	void
98	benchmarker::run_if_toggled(const benchmark_toggles& toggles)	×
99	{
100	const auto s = enabled(toggles);	×
101	if (s != strategy::skip) {	×
102	run(s);	×
103	}
104	}
105
106	/** Called before `setup` to perform any per batch setup */
107	void
108	benchmarker::setup(){};	×
109
110	strategy
111	benchmarker::enabled(const benchmark_toggles& toggles) const	×
112	{
113	if (toggles.defaults()) {	×
114	return strategy::benchmark;
115	}
116
117	for (const auto& toggle : m_toggles) {	×
118	if (toggles.is_set(toggle)) {	×
119	return strategy::benchmark;	×
120	}
121	}
122
123	return m_required ? strategy::passthrough : strategy::skip;	×
124	}
125
126	void
127	benchmarker::run(const strategy s)	×
128	{
129	log::cerr() << "\r\033[KBenchmarking: " << summarize(0);	×
130
131	if (s != strategy::passthrough) {	×
132	for (size_t i = 1; i <= BENCHMARK_BURN_IN; ++i) {	×
133	setup();	×
134	execute();	×
135
136	log::cerr() << std::fixed << std::setprecision(5)	×
137	<< "\rBenchmarking: " << m_description << " burn-in loop "	×
138	<< i << " completed";	×
139	}
140	}
141
142	size_t loops = 0;
143	do {	×
144	uint64_t elapsed =	×
145	std::accumulate(m_durations.begin(), m_durations.end(), uint64_t());	×
146
147	do {	×
148	setup();	×
149	const auto start = clock::now();	×
150	execute();	×
151	const auto duration = (clock::now() - start).count();	×
152
153	loops++;	×
154	elapsed += duration;	×
155	m_durations.push_back(duration);	×
156
157	log::cerr() << "\r\033[KBenchmarking: " << summarize(loops);	×
158	} while (s == strategy::benchmark &&	×
159	m_durations.size() < BENCHMARK_MAX_LOOPS &&	×
160	(m_durations.size() < BENCHMARK_MIN_LOOPS \|\|	×
161	(elapsed < BENCHMARK_MIN_TIME_NS &&	×
162	elapsed / m_durations.size() >= BENCHMARK_CUTOFF_TIME_NS)));	×
163	} while (s == strategy::benchmark && (loops < 2 * m_durations.size()) &&	×
164	grubbs_test_prune(m_durations));	×
165
166	log::cerr() << "\r\033[K";	×
167	log::info() << " " << summarize(loops);	×
168	}
169
170	std::string
171	benchmarker::summarize(size_t loops) const	×
172	{
173	std::ostringstream ss;	×
174	if (loops) {	×
175	ss << m_durations.size();	×
176	if (loops > m_durations.size()) {	×
177	ss << " + " << loops - m_durations.size();	×
178	}
179
180	ss << (loops != 1 ? " loops of " : " loop of ");	×
181	}
182
183	ss << m_description;	×
184
185	if (!m_durations.empty()) {	×
186	ss << " in " << std::fixed << std::setprecision(5)	×
187	<< arithmetic_mean(m_durations) / 1e9;	×
188
189	if (m_durations.size() > 1) {	×
190	ss << " +/- " << std::setprecision(6)	×
191	<< standard_deviation(m_durations) / 1e9;	×
192	}
193
194	ss << " seconds";	×
195	}
196
197	return ss.str();	×
198	}
199
200	} // namespace adapterremoval

MikkelSchubert / adapterremoval / #45

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous