MikkelSchubert / adapterremoval / #79

Committed 03 Apr 2025 05:22PM UTC coverage: 27.703%. Remained the same

Build # #79

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

use common constants for /dev/ paths and similiar (#102)

This replaces multiple hardcoded strings like "/dev/stdin" and "-" with a constants

Run Details

2 of 11 new or added lines in 2 files covered. (18.18%)

1 existing line in 1 file now uncovered.

2701 of 9750 relevant lines covered (27.7%)

4134.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.77

/src/managed_io.cpp

// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: 2015 Mikkel Schubert <mikkelsch@gmail.com>
#include "managed_io.hpp"  // declarations
#include "commontypes.hpp" // for DEV_STDOUT, DEV_STDERR, ...
#include "debug.hpp"       // for AR_REQUIRE
#include "errors.hpp"      // for io_error
#include "logging.hpp"     // for log::warn
#include "strutils.hpp"    // for log_escape
#include <algorithm>       // for any_of
#include <cerrno>          // for EMFILE, errno
#include <cstddef>         // for size_t
#include <cstdio>          // for fopen, fread, fwrite, ...
#include <exception>       // for std::exception
#include <fcntl.h>         // for posix_fadvise
#include <mutex>           // for mutex, lock_guard
#include <string>          // for string
#include <sys/stat.h>      // for fstat

namespace adapterremoval {

class io_manager
{
public:
  static void open(managed_reader* reader)
  {
    if (reader->m_file) {
      return;
    } else if (reader->filename() == DEV_STDIN) {
      reader->m_file = stdin;
    } else if (reader->filename() != DEV_PIPE) {
      reader->m_file = io_manager::fopen(reader->filename(), "rb");

#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 600) ||                        \
  (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L)
      posix_fadvise(fileno(reader->m_file), 0, 0, POSIX_FADV_WILLNEED);
      posix_fadvise(fileno(reader->m_file), 0, 0, POSIX_FADV_SEQUENTIAL);
#endif
    } else {
      // Merged I/O depends on filenames being identical
      AR_FAIL("unhandled STDIN marker");
    }
  }

  static void open(managed_writer* writer)
  {
    if (writer->m_file) {
      return;
    } else if (writer->filename() == DEV_STDOUT) {
      writer->m_file = stdout;
    } else if (writer->filename() == DEV_STDERR) {
      // Not sure why anyone would do this, but ¯\_(ツ)_/¯
      writer->m_file = stderr;
    } else if (writer->filename() != DEV_PIPE) {
      writer->m_file =
        io_manager::fopen(writer->filename(), writer->m_created ? "ab" : "wb");
    } else {
      // Merged I/O depends on filenames being identical
      AR_FAIL("unhandled STDOUT marker");
    }

    writer->m_created = true;
    writer->m_stream =
      io_manager::is_stream(writer->filename(), writer->m_file);
  }

  /** Adds writer to list of inactive writers */
  static void add(managed_writer* writer)
  {
    std::lock_guard<std::mutex> lock(m_lock);

    AR_REQUIRE(!writer->m_prev);
    AR_REQUIRE(!writer->m_next);
    AR_REQUIRE(!m_head == !m_tail);
    if (m_head) {
      writer->m_next = m_head;
      m_head->m_prev = writer;
    }

    m_head = writer;

    if (!m_tail) {
      m_tail = writer;
    }

    AR_REQUIRE(m_head && m_tail);
    AR_REQUIRE(!m_head->m_prev);
    AR_REQUIRE(!m_tail->m_next);
  }

  /* Removes the writer from the list of inactive writers */
  static void remove(managed_writer* writer)
  {
    std::lock_guard<std::mutex> lock(m_lock);

    AR_REQUIRE(!m_head == !m_tail);
    AR_REQUIRE(!m_head || !m_head->m_prev);
    AR_REQUIRE(!m_tail || !m_tail->m_next);

    if (writer == m_head) {
      m_head = writer->m_next;
    }

    if (writer == m_tail) {
      m_tail = writer->m_prev;
    }

    AR_REQUIRE(!m_head == !m_tail);

    if (writer->m_prev) {
      writer->m_prev->m_next = writer->m_next;
    }

    if (writer->m_next) {
      writer->m_next->m_prev = writer->m_prev;
    }

    writer->m_prev = nullptr;
    writer->m_next = nullptr;

    AR_REQUIRE(writer != m_head);
    AR_REQUIRE(writer != m_tail);
    AR_REQUIRE(!writer->m_prev);
    AR_REQUIRE(!writer->m_next);
    AR_REQUIRE(!m_head || !m_head->m_prev);
    AR_REQUIRE(!m_tail || !m_tail->m_next);
  }

  io_manager() = delete;
  ~io_manager() = delete;
  io_manager(const io_manager&) = delete;
  io_manager(io_manager&&) = delete;
  io_manager& operator=(const io_manager&) = delete;
  io_manager& operator=(io_manager&&) = delete;

private:
  static FILE* fopen(const std::string& filename, const char* mode)
  {
    while (true) {
      FILE* handle = ::fopen(filename.c_str(), mode);

      if (handle) {
        AR_REQUIRE(!::ferror(handle));
        return handle;
      } else if (errno == EMFILE) {
        close_one();
      } else {
        throw io_error("failed to open file", errno);
      }
    }
  }

  static bool is_stream(const std::string& filename, FILE* handle)
  {
    if (handle == stdin || handle == stdout || handle == stderr) {
      return true;
    }

    struct stat statbuf = {};
    if (fstat(fileno(handle), &statbuf) == 0) {
      return S_ISFIFO(statbuf.st_mode);
    }

    log::warn() << "Could not fstat " << log_escape(filename);

    // Assumed to be a stream to be safe
    return true;
  }

  /** Try to close the least recently used writer */
  static void close_one()
  {
    AR_REQUIRE(!m_head == !m_tail);
    if (!m_warning_printed) {
      log::warn() << "Number of available file-handles (ulimit -n) is too low. "
                  << "AdapterRemoval will dynamically close/re-open files as "
                  << "required, but performance may suffer as a result.";

      m_warning_printed = true;
    }

    if (m_tail) {
      if (fclose(m_tail->m_file)) {
        m_tail->m_file = nullptr;
        throw io_error("failed to close file", errno);
      }
      m_tail->m_file = nullptr;

      remove(m_tail);
    } else {
      throw io_error(
        "available number of file-handles too low; could not open any files");
    }
  }

  //! Indicates if a performance warning has been printed
  static bool m_warning_printed;
  //! Most recently used managed_writer
  static managed_writer* m_head;
  //! Least recently used managed_writer
  static managed_writer* m_tail;
  //! Lock used to control access to internal state
  static std::mutex m_lock;
};

bool io_manager::m_warning_printed = false;
managed_writer* io_manager::m_head = nullptr;
managed_writer* io_manager::m_tail = nullptr;
std::mutex io_manager::m_lock{};

///////////////////////////////////////////////////////////////////////////////

managed_reader::managed_reader(FILE* handle)
  : m_filename("<unknown file>")
  , m_file(handle)
{
  AR_REQUIRE(handle);
}

managed_reader::managed_reader(std::string filename)
  : m_filename(std::move(filename))
{
  io_manager::open(this);
}

managed_reader::~managed_reader()
{
  if (m_file && ::fclose(m_file) != 0) {
    AR_FAIL(format_io_error("error closing " + log_escape(m_filename), errno));
  }
}

void
managed_reader::close()
{
  if (m_file) {
    if (::fclose(m_file) != 0) {
      m_file = nullptr;
      throw io_error("error closing " + log_escape(m_filename), errno);
    }

    m_file = nullptr;
  }
}

size_t
managed_reader::read(void* buffer, size_t size)
{
  AR_REQUIRE(buffer);
  const auto nread = ::fread(buffer, 1, size, m_file);
  if (ferror(m_file)) {
    throw io_error("error reading " + log_escape(m_filename), errno);
  }

  return nread;
}

///////////////////////////////////////////////////////////////////////////////

/** Locker  */
class writer_lock
{
public:
  explicit writer_lock(managed_writer* writer)
    : m_writer(writer)
  {
    AR_REQUIRE(m_writer);

    // Remove from global queue to prevent other threads from manipulating it
    io_manager::remove(m_writer);
    io_manager::open(m_writer);
  };

  ~writer_lock()
  {
    // Streams cannot be managed, since they cannot be reopened
    if (m_writer->m_file && !m_writer->m_stream) {
      // Allow this writer to be closed if we run out of file handles
      io_manager::add(m_writer);
    }
  }

  void write(const void* buffer, size_t size)
  {
    AR_REQUIRE(buffer || size == 0);
    AR_REQUIRE(m_writer && m_writer->m_file);
    if (size) {
      const auto ret = ::fwrite(buffer, 1, size, m_writer->m_file);
      if (ret != size) {
        throw io_error("error writing to " + log_escape(m_writer->m_filename),
                       errno);
      }
    }
  }

  void flush()
  {
    AR_REQUIRE(m_writer && m_writer->m_file);
    if (::fflush(m_writer->m_file)) {
      throw io_error("error flushing file " + log_escape(m_writer->filename()),
                     errno);
    }
  }

  writer_lock(const writer_lock&) = delete;
  writer_lock(writer_lock&&) = delete;
  writer_lock& operator=(const writer_lock&) = delete;
  writer_lock& operator=(writer_lock&&) = delete;

private:
  managed_writer* m_writer;
};

///////////////////////////////////////////////////////////////////////////////

namespace {

bool
any_nonempty_buffers(const buffer_vec& buffers)
{
  return std::any_of(buffers.begin(), buffers.end(), [](const auto& it) {
    return it.size() != 0;
  });
}

} // namespace

managed_writer::managed_writer(std::string filename)
  : m_filename(std::move(filename))
{
}

managed_writer::~managed_writer()
{
  try {
    close();
  } catch (const std::exception&) {
    AR_FAIL("unhandled exception");
  }
}

void
managed_writer::write(const buffer& buf, const flush mode)
{
  if (buf.size() || mode == flush::on) {
    writer_lock writer{ this };

    writer.write(buf.data(), buf.size());

    if (mode == flush::on) {
      writer.flush();
    }
  }
}

void
managed_writer::write(const buffer_vec& buffers, const flush mode)
{
  if (mode == flush::on || any_nonempty_buffers(buffers)) {
    writer_lock writer{ this };

    for (const auto& buf : buffers) {
      writer.write(buf.data(), buf.size());
    }

    if (mode == flush::on) {
      writer.flush();
    }
  }
}

void
managed_writer::write(const std::string& buf, const flush mode)
{
  if (!buf.empty() || mode == flush::on) {
    writer_lock writer{ this };

    writer.write(buf.data(), buf.length());

    if (mode == flush::on) {
      writer.flush();
    }
  }
}

void
managed_writer::close()
{
  io_manager::remove(this);
  if (m_file) {
    if (fclose(m_file)) {
      m_file = nullptr;
      throw io_error("failed to close file", errno);
    }

    m_file = nullptr;
  }
}

} // namespace adapterremoval

1	// SPDX-License-Identifier: GPL-3.0-or-later
2	// SPDX-FileCopyrightText: 2015 Mikkel Schubert <mikkelsch@gmail.com>
3	#include "managed_io.hpp" // declarations
4	#include "commontypes.hpp" // for DEV_STDOUT, DEV_STDERR, ...
5	#include "debug.hpp" // for AR_REQUIRE
6	#include "errors.hpp" // for io_error
7	#include "logging.hpp" // for log::warn
8	#include "strutils.hpp" // for log_escape
9	#include <algorithm> // for any_of
10	#include <cerrno> // for EMFILE, errno
11	#include <cstddef> // for size_t
12	#include <cstdio> // for fopen, fread, fwrite, ...
13	#include <exception> // for std::exception
14	#include <fcntl.h> // for posix_fadvise
15	#include <mutex> // for mutex, lock_guard
16	#include <string> // for string
17	#include <sys/stat.h> // for fstat
18
19	namespace adapterremoval {
20
21	class io_manager
22	{
23	public:
24	static void open(managed_reader* reader)	1✔
25	{
26	if (reader->m_file) {	1✔
27	return;
28	} else if (reader->filename() == DEV_STDIN) {	2✔
29	reader->m_file = stdin;	×
30	} else if (reader->filename() != DEV_PIPE) {	3✔
31	reader->m_file = io_manager::fopen(reader->filename(), "rb");	2✔
32
33	#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 600) \|\| \
34	(defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L)
35	posix_fadvise(fileno(reader->m_file), 0, 0, POSIX_FADV_WILLNEED);	×
36	posix_fadvise(fileno(reader->m_file), 0, 0, POSIX_FADV_SEQUENTIAL);	×
37	#endif
38	} else {
39	// Merged I/O depends on filenames being identical
40	AR_FAIL("unhandled STDIN marker");	×
41	}
42	}
43
44	static void open(managed_writer* writer)
45	{
46	if (writer->m_file) {	×
47	return;
NEW 48	} else if (writer->filename() == DEV_STDOUT) {	×
49	writer->m_file = stdout;	×
NEW 50	} else if (writer->filename() == DEV_STDERR) {	×
51	// Not sure why anyone would do this, but ¯\_(ツ)_/¯
52	writer->m_file = stderr;	×
NEW 53	} else if (writer->filename() != DEV_PIPE) {	×
54	writer->m_file =	×
55	io_manager::fopen(writer->filename(), writer->m_created ? "ab" : "wb");	×
56	} else {
57	// Merged I/O depends on filenames being identical
58	AR_FAIL("unhandled STDOUT marker");	×
59	}
60
61	writer->m_created = true;	×
62	writer->m_stream =	×
63	io_manager::is_stream(writer->filename(), writer->m_file);	×
64	}
65
66	/** Adds writer to list of inactive writers */
67	static void add(managed_writer* writer)
68	{
69	std::lock_guard<std::mutex> lock(m_lock);	×
70
71	AR_REQUIRE(!writer->m_prev);	×
72	AR_REQUIRE(!writer->m_next);	×
73	AR_REQUIRE(!m_head == !m_tail);	×
74	if (m_head) {	×
75	writer->m_next = m_head;	×
76	m_head->m_prev = writer;	×
77	}
78
79	m_head = writer;	×
80
81	if (!m_tail) {	×
82	m_tail = writer;	×
83	}
84
85	AR_REQUIRE(m_head && m_tail);	×
86	AR_REQUIRE(!m_head->m_prev);	×
87	AR_REQUIRE(!m_tail->m_next);	×
88	}
89
90	/* Removes the writer from the list of inactive writers */
91	static void remove(managed_writer* writer)
92	{
93	std::lock_guard<std::mutex> lock(m_lock);	×
94
95	AR_REQUIRE(!m_head == !m_tail);	×
96	AR_REQUIRE(!m_head \|\| !m_head->m_prev);	×
97	AR_REQUIRE(!m_tail \|\| !m_tail->m_next);	×
98
99	if (writer == m_head) {	×
100	m_head = writer->m_next;	×
101	}
102
103	if (writer == m_tail) {	×
104	m_tail = writer->m_prev;	×
105	}
106
107	AR_REQUIRE(!m_head == !m_tail);	×
108
109	if (writer->m_prev) {	×
110	writer->m_prev->m_next = writer->m_next;	×
111	}
112
113	if (writer->m_next) {	×
114	writer->m_next->m_prev = writer->m_prev;	×
115	}
116
117	writer->m_prev = nullptr;	×
118	writer->m_next = nullptr;	×
119
120	AR_REQUIRE(writer != m_head);	×
121	AR_REQUIRE(writer != m_tail);	×
122	AR_REQUIRE(!writer->m_prev);	×
123	AR_REQUIRE(!writer->m_next);	×
124	AR_REQUIRE(!m_head \|\| !m_head->m_prev);	×
125	AR_REQUIRE(!m_tail \|\| !m_tail->m_next);	×
126	}
127
128	io_manager() = delete;
129	~io_manager() = delete;
130	io_manager(const io_manager&) = delete;
131	io_manager(io_manager&&) = delete;
132	io_manager& operator=(const io_manager&) = delete;
133	io_manager& operator=(io_manager&&) = delete;
134
135	private:
136	static FILE* fopen(const std::string& filename, const char* mode)	1✔
137	{
138	while (true) {	1✔
139	FILE* handle = ::fopen(filename.c_str(), mode);	2✔
140
141	if (handle) {	1✔
142	AR_REQUIRE(!::ferror(handle));	×
143	return handle;	×
144	} else if (errno == EMFILE) {	1✔
145	close_one();	×
146	} else {
147	throw io_error("failed to open file", errno);	2✔
148	}
149	}
150	}
151
152	static bool is_stream(const std::string& filename, FILE* handle)
153	{
154	if (handle == stdin \|\| handle == stdout \|\| handle == stderr) {	×
155	return true;
156	}
157
158	struct stat statbuf = {};	×
159	if (fstat(fileno(handle), &statbuf) == 0) {	×
160	return S_ISFIFO(statbuf.st_mode);	×
161	}
162
163	log::warn() << "Could not fstat " << log_escape(filename);	×
164
165	// Assumed to be a stream to be safe
166	return true;	×
167	}
168
169	/** Try to close the least recently used writer */
170	static void close_one()
171	{
172	AR_REQUIRE(!m_head == !m_tail);	×
173	if (!m_warning_printed) {	×
174	log::warn() << "Number of available file-handles (ulimit -n) is too low. "	×
175	<< "AdapterRemoval will dynamically close/re-open files as "	×
176	<< "required, but performance may suffer as a result.";	×
177
178	m_warning_printed = true;	×
179	}
180
181	if (m_tail) {	×
182	if (fclose(m_tail->m_file)) {	×
183	m_tail->m_file = nullptr;	×
184	throw io_error("failed to close file", errno);	×
185	}
186	m_tail->m_file = nullptr;	×
187
188	remove(m_tail);	×
189	} else {
190	throw io_error(	×
191	"available number of file-handles too low; could not open any files");	×
192	}
193	}
194
195	//! Indicates if a performance warning has been printed
196	static bool m_warning_printed;
197	//! Most recently used managed_writer
198	static managed_writer* m_head;
199	//! Least recently used managed_writer
200	static managed_writer* m_tail;
201	//! Lock used to control access to internal state
202	static std::mutex m_lock;
203	};
204
205	bool io_manager::m_warning_printed = false;
206	managed_writer* io_manager::m_head = nullptr;
207	managed_writer* io_manager::m_tail = nullptr;
208	std::mutex io_manager::m_lock{};
209
210	///////////////////////////////////////////////////////////////////////////////
211
212	managed_reader::managed_reader(FILE* handle)	14✔
213	: m_filename("<unknown file>")	29✔
214	, m_file(handle)	14✔
215	{
216	AR_REQUIRE(handle);	17✔
217	}	14✔
218
219	managed_reader::managed_reader(std::string filename)	1✔
220	: m_filename(std::move(filename))	2✔
221	{
222	io_manager::open(this);	1✔
223	}	1✔
224
225	managed_reader::~managed_reader()	13✔
226	{
227	if (m_file && ::fclose(m_file) != 0) {	13✔
228	AR_FAIL(format_io_error("error closing " + log_escape(m_filename), errno));	×
229	}
230	}	13✔
231
232	void
233	managed_reader::close()	×
234	{
235	if (m_file) {	×
236	if (::fclose(m_file) != 0) {	×
237	m_file = nullptr;	×
238	throw io_error("error closing " + log_escape(m_filename), errno);	×
239	}
240
241	m_file = nullptr;	×
242	}
243	}
244
245	size_t
246	managed_reader::read(void* buffer, size_t size)	33✔
247	{
248	AR_REQUIRE(buffer);	33✔
249	const auto nread = ::fread(buffer, 1, size, m_file);	33✔
250	if (ferror(m_file)) {	33✔
251	throw io_error("error reading " + log_escape(m_filename), errno);	×
252	}
253
254	return nread;	33✔
255	}
256
257	///////////////////////////////////////////////////////////////////////////////
258
259	/** Locker */
260	class writer_lock
261	{
262	public:
263	explicit writer_lock(managed_writer* writer)	×
264	: m_writer(writer)	×
265	{
266	AR_REQUIRE(m_writer);	×
267
268	// Remove from global queue to prevent other threads from manipulating it
269	io_manager::remove(m_writer);	×
270	io_manager::open(m_writer);	×
271	};
272
273	~writer_lock()	×
274	{
275	// Streams cannot be managed, since they cannot be reopened
276	if (m_writer->m_file && !m_writer->m_stream) {	×
277	// Allow this writer to be closed if we run out of file handles
278	io_manager::add(m_writer);	×
279	}
280	}
281
282	void write(const void* buffer, size_t size)	×
283	{
284	AR_REQUIRE(buffer \|\| size == 0);	×
285	AR_REQUIRE(m_writer && m_writer->m_file);	×
286	if (size) {	×
287	const auto ret = ::fwrite(buffer, 1, size, m_writer->m_file);	×
288	if (ret != size) {	×
289	throw io_error("error writing to " + log_escape(m_writer->m_filename),	×
290	errno);	×
291	}
292	}
293	}
294
295	void flush()	×
296	{
297	AR_REQUIRE(m_writer && m_writer->m_file);	×
298	if (::fflush(m_writer->m_file)) {	×
299	throw io_error("error flushing file " + log_escape(m_writer->filename()),	×
300	errno);	×
301	}
302	}
303
304	writer_lock(const writer_lock&) = delete;
305	writer_lock(writer_lock&&) = delete;
306	writer_lock& operator=(const writer_lock&) = delete;
307	writer_lock& operator=(writer_lock&&) = delete;
308
309	private:
310	managed_writer* m_writer;
311	};
312
313	///////////////////////////////////////////////////////////////////////////////
314
315	namespace {
316
317	bool
318	any_nonempty_buffers(const buffer_vec& buffers)	×
319	{
320	return std::any_of(buffers.begin(), buffers.end(), [](const auto& it) {	×
321	return it.size() != 0;	×
322	});	×
323	}
324
325	} // namespace
326
327	managed_writer::managed_writer(std::string filename)	×
328	: m_filename(std::move(filename))	×
329	{
330	}
331
332	managed_writer::~managed_writer()	×
333	{
334	try {	×
335	close();	×
336	} catch (const std::exception&) {	×
337	AR_FAIL("unhandled exception");	×
338	}
339	}
340
341	void
342	managed_writer::write(const buffer& buf, const flush mode)	×
343	{
344	if (buf.size() \|\| mode == flush::on) {	×
345	writer_lock writer{ this };	×
346
347	writer.write(buf.data(), buf.size());	×
348
349	if (mode == flush::on) {	×
350	writer.flush();	×
351	}
352	}
353	}
354
355	void
356	managed_writer::write(const buffer_vec& buffers, const flush mode)	×
357	{
358	if (mode == flush::on \|\| any_nonempty_buffers(buffers)) {	×
359	writer_lock writer{ this };	×
360
361	for (const auto& buf : buffers) {	×
362	writer.write(buf.data(), buf.size());	×
363	}
364
365	if (mode == flush::on) {	×
366	writer.flush();	×
367	}
368	}
369	}
370
371	void
372	managed_writer::write(const std::string& buf, const flush mode)	×
373	{
374	if (!buf.empty() \|\| mode == flush::on) {	×
375	writer_lock writer{ this };	×
376
377	writer.write(buf.data(), buf.length());	×
378
379	if (mode == flush::on) {	×
380	writer.flush();	×
381	}
382	}
383	}
384
385	void
386	managed_writer::close()	×
387	{
388	io_manager::remove(this);	×
389	if (m_file) {	×
390	if (fclose(m_file)) {	×
391	m_file = nullptr;	×
392	throw io_error("failed to close file", errno);	×
393	}
394
395	m_file = nullptr;	×
396	}
397	}
398
399	} // namespace adapterremoval

MikkelSchubert / adapterremoval / #79

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous