18743945403

Committed 23 Oct 2025 09:29AM UTC coverage: 65.845% (+0.02%) from 65.829%

Build # 18743945403

Build Type

Pull #16356

github

Committed by

web-flow

Commit Message

Merge 8a2027ef1 into efa3637e8

Pull Request Pull Request #16356: auth 5.0: backport "pdnsutil: fix b2b-migrate to from sql to non-sql"

Run Details

42073 of 92452 branches covered (45.51%)

Branch coverage included in aggregate %.

128008 of 165855 relevant lines covered (77.18%)

6379935.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.93

/pdns/recursordist/nod.cc

/*
 * This file is part of PowerDNS or dnsdist.
 * Copyright -- PowerDNS.COM B.V. and its contributors
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * In addition, for the avoidance of any doubt, permission is granted to
 * link this program with OpenSSL and to (re)distribute the binaries
 * produced as the result of such linking.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include "nod.hh"
#include <fstream>
#include "pdnsexception.hh"
#include <iostream>
#include <ctime>
#include <thread>
#include "threadname.hh"
#include <cstdlib>
#include "logging.hh"
#include "misc.hh"

using namespace nod;
namespace filesystem = boost::filesystem;

// PersistentSBF Implementation

std::mutex PersistentSBF::d_cachedir_mutex;

void PersistentSBF::remove_tmp_files(const filesystem::path& path, std::scoped_lock<std::mutex>& /* lock */)
{
  Regex file_regex(d_prefix + ".*\\." + bf_suffix + "\\..{8}$");
  for (const auto& file : filesystem::directory_iterator(path)) {
    if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {
      filesystem::remove(file);
    }
  }
}

// This looks for the newest (per-thread) snapshot it can find and it restores from that. Then
// immediately snapshots with the current thread id, before removing the old snapshot.
// In this way, we can have per-thread SBFs, but still snapshot and restore.  The mutex has to be
// static because we can't have multiple (i.e. per-thread) instances iterating and writing to the
// cache dir at the same time
bool PersistentSBF::init(bool ignore_pid)
{
  auto log = g_slog->withName("nod");
  std::scoped_lock<std::mutex> lock(d_cachedir_mutex);
  if (d_cachedir.length() != 0) {
    filesystem::path path(d_cachedir);
    try {
      if (filesystem::exists(path) && filesystem::is_directory(path)) {
        remove_tmp_files(path, lock);
        filesystem::path newest_file;
        std::time_t newest_time = 0;
        Regex file_regex(d_prefix + ".*\\." + bf_suffix + "$");
        for (const auto& file : filesystem::directory_iterator(path)) {
          if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {
            if (ignore_pid || (file.path().filename().string().find(std::to_string(getpid())) == std::string::npos)) {
              // look for the newest file matching the regex
              if (last_write_time(file.path()) > newest_time) {
                newest_time = last_write_time(file.path());
                newest_file = file.path();
              }
            }
          }
        }
        if (!newest_file.empty() && filesystem::exists(newest_file)) {
          const std::string& filename = newest_file.string();
          std::ifstream infile;
          try {
            infile.open(filename, std::ios::in | std::ios::binary);
            SLOG(g_log << Logger::Warning << "Found SBF file " << filename << endl,
                 log->info(Logr::Warning, "Found SBF File", "file", Logging::Loggable(filename)));
            // read the file into the sbf
            d_sbf.lock()->restore(infile);
            infile.close();
            // now dump it out again with new thread id & process id
            snapshotCurrent(std::this_thread::get_id());
            // Remove the old file we just read to stop proliferation
            filesystem::remove(newest_file);
          }
          catch (const std::runtime_error& e) {
            infile.close();
            filesystem::remove(newest_file);
            SLOG(g_log << Logger::Warning << "NODDB init: Cannot parse file: " << filename << ": " << e.what() << "; removed" << endl,
                 log->error(Logr::Warning, e.what(), "NODDB init: Cannot parse file, removed", "file", Logging::Loggable(filename)));
          }
        }
      }
    }
    catch (const filesystem::filesystem_error& e) {
      SLOG(g_log << Logger::Warning << "NODDB init failed: " << e.what() << endl,
           log->error(Logr::Warning, e.what(), "NODDB init failed", "exception", Logging::Loggable("filesystem::filesystem_error")));
      return false;
    }
  }
  return true;
}

void PersistentSBF::setCacheDir(const std::string& cachedir)
{
  filesystem::path path(cachedir);
  if (!exists(path)) {
    throw PDNSException("NODDB setCacheDir specified nonexistent directory: " + cachedir);
  }
  if (!is_directory(path)) {
    throw PDNSException("NODDB setCacheDir specified a file not a directory: " + cachedir);
  }
  d_cachedir = cachedir;
}

// Dump the SBF to a file
// To spend the least amount of time inside the mutex, we dump to an
// intermediate stringstream, otherwise the lock would be waiting for
// file IO to complete
bool PersistentSBF::snapshotCurrent(std::thread::id tid)
{
  auto log = g_slog->withName("nod");
  if (d_cachedir.length() != 0) {
    filesystem::path path(d_cachedir);
    filesystem::path file(d_cachedir);
    std::stringstream strStream;
    strStream << d_prefix << "_" << tid;
    file /= strStream.str() + "_" + std::to_string(getpid()) + "." + bf_suffix;
    if (filesystem::exists(path) && filesystem::is_directory(path)) {
      try {
        std::ostringstream oss;
        {
          // only lock while dumping to a stringstream
          d_sbf.lock()->dump(oss);
        }
        // Now write it out to the file
        std::string ftmp = file.string() + ".XXXXXXXX";
        auto fileDesc = FDWrapper(mkstemp(ftmp.data()));
        if (fileDesc == -1) {
          throw std::runtime_error("Cannot create temp file: " + stringerror());
        }
        const std::string str = oss.str(); // XXX creates a copy, with c++20 we can use view()
        ssize_t len = write(fileDesc, str.data(), str.length());
        if (len != static_cast<ssize_t>(str.length())) {
          filesystem::remove(ftmp.c_str());
          throw std::runtime_error("Failed to write to file:" + ftmp);
        }
        if (fileDesc.reset() != 0) {
          filesystem::remove(ftmp);
          throw std::runtime_error("Failed to write to file:" + ftmp);
        }
        try {
          filesystem::rename(ftmp, file);
        }
        catch (const std::runtime_error& e) {
          SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot rename file: " << e.what() << endl,
               log->error(Logr::Warning, e.what(), "NODDB snapshot: Cannot rename file", "exception", Logging::Loggable("std::runtime_error")));
          filesystem::remove(ftmp);
          throw;
        }
        return true;
      }
      catch (const std::runtime_error& e) {
        SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot write file: " << e.what() << endl,
             log->error(Logr::Warning, e.what(), "NODDB snapshot: Cannot write file", "exception", Logging::Loggable("std::runtime_error")));
      }
    }
    else {
      SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot write file: " << file.string() << endl,
           log->info(Logr::Warning, "NODDB snapshot: Cannot write file", "file", Logging::Loggable(file.string())));
    }
  }
  return false;
}

// NODDB Implementation

void NODDB::housekeepingThread(std::thread::id tid)
{
  setThreadName("rec/nod-hk");
  for (;;) {
    std::this_thread::sleep_for(std::chrono::seconds(d_snapshot_interval));
    snapshotCurrent(tid);
  }
}

bool NODDB::isNewDomain(const std::string& domain)
{
  DNSName dname(domain);
  return isNewDomain(dname);
}

bool NODDB::isNewDomain(const DNSName& dname)
{
  std::string dname_lc = dname.toDNSStringLC();
  // the result is always the inverse of what is returned by the SBF
  return !d_psbf.testAndAdd(dname_lc);
}

bool NODDB::isNewDomainWithParent(const std::string& domain, std::string& observed)
{
  DNSName dname(domain);
  return isNewDomainWithParent(dname, observed);
}

bool NODDB::isNewDomainWithParent(const DNSName& dname, std::string& observed)
{
  bool ret = isNewDomain(dname);
  if (ret) {
    DNSName mdname = dname;
    while (mdname.chopOff()) {
      if (!isNewDomain(mdname)) {
        observed = mdname.toString();
        break;
      }
    }
  }
  return ret;
}

void NODDB::addDomain(const DNSName& dname)
{
  std::string native_domain = dname.toDNSStringLC();
  d_psbf.add(native_domain);
}

void NODDB::addDomain(const std::string& domain)
{
  DNSName dname(domain);
  addDomain(dname);
}

// UniqueResponseDB Implementation
bool UniqueResponseDB::isUniqueResponse(const std::string& response)
{
  return !d_psbf.testAndAdd(response);
}

void UniqueResponseDB::addResponse(const std::string& response)
{
  d_psbf.add(response);
}

void UniqueResponseDB::housekeepingThread(std::thread::id tid)
{
  setThreadName("rec/udr-hk");
  for (;;) {
    std::this_thread::sleep_for(std::chrono::seconds(d_snapshot_interval));
    snapshotCurrent(tid);
  }
}

1	/*
2	* This file is part of PowerDNS or dnsdist.
3	* Copyright -- PowerDNS.COM B.V. and its contributors
4	*
5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of version 2 of the GNU General Public License as
7	* published by the Free Software Foundation.
8	*
9	* In addition, for the avoidance of any doubt, permission is granted to
10	* link this program with OpenSSL and to (re)distribute the binaries
11	* produced as the result of such linking.
12	*
13	* This program is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	* GNU General Public License for more details.
17	*
18	* You should have received a copy of the GNU General Public License
19	* along with this program; if not, write to the Free Software
20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21	*/
22
23	#include "nod.hh"
24	#include <fstream>
25	#include "pdnsexception.hh"
26	#include <iostream>
27	#include <ctime>
28	#include <thread>
29	#include "threadname.hh"
30	#include <cstdlib>
31	#include "logging.hh"
32	#include "misc.hh"
33
34	using namespace nod;
35	namespace filesystem = boost::filesystem;
36
37	// PersistentSBF Implementation
38
39	std::mutex PersistentSBF::d_cachedir_mutex;
40
41	void PersistentSBF::remove_tmp_files(const filesystem::path& path, std::scoped_lock<std::mutex>& /* lock */)
42	{	9✔
43	Regex file_regex(d_prefix + ".*\\." + bf_suffix + "\\..{8}$");	9✔
44	for (const auto& file : filesystem::directory_iterator(path)) {	103✔
45	if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {	103!
46	filesystem::remove(file);	×
47	}	×
48	}	103✔
49	}	9✔
50
51	// This looks for the newest (per-thread) snapshot it can find and it restores from that. Then
52	// immediately snapshots with the current thread id, before removing the old snapshot.
53	// In this way, we can have per-thread SBFs, but still snapshot and restore. The mutex has to be
54	// static because we can't have multiple (i.e. per-thread) instances iterating and writing to the
55	// cache dir at the same time
56	bool PersistentSBF::init(bool ignore_pid)
57	{	9✔
58	auto log = g_slog->withName("nod");	9✔
59	std::scoped_lock<std::mutex> lock(d_cachedir_mutex);	9✔
60	if (d_cachedir.length() != 0) {	9!
61	filesystem::path path(d_cachedir);	9✔
62	try {	9✔
63	if (filesystem::exists(path) && filesystem::is_directory(path)) {	9!
64	remove_tmp_files(path, lock);	9✔
65	filesystem::path newest_file;	9✔
66	std::time_t newest_time = 0;	9✔
67	Regex file_regex(d_prefix + ".*\\." + bf_suffix + "$");	9✔
68	for (const auto& file : filesystem::directory_iterator(path)) {	103✔
69	if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {	103✔
70	if (ignore_pid \|\| (file.path().filename().string().find(std::to_string(getpid())) == std::string::npos)) {	1!
71	// look for the newest file matching the regex
72	if (last_write_time(file.path()) > newest_time) {	1!
73	newest_time = last_write_time(file.path());	1✔
74	newest_file = file.path();	1✔
75	}	1✔
76	}	1✔
77	}	1✔
78	}	103✔
79	if (!newest_file.empty() && filesystem::exists(newest_file)) {	9!
80	const std::string& filename = newest_file.string();	1✔
81	std::ifstream infile;	1✔
82	try {	1✔
83	infile.open(filename, std::ios::in \| std::ios::binary);	1✔
84	SLOG(g_log << Logger::Warning << "Found SBF file " << filename << endl,	1✔
85	log->info(Logr::Warning, "Found SBF File", "file", Logging::Loggable(filename)));	1✔
86	// read the file into the sbf
87	d_sbf.lock()->restore(infile);	1✔
88	infile.close();	1✔
89	// now dump it out again with new thread id & process id
90	snapshotCurrent(std::this_thread::get_id());	1✔
91	// Remove the old file we just read to stop proliferation
92	filesystem::remove(newest_file);	1✔
93	}	1✔
94	catch (const std::runtime_error& e) {	1✔
95	infile.close();	×
96	filesystem::remove(newest_file);	×
97	SLOG(g_log << Logger::Warning << "NODDB init: Cannot parse file: " << filename << ": " << e.what() << "; removed" << endl,	×
98	log->error(Logr::Warning, e.what(), "NODDB init: Cannot parse file, removed", "file", Logging::Loggable(filename)));	×
99	}	×
100	}	1✔
101	}	9✔
102	}	9✔
103	catch (const filesystem::filesystem_error& e) {	9✔
104	SLOG(g_log << Logger::Warning << "NODDB init failed: " << e.what() << endl,	×
105	log->error(Logr::Warning, e.what(), "NODDB init failed", "exception", Logging::Loggable("filesystem::filesystem_error")));	×
106	return false;	×
107	}	×
108	}	9✔
109	return true;	9✔
110	}	9✔
111
112	void PersistentSBF::setCacheDir(const std::string& cachedir)
113	{	10✔
114	filesystem::path path(cachedir);	10✔
115	if (!exists(path)) {	10✔
116	throw PDNSException("NODDB setCacheDir specified nonexistent directory: " + cachedir);	1✔
117	}	1✔
118	if (!is_directory(path)) {	9!
119	throw PDNSException("NODDB setCacheDir specified a file not a directory: " + cachedir);	×
120	}	×
121	d_cachedir = cachedir;	9✔
122	}	9✔
123
124	// Dump the SBF to a file
125	// To spend the least amount of time inside the mutex, we dump to an
126	// intermediate stringstream, otherwise the lock would be waiting for
127	// file IO to complete
128	bool PersistentSBF::snapshotCurrent(std::thread::id tid)
129	{	2✔
130	auto log = g_slog->withName("nod");	2✔
131	if (d_cachedir.length() != 0) {	2!
132	filesystem::path path(d_cachedir);	2✔
133	filesystem::path file(d_cachedir);	2✔
134	std::stringstream strStream;	2✔
135	strStream << d_prefix << "_" << tid;	2✔
136	file /= strStream.str() + "_" + std::to_string(getpid()) + "." + bf_suffix;	2✔
137	if (filesystem::exists(path) && filesystem::is_directory(path)) {	2!
138	try {	2✔
139	std::ostringstream oss;	2✔
140	{	2✔
141	// only lock while dumping to a stringstream
142	d_sbf.lock()->dump(oss);	2✔
143	}	2✔
144	// Now write it out to the file
145	std::string ftmp = file.string() + ".XXXXXXXX";	2✔
146	auto fileDesc = FDWrapper(mkstemp(ftmp.data()));	2✔
147	if (fileDesc == -1) {	2!
148	throw std::runtime_error("Cannot create temp file: " + stringerror());	×
149	}	×
150	const std::string str = oss.str(); // XXX creates a copy, with c++20 we can use view()	2✔
151	ssize_t len = write(fileDesc, str.data(), str.length());	2✔
152	if (len != static_cast<ssize_t>(str.length())) {	2!
153	filesystem::remove(ftmp.c_str());	×
154	throw std::runtime_error("Failed to write to file:" + ftmp);	×
155	}	×
156	if (fileDesc.reset() != 0) {	2!
157	filesystem::remove(ftmp);	×
158	throw std::runtime_error("Failed to write to file:" + ftmp);	×
159	}	×
160	try {	2✔
161	filesystem::rename(ftmp, file);	2✔
162	}	2✔
163	catch (const std::runtime_error& e) {	2✔
164	SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot rename file: " << e.what() << endl,	×
165	log->error(Logr::Warning, e.what(), "NODDB snapshot: Cannot rename file", "exception", Logging::Loggable("std::runtime_error")));	×
166	filesystem::remove(ftmp);	×
167	throw;	×
168	}	×
169	return true;	2✔
170	}	2✔
171	catch (const std::runtime_error& e) {	2✔
172	SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot write file: " << e.what() << endl,	×
173	log->error(Logr::Warning, e.what(), "NODDB snapshot: Cannot write file", "exception", Logging::Loggable("std::runtime_error")));	×
174	}	×
175	}	2✔
176	else {	×
177	SLOG(g_log << Logger::Warning << "NODDB snapshot: Cannot write file: " << file.string() << endl,	×
178	log->info(Logr::Warning, "NODDB snapshot: Cannot write file", "file", Logging::Loggable(file.string())));	×
179	}	×
180	}	2✔
181	return false;	×
182	}	2✔
183
184	// NODDB Implementation
185
186	void NODDB::housekeepingThread(std::thread::id tid)
187	{	3✔
188	setThreadName("rec/nod-hk");	3✔
189	for (;;) {	3✔
190	std::this_thread::sleep_for(std::chrono::seconds(d_snapshot_interval));	3✔
191	snapshotCurrent(tid);	3✔
192	}	3✔
193	}	3✔
194
195	bool NODDB::isNewDomain(const std::string& domain)
196	{	1,000,000✔
197	DNSName dname(domain);	1,000,000✔
198	return isNewDomain(dname);	1,000,000✔
199	}	1,000,000✔
200
201	bool NODDB::isNewDomain(const DNSName& dname)
202	{	1,000,015✔
203	std::string dname_lc = dname.toDNSStringLC();	1,000,015✔
204	// the result is always the inverse of what is returned by the SBF
205	return !d_psbf.testAndAdd(dname_lc);	1,000,015✔
206	}	1,000,015✔
207
208	bool NODDB::isNewDomainWithParent(const std::string& domain, std::string& observed)
209	{	×
210	DNSName dname(domain);	×
211	return isNewDomainWithParent(dname, observed);	×
212	}	×
213
214	bool NODDB::isNewDomainWithParent(const DNSName& dname, std::string& observed)
215	{	1✔
216	bool ret = isNewDomain(dname);	1✔
217	if (ret) {	1!
218	DNSName mdname = dname;	1✔
219	while (mdname.chopOff()) {	1!
220	if (!isNewDomain(mdname)) {	1!
221	observed = mdname.toString();	1✔
222	break;	1✔
223	}	1✔
224	}	1✔
225	}	1✔
226	return ret;	1✔
227	}	1✔
228
229	void NODDB::addDomain(const DNSName& dname)
230	{	1✔
231	std::string native_domain = dname.toDNSStringLC();	1✔
232	d_psbf.add(native_domain);	1✔
233	}	1✔
234
235	void NODDB::addDomain(const std::string& domain)
236	{	1✔
237	DNSName dname(domain);	1✔
238	addDomain(dname);	1✔
239	}	1✔
240
241	// UniqueResponseDB Implementation
242	bool UniqueResponseDB::isUniqueResponse(const std::string& response)
243	{	3✔
244	return !d_psbf.testAndAdd(response);	3✔
245	}	3✔
246
247	void UniqueResponseDB::addResponse(const std::string& response)
248	{	×
249	d_psbf.add(response);	×
250	}	×
251
252	void UniqueResponseDB::housekeepingThread(std::thread::id tid)
253	{	3✔
254	setThreadName("rec/udr-hk");	3✔
255	for (;;) {	3✔
256	std::this_thread::sleep_for(std::chrono::seconds(d_snapshot_interval));	3✔
257	snapshotCurrent(tid);	3✔
258	}	3✔
259	}	3✔

PowerDNS / pdns / 18743945403

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous