1771

Committed 20 Oct 2023 08:58AM UTC coverage: 91.567% (-0.009%) from 91.576%

Build # 1771

Build Type

push

Evergreen

Committed by

web-flow

Commit Message

Fix blocked DB::open on multiprocess access on exFAT filesystem (#6959)

Fix double file lock and DB::open being blocked with multiple concurrent realm access on fat32/exfat file systems.

When file is truncated on fat32/exfat its uid is available for other files. With multiple processes opening and truncating the same set of files could lead to the situation when within one process get_unique_id will return the same value for different files in timing is right. This breaks proper initialization of static data for interprocess mutexes, so that subsequent locks will hang by trying to lock essentially the same file twice.

Run Details

94304 of 173552 branches covered (0.0%)

59 of 82 new or added lines in 5 files covered. (71.95%)

53 existing lines in 13 files now uncovered.

230544 of 251776 relevant lines covered (91.57%)

6594884.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.77

/test/util/test_path.cpp

/*************************************************************************
 *
 * Copyright 2016 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#include "test_path.hpp"

#include "misc.hpp"
#include "spawned_process.hpp"

#include <realm/util/file.hpp>
#include <realm/db.hpp>
#include <realm/history.hpp>

#include <algorithm>
#include <string>

#if REALM_PLATFORM_APPLE
#include <realm/util/cf_ptr.hpp>

#include <CoreFoundation/CoreFoundation.h>
#include <sys/mount.h>
#include <sys/param.h>
#elif defined(_WIN32)
#include <Windows.h>
// PathCchRemoveFileSpec()
#include <pathcch.h>
#pragma comment(lib, "Pathcch.lib")
#else
#include <unistd.h>
#include <libgen.h>
#endif

using namespace realm::util;

namespace {

bool g_keep_files = false;

std::string g_path_prefix;
std::string g_resource_path;
std::string g_exe_name;

#ifdef _WIN32
std::string sanitize_for_file_name(std::string str)
{
    static const std::string invalid("<>:\"|?*\\/");
    std::transform(str.begin(), str.end(), str.begin(), [](char c) {
        if (invalid.find(c) != std::string::npos)
            return '-';
        return c;
    });
    return str;
}
#else
std::string sanitize_for_file_name(const std::string& str)
{
    return str;
}
#endif

#if REALM_PLATFORM_APPLE
std::string url_to_path(CFURLRef url)
{
    auto absolute = adoptCF(CFURLCopyAbsoluteURL(url));
    auto path = adoptCF(CFURLCopyPath(absolute.get()));
    auto length = CFStringGetLength(path.get());
    std::string ret;
    ret.resize(CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8));
    CFIndex bytes_written;
    CFStringGetBytes(path.get(), {0, length}, kCFStringEncodingUTF8, 0, false, reinterpret_cast<uint8_t*>(ret.data()),
                     ret.size(), &bytes_written);
    REALM_ASSERT(bytes_written);
    ret.resize(bytes_written);
    return ret;
}
#endif

} // anonymous namespace

namespace realm::test_util {

void keep_test_files()
{
    g_keep_files = true;
}

std::string get_test_path(const std::string& test_name, const std::string& suffix)
{
    return g_path_prefix + sanitize_for_file_name(test_name) + suffix;
}

std::string get_test_path_prefix()
{
    return g_path_prefix;
}

bool initialize_test_path(int argc, const char* argv[])
{
#if REALM_PLATFORM_APPLE
    // On Apple platforms we copy everything into a read-only bundle containing
    // the test executable and resource files, and have to create test files in
    // a temporary directory.
#if REALM_APPLE_DEVICE || TARGET_OS_SIMULATOR
    auto home = adoptCF(CFCopyHomeDirectoryURL());
    g_path_prefix = url_to_path(home.get()) + "Documents/";
#else
    g_path_prefix = util::make_temp_dir() + "/";
#endif

    auto resources_url = adoptCF(CFBundleCopyResourcesDirectoryURL(CFBundleGetMainBundle()));
    g_resource_path = url_to_path(resources_url.get());

    // On other platforms we can write to the executable's directory, so we use
    // that as the base path.
#elif defined(_MSC_VER)
    wchar_t path[MAX_PATH];
    if (GetModuleFileName(NULL, path, MAX_PATH) == 0) {
        fprintf(stderr, "Failed to retrieve path to exectuable.\n");
        return false;
    }
    PathCchRemoveFileSpec(path, MAX_PATH);
    SetCurrentDirectory(path);
    g_path_prefix = std::filesystem::path(path).u8string();
    g_resource_path = g_path_prefix + "\\resources\\";
#else
    char executable[PATH_MAX];
    if (realpath(argv[0], executable) == nullptr) {
        fprintf(stderr, "Failed to retrieve path to exectuable.\n");
        return false;
    }
    const char* directory = dirname(executable);
    if (chdir(directory) < 0) {
        fprintf(stderr, "Failed to change directory.\n");
        return false;
    }
    g_resource_path = File::resolve("resources", directory) + "/";
    g_path_prefix = directory;
#endif

    if (argc > 0) {
        g_exe_name = argv[0];
    }

    if (argc > 1) {
        g_path_prefix = argv[1];
    }

    return true;
}

bool test_dir_is_exfat()
{
#if REALM_PLATFORM_APPLE
    if (test_util::get_test_path_prefix().empty())
        return false;

    struct statfs fsbuf;
    int ret = statfs(test_util::get_test_path_prefix().c_str(), &fsbuf);
    REALM_ASSERT_RELEASE(ret == 0);
    // The documentation and headers helpfully don't list any of the values of
    // f_type or provide constants for them
    std::string fs_typename = fsbuf.f_fstypename;
    std::transform(fs_typename.begin(), fs_typename.end(), fs_typename.begin(), toLowerAscii);
    return fs_typename.find(std::string("exfat")) != std::string::npos ||
           fs_typename.find(std::string("msdos")) != std::string::npos;
#else
    return false;
#endif
}

std::string get_test_resource_path()
{
    return g_resource_path;
}

std::string get_test_exe_name()
{
    return g_exe_name;
}

TestPathGuard::TestPathGuard(const std::string& path)
    : m_path(path)
    , m_do_remove(test_util::SpawnedProcess::is_parent())
{
    if (m_do_remove) {
        File::try_remove(m_path);
    }
}

TestPathGuard::~TestPathGuard() noexcept
{
    if (g_keep_files)
        return;
    if (!m_do_remove)
        return;
    try {
        if (!m_path.empty())
            File::try_remove(m_path);
    }
    catch (...) {
        // Exception deliberately ignored
    }
}

TestPathGuard::TestPathGuard(TestPathGuard&& other) noexcept
    : m_path(std::move(other.m_path))
{
    other.m_path.clear();
}

TestPathGuard& TestPathGuard::operator=(TestPathGuard&& other) noexcept
{
    m_path = std::move(other.m_path);
    other.m_path.clear();
    return *this;
}


TestDirGuard::TestDirGuard(const std::string& path, bool init_clean)
    : m_path(path)
{
    if (!try_make_dir(path)) {
        if (init_clean)
            clean_dir(path);
    }
}

TestDirGuard::~TestDirGuard() noexcept
{
    if (g_keep_files)
        return;

    if (!do_remove)
        return;

    try {
        clean_dir(m_path);
        remove_dir(m_path);
    }
    catch (...) {
        // Exception deliberately ignored
    }
}

namespace {
void do_clean_dir(const std::string& path, const std::string& guard_string)
{
    DirScanner ds(path, true);
    std::string name;
    while (ds.next(name)) {
        std::string subpath = File::resolve(name, path);
        if (File::is_dir(subpath)) {
            do_clean_dir(subpath, guard_string);
            remove_dir(subpath);
        }
        else {
            // Try to avoid accidental removal of precious files due to bugs in
            // TestDirGuard or TEST_DIR macro.
            if (subpath.find(guard_string) == std::string::npos)
                throw std::runtime_error("Bad test dir path: " + path + ", guard: " + guard_string);
            File::remove(subpath);
        }
    }
}
}

void TestDirGuard::clean_dir(const std::string& path)
{
    do_clean_dir(path, ".test-dir");
}


DBTestPathGuard::DBTestPathGuard(const std::string& path)
    : TestPathGuard(path)
{
    cleanup();
}

DBTestPathGuard::~DBTestPathGuard() noexcept
{
    if (!g_keep_files && !m_path.empty())
        cleanup();
}

void DBTestPathGuard::cleanup() const noexcept
{
    if (!m_do_remove)
        return;
    try {
        do_clean_dir(m_path + ".management", ".management");
        if (File::is_dir(m_path + ".management"))
            remove_dir(m_path + ".management");
        File::try_remove(get_lock_path());
    }
    catch (...) {
        // Exception deliberately ignored
    }
}

TestDirNameGenerator::TestDirNameGenerator(std::string path)
    : m_path{std::move(path)}
{
}

std::string TestDirNameGenerator::next()
{
    return m_path + "/" + std::to_string(m_counter++);
}

std::shared_ptr<DB> get_test_db(const std::string& path, const char* crypt_key)
{
    const char* str = getenv("UNITTEST_LOG_LEVEL");
    realm::util::Logger::Level core_log_level = realm::util::Logger::Level::off;
    if (str && strlen(str) != 0) {
        std::istringstream in(str);
        in.imbue(std::locale::classic());
        in.flags(in.flags() & ~std::ios_base::skipws); // Do not accept white space
        in >> core_log_level;
    }

    DBOptions options;
    options.logger = std::make_shared<util::StderrLogger>(core_log_level);
    options.encryption_key = crypt_key;
    return DB::create(make_in_realm_history(), path, options);
}

} // namespace realm::test_util

1	/*************************************************************************
2	*
3	* Copyright 2016 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#include "test_path.hpp"
20
21	#include "misc.hpp"
22	#include "spawned_process.hpp"
23
24	#include <realm/util/file.hpp>
25	#include <realm/db.hpp>
26	#include <realm/history.hpp>
27
28	#include <algorithm>
29	#include <string>
30
31	#if REALM_PLATFORM_APPLE
32	#include <realm/util/cf_ptr.hpp>
33
34	#include <CoreFoundation/CoreFoundation.h>
35	#include <sys/mount.h>
36	#include <sys/param.h>
37	#elif defined(_WIN32)
38	#include <Windows.h>
39	// PathCchRemoveFileSpec()
40	#include <pathcch.h>
41	#pragma comment(lib, "Pathcch.lib")
42	#else
43	#include <unistd.h>
44	#include <libgen.h>
45	#endif
46
47	using namespace realm::util;
48
49	namespace {
50
51	bool g_keep_files = false;
52
53	std::string g_path_prefix;
54	std::string g_resource_path;
55	std::string g_exe_name;
56
57	#ifdef _WIN32
58	std::string sanitize_for_file_name(std::string str)
59	{
60	static const std::string invalid("<>:\"\|?*\\/");
61	std::transform(str.begin(), str.end(), str.begin(), [](char c) {
62	if (invalid.find(c) != std::string::npos)
63	return '-';
64	return c;
65	});
66	return str;
67	}
68	#else
69	std::string sanitize_for_file_name(const std::string& str)
70	{	9,099✔
71	return str;	9,099✔
72	}	9,099✔
73	#endif
74
75	#if REALM_PLATFORM_APPLE
76	std::string url_to_path(CFURLRef url)
77	{	12✔
78	auto absolute = adoptCF(CFURLCopyAbsoluteURL(url));	12✔
79	auto path = adoptCF(CFURLCopyPath(absolute.get()));	12✔
80	auto length = CFStringGetLength(path.get());	12✔
81	std::string ret;	12✔
82	ret.resize(CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8));	12✔
83	CFIndex bytes_written;	12✔
84	CFStringGetBytes(path.get(), {0, length}, kCFStringEncodingUTF8, 0, false, reinterpret_cast<uint8_t*>(ret.data()),	12✔
85	ret.size(), &bytes_written);	12✔
86	REALM_ASSERT(bytes_written);	12✔
87	ret.resize(bytes_written);	12✔
88	return ret;	12✔
89	}	12✔
90	#endif
91
92	} // anonymous namespace
93
94	namespace realm::test_util {
95
96	void keep_test_files()
97	{	×
98	g_keep_files = true;	×
99	}	×
100
101	std::string get_test_path(const std::string& test_name, const std::string& suffix)
102	{	9,099✔
103	return g_path_prefix + sanitize_for_file_name(test_name) + suffix;	9,099✔
104	}	9,099✔
105
106	std::string get_test_path_prefix()
107	{	816✔
108	return g_path_prefix;	816✔
109	}	816✔
110
111	bool initialize_test_path(int argc, const char* argv[])
112	{	24✔
113	#if REALM_PLATFORM_APPLE	12✔
114	// On Apple platforms we copy everything into a read-only bundle containing
115	// the test executable and resource files, and have to create test files in
116	// a temporary directory.
117	#if REALM_APPLE_DEVICE \|\| TARGET_OS_SIMULATOR
118	auto home = adoptCF(CFCopyHomeDirectoryURL());
119	g_path_prefix = url_to_path(home.get()) + "Documents/";
120	#else
121	g_path_prefix = util::make_temp_dir() + "/";	12✔
122	#endif	12✔
123
124	auto resources_url = adoptCF(CFBundleCopyResourcesDirectoryURL(CFBundleGetMainBundle()));	12✔
125	g_resource_path = url_to_path(resources_url.get());	12✔
126
127	// On other platforms we can write to the executable's directory, so we use
128	// that as the base path.
129	#elif defined(_MSC_VER)
130	wchar_t path[MAX_PATH];
131	if (GetModuleFileName(NULL, path, MAX_PATH) == 0) {
132	fprintf(stderr, "Failed to retrieve path to exectuable.\n");
133	return false;
134	}
135	PathCchRemoveFileSpec(path, MAX_PATH);
136	SetCurrentDirectory(path);
137	g_path_prefix = std::filesystem::path(path).u8string();
138	g_resource_path = g_path_prefix + "\\resources\\";
139	#else
140	char executable[PATH_MAX];	12✔
141	if (realpath(argv[0], executable) == nullptr) {	12✔
142	fprintf(stderr, "Failed to retrieve path to exectuable.\n");
143	return false;
144	}
145	const char* directory = dirname(executable);	12✔
146	if (chdir(directory) < 0) {	12✔
147	fprintf(stderr, "Failed to change directory.\n");
148	return false;
149	}
150	g_resource_path = File::resolve("resources", directory) + "/";	12✔
151	g_path_prefix = directory;	12✔
152	#endif	12✔
153		12✔
154	if (argc > 0) {	24✔
155	g_exe_name = argv[0];	24✔
156	}	24✔
157		12✔
158	if (argc > 1) {	24✔
159	g_path_prefix = argv[1];	×
160	}	×
161		12✔
162	return true;	24✔
163	}	24✔
164
165	bool test_dir_is_exfat()
166	{	12✔
167	#if REALM_PLATFORM_APPLE	6✔
168	if (test_util::get_test_path_prefix().empty())	6✔
169	return false;
170
171	struct statfs fsbuf;	6✔
172	int ret = statfs(test_util::get_test_path_prefix().c_str(), &fsbuf);	6✔
173	REALM_ASSERT_RELEASE(ret == 0);	6✔
174	// The documentation and headers helpfully don't list any of the values of
175	// f_type or provide constants for them
176	std::string fs_typename = fsbuf.f_fstypename;	6✔
177	std::transform(fs_typename.begin(), fs_typename.end(), fs_typename.begin(), toLowerAscii);	6✔
178	return fs_typename.find(std::string("exfat")) != std::string::npos \|\|	6✔
179	fs_typename.find(std::string("msdos")) != std::string::npos;	6✔
180	#else
181	return false;	6✔
182	#endif	6✔
183	}	12✔
184
185	std::string get_test_resource_path()
186	{	3,759✔
187	return g_resource_path;	3,759✔
188	}	3,759✔
189
190	std::string get_test_exe_name()
191	{	738✔
192	return g_exe_name;	738✔
193	}	738✔
194
195	TestPathGuard::TestPathGuard(const std::string& path)
196	: m_path(path)
197	, m_do_remove(test_util::SpawnedProcess::is_parent())
198	{	8,346✔
199	if (m_do_remove) {	8,346✔
200	File::try_remove(m_path);	8,346✔
201	}	8,346✔
202	}	8,346✔
203
204	TestPathGuard::~TestPathGuard() noexcept
205	{	8,346✔
206	if (g_keep_files)	8,346✔
207	return;	×
208	if (!m_do_remove)	8,346✔
209	return;	×
210	try {	8,346✔
211	if (!m_path.empty())	8,346✔
212	File::try_remove(m_path);	8,346✔
213	}	8,346✔
214	catch (...) {	4,176✔
215	// Exception deliberately ignored
216	}	×
217	}	8,346✔
218
219	TestPathGuard::TestPathGuard(TestPathGuard&& other) noexcept
220	: m_path(std::move(other.m_path))
221	{	×
222	other.m_path.clear();	×
223	}	×
224
225	TestPathGuard& TestPathGuard::operator=(TestPathGuard&& other) noexcept
226	{	×
227	m_path = std::move(other.m_path);	×
228	other.m_path.clear();	×
229	return *this;	×
230	}	×
231
232
233	TestDirGuard::TestDirGuard(const std::string& path, bool init_clean)
234	: m_path(path)
235	{	762✔
236	if (!try_make_dir(path)) {	762✔
NEW 237	if (init_clean)	×
NEW 238	clean_dir(path);	×
UNCOV 239	}	×
240	}	762✔
241
242	TestDirGuard::~TestDirGuard() noexcept
243	{	762✔
244	if (g_keep_files)	762✔
245	return;	×
246		384✔
247	if (!do_remove)	762✔
NEW 248	return;	×
249		384✔
250	try {	762✔
251	clean_dir(m_path);	762✔
252	remove_dir(m_path);	762✔
253	}	762✔
254	catch (...) {	384✔
255	// Exception deliberately ignored
256	}	×
257	}	762✔
258
259	namespace {
260	void do_clean_dir(const std::string& path, const std::string& guard_string)
261	{	18,270✔
262	DirScanner ds(path, true);	18,270✔
263	std::string name;	18,270✔
264	while (ds.next(name)) {	43,149✔
265	std::string subpath = File::resolve(name, path);	24,879✔
266	if (File::is_dir(subpath)) {	24,879✔
267	do_clean_dir(subpath, guard_string);	1,464✔
268	remove_dir(subpath);	1,464✔
269	}	1,464✔
270	else {	23,415✔
271	// Try to avoid accidental removal of precious files due to bugs in	825✔
272	// TestDirGuard or TEST_DIR macro.	825✔
273	if (subpath.find(guard_string) == std::string::npos)	23,415✔
NEW 274	throw std::runtime_error("Bad test dir path: " + path + ", guard: " + guard_string);	×
275	File::remove(subpath);	23,415✔
276	}	23,415✔
277	}	24,879✔
278	}	18,270✔
279	}
280
281	void TestDirGuard::clean_dir(const std::string& path)
282	{	822✔
283	do_clean_dir(path, ".test-dir");	822✔
284	}	822✔
285
286
287	DBTestPathGuard::DBTestPathGuard(const std::string& path)
288	: TestPathGuard(path)
289	{	7,992✔
290	cleanup();	7,992✔
291	}	7,992✔
292
293	DBTestPathGuard::~DBTestPathGuard() noexcept
294	{	7,992✔
295	if (!g_keep_files && !m_path.empty())	7,992✔
296	cleanup();	7,992✔
297	}	7,992✔
298
299	void DBTestPathGuard::cleanup() const noexcept
300	{	15,984✔
301	if (!m_do_remove)	15,984✔
302	return;	×
303	try {	15,984✔
304	do_clean_dir(m_path + ".management", ".management");	15,984✔
305	if (File::is_dir(m_path + ".management"))	15,984✔
306	remove_dir(m_path + ".management");	7,926✔
307	File::try_remove(get_lock_path());	15,984✔
308	}	15,984✔
309	catch (...) {	7,998✔
310	// Exception deliberately ignored
311	}	×
312	}	15,984✔
313
314	TestDirNameGenerator::TestDirNameGenerator(std::string path)
315	: m_path{std::move(path)}
316	{	×
317	}	×
318
319	std::string TestDirNameGenerator::next()
320	{	×
321	return m_path + "/" + std::to_string(m_counter++);	×
322	}	×
323
324	std::shared_ptr<DB> get_test_db(const std::string& path, const char* crypt_key)
325	{	114✔
326	const char* str = getenv("UNITTEST_LOG_LEVEL");	114✔
327	realm::util::Logger::Level core_log_level = realm::util::Logger::Level::off;	114✔
328	if (str && strlen(str) != 0) {	114!
329	std::istringstream in(str);	×
330	in.imbue(std::locale::classic());	×
331	in.flags(in.flags() & ~std::ios_base::skipws); // Do not accept white space	×
332	in >> core_log_level;	×
333	}	×
334		57✔
335	DBOptions options;	114✔
336	options.logger = std::make_shared<util::StderrLogger>(core_log_level);	114✔
337	options.encryption_key = crypt_key;	114✔
338	return DB::create(make_in_realm_history(), path, options);	114✔
339	}	114✔
340
341	} // namespace realm::test_util

realm / realm-core / 1771

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous