1771

Committed 20 Oct 2023 08:58AM UTC coverage: 91.567% (-0.009%) from 91.576%

Build # 1771

Build Type

push

Evergreen

Committed by

web-flow

Commit Message

Fix blocked DB::open on multiprocess access on exFAT filesystem (#6959)

Fix double file lock and DB::open being blocked with multiple concurrent realm access on fat32/exfat file systems.

When file is truncated on fat32/exfat its uid is available for other files. With multiple processes opening and truncating the same set of files could lead to the situation when within one process get_unique_id will return the same value for different files in timing is right. This breaks proper initialization of static data for interprocess mutexes, so that subsequent locks will hang by trying to lock essentially the same file twice.

Run Details

94304 of 173552 branches covered (0.0%)

59 of 82 new or added lines in 5 files covered. (71.95%)

53 existing lines in 13 files now uncovered.

230544 of 251776 relevant lines covered (91.57%)

6594884.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.2

/test/test_file.cpp

/*************************************************************************
 *
 * Copyright 2016 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#include "testsettings.hpp"
#ifdef TEST_FILE

#include <map>
#include <ostream>
#include <sstream>

#include <realm/util/file.hpp>
#include <realm/util/file_mapper.hpp>

#include "test.hpp"

#if REALM_PLATFORM_APPLE
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#endif

using namespace realm;
using namespace realm::util;
using namespace realm::test_util;


// Test independence and thread-safety
// -----------------------------------
//
// All tests must be thread safe and independent of each other. This
// is required because it allows for both shuffling of the execution
// order and for parallelized testing.
//
// In particular, avoid using std::rand() since it is not guaranteed
// to be thread safe. Instead use the API offered in
// `test/util/random.hpp`.
//
// All files created in tests must use the TEST_PATH macro (or one of
// its friends) to obtain a suitable file system path. See
// `test/util/test_path.hpp`.
//
//
// Debugging and the ONLY() macro
// ------------------------------
//
// A simple way of disabling all tests except one called `Foo`, is to
// replace TEST(Foo) with ONLY(Foo) and then recompile and rerun the
// test suite. Note that you can also use filtering by setting the
// environment varible `UNITTEST_FILTER`. See `README.md` for more on
// this.
//
// Another way to debug a particular test, is to copy that test into
// `experiments/testcase.cpp` and then run `sh build.sh
// check-testcase` (or one of its friends) from the command line.


TEST(File_ExistsAndRemove)
{
    TEST_PATH(path);
    File(path, File::mode_Write);
    CHECK(File::exists(path));
    CHECK(File::try_remove(path));
    CHECK(!File::exists(path));
    CHECK(!File::try_remove(path));
}

TEST(File_IsSame)
{
    TEST_PATH(path_1);
    TEST_PATH(path_2);

    // exFAT does not allocate inode numbers until the file is first non-empty,
    // so all never-written-to files appear to be the same file
    File(path_1, File::mode_Write).resize(1);
    File(path_2, File::mode_Write).resize(1);

    File f1(path_1, File::mode_Append);
    File f2(path_1, File::mode_Read);
    File f3(path_2, File::mode_Append);

    CHECK(f1.is_same_file(f1));
    CHECK(f1.is_same_file(f2));
    CHECK(!f1.is_same_file(f3));
    CHECK(!f2.is_same_file(f3));
}


TEST(File_Streambuf)
{
    TEST_PATH(path);
    {
        File f(path, File::mode_Write);
        File::Streambuf b(&f);
        std::ostream out(&b);
        out << "Line " << 1 << std::endl;
        out << "Line " << 2 << std::endl;
    }
    {
        File f(path, File::mode_Read);
        char buffer[256];
        size_t n = f.read(buffer);
        std::string s_1(buffer, buffer + n);
        std::ostringstream out;
        out << "Line " << 1 << std::endl;
        out << "Line " << 2 << std::endl;
        std::string s_2 = out.str();
        CHECK(s_1 == s_2);
    }
}


TEST(File_Map)
{
    TEST_PATH(path);
    const char data[4096] = "12345678901234567890";
    size_t len = strlen(data);
    {
        File f(path, File::mode_Write);
        f.set_encryption_key(crypt_key());
        f.resize(len);

        File::Map<char> map(f, File::access_ReadWrite, len);
        realm::util::encryption_read_barrier(map, 0, len);
        memcpy(map.get_addr(), data, len);
        realm::util::encryption_write_barrier(map, 0, len);
    }
    {
        File f(path, File::mode_Read);
        f.set_encryption_key(crypt_key());
        File::Map<char> map(f, File::access_ReadOnly, len);
        realm::util::encryption_read_barrier(map, 0, len);
        CHECK(memcmp(map.get_addr(), data, len) == 0);
    }
}


TEST(File_MapMultiplePages)
{
    // two blocks of IV tables
    const size_t count = 4096 / sizeof(size_t) * 256 * 2;

    TEST_PATH(path);
    {
        File f(path, File::mode_Write);
        f.set_encryption_key(crypt_key());
        f.resize(count * sizeof(size_t));

        File::Map<size_t> map(f, File::access_ReadWrite, count * sizeof(size_t));
        realm::util::encryption_read_barrier(map, 0, count);
        for (size_t i = 0; i < count; ++i)
            map.get_addr()[i] = i;
        realm::util::encryption_write_barrier(map, 0, count);
    }
    {
        File f(path, File::mode_Read);
        f.set_encryption_key(crypt_key());
        File::Map<size_t> map(f, File::access_ReadOnly, count * sizeof(size_t));
        realm::util::encryption_read_barrier(map, 0, count);
        for (size_t i = 0; i < count; ++i) {
            CHECK_EQUAL(map.get_addr()[i], i);
            if (map.get_addr()[i] != i)
                return;
        }
    }
}

TEST(File_ReaderAndWriter)
{
    const size_t count = 4096 / sizeof(size_t) * 256 * 2;

    TEST_PATH(path);

    File writer(path, File::mode_Write);
    writer.set_encryption_key(crypt_key());
    writer.resize(count * sizeof(size_t));

    File reader(path, File::mode_Read);
    reader.set_encryption_key(crypt_key());
    CHECK_EQUAL(writer.get_size(), reader.get_size());

    File::Map<size_t> write(writer, File::access_ReadWrite, count * sizeof(size_t));
    File::Map<size_t> read(reader, File::access_ReadOnly, count * sizeof(size_t));

    for (size_t i = 0; i < count; i += 100) {
        realm::util::encryption_read_barrier(write, i, 1);
        write.get_addr()[i] = i;
        realm::util::encryption_write_barrier(write, i);
        realm::util::encryption_read_barrier(read, i);
        CHECK_EQUAL(read.get_addr()[i], i);
        if (read.get_addr()[i] != i)
            return;
    }
}

TEST(File_Offset)
{
    const size_t size = page_size();
    const size_t count_per_page = size / sizeof(size_t);
    // two blocks of IV tables
    const size_t page_count = 256 * 2 / (size / 4096);

    TEST_PATH(path);
    {
        File f(path, File::mode_Write);
        f.set_encryption_key(crypt_key());
        f.resize(page_count * size);

        for (size_t i = 0; i < page_count; ++i) {
            File::Map<size_t> map(f, i * size, File::access_ReadWrite, size);
            for (size_t j = 0; j < count_per_page; ++j) {
                realm::util::encryption_read_barrier(map, j, 1);
                map.get_addr()[j] = i * size + j;
                realm::util::encryption_write_barrier(map, j);
            }
        }
    }
    {
        File f(path, File::mode_Read);
        f.set_encryption_key(crypt_key());
        for (size_t i = 0; i < page_count; ++i) {
            File::Map<size_t> map(f, i * size, File::access_ReadOnly, size);
            for (size_t j = 0; j < count_per_page; ++j) {
                realm::util::encryption_read_barrier(map, j);
                CHECK_EQUAL(map.get_addr()[j], i * size + j);
                if (map.get_addr()[j] != i * size + j)
                    return;
            }
        }
    }
}


TEST(File_MultipleWriters)
{
    const size_t count = 4096 / sizeof(size_t) * 256 * 2;
#if defined(_WIN32) && defined(REALM_ENABLE_ENCRYPTION)
    // This test runs really slow on Windows with encryption
    const size_t increments = 3000;
#else
    const size_t increments = 100;
#endif
    TEST_PATH(path);

    {
        File w1(path, File::mode_Write);
        w1.set_encryption_key(crypt_key());
        w1.resize(count * sizeof(size_t));

        File w2(path, File::mode_Write);
        w2.set_encryption_key(crypt_key());
        w2.resize(count * sizeof(size_t));

        File::Map<size_t> map1(w1, File::access_ReadWrite, count * sizeof(size_t));
        File::Map<size_t> map2(w2, File::access_ReadWrite, count * sizeof(size_t));

        // Place zeroes in selected places
        for (size_t i = 0; i < count; i += increments) {
            realm::util::encryption_read_barrier(map1, i);
            map1.get_addr()[i] = 0;
            realm::util::encryption_write_barrier(map1, i);
        }

        for (size_t i = 0; i < count; i += increments) {
            realm::util::encryption_read_barrier(map1, i, 1);
            ++map1.get_addr()[i];
            realm::util::encryption_write_barrier(map1, i);
            realm::util::encryption_read_barrier(map2, i, 1);
            ++map2.get_addr()[i];
            realm::util::encryption_write_barrier(map2, i);
        }
    }

    File reader(path, File::mode_Read);
    reader.set_encryption_key(crypt_key());

    File::Map<size_t> read(reader, File::access_ReadOnly, count * sizeof(size_t));
    realm::util::encryption_read_barrier(read, 0, count);
    for (size_t i = 0; i < count; i += increments) {
        CHECK_EQUAL(read.get_addr()[i], 2);
        if (read.get_addr()[i] != 2)
            return;
    }
}


TEST(File_SetEncryptionKey)
{
    TEST_PATH(path);
    File f(path, File::mode_Write);
    const char key[64] = {0};

#if REALM_ENABLE_ENCRYPTION
    f.set_encryption_key(key); // should not throw
#else
    CHECK_THROW_EX(f.set_encryption_key(key), Exception, (e.code() == ErrorCodes::NotSupported));
#endif
}


TEST(File_ReadWrite)
{
    TEST_PATH(path);
    File f(path, File::mode_Write);
    f.set_encryption_key(crypt_key());
    f.resize(100);

    for (char i = 0; i < 100; ++i)
        f.write(&i, 1);
    f.seek(0);
    for (char i = 0; i < 100; ++i) {
        char read;
        f.read(&read, 1);
        CHECK_EQUAL(i, read);
    }
}


TEST(File_Resize)
{
    TEST_PATH(path);
    File f(path, File::mode_Write);
    f.set_encryption_key(crypt_key());

    f.resize(page_size() * 2);
    CHECK_EQUAL(page_size() * 2, f.get_size());
    {
        File::Map<unsigned char> m(f, File::access_ReadWrite, page_size() * 2);
        for (unsigned int i = 0; i < page_size() * 2; ++i) {
            realm::util::encryption_read_barrier(m, i, 1);
            m.get_addr()[i] = static_cast<unsigned char>(i);
            realm::util::encryption_write_barrier(m, i);
        }

        // Resizing away the first write is indistinguishable in encrypted files
        // from the process being interrupted before it does the first write,
        // but with subsequent writes it can tell that there was once valid
        // encrypted data there, so flush and write a second time
        m.sync();
        for (unsigned int i = 0; i < page_size() * 2; ++i) {
            realm::util::encryption_read_barrier(m, i, 1);
            m.get_addr()[i] = static_cast<unsigned char>(i);
            realm::util::encryption_write_barrier(m, i);
        }
    }

    f.resize(page_size());
    CHECK_EQUAL(page_size(), f.get_size());
    {
        File::Map<unsigned char> m(f, File::access_ReadOnly, page_size());
        for (unsigned int i = 0; i < page_size(); ++i) {
            realm::util::encryption_read_barrier(m, i);
            CHECK_EQUAL(static_cast<unsigned char>(i), m.get_addr()[i]);
            if (static_cast<unsigned char>(i) != m.get_addr()[i])
                return;
        }
    }

    f.resize(page_size() * 2);
    CHECK_EQUAL(page_size() * 2, f.get_size());
    {
        File::Map<unsigned char> m(f, File::access_ReadWrite, page_size() * 2);
        for (unsigned int i = 0; i < page_size() * 2; ++i) {
            realm::util::encryption_read_barrier(m, i, 1);
            m.get_addr()[i] = static_cast<unsigned char>(i);
            realm::util::encryption_write_barrier(m, i);
        }
    }
    {
        File::Map<unsigned char> m(f, File::access_ReadOnly, page_size() * 2);
        for (unsigned int i = 0; i < page_size() * 2; ++i) {
            realm::util::encryption_read_barrier(m, i);
            CHECK_EQUAL(static_cast<unsigned char>(i), m.get_addr()[i]);
            if (static_cast<unsigned char>(i) != m.get_addr()[i])
                return;
        }
    }
}


TEST(File_NotFound)
{
    TEST_PATH(path);
    File file;
    CHECK_THROW_EX(file.open(path), FileAccessError, e.get_path() == std::string(path));
}


TEST(File_PathNotFound)
{
    File file;
    CHECK_THROW_EX(file.open(""), FileAccessError, e.code() == ErrorCodes::FileNotFound);
}


TEST(File_Exists)
{
    TEST_PATH(path);
    File file;
    file.open(path, File::mode_Write); // Create the file
    file.close();
    CHECK_THROW_EX(file.open(path, File::access_ReadWrite, File::create_Must, File::flag_Trunc), FileAccessError,
                   e.get_path() == std::string(path) && e.code() == ErrorCodes::FileAlreadyExists);
}


TEST(File_Move)
{
    TEST_PATH(path);
    File file_1(path, File::mode_Write);
    CHECK(file_1.is_attached());
    File file_2(std::move(file_1));
    CHECK_NOT(file_1.is_attached());
    CHECK(file_2.is_attached());
    file_1 = std::move(file_2);
    CHECK(file_1.is_attached());
    CHECK_NOT(file_2.is_attached());
}

#if 0
TEST(File_PreallocResizing)
{
    TEST_PATH(path);
    File file(path, File::mode_Write);
    CHECK(file.is_attached());
    // we cannot test this with encryption...prealloc always allocates a full page
    file.prealloc(0); // this is allowed
    CHECK_EQUAL(file.get_size(), 0);
    file.prealloc(100);
    CHECK_EQUAL(file.get_size(), 100);
    file.prealloc(50);
    CHECK_EQUAL(file.get_size(), 100); // prealloc does not reduce size

    // To expose the preallocation bug, we need to iterate over a large numbers, less than 4096.
    // If the bug is present, we will allocate additional space to the file on every call, but if it is
    // not present, the OS will preallocate 4096 only on the first call.
    constexpr size_t init_size = 2048;
    constexpr size_t dest_size = 3000;
    for (size_t prealloc_space = init_size; prealloc_space <= dest_size; ++prealloc_space) {
        file.prealloc(prealloc_space);
        CHECK_EQUAL(file.get_size(), prealloc_space);
    }

#if REALM_PLATFORM_APPLE
    int fd = ::open(path.c_str(), O_RDONLY);
    CHECK(fd >= 0);
    struct stat statbuf;
    CHECK(fstat(fd, &statbuf) == 0);
    size_t allocated_size = statbuf.st_blocks;
    CHECK_EQUAL(statbuf.st_size, dest_size);
    CHECK(!int_multiply_with_overflow_detect(allocated_size, S_BLKSIZE));

    // When performing prealloc, the OS has the option to preallocate more than the requeted space
    // but we need to check that the preallocated space is within a reasonable bound.
    // If space is being incorrectly preallocated (growing on each call) then we will have more than 3000KB
    // of preallocated space, but if it is being allocated correctly (only when we need to expand) then we'll have
    // a multiple of the optimal file system I/O operation (`stat -f %k .`) which is 4096 on HSF+.
    // To give flexibility for file system prealloc implementations we check that the preallocated space is within
    // at least 16 times the nominal requested size.
    CHECK_LESS(allocated_size, 4096 * 16);

    CHECK(::close(fd) == 0);
#endif
}
#endif

TEST(File_PreallocResizingAPFSBug)
{
    TEST_PATH(path);
    File file(path, File::mode_Write);
    CHECK(file.is_attached());
    file.write("aaaaaaaaaaaaaaaaaaaa"); // 20 a's
    // calling prealloc on a newly created file would sometimes fail on APFS with EINVAL via fcntl(F_PREALLOCATE)
    // this may not be the only way to trigger the error, but it does seem to be timing dependant.
    file.prealloc(100);
    CHECK_EQUAL(file.get_size(), 100);

    // let's write past the first prealloc block (@ 4096) and verify it reads correctly too.
    file.write("aaaaa");
    // this will change the file size, but likely won't preallocate more space since the first call to prealloc
    // will probably have allocated a whole 4096 block.
    file.prealloc(200);
    CHECK_EQUAL(file.get_size(), 200);
    file.write("aa");
    file.prealloc(5020); // expands to another 4096 block
    constexpr size_t insert_pos = 5000;
    const char* insert_str = "hello";
    file.seek(insert_pos);
    file.write(insert_str);
    file.seek(insert_pos);
    CHECK_EQUAL(file.get_size(), 5020);
    constexpr size_t input_size = 6;
    char input[input_size];
    file.read(input, input_size);
    CHECK_EQUAL(strncmp(input, insert_str, input_size), 0);
}

TEST(File_parent_dir)
{
    std::map<std::string, std::string> mappings = {{"Unicorn🦄/file.cpp", "Unicorn🦄"},
                                                   {"", ""},
                                                   {"asdf", ""},
                                                   {"file.cpp", ""},
                                                   {"Unicorn🦄", ""},
                                                   {"parent/file.cpp", "parent"},
                                                   {"parent//file.cpp", "parent"},
                                                   {"parent///file.cpp", "parent"},
                                                   {"parent////file.cpp", "parent"},
                                                   {"1/2/3/4.cpp", "1/2/3"},
                                                   {"/1/2/3/4", "/1/2/3"}};
    for (auto [input, expected] : mappings) {
        std::string actual = File::parent_dir(input);
        CHECK_EQUAL(actual, expected);
        if (actual != expected) {
            realm::util::format(std::cout, "unexpected result '%1' for input '%2'", actual, input);
        }
    }
}

TEST(File_GetUniqueID)
{
    TEST_PATH(path_1);
    TEST_PATH(path_2);
    TEST_PATH(path_3);

    File file1_1;
    File file1_2;
    File file2_1;
    file1_1.open(path_1, File::mode_Write);
    file1_2.open(path_1, File::mode_Read);
    file2_1.open(path_2, File::mode_Write);

    // exFAT does not allocate inode numbers until the file is first non-empty
    file1_1.resize(1);
    file2_1.resize(1);

    File::UniqueID uid1_1 = file1_1.get_unique_id();
    File::UniqueID uid1_2 = file1_2.get_unique_id();
    File::UniqueID uid2_1 = file2_1.get_unique_id();
    std::optional<File::UniqueID> uid2_2;
    CHECK(uid2_2 = File::get_unique_id(path_2));

    CHECK(uid1_1 == uid1_2);
    CHECK(uid2_1 == *uid2_2);
    CHECK(uid1_1 != uid2_1);

    // Path doesn't exist
    CHECK_NOT(File::get_unique_id(path_3));

    // Test operator<
    File::UniqueID uid4_1{0, 5};
    File::UniqueID uid4_2{1, 42};
    CHECK(uid4_1 < uid4_2);
    CHECK_NOT(uid4_2 < uid4_1);

    uid4_1 = {0, 1};
    uid4_2 = {0, 2};
    CHECK(uid4_1 < uid4_2);
    CHECK_NOT(uid4_2 < uid4_1);

    uid4_1 = uid4_2;
    CHECK_NOT(uid4_1 < uid4_2);
    CHECK_NOT(uid4_2 < uid4_1);

    file1_1.resize(0);
    file2_1.resize(0);
    file2_1.resize(1);
    file1_1.resize(1);
    if (!test_util::test_dir_is_exfat()) {
        CHECK(uid1_1 == file1_1.get_unique_id());
        CHECK(uid2_1 == file2_1.get_unique_id());
    }
    else {
        CHECK(uid1_1 == file2_1.get_unique_id());
        CHECK(uid2_1 == file1_1.get_unique_id());
    }
}

TEST(File_Temp)
{
    auto tmp_file_name = make_temp_file("foo");
    {
        File file1;
        file1.open(tmp_file_name, File::mode_Write);
        CHECK(file1.is_attached());
    }
    remove(tmp_file_name.c_str());
}

#endif // TEST_FILE

1	/*************************************************************************
2	*
3	* Copyright 2016 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#include "testsettings.hpp"
20	#ifdef TEST_FILE
21
22	#include <map>
23	#include <ostream>
24	#include <sstream>
25
26	#include <realm/util/file.hpp>
27	#include <realm/util/file_mapper.hpp>
28
29	#include "test.hpp"
30
31	#if REALM_PLATFORM_APPLE
32	#include <fcntl.h>
33	#include <sys/stat.h>
34	#include <unistd.h>
35	#endif
36
37	using namespace realm;
38	using namespace realm::util;
39	using namespace realm::test_util;
40
41
42	// Test independence and thread-safety
43	// -----------------------------------
44	//
45	// All tests must be thread safe and independent of each other. This
46	// is required because it allows for both shuffling of the execution
47	// order and for parallelized testing.
48	//
49	// In particular, avoid using std::rand() since it is not guaranteed
50	// to be thread safe. Instead use the API offered in
51	// `test/util/random.hpp`.
52	//
53	// All files created in tests must use the TEST_PATH macro (or one of
54	// its friends) to obtain a suitable file system path. See
55	// `test/util/test_path.hpp`.
56	//
57	//
58	// Debugging and the ONLY() macro
59	// ------------------------------
60	//
61	// A simple way of disabling all tests except one called `Foo`, is to
62	// replace TEST(Foo) with ONLY(Foo) and then recompile and rerun the
63	// test suite. Note that you can also use filtering by setting the
64	// environment varible `UNITTEST_FILTER`. See `README.md` for more on
65	// this.
66	//
67	// Another way to debug a particular test, is to copy that test into
68	// `experiments/testcase.cpp` and then run `sh build.sh
69	// check-testcase` (or one of its friends) from the command line.
70
71
72	TEST(File_ExistsAndRemove)
73	{	2✔
74	TEST_PATH(path);	2✔
75	File(path, File::mode_Write);	2✔
76	CHECK(File::exists(path));	2✔
77	CHECK(File::try_remove(path));	2✔
78	CHECK(!File::exists(path));	2✔
79	CHECK(!File::try_remove(path));	2✔
80	}	2✔
81
82	TEST(File_IsSame)
83	{	2✔
84	TEST_PATH(path_1);	2✔
85	TEST_PATH(path_2);	2✔
86		1✔
87	// exFAT does not allocate inode numbers until the file is first non-empty,	1✔
88	// so all never-written-to files appear to be the same file	1✔
89	File(path_1, File::mode_Write).resize(1);	2✔
90	File(path_2, File::mode_Write).resize(1);	2✔
91		1✔
92	File f1(path_1, File::mode_Append);	2✔
93	File f2(path_1, File::mode_Read);	2✔
94	File f3(path_2, File::mode_Append);	2✔
95		1✔
96	CHECK(f1.is_same_file(f1));	2✔
97	CHECK(f1.is_same_file(f2));	2✔
98	CHECK(!f1.is_same_file(f3));	2✔
99	CHECK(!f2.is_same_file(f3));	2✔
100	}	2✔
101
102
103	TEST(File_Streambuf)
104	{	2✔
105	TEST_PATH(path);	2✔
106	{	2✔
107	File f(path, File::mode_Write);	2✔
108	File::Streambuf b(&f);	2✔
109	std::ostream out(&b);	2✔
110	out << "Line " << 1 << std::endl;	2✔
111	out << "Line " << 2 << std::endl;	2✔
112	}	2✔
113	{	2✔
114	File f(path, File::mode_Read);	2✔
115	char buffer[256];	2✔
116	size_t n = f.read(buffer);	2✔
117	std::string s_1(buffer, buffer + n);	2✔
118	std::ostringstream out;	2✔
119	out << "Line " << 1 << std::endl;	2✔
120	out << "Line " << 2 << std::endl;	2✔
121	std::string s_2 = out.str();	2✔
122	CHECK(s_1 == s_2);	2✔
123	}	2✔
124	}	2✔
125
126
127	TEST(File_Map)
128	{	2✔
129	TEST_PATH(path);	2✔
130	const char data[4096] = "12345678901234567890";	2✔
131	size_t len = strlen(data);	2✔
132	{	2✔
133	File f(path, File::mode_Write);	2✔
134	f.set_encryption_key(crypt_key());	2✔
135	f.resize(len);	2✔
136		1✔
137	File::Map<char> map(f, File::access_ReadWrite, len);	2✔
138	realm::util::encryption_read_barrier(map, 0, len);	2✔
139	memcpy(map.get_addr(), data, len);	2✔
140	realm::util::encryption_write_barrier(map, 0, len);	2✔
141	}	2✔
142	{	2✔
143	File f(path, File::mode_Read);	2✔
144	f.set_encryption_key(crypt_key());	2✔
145	File::Map<char> map(f, File::access_ReadOnly, len);	2✔
146	realm::util::encryption_read_barrier(map, 0, len);	2✔
147	CHECK(memcmp(map.get_addr(), data, len) == 0);	2✔
148	}	2✔
149	}	2✔
150
151
152	TEST(File_MapMultiplePages)
153	{	2✔
154	// two blocks of IV tables	1✔
155	const size_t count = 4096 / sizeof(size_t) * 256 * 2;	2✔
156		1✔
157	TEST_PATH(path);	2✔
158	{	2✔
159	File f(path, File::mode_Write);	2✔
160	f.set_encryption_key(crypt_key());	2✔
161	f.resize(count * sizeof(size_t));	2✔
162		1✔
163	File::Map<size_t> map(f, File::access_ReadWrite, count * sizeof(size_t));	2✔
164	realm::util::encryption_read_barrier(map, 0, count);	2✔
165	for (size_t i = 0; i < count; ++i)	524,290✔
166	map.get_addr()[i] = i;	524,288✔
167	realm::util::encryption_write_barrier(map, 0, count);	2✔
168	}	2✔
169	{	2✔
170	File f(path, File::mode_Read);	2✔
171	f.set_encryption_key(crypt_key());	2✔
172	File::Map<size_t> map(f, File::access_ReadOnly, count * sizeof(size_t));	2✔
173	realm::util::encryption_read_barrier(map, 0, count);	2✔
174	for (size_t i = 0; i < count; ++i) {	524,290✔
175	CHECK_EQUAL(map.get_addr()[i], i);	524,288✔
176	if (map.get_addr()[i] != i)	524,288✔
177	return;	×
178	}	524,288✔
179	}	2✔
180	}	2✔
181
182	TEST(File_ReaderAndWriter)
183	{	2✔
184	const size_t count = 4096 / sizeof(size_t) * 256 * 2;	2✔
185		1✔
186	TEST_PATH(path);	2✔
187		1✔
188	File writer(path, File::mode_Write);	2✔
189	writer.set_encryption_key(crypt_key());	2✔
190	writer.resize(count * sizeof(size_t));	2✔
191		1✔
192	File reader(path, File::mode_Read);	2✔
193	reader.set_encryption_key(crypt_key());	2✔
194	CHECK_EQUAL(writer.get_size(), reader.get_size());	2✔
195		1✔
196	File::Map<size_t> write(writer, File::access_ReadWrite, count * sizeof(size_t));	2✔
197	File::Map<size_t> read(reader, File::access_ReadOnly, count * sizeof(size_t));	2✔
198		1✔
199	for (size_t i = 0; i < count; i += 100) {	5,246✔
200	realm::util::encryption_read_barrier(write, i, 1);	5,244✔
201	write.get_addr()[i] = i;	5,244✔
202	realm::util::encryption_write_barrier(write, i);	5,244✔
203	realm::util::encryption_read_barrier(read, i);	5,244✔
204	CHECK_EQUAL(read.get_addr()[i], i);	5,244✔
205	if (read.get_addr()[i] != i)	5,244✔
206	return;	×
207	}	5,244✔
208	}	2✔
209
210	TEST(File_Offset)
211	{	2✔
212	const size_t size = page_size();	2✔
213	const size_t count_per_page = size / sizeof(size_t);	2✔
214	// two blocks of IV tables	1✔
215	const size_t page_count = 256 * 2 / (size / 4096);	2✔
216		1✔
217	TEST_PATH(path);	2✔
218	{	2✔
219	File f(path, File::mode_Write);	2✔
220	f.set_encryption_key(crypt_key());	2✔
221	f.resize(page_count * size);	2✔
222		1✔
223	for (size_t i = 0; i < page_count; ++i) {	642✔
224	File::Map<size_t> map(f, i * size, File::access_ReadWrite, size);	640✔
225	for (size_t j = 0; j < count_per_page; ++j) {	524,928✔
226	realm::util::encryption_read_barrier(map, j, 1);	524,288✔
227	map.get_addr()[j] = i * size + j;	524,288✔
228	realm::util::encryption_write_barrier(map, j);	524,288✔
229	}	524,288✔
230	}	640✔
231	}	2✔
232	{	2✔
233	File f(path, File::mode_Read);	2✔
234	f.set_encryption_key(crypt_key());	2✔
235	for (size_t i = 0; i < page_count; ++i) {	642✔
236	File::Map<size_t> map(f, i * size, File::access_ReadOnly, size);	640✔
237	for (size_t j = 0; j < count_per_page; ++j) {	524,928✔
238	realm::util::encryption_read_barrier(map, j);	524,288✔
239	CHECK_EQUAL(map.get_addr()[j], i * size + j);	524,288✔
240	if (map.get_addr()[j] != i * size + j)	524,288✔
241	return;	×
242	}	524,288✔
243	}	640✔
244	}	2✔
245	}	2✔
246
247
248	TEST(File_MultipleWriters)
249	{	2✔
250	const size_t count = 4096 / sizeof(size_t) * 256 * 2;	2✔
251	#if defined(_WIN32) && defined(REALM_ENABLE_ENCRYPTION)
252	// This test runs really slow on Windows with encryption
253	const size_t increments = 3000;
254	#else
255	const size_t increments = 100;	2✔
256	#endif	2✔
257	TEST_PATH(path);	2✔
258		1✔
259	{	2✔
260	File w1(path, File::mode_Write);	2✔
261	w1.set_encryption_key(crypt_key());	2✔
262	w1.resize(count * sizeof(size_t));	2✔
263		1✔
264	File w2(path, File::mode_Write);	2✔
265	w2.set_encryption_key(crypt_key());	2✔
266	w2.resize(count * sizeof(size_t));	2✔
267		1✔
268	File::Map<size_t> map1(w1, File::access_ReadWrite, count * sizeof(size_t));	2✔
269	File::Map<size_t> map2(w2, File::access_ReadWrite, count * sizeof(size_t));	2✔
270		1✔
271	// Place zeroes in selected places	1✔
272	for (size_t i = 0; i < count; i += increments) {	5,246✔
273	realm::util::encryption_read_barrier(map1, i);	5,244✔
274	map1.get_addr()[i] = 0;	5,244✔
275	realm::util::encryption_write_barrier(map1, i);	5,244✔
276	}	5,244✔
277		1✔
278	for (size_t i = 0; i < count; i += increments) {	5,246✔
279	realm::util::encryption_read_barrier(map1, i, 1);	5,244✔
280	++map1.get_addr()[i];	5,244✔
281	realm::util::encryption_write_barrier(map1, i);	5,244✔
282	realm::util::encryption_read_barrier(map2, i, 1);	5,244✔
283	++map2.get_addr()[i];	5,244✔
284	realm::util::encryption_write_barrier(map2, i);	5,244✔
285	}	5,244✔
286	}	2✔
287		1✔
288	File reader(path, File::mode_Read);	2✔
289	reader.set_encryption_key(crypt_key());	2✔
290		1✔
291	File::Map<size_t> read(reader, File::access_ReadOnly, count * sizeof(size_t));	2✔
292	realm::util::encryption_read_barrier(read, 0, count);	2✔
293	for (size_t i = 0; i < count; i += increments) {	5,246✔
294	CHECK_EQUAL(read.get_addr()[i], 2);	5,244✔
295	if (read.get_addr()[i] != 2)	5,244✔
296	return;	×
297	}	5,244✔
298	}	2✔
299
300
301	TEST(File_SetEncryptionKey)
302	{	2✔
303	TEST_PATH(path);	2✔
304	File f(path, File::mode_Write);	2✔
305	const char key[64] = {0};	2✔
306		1✔
307	#if REALM_ENABLE_ENCRYPTION	2✔
308	f.set_encryption_key(key); // should not throw	2✔
309	#else
310	CHECK_THROW_EX(f.set_encryption_key(key), Exception, (e.code() == ErrorCodes::NotSupported));
311	#endif
312	}	2✔
313
314
315	TEST(File_ReadWrite)
316	{	2✔
317	TEST_PATH(path);	2✔
318	File f(path, File::mode_Write);	2✔
319	f.set_encryption_key(crypt_key());	2✔
320	f.resize(100);	2✔
321		1✔
322	for (char i = 0; i < 100; ++i)	202✔
323	f.write(&i, 1);	200✔
324	f.seek(0);	2✔
325	for (char i = 0; i < 100; ++i) {	202✔
326	char read;	200✔
327	f.read(&read, 1);	200✔
328	CHECK_EQUAL(i, read);	200✔
329	}	200✔
330	}	2✔
331
332
333	TEST(File_Resize)
334	{	2✔
335	TEST_PATH(path);	2✔
336	File f(path, File::mode_Write);	2✔
337	f.set_encryption_key(crypt_key());	2✔
338		1✔
339	f.resize(page_size() * 2);	2✔
340	CHECK_EQUAL(page_size() * 2, f.get_size());	2✔
341	{	2✔
342	File::Map<unsigned char> m(f, File::access_ReadWrite, page_size() * 2);	2✔
343	for (unsigned int i = 0; i < page_size() * 2; ++i) {	40,962✔
344	realm::util::encryption_read_barrier(m, i, 1);	40,960✔
345	m.get_addr()[i] = static_cast<unsigned char>(i);	40,960✔
346	realm::util::encryption_write_barrier(m, i);	40,960✔
347	}	40,960✔
348		1✔
349	// Resizing away the first write is indistinguishable in encrypted files	1✔
350	// from the process being interrupted before it does the first write,	1✔
351	// but with subsequent writes it can tell that there was once valid	1✔
352	// encrypted data there, so flush and write a second time	1✔
353	m.sync();	2✔
354	for (unsigned int i = 0; i < page_size() * 2; ++i) {	40,962✔
355	realm::util::encryption_read_barrier(m, i, 1);	40,960✔
356	m.get_addr()[i] = static_cast<unsigned char>(i);	40,960✔
357	realm::util::encryption_write_barrier(m, i);	40,960✔
358	}	40,960✔
359	}	2✔
360		1✔
361	f.resize(page_size());	2✔
362	CHECK_EQUAL(page_size(), f.get_size());	2✔
363	{	2✔
364	File::Map<unsigned char> m(f, File::access_ReadOnly, page_size());	2✔
365	for (unsigned int i = 0; i < page_size(); ++i) {	20,482✔
366	realm::util::encryption_read_barrier(m, i);	20,480✔
367	CHECK_EQUAL(static_cast<unsigned char>(i), m.get_addr()[i]);	20,480✔
368	if (static_cast<unsigned char>(i) != m.get_addr()[i])	20,480✔
369	return;	×
370	}	20,480✔
371	}	2✔
372		1✔
373	f.resize(page_size() * 2);	2✔
374	CHECK_EQUAL(page_size() * 2, f.get_size());	2✔
375	{	2✔
376	File::Map<unsigned char> m(f, File::access_ReadWrite, page_size() * 2);	2✔
377	for (unsigned int i = 0; i < page_size() * 2; ++i) {	40,962✔
378	realm::util::encryption_read_barrier(m, i, 1);	40,960✔
379	m.get_addr()[i] = static_cast<unsigned char>(i);	40,960✔
380	realm::util::encryption_write_barrier(m, i);	40,960✔
381	}	40,960✔
382	}	2✔
383	{	2✔
384	File::Map<unsigned char> m(f, File::access_ReadOnly, page_size() * 2);	2✔
385	for (unsigned int i = 0; i < page_size() * 2; ++i) {	40,962✔
386	realm::util::encryption_read_barrier(m, i);	40,960✔
387	CHECK_EQUAL(static_cast<unsigned char>(i), m.get_addr()[i]);	40,960✔
388	if (static_cast<unsigned char>(i) != m.get_addr()[i])	40,960✔
389	return;	×
390	}	40,960✔
391	}	2✔
392	}	2✔
393
394
395	TEST(File_NotFound)
396	{	2✔
397	TEST_PATH(path);	2✔
398	File file;	2✔
399	CHECK_THROW_EX(file.open(path), FileAccessError, e.get_path() == std::string(path));	2✔
400	}	2✔
401
402
403	TEST(File_PathNotFound)
404	{	2✔
405	File file;	2✔
406	CHECK_THROW_EX(file.open(""), FileAccessError, e.code() == ErrorCodes::FileNotFound);	2✔
407	}	2✔
408
409
410	TEST(File_Exists)
411	{	2✔
412	TEST_PATH(path);	2✔
413	File file;	2✔
414	file.open(path, File::mode_Write); // Create the file	2✔
415	file.close();	2✔
416	CHECK_THROW_EX(file.open(path, File::access_ReadWrite, File::create_Must, File::flag_Trunc), FileAccessError,	2✔
417	e.get_path() == std::string(path) && e.code() == ErrorCodes::FileAlreadyExists);	2✔
418	}	2✔
419
420
421	TEST(File_Move)
422	{	2✔
423	TEST_PATH(path);	2✔
424	File file_1(path, File::mode_Write);	2✔
425	CHECK(file_1.is_attached());	2✔
426	File file_2(std::move(file_1));	2✔
427	CHECK_NOT(file_1.is_attached());	2✔
428	CHECK(file_2.is_attached());	2✔
429	file_1 = std::move(file_2);	2✔
430	CHECK(file_1.is_attached());	2✔
431	CHECK_NOT(file_2.is_attached());	2✔
432	}	2✔
433
434	#if 0
435	TEST(File_PreallocResizing)
436	{
437	TEST_PATH(path);
438	File file(path, File::mode_Write);
439	CHECK(file.is_attached());
440	// we cannot test this with encryption...prealloc always allocates a full page
441	file.prealloc(0); // this is allowed
442	CHECK_EQUAL(file.get_size(), 0);
443	file.prealloc(100);
444	CHECK_EQUAL(file.get_size(), 100);
445	file.prealloc(50);
446	CHECK_EQUAL(file.get_size(), 100); // prealloc does not reduce size
447
448	// To expose the preallocation bug, we need to iterate over a large numbers, less than 4096.
449	// If the bug is present, we will allocate additional space to the file on every call, but if it is
450	// not present, the OS will preallocate 4096 only on the first call.
451	constexpr size_t init_size = 2048;
452	constexpr size_t dest_size = 3000;
453	for (size_t prealloc_space = init_size; prealloc_space <= dest_size; ++prealloc_space) {
454	file.prealloc(prealloc_space);
455	CHECK_EQUAL(file.get_size(), prealloc_space);
456	}
457
458	#if REALM_PLATFORM_APPLE
459	int fd = ::open(path.c_str(), O_RDONLY);
460	CHECK(fd >= 0);
461	struct stat statbuf;
462	CHECK(fstat(fd, &statbuf) == 0);
463	size_t allocated_size = statbuf.st_blocks;
464	CHECK_EQUAL(statbuf.st_size, dest_size);
465	CHECK(!int_multiply_with_overflow_detect(allocated_size, S_BLKSIZE));
466
467	// When performing prealloc, the OS has the option to preallocate more than the requeted space
468	// but we need to check that the preallocated space is within a reasonable bound.
469	// If space is being incorrectly preallocated (growing on each call) then we will have more than 3000KB
470	// of preallocated space, but if it is being allocated correctly (only when we need to expand) then we'll have
471	// a multiple of the optimal file system I/O operation (`stat -f %k .`) which is 4096 on HSF+.
472	// To give flexibility for file system prealloc implementations we check that the preallocated space is within
473	// at least 16 times the nominal requested size.
474	CHECK_LESS(allocated_size, 4096 * 16);
475
476	CHECK(::close(fd) == 0);
477	#endif
478	}
479	#endif
480
481	TEST(File_PreallocResizingAPFSBug)
482	{	2✔
483	TEST_PATH(path);	2✔
484	File file(path, File::mode_Write);	2✔
485	CHECK(file.is_attached());	2✔
486	file.write("aaaaaaaaaaaaaaaaaaaa"); // 20 a's	2✔
487	// calling prealloc on a newly created file would sometimes fail on APFS with EINVAL via fcntl(F_PREALLOCATE)	1✔
488	// this may not be the only way to trigger the error, but it does seem to be timing dependant.	1✔
489	file.prealloc(100);	2✔
490	CHECK_EQUAL(file.get_size(), 100);	2✔
491		1✔
492	// let's write past the first prealloc block (@ 4096) and verify it reads correctly too.	1✔
493	file.write("aaaaa");	2✔
494	// this will change the file size, but likely won't preallocate more space since the first call to prealloc	1✔
495	// will probably have allocated a whole 4096 block.	1✔
496	file.prealloc(200);	2✔
497	CHECK_EQUAL(file.get_size(), 200);	2✔
498	file.write("aa");	2✔
499	file.prealloc(5020); // expands to another 4096 block	2✔
500	constexpr size_t insert_pos = 5000;	2✔
501	const char* insert_str = "hello";	2✔
502	file.seek(insert_pos);	2✔
503	file.write(insert_str);	2✔
504	file.seek(insert_pos);	2✔
505	CHECK_EQUAL(file.get_size(), 5020);	2✔
506	constexpr size_t input_size = 6;	2✔
507	char input[input_size];	2✔
508	file.read(input, input_size);	2✔
509	CHECK_EQUAL(strncmp(input, insert_str, input_size), 0);	2✔
510	}	2✔
511
512	TEST(File_parent_dir)
513	{	2✔
514	std::map<std::string, std::string> mappings = {{"Unicorn🦄/file.cpp", "Unicorn🦄"},	2✔
515	{"", ""},	2✔
516	{"asdf", ""},	2✔
517	{"file.cpp", ""},	2✔
518	{"Unicorn🦄", ""},	2✔
519	{"parent/file.cpp", "parent"},	2✔
520	{"parent//file.cpp", "parent"},	2✔
521	{"parent///file.cpp", "parent"},	2✔
522	{"parent////file.cpp", "parent"},	2✔
523	{"1/2/3/4.cpp", "1/2/3"},	2✔
524	{"/1/2/3/4", "/1/2/3"}};	2✔
525	for (auto [input, expected] : mappings) {	22✔
526	std::string actual = File::parent_dir(input);	22✔
527	CHECK_EQUAL(actual, expected);	22✔
528	if (actual != expected) {	22✔
529	realm::util::format(std::cout, "unexpected result '%1' for input '%2'", actual, input);	×
530	}	×
531	}	22✔
532	}	2✔
533
534	TEST(File_GetUniqueID)
535	{	2✔
536	TEST_PATH(path_1);	2✔
537	TEST_PATH(path_2);	2✔
538	TEST_PATH(path_3);	2✔
539		1✔
540	File file1_1;	2✔
541	File file1_2;	2✔
542	File file2_1;	2✔
543	file1_1.open(path_1, File::mode_Write);	2✔
544	file1_2.open(path_1, File::mode_Read);	2✔
545	file2_1.open(path_2, File::mode_Write);	2✔
546		1✔
547	// exFAT does not allocate inode numbers until the file is first non-empty	1✔
548	file1_1.resize(1);	2✔
549	file2_1.resize(1);	2✔
550		1✔
551	File::UniqueID uid1_1 = file1_1.get_unique_id();	2✔
552	File::UniqueID uid1_2 = file1_2.get_unique_id();	2✔
553	File::UniqueID uid2_1 = file2_1.get_unique_id();	2✔
554	std::optional<File::UniqueID> uid2_2;	2✔
555	CHECK(uid2_2 = File::get_unique_id(path_2));	2✔
556		1✔
557	CHECK(uid1_1 == uid1_2);	2✔
558	CHECK(uid2_1 == *uid2_2);	2✔
559	CHECK(uid1_1 != uid2_1);	2✔
560		1✔
561	// Path doesn't exist	1✔
562	CHECK_NOT(File::get_unique_id(path_3));	2✔
563		1✔
564	// Test operator<	1✔
565	File::UniqueID uid4_1{0, 5};	2✔
566	File::UniqueID uid4_2{1, 42};	2✔
567	CHECK(uid4_1 < uid4_2);	2✔
568	CHECK_NOT(uid4_2 < uid4_1);	2✔
569		1✔
570	uid4_1 = {0, 1};	2✔
571	uid4_2 = {0, 2};	2✔
572	CHECK(uid4_1 < uid4_2);	2✔
573	CHECK_NOT(uid4_2 < uid4_1);	2✔
574		1✔
575	uid4_1 = uid4_2;	2✔
576	CHECK_NOT(uid4_1 < uid4_2);	2✔
577	CHECK_NOT(uid4_2 < uid4_1);	2✔
578		1✔
579	file1_1.resize(0);	2✔
580	file2_1.resize(0);	2✔
581	file2_1.resize(1);	2✔
582	file1_1.resize(1);	2✔
583	if (!test_util::test_dir_is_exfat()) {	2✔
584	CHECK(uid1_1 == file1_1.get_unique_id());	2✔
585	CHECK(uid2_1 == file2_1.get_unique_id());	2✔
586	}	2✔
NEW 587	else {	×
NEW 588	CHECK(uid1_1 == file2_1.get_unique_id());	×
NEW 589	CHECK(uid2_1 == file1_1.get_unique_id());	×
NEW 590	}	×
591	}	2✔
592
593	TEST(File_Temp)
594	{	2✔
595	auto tmp_file_name = make_temp_file("foo");	2✔
596	{	2✔
597	File file1;	2✔
598	file1.open(tmp_file_name, File::mode_Write);	2✔
599	CHECK(file1.is_attached());	2✔
600	}	2✔
601	remove(tmp_file_name.c_str());	2✔
602	}	2✔
603
604	#endif // TEST_FILE

realm / realm-core / 1771

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous