nikola.irinchev_478

Committed 10 Jun 2024 05:02PM UTC coverage: 90.821% (-0.2%) from 90.975%

Build # nikola.irinchev_478

Build Type

Pull #7792

Evergreen

Committed by

nirinchev

Commit Message

Replace app_config_get_sync_client_config with _set_sync_client_config

Pull Request Pull Request #7792: Replace app_config_get_sync_client_config with _set_sync_client_config

Run Details

101696 of 180086 branches covered (56.47%)

0 of 3 new or added lines in 1 file covered. (0.0%)

1872 existing lines in 57 files now uncovered.

214568 of 236253 relevant lines covered (90.82%)

5116114.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.35

/test/test_compaction.cpp

/*************************************************************************
 *
 * Copyright 2022 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#include <realm.hpp>
#include <realm/util/scope_exit.hpp>
#include "test.hpp"

#include <iostream>
#include <chrono>

// #include <valgrind/callgrind.h>

#ifndef CALLGRIND_START_INSTRUMENTATION
#define CALLGRIND_START_INSTRUMENTATION
#endif

#ifndef CALLGRIND_STOP_INSTRUMENTATION
#define CALLGRIND_STOP_INSTRUMENTATION
#endif

// valgrind --tool=callgrind --instr-atstart=no realm-tests

using namespace std::chrono;

using namespace realm;
using namespace realm::util;
using namespace realm::test_util;
using unit_test::TestContext;

TEST(Compaction_WhileGrowing)
{
    Random random(random_int<unsigned long>());
    SHARED_GROUP_TEST_PATH(path);
    DBRef db = DB::create(make_in_realm_history(), path);
    size_t free_space, used_space;

    auto tr = db->start_write();
    auto table1 = tr->add_table("Binaries");
    auto col_bin1 = table1->add_column(type_Binary, "str", true);
    auto table2 = tr->add_table("Integers");
    auto col_bin2 = table2->add_column(type_Binary, "str", true);
    tr->commit_and_continue_as_read();
    char w[5000];
    for (int i = 0; i < 5000; ++i) {
        w[i] = '0' + (i % 64);
    }
    int num = (REALM_MAX_BPNODE_SIZE == 1000) ? 1400 : 1300;
    tr->promote_to_write();
    CHECK(db->get_evacuation_stage() == DB::EvacStage::idle);
    for (int j = 0; j < num; ++j) {
        table1->create_object().set(col_bin1, BinaryData(w, 450));
        table2->create_object().set(col_bin2, BinaryData(w, 200));
        if (j % 10 == 0) {
            tr->commit_and_continue_as_read();
            tr->promote_to_write();
        }
    }
    tr->commit_and_continue_as_read();

    tr->promote_to_write();
    auto objp = table1->begin();
    for (int j = 0; j < num - 30; ++j, ++objp) {
        objp->set(col_bin1, BinaryData());
        if (j % 10 == 0) {
            tr->commit_and_continue_as_read();
            tr->promote_to_write();
        }
        if (db->get_evacuation_stage() == DB::EvacStage::evacuating)
            break;
    }

    CHECK(db->get_evacuation_stage() == DB::EvacStage::evacuating);
    tr->commit_and_continue_as_read();
    db->get_stats(free_space, used_space);
    // The file is now subject to compaction
    if (!CHECK(free_space > 2 * used_space)) {
        std::cout << "Free space: " << free_space << std::endl;
        std::cout << "Used space: " << used_space << std::endl;
    }

    // During the following, the space kept in "m_under_evacuation" will be used
    // before all elements have been moved, which will terminate that session
    tr->promote_to_write();
    table1->create_object().set(col_bin1, BinaryData(w, 4500));
    table1->create_object().set(col_bin1, BinaryData(w, 4500));
    tr->commit_and_continue_as_read();

    CHECK(db->get_evacuation_stage() == DB::EvacStage::blocked);

    // db->get_stats(free_space, used_space);
    // std::cout << "Total: " << free_space + used_space << ", "
    //           << "Free: " << free_space << ", "
    //           << "Used: " << used_space << std::endl;

    tr->promote_to_write();
    table1->clear();
    table2->clear();
    tr->commit_and_continue_as_read();
    // Now there should be room for compaction

    auto n = 20; // Ensure that test will end
    do {
        tr->promote_to_write();
        tr->commit_and_continue_as_read();
        db->get_stats(free_space, used_space);
    } while (db->get_evacuation_stage() != DB::EvacStage::idle && --n > 0);
    CHECK_LESS(free_space, 0x10000);
}

TEST_TYPES(Compaction_Large, std::true_type, std::false_type)
{
    using type = typename TEST_TYPE::type;
    SHARED_GROUP_TEST_PATH(path);
    int64_t total;
    {
        DBRef db;
        if constexpr (type::value) {
            db = DB::create(make_in_realm_history(), path);
        }
        else {
            db = DB::create(make_in_realm_history());
        }
        {
            auto tr = db->start_write();
            auto t = tr->add_table("the_table");
            auto c = t->add_column(type_Binary, "str", true);
            char w[1000];
            for (int i = 0; i < 1000; ++i) {
                w[i] = '0' + (i % 10);
            }
            size_t num = 100000;
            for (size_t j = 0; j < num; ++j) {
                BinaryData sd(w, 500 + (j % 500));
                t->create_object().set(c, sd);
            }
            tr->commit_and_continue_as_read();

            tr->promote_to_write();
            int j = 0;
            for (auto o : *t) {
                BinaryData sd(w, j % 500);
                o.set(c, sd);
                ++j;
            }
            tr->commit_and_continue_as_read();

            tr->promote_to_write();
            // This will likely make the table names reside in the upper end of the file
            tr->add_table("another_table");
            tr->commit_and_continue_as_read();
        }

        auto worker = [db] {
            Random random(random_int<unsigned long>());
            size_t free_space, used_space;
            auto tr = db->start_read();
            auto t = tr->get_table("the_table");
            auto c = t->get_column_key("str");
            std::string data("abcdefghij");
            do {
                tr->promote_to_write();
                for (int j = 0; j < 500; j++) {
                    int index = random.draw_int_mod(10000);
                    auto obj = t->get_object(index);
                    obj.set(c, BinaryData(data.data(), j % 10));
                }
                tr->commit_and_continue_as_read();
                db->get_stats(free_space, used_space);
            } while (free_space > used_space);
        };

        std::thread t1(worker);
        std::thread t2(worker);
        t1.join();
        t2.join();
        size_t free_space, used_space;
        db->get_stats(free_space, used_space);
        total = free_space + used_space;
    }
    if constexpr (type::value) {
        File f(path);
        // std::cout << "Size : " << f.get_size() << std::endl;
        {
            DB::create(make_in_realm_history(), path);
        }
        // std::cout << "Size : " << f.get_size() << std::endl;
        CHECK(f.get_size() == total);
    }
}

NONCONCURRENT_TEST(Compaction_Performance)
{
    auto old_disable_sync_to_disk = get_disable_sync_to_disk();
    disable_sync_to_disk(false);
    util::ScopeExit on_exit([old_disable_sync_to_disk]() noexcept {
        disable_sync_to_disk(old_disable_sync_to_disk);
    });
    SHARED_GROUP_TEST_PATH(path);
    DBRef db = DB::create(make_in_realm_history(), path);
    auto tr = db->start_write();
    auto table_foo = tr->add_table("foo");
    auto col_bin = table_foo->add_column(type_Binary, "bin", true);
    std::string big_string(0x10000, 'a');
    for (int i = 0; i < 1200; ++i) {
        table_foo->create_object().set(col_bin, BinaryData(big_string.data(), 0x10000));
    }
    tr->commit_and_continue_as_read();

    tr->promote_to_write();
    auto table_bar = tr->add_table("bar");
    auto col_str = table_bar->add_column(type_String, "str");
    std::string str_b(512, 'b');
    std::string str_c(512, 'c');
    std::string str_d(512, 'd');
    for (int i = 0; i < 10000; ++i) {
        table_bar->create_object().set(col_str, str_b);
    }
    tr->commit_and_continue_as_read();

    auto objp = table_bar->begin();

    auto run = [&](std::string& s) {
        auto t1 = steady_clock::now();

        // CALLGRIND_START_INSTRUMENTATION;

        for (size_t i = 0; i < 10; i++) {
            tr->promote_to_write();
            for (size_t t = 0; t < 100; t++) {
                objp->set(col_str, s);
                ++objp;
            }
            tr->commit_and_continue_as_read();
        }

        // CALLGRIND_STOP_INSTRUMENTATION;

        auto t2 = steady_clock::now();

        return duration_cast<microseconds>(t2 - t1).count();
    };

    auto t0 = run(str_c);
    tr->promote_to_write();
    table_foo->clear();
    tr->commit_and_continue_as_read();
    auto t1 = run(str_d);
    std::cout << "Normal: " << t0 << " us" << std::endl;
    std::cout << "Compacting: " << t1 << " us" << std::endl;
}

1	/*************************************************************************
2	*
3	* Copyright 2022 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#include <realm.hpp>
20	#include <realm/util/scope_exit.hpp>
21	#include "test.hpp"
22
23	#include <iostream>
24	#include <chrono>
25
26	// #include <valgrind/callgrind.h>
27
28	#ifndef CALLGRIND_START_INSTRUMENTATION
29	#define CALLGRIND_START_INSTRUMENTATION
30	#endif
31
32	#ifndef CALLGRIND_STOP_INSTRUMENTATION
33	#define CALLGRIND_STOP_INSTRUMENTATION
34	#endif
35
36	// valgrind --tool=callgrind --instr-atstart=no realm-tests
37
38	using namespace std::chrono;
39
40	using namespace realm;
41	using namespace realm::util;
42	using namespace realm::test_util;
43	using unit_test::TestContext;
44
45	TEST(Compaction_WhileGrowing)
46	{	2✔
47	Random random(random_int<unsigned long>());	2✔
48	SHARED_GROUP_TEST_PATH(path);	2✔
49	DBRef db = DB::create(make_in_realm_history(), path);	2✔
50	size_t free_space, used_space;	2✔
51
52	auto tr = db->start_write();	2✔
53	auto table1 = tr->add_table("Binaries");	2✔
54	auto col_bin1 = table1->add_column(type_Binary, "str", true);	2✔
55	auto table2 = tr->add_table("Integers");	2✔
56	auto col_bin2 = table2->add_column(type_Binary, "str", true);	2✔
57	tr->commit_and_continue_as_read();	2✔
58	char w[5000];	2✔
59	for (int i = 0; i < 5000; ++i) {	10,002✔
60	w[i] = '0' + (i % 64);	10,000✔
61	}	10,000✔
62	int num = (REALM_MAX_BPNODE_SIZE == 1000) ? 1400 : 1300;	2✔
63	tr->promote_to_write();	2✔
64	CHECK(db->get_evacuation_stage() == DB::EvacStage::idle);	2✔
65	for (int j = 0; j < num; ++j) {	2,802✔
66	table1->create_object().set(col_bin1, BinaryData(w, 450));	2,800✔
67	table2->create_object().set(col_bin2, BinaryData(w, 200));	2,800✔
68	if (j % 10 == 0) {	2,800✔
69	tr->commit_and_continue_as_read();	280✔
70	tr->promote_to_write();	280✔
71	}	280✔
72	}	2,800✔
73	tr->commit_and_continue_as_read();	2✔
74
75	tr->promote_to_write();	2✔
76	auto objp = table1->begin();	2✔
77	for (int j = 0; j < num - 30; ++j, ++objp) {	2,642✔
78	objp->set(col_bin1, BinaryData());	2,642✔
79	if (j % 10 == 0) {	2,642✔
80	tr->commit_and_continue_as_read();	266✔
81	tr->promote_to_write();	266✔
82	}	266✔
83	if (db->get_evacuation_stage() == DB::EvacStage::evacuating)	2,642✔
84	break;	2✔
85	}	2,642✔
86
87	CHECK(db->get_evacuation_stage() == DB::EvacStage::evacuating);	2✔
88	tr->commit_and_continue_as_read();	2✔
89	db->get_stats(free_space, used_space);	2✔
90	// The file is now subject to compaction
91	if (!CHECK(free_space > 2 * used_space)) {	2✔
UNCOV 92	std::cout << "Free space: " << free_space << std::endl;	×
93	std::cout << "Used space: " << used_space << std::endl;	×
94	}	×
95
96	// During the following, the space kept in "m_under_evacuation" will be used
97	// before all elements have been moved, which will terminate that session
98	tr->promote_to_write();	2✔
99	table1->create_object().set(col_bin1, BinaryData(w, 4500));	2✔
100	table1->create_object().set(col_bin1, BinaryData(w, 4500));	2✔
101	tr->commit_and_continue_as_read();	2✔
102
103	CHECK(db->get_evacuation_stage() == DB::EvacStage::blocked);	2✔
104
105	// db->get_stats(free_space, used_space);
106	// std::cout << "Total: " << free_space + used_space << ", "
107	// << "Free: " << free_space << ", "
108	// << "Used: " << used_space << std::endl;
109
110	tr->promote_to_write();	2✔
111	table1->clear();	2✔
112	table2->clear();	2✔
113	tr->commit_and_continue_as_read();	2✔
114	// Now there should be room for compaction
115
116	auto n = 20; // Ensure that test will end	2✔
117	do {	24✔
118	tr->promote_to_write();	24✔
119	tr->commit_and_continue_as_read();	24✔
120	db->get_stats(free_space, used_space);	24✔
121	} while (db->get_evacuation_stage() != DB::EvacStage::idle && --n > 0);	24✔
122	CHECK_LESS(free_space, 0x10000);	2✔
123	}	2✔
124
125	TEST_TYPES(Compaction_Large, std::true_type, std::false_type)
126	{	4✔
127	using type = typename TEST_TYPE::type;	4✔
128	SHARED_GROUP_TEST_PATH(path);	4✔
129	int64_t total;	4✔
130	{	4✔
131	DBRef db;	4✔
132	if constexpr (type::value) {	4✔
133	db = DB::create(make_in_realm_history(), path);	2✔
134	}	1✔
135	else {	2✔
136	db = DB::create(make_in_realm_history());	2✔
137	}	2✔
138	{	4✔
139	auto tr = db->start_write();	4✔
140	auto t = tr->add_table("the_table");	4✔
141	auto c = t->add_column(type_Binary, "str", true);	4✔
142	char w[1000];	4✔
143	for (int i = 0; i < 1000; ++i) {	4,004✔
144	w[i] = '0' + (i % 10);	4,000✔
145	}	4,000✔
146	size_t num = 100000;	4✔
147	for (size_t j = 0; j < num; ++j) {	400,004✔
148	BinaryData sd(w, 500 + (j % 500));	400,000✔
149	t->create_object().set(c, sd);	400,000✔
150	}	400,000✔
151	tr->commit_and_continue_as_read();	4✔
152
153	tr->promote_to_write();	4✔
154	int j = 0;	4✔
155	for (auto o : *t) {	400,000✔
156	BinaryData sd(w, j % 500);	400,000✔
157	o.set(c, sd);	400,000✔
158	++j;	400,000✔
159	}	400,000✔
160	tr->commit_and_continue_as_read();	4✔
161
162	tr->promote_to_write();	4✔
163	// This will likely make the table names reside in the upper end of the file
164	tr->add_table("another_table");	4✔
165	tr->commit_and_continue_as_read();	4✔
166	}	4✔
167
168	auto worker = [db] {	7✔
169	Random random(random_int<unsigned long>());	7✔
170	size_t free_space, used_space;	7✔
171	auto tr = db->start_read();	7✔
172	auto t = tr->get_table("the_table");	7✔
173	auto c = t->get_column_key("str");	7✔
174	std::string data("abcdefghij");	7✔
175	do {	1,731✔
176	tr->promote_to_write();	1,731✔
177	for (int j = 0; j < 500; j++) {	867,731✔
178	int index = random.draw_int_mod(10000);	866,000✔
179	auto obj = t->get_object(index);	866,000✔
180	obj.set(c, BinaryData(data.data(), j % 10));	866,000✔
181	}	866,000✔
182	tr->commit_and_continue_as_read();	1,731✔
183	db->get_stats(free_space, used_space);	1,731✔
184	} while (free_space > used_space);	1,731✔
185	};	7✔
186
187	std::thread t1(worker);	4✔
188	std::thread t2(worker);	4✔
189	t1.join();	4✔
190	t2.join();	4✔
191	size_t free_space, used_space;	4✔
192	db->get_stats(free_space, used_space);	4✔
193	total = free_space + used_space;	4✔
194	}	4✔
195	if constexpr (type::value) {	4✔
196	File f(path);	2✔
197	// std::cout << "Size : " << f.get_size() << std::endl;
198	{	2✔
199	DB::create(make_in_realm_history(), path);	2✔
200	}	2✔
201	// std::cout << "Size : " << f.get_size() << std::endl;
202	CHECK(f.get_size() == total);	2✔
203	}	2✔
204	}	4✔
205
206	NONCONCURRENT_TEST(Compaction_Performance)
207	{	2✔
208	auto old_disable_sync_to_disk = get_disable_sync_to_disk();	2✔
209	disable_sync_to_disk(false);	2✔
210	util::ScopeExit on_exit([old_disable_sync_to_disk]() noexcept {	2✔
211	disable_sync_to_disk(old_disable_sync_to_disk);	2✔
212	});	2✔
213	SHARED_GROUP_TEST_PATH(path);	2✔
214	DBRef db = DB::create(make_in_realm_history(), path);	2✔
215	auto tr = db->start_write();	2✔
216	auto table_foo = tr->add_table("foo");	2✔
217	auto col_bin = table_foo->add_column(type_Binary, "bin", true);	2✔
218	std::string big_string(0x10000, 'a');	2✔
219	for (int i = 0; i < 1200; ++i) {	2,402✔
220	table_foo->create_object().set(col_bin, BinaryData(big_string.data(), 0x10000));	2,400✔
221	}	2,400✔
222	tr->commit_and_continue_as_read();	2✔
223
224	tr->promote_to_write();	2✔
225	auto table_bar = tr->add_table("bar");	2✔
226	auto col_str = table_bar->add_column(type_String, "str");	2✔
227	std::string str_b(512, 'b');	2✔
228	std::string str_c(512, 'c');	2✔
229	std::string str_d(512, 'd');	2✔
230	for (int i = 0; i < 10000; ++i) {	20,002✔
231	table_bar->create_object().set(col_str, str_b);	20,000✔
232	}	20,000✔
233	tr->commit_and_continue_as_read();	2✔
234
235	auto objp = table_bar->begin();	2✔
236
237	auto run = [&](std::string& s) {	4✔
238	auto t1 = steady_clock::now();	4✔
239
240	// CALLGRIND_START_INSTRUMENTATION;
241
242	for (size_t i = 0; i < 10; i++) {	44✔
243	tr->promote_to_write();	40✔
244	for (size_t t = 0; t < 100; t++) {	4,040✔
245	objp->set(col_str, s);	4,000✔
246	++objp;	4,000✔
247	}	4,000✔
248	tr->commit_and_continue_as_read();	40✔
249	}	40✔
250
251	// CALLGRIND_STOP_INSTRUMENTATION;
252
253	auto t2 = steady_clock::now();	4✔
254
255	return duration_cast<microseconds>(t2 - t1).count();	4✔
256	};	4✔
257
258	auto t0 = run(str_c);	2✔
259	tr->promote_to_write();	2✔
260	table_foo->clear();	2✔
261	tr->commit_and_continue_as_read();	2✔
262	auto t1 = run(str_d);	2✔
263	std::cout << "Normal: " << t0 << " us" << std::endl;	2✔
264	std::cout << "Compacting: " << t1 << " us" << std::endl;	2✔
265	}	2✔

realm / realm-core / nikola.irinchev_478

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous