1880

Committed 01 Dec 2023 01:56AM UTC coverage: 91.679% (+0.02%) from 91.661%

Build # 1880

Build Type

push

Evergreen

Committed by

web-flow

Commit Message

Js/device testing (#6964)

* add a combined test executable

* disable a recently added test on mobile

* simplify link dependencies

* add missing namespace

* fix merge

* fix object-store benchmarks on android

Run Details

92350 of 169306 branches covered (0.0%)

433 of 500 new or added lines in 5 files covered. (86.6%)

35 existing lines in 13 files now uncovered.

231795 of 252834 relevant lines covered (91.68%)

6397738.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.13

/test/object-store/test_runner.cpp

////////////////////////////////////////////////////////////////////////////
//
// Copyright 2023 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#include <util/crypt_key.hpp>
#include <util/test_path.hpp>

#include <realm/util/features.h>
#include <realm/util/to_string.hpp>

#if TEST_SCHEDULER_UV
#include <realm/object-store/util/uv/scheduler.hpp>
#endif

#include <catch2/catch_all.hpp>
#include <catch2/reporters/catch_reporter_cumulative_base.hpp>
#include <catch2/catch_test_case_info.hpp>
#include <catch2/reporters/catch_reporter_registrars.hpp>
#include <external/json/json.hpp>

#include <chrono>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <limits.h>


int run_object_store_tests(int argc, const char** argv);

int run_object_store_tests(int argc, const char** argv)
{
    auto t1 = std::chrono::steady_clock::now();

    realm::test_util::initialize_test_path(1, argv);

    Catch::ConfigData config;

    if (const char* str = getenv("UNITTEST_EVERGREEN_TEST_RESULTS"); str && strlen(str) != 0) {
        std::cout << "Configuring evergreen reporter to store test results in " << str << std::endl;
        // If the output file already exists, make a copy so these results can be appended to it
        std::map<std::string, std::string> custom_options;
        if (std::filesystem::exists(str)) {
            std::string results_copy = realm::util::format("%1.bak", str);
            std::filesystem::copy(str, results_copy, std::filesystem::copy_options::overwrite_existing);
            custom_options["json_file"] = results_copy;
            std::cout << "Existing results file copied to " << results_copy << std::endl;
        }
        config.showDurations = Catch::ShowDurations::Always; // this is to help debug hangs in Evergreen
        config.reporterSpecifications.push_back(Catch::ReporterSpec{"console", {}, {}, {}});
        config.reporterSpecifications.push_back(
            Catch::ReporterSpec{"evergreen", {str}, {}, std::move(custom_options)});
    }
    else if (const char* str = getenv("UNITTEST_XML"); str && strlen(str) != 0) {
        std::cout << "Configuring jUnit reporter to store test results in " << str << std::endl;
        config.showDurations = Catch::ShowDurations::Always; // this is to help debug hangs in Jenkins
        config.reporterSpecifications.push_back(Catch::ReporterSpec{"console", {}, {}, {}});
        config.reporterSpecifications.push_back(Catch::ReporterSpec{"junit", {str}, {}, {}});
    }

    if (const char* env = getenv("UNITTEST_ENCRYPT_ALL")) {
        std::string str(env);
        for (auto& c : str) {
            c = tolower(c);
        }
        if (str == "1" || str == "on" || str == "yes") {
            realm::test_util::enable_always_encrypt();
        }
    }

#ifdef TEST_TIMEOUT_EXTRA
    std::cout << "Test wait timeouts extended by " << TEST_TIMEOUT_EXTRA << " seconds" << std::endl;
#endif

#if TEST_SCHEDULER_UV
    realm::util::Scheduler::set_default_factory([]() {
        return std::make_shared<realm::util::UvMainLoopScheduler>();
    });
#endif

    Catch::Session session;
    session.useConfigData(config);
    int result = session.run(argc, argv);

    auto t2 = std::chrono::steady_clock::now();
    auto ms_int = std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1);
    std::cout << "Test time: " << (ms_int.count() / 1000.0) << "s" << std::endl << std::endl;
    return result < 0xff ? result : 0xff;
}

namespace Catch {
class EvergreenReporter : public CumulativeReporterBase {
public:
    struct TestResult {
        TestResult()
            : start_time{std::chrono::system_clock::now()}
            , end_time{}
            , status{"unknown"}
        {
        }
        std::chrono::system_clock::time_point start_time;
        std::chrono::system_clock::time_point end_time;
        std::string status;
    };

    using Base = CumulativeReporterBase;
    using CumulativeReporterBase::CumulativeReporterBase;
    static std::string getDescription()
    {
        return "Reports test results in a format consumable by Evergreen.";
    }

    void assertionEnded(AssertionStats const& assertionStats) override
    {
        if (!assertionStats.assertionResult.isOk()) {
            std::cerr << "Assertion failure: " << assertionStats.assertionResult.getSourceInfo() << std::endl;
            std::cerr << "\t from expresion: '" << assertionStats.assertionResult.getExpression() << "'" << std::endl;
            std::cerr << "\t with expansion: '" << assertionStats.assertionResult.getExpandedExpression() << "'"
                      << std::endl;
            for (auto& message : assertionStats.infoMessages) {
                std::cerr << "\t message: " << message.message << std::endl;
            }
            std::cerr << std::endl;
        }
    }
    void testCaseStarting(TestCaseInfo const& testCaseInfo) override
    {
        m_results.emplace(std::make_pair(testCaseInfo.name, TestResult{}));
        Base::testCaseStarting(testCaseInfo);
    }
    void testCaseEnded(TestCaseStats const& testCaseStats) override
    {
        auto it = m_results.find(testCaseStats.testInfo->name);
        if (it == m_results.end()) {
            throw std::runtime_error("logic error in Evergreen section reporter, could not end test case '" +
                                     testCaseStats.testInfo->name + "' which was never tracked as started.");
        }
        if (testCaseStats.totals.assertions.allPassed()) {
            it->second.status = "pass";
        }
        else {
            it->second.status = "fail";
        }
        it->second.end_time = std::chrono::system_clock::now();
        Base::testCaseEnded(testCaseStats);
    }
    void sectionStarting(SectionInfo const& sectionInfo) override
    {
        if (m_pending_name.empty()) {
            m_pending_name = sectionInfo.name;
        }
        else {
            m_pending_name += "::" + sectionInfo.name;
        }
        m_pending_test = {};
        Base::sectionStarting(sectionInfo);
    }
    void sectionEnded(SectionStats const& sectionStats) override
    {
        if (!m_pending_name.empty()) {
            if (sectionStats.assertions.allPassed()) {
                m_pending_test.status = "pass";
            }
            else {
                m_pending_test.status = "fail";
            }
            m_pending_test.end_time = std::chrono::system_clock::now();
            m_results.emplace(std::make_pair(m_pending_name, m_pending_test));
            m_pending_name = "";
        }
        Base::sectionEnded(sectionStats);
    }
    void testRunEndedCumulative() override
    {
        auto& options = m_customOptions;
        auto& json_file = options["json_file"];
        nlohmann::json results_arr = nlohmann::json::array();
        // If the results file already exists, include the results from that file
        try {
            if (!json_file.empty() && std::filesystem::exists(json_file)) {
                std::ifstream f(json_file);
                // Make sure the file was successfully opened and is not empty
                if (f.is_open() && !f.eof()) {
                    nlohmann::json existing_data = nlohmann::json::parse(f);
                    auto results = existing_data.find("results");
                    if (results != existing_data.end() && results->is_array()) {
                        std::cout << "Appending tests from previous results" << std::endl;
                        results_arr = *results;
                    }
                }
            }
        }
        catch (nlohmann::json::exception&) {
            // json parse error, ignore the entries
        }
        catch (std::exception&) {
            // unable to open/read file
        }
        for (const auto& [test_name, cur_result] : m_results) {
            auto to_millis = [](const auto& tp) -> double {
                return static_cast<double>(
                    std::chrono::duration_cast<std::chrono::milliseconds>(tp.time_since_epoch()).count());
            };
            double start_secs = to_millis(cur_result.start_time) / 1000;
            double end_secs = to_millis(cur_result.end_time) / 1000;
            int exit_code = 0;
            if (cur_result.status != "pass") {
                exit_code = 1;
            }

            nlohmann::json cur_result_obj = {{"test_file", test_name}, {"status", cur_result.status},
                                             {"exit_code", exit_code}, {"start", start_secs},
                                             {"end", end_secs},        {"elapsed", end_secs - start_secs}};
            results_arr.push_back(std::move(cur_result_obj));
        }
        auto result_file_obj = nlohmann::json{{"results", std::move(results_arr)}};
        m_stream << result_file_obj << std::endl;
        if (!json_file.empty() && std::filesystem::exists(json_file)) {
            // Delete the old results file
            std::filesystem::remove(json_file);
        }
    }

    TestResult m_pending_test;
    std::string m_pending_name;
    std::vector<std::pair<std::string, TestResult>> m_current_stack;
    std::map<std::string, TestResult> m_results;
};

CATCH_REGISTER_REPORTER("evergreen", EvergreenReporter)

} // end namespace Catch

1	////////////////////////////////////////////////////////////////////////////
2	//
3	// Copyright 2023 Realm Inc.
4	//
5	// Licensed under the Apache License, Version 2.0 (the "License");
6	// you may not use this file except in compliance with the License.
7	// You may obtain a copy of the License at
8	//
9	// http://www.apache.org/licenses/LICENSE-2.0
10	//
11	// Unless required by applicable law or agreed to in writing, software
12	// distributed under the License is distributed on an "AS IS" BASIS,
13	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	// See the License for the specific language governing permissions and
15	// limitations under the License.
16	//
17	////////////////////////////////////////////////////////////////////////////
18
19	#include <util/crypt_key.hpp>
20	#include <util/test_path.hpp>
21
22	#include <realm/util/features.h>
23	#include <realm/util/to_string.hpp>
24
25	#if TEST_SCHEDULER_UV
26	#include <realm/object-store/util/uv/scheduler.hpp>
27	#endif
28
29	#include <catch2/catch_all.hpp>
30	#include <catch2/reporters/catch_reporter_cumulative_base.hpp>
31	#include <catch2/catch_test_case_info.hpp>
32	#include <catch2/reporters/catch_reporter_registrars.hpp>
33	#include <external/json/json.hpp>
34
35	#include <chrono>
36	#include <filesystem>
37	#include <fstream>
38	#include <iostream>
39	#include <limits.h>
40
41
42	int run_object_store_tests(int argc, const char** argv);
43
44	int run_object_store_tests(int argc, const char** argv)
45	{	4✔
46	auto t1 = std::chrono::steady_clock::now();	4✔
47		2✔
48	realm::test_util::initialize_test_path(1, argv);	4✔
49		2✔
50	Catch::ConfigData config;	4✔
51		2✔
52	if (const char* str = getenv("UNITTEST_EVERGREEN_TEST_RESULTS"); str && strlen(str) != 0) {	4✔
53	std::cout << "Configuring evergreen reporter to store test results in " << str << std::endl;	4✔
54	// If the output file already exists, make a copy so these results can be appended to it	2✔
55	std::map<std::string, std::string> custom_options;	4✔
56	if (std::filesystem::exists(str)) {	4✔
NEW 57	std::string results_copy = realm::util::format("%1.bak", str);	×
NEW 58	std::filesystem::copy(str, results_copy, std::filesystem::copy_options::overwrite_existing);	×
NEW 59	custom_options["json_file"] = results_copy;	×
NEW 60	std::cout << "Existing results file copied to " << results_copy << std::endl;	×
NEW 61	}	×
62	config.showDurations = Catch::ShowDurations::Always; // this is to help debug hangs in Evergreen	4✔
63	config.reporterSpecifications.push_back(Catch::ReporterSpec{"console", {}, {}, {}});	4✔
64	config.reporterSpecifications.push_back(	4✔
65	Catch::ReporterSpec{"evergreen", {str}, {}, std::move(custom_options)});	4✔
66	}	4✔
NEW 67	else if (const char* str = getenv("UNITTEST_XML"); str && strlen(str) != 0) {	×
NEW 68	std::cout << "Configuring jUnit reporter to store test results in " << str << std::endl;	×
NEW 69	config.showDurations = Catch::ShowDurations::Always; // this is to help debug hangs in Jenkins	×
NEW 70	config.reporterSpecifications.push_back(Catch::ReporterSpec{"console", {}, {}, {}});	×
NEW 71	config.reporterSpecifications.push_back(Catch::ReporterSpec{"junit", {str}, {}, {}});	×
NEW 72	}	×
73		2✔
74	if (const char* env = getenv("UNITTEST_ENCRYPT_ALL")) {	4✔
NEW 75	std::string str(env);	×
NEW 76	for (auto& c : str) {	×
NEW 77	c = tolower(c);	×
NEW 78	}	×
NEW 79	if (str == "1" \|\| str == "on" \|\| str == "yes") {	×
NEW 80	realm::test_util::enable_always_encrypt();	×
NEW 81	}	×
NEW 82	}	×
83		2✔
84	#ifdef TEST_TIMEOUT_EXTRA
85	std::cout << "Test wait timeouts extended by " << TEST_TIMEOUT_EXTRA << " seconds" << std::endl;
86	#endif
87		2✔
88	#if TEST_SCHEDULER_UV	2✔
89	realm::util::Scheduler::set_default_factory([]() {	11,977✔
90	return std::make_shared<realm::util::UvMainLoopScheduler>();	11,977✔
91	});	11,977✔
92	#endif	2✔
93		2✔
94	Catch::Session session;	4✔
95	session.useConfigData(config);	4✔
96	int result = session.run(argc, argv);	4✔
97		2✔
98	auto t2 = std::chrono::steady_clock::now();	4✔
99	auto ms_int = std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1);	4✔
100	std::cout << "Test time: " << (ms_int.count() / 1000.0) << "s" << std::endl << std::endl;	4✔
101	return result < 0xff ? result : 0xff;	4✔
102	}	4✔
103
104	namespace Catch {
105	class EvergreenReporter : public CumulativeReporterBase {
106	public:
107	struct TestResult {
108	TestResult()
109	: start_time{std::chrono::system_clock::now()}
110	, end_time{}
111	, status{"unknown"}
112	{	41,562✔
113	}	41,562✔
114	std::chrono::system_clock::time_point start_time;
115	std::chrono::system_clock::time_point end_time;
116	std::string status;
117	};
118
119	using Base = CumulativeReporterBase;
120	using CumulativeReporterBase::CumulativeReporterBase;
121	static std::string getDescription()
NEW 122	{	×
NEW 123	return "Reports test results in a format consumable by Evergreen.";	×
NEW 124	}	×
125
126	void assertionEnded(AssertionStats const& assertionStats) override
NEW 127	{	×
NEW 128	if (!assertionStats.assertionResult.isOk()) {	×
NEW 129	std::cerr << "Assertion failure: " << assertionStats.assertionResult.getSourceInfo() << std::endl;	×
NEW 130	std::cerr << "\t from expresion: '" << assertionStats.assertionResult.getExpression() << "'" << std::endl;	×
NEW 131	std::cerr << "\t with expansion: '" << assertionStats.assertionResult.getExpandedExpression() << "'"	×
NEW 132	<< std::endl;	×
NEW 133	for (auto& message : assertionStats.infoMessages) {	×
NEW 134	std::cerr << "\t message: " << message.message << std::endl;	×
NEW 135	}	×
NEW 136	std::cerr << std::endl;	×
NEW 137	}	×
NEW 138	}	×
139	void testCaseStarting(TestCaseInfo const& testCaseInfo) override
140	{	1,003✔
141	m_results.emplace(std::make_pair(testCaseInfo.name, TestResult{}));	1,003✔
142	Base::testCaseStarting(testCaseInfo);	1,003✔
143	}	1,003✔
144	void testCaseEnded(TestCaseStats const& testCaseStats) override
145	{	1,003✔
146	auto it = m_results.find(testCaseStats.testInfo->name);	1,003✔
147	if (it == m_results.end()) {	1,003✔
NEW 148	throw std::runtime_error("logic error in Evergreen section reporter, could not end test case '" +	×
NEW 149	testCaseStats.testInfo->name + "' which was never tracked as started.");	×
NEW 150	}	×
151	if (testCaseStats.totals.assertions.allPassed()) {	1,003✔
152	it->second.status = "pass";	1,003✔
153	}	1,003✔
NEW 154	else {	×
NEW 155	it->second.status = "fail";	×
NEW 156	}	×
157	it->second.end_time = std::chrono::system_clock::now();	1,003✔
158	Base::testCaseEnded(testCaseStats);	1,003✔
159	}	1,003✔
160	void sectionStarting(SectionInfo const& sectionInfo) override
161	{	40,555✔
162	if (m_pending_name.empty()) {	40,555✔
163	m_pending_name = sectionInfo.name;	15,071✔
164	}	15,071✔
165	else {	25,484✔
166	m_pending_name += "::" + sectionInfo.name;	25,484✔
167	}	25,484✔
168	m_pending_test = {};	40,555✔
169	Base::sectionStarting(sectionInfo);	40,555✔
170	}	40,555✔
171	void sectionEnded(SectionStats const& sectionStats) override
172	{	40,555✔
173	if (!m_pending_name.empty()) {	40,555✔
174	if (sectionStats.assertions.allPassed()) {	15,071✔
175	m_pending_test.status = "pass";	15,071✔
176	}	15,071✔
NEW 177	else {	×
NEW 178	m_pending_test.status = "fail";	×
NEW 179	}	×
180	m_pending_test.end_time = std::chrono::system_clock::now();	15,071✔
181	m_results.emplace(std::make_pair(m_pending_name, m_pending_test));	15,071✔
182	m_pending_name = "";	15,071✔
183	}	15,071✔
184	Base::sectionEnded(sectionStats);	40,555✔
185	}	40,555✔
186	void testRunEndedCumulative() override
187	{	4✔
188	auto& options = m_customOptions;	4✔
189	auto& json_file = options["json_file"];	4✔
190	nlohmann::json results_arr = nlohmann::json::array();	4✔
191	// If the results file already exists, include the results from that file	2✔
192	try {	4✔
193	if (!json_file.empty() && std::filesystem::exists(json_file)) {	4!
NEW 194	std::ifstream f(json_file);	×
195	// Make sure the file was successfully opened and is not empty
NEW 196	if (f.is_open() && !f.eof()) {	×
NEW 197	nlohmann::json existing_data = nlohmann::json::parse(f);	×
NEW 198	auto results = existing_data.find("results");	×
NEW 199	if (results != existing_data.end() && results->is_array()) {	×
NEW 200	std::cout << "Appending tests from previous results" << std::endl;	×
NEW 201	results_arr = *results;	×
NEW 202	}	×
NEW 203	}	×
NEW 204	}	×
205	}	4✔
206	catch (nlohmann::json::exception&) {	2✔
207	// json parse error, ignore the entries
NEW 208	}	×
209	catch (std::exception&) {	2✔
210	// unable to open/read file
NEW 211	}	×
212	for (const auto& [test_name, cur_result] : m_results) {	14,144✔
213	auto to_millis = [](const auto& tp) -> double {	28,288✔
214	return static_cast<double>(	28,288✔
215	std::chrono::duration_cast<std::chrono::milliseconds>(tp.time_since_epoch()).count());	28,288✔
216	};	28,288✔
217	double start_secs = to_millis(cur_result.start_time) / 1000;	14,144✔
218	double end_secs = to_millis(cur_result.end_time) / 1000;	14,144✔
219	int exit_code = 0;	14,144✔
220	if (cur_result.status != "pass") {	14,144✔
NEW 221	exit_code = 1;	×
NEW 222	}	×
223		7,034✔
224	nlohmann::json cur_result_obj = {{"test_file", test_name}, {"status", cur_result.status},	14,144✔
225	{"exit_code", exit_code}, {"start", start_secs},	14,144✔
226	{"end", end_secs}, {"elapsed", end_secs - start_secs}};	14,144✔
227	results_arr.push_back(std::move(cur_result_obj));	14,144✔
228	}	14,144✔
229	auto result_file_obj = nlohmann::json{{"results", std::move(results_arr)}};	4✔
230	m_stream << result_file_obj << std::endl;	4✔
231	if (!json_file.empty() && std::filesystem::exists(json_file)) {	4!
232	// Delete the old results file
NEW 233	std::filesystem::remove(json_file);	×
NEW 234	}	×
235	}	4✔
236
237	TestResult m_pending_test;
238	std::string m_pending_name;
239	std::vector<std::pair<std::string, TestResult>> m_current_stack;
240	std::map<std::string, TestResult> m_results;
241	};
242
243	CATCH_REGISTER_REPORTER("evergreen", EvergreenReporter)
244
245	} // end namespace Catch

realm / realm-core / 1880

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous