github_pull_request_281750

Committed 30 Oct 2023 03:37PM UTC coverage: 90.528% (-1.0%) from 91.571%

Build # github_pull_request_281750

Build Type

Pull #6073

Evergreen

Committed by

jedelbo

Commit Message

Log free space and history sizes when opening file

Pull Request Pull Request #6073: Merge next-major

Run Details

95488 of 175952 branches covered (0.0%)

8973 of 12277 new or added lines in 149 files covered. (73.09%)

622 existing lines in 51 files now uncovered.

233503 of 257934 relevant lines covered (90.53%)

6533720.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.47

/src/realm/backup_restore.cpp

/*************************************************************************
 *
 * Copyright 2016 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#include <realm/backup_restore.hpp>
#include <realm/util/file.hpp>
#include <realm/util/time.hpp>

#include <vector>
#include <chrono>

namespace realm {

/*
 * IMPORTANT: The following two arrays must be kept updated
 * as new versions are released or if rollback is ever done.
 */

using VersionList = BackupHandler::VersionList;
using VersionTimeList = BackupHandler::VersionTimeList;

// Note: accepted versions should have new versions added at front
const VersionList BackupHandler::accepted_versions_ = {24, 23, 22, 21, 20, 11, 10};

// the pair is <version, age-in-seconds>
// we keep backup files in 3 months.
static constexpr int three_months = 3 * 31 * 24 * 60 * 60;
const VersionTimeList BackupHandler::delete_versions_{{23, three_months}, {22, three_months}, {21, three_months},
                                                      {20, three_months}, {11, three_months}, {10, three_months}};


// helper functions
static std::string backup_name(const std::string& prefix, int version)
{
    return util::format("%1v%2.backup.realm", prefix, version);
}

static bool backup_exists(const std::string& prefix, int version)
{
    std::string fname = backup_name(prefix, version);
    return util::File::exists(fname);
}

std::string BackupHandler::get_prefix_from_path(const std::string& path)
{
    auto size = path.size();

    // remove a suffix ".realm" but add back the "."
    if (size > 6 && path.substr(size - 6, 6) == ".realm") {
        return path.substr(0, size - 5); // include '.'
    }

    // if no ".realm" suffix, at least ensure a terminating "."
    if (path[size - 1] == '.') {
        return path;
    }
    return path + ".";
}

BackupHandler::BackupHandler(const std::string& path, const VersionList& accepted,
                             const VersionTimeList& to_be_deleted)
{
    m_path = path;
    m_prefix = get_prefix_from_path(path);
    m_accepted_versions = accepted;
    m_delete_versions = to_be_deleted;
}

bool BackupHandler::must_restore_from_backup(int current_file_format_version) const
{
    if (current_file_format_version == 0)
        return false;
    if (is_accepted_file_format(current_file_format_version))
        return false;
    for (auto v : m_accepted_versions) {
        if (backup_exists(m_prefix, v))
            return true;
    }
    return false;
}

bool BackupHandler::is_accepted_file_format(int version) const noexcept
{
    for (auto v : m_accepted_versions) {
        if (v == version)
            return true;
    }
    return false;
}

void BackupHandler::restore_from_backup()
{
    for (auto v : m_accepted_versions) {
        if (backup_exists(m_prefix, v)) {
            prep_logging();
            auto backup_nm = backup_name(m_prefix, v);
            m_logger->info(util::LogCategory::storage, "%1 : Restoring from backup: %2", m_time_buf, backup_nm);
            util::File::move(backup_nm, m_path);
            return;
        }
    }
}

void BackupHandler::cleanup_backups()
{
    auto now = time(nullptr);
    for (auto v : m_delete_versions) {
        try {
            if (backup_exists(m_prefix, v.first)) {
                std::string fn = backup_name(m_prefix, v.first);
                // Assuming time_t is in seconds (should be on posix, but...)
                auto last_modified = util::File::last_write_time(fn);
                double diff = difftime(now, last_modified);
                if (diff > v.second) {
                    prep_logging();
                    m_logger->info(util::LogCategory::storage, "%1 : Removing old backup: %2   (age %3)", m_time_buf,
                                   fn, diff);
                    util::File::remove(fn);
                }
            }
        }
        catch (...) // ignore any problems, just leave the files
        {
        }
    }
}

void BackupHandler::ensure_logger()
{
    m_logger = std::make_unique<util::AppendToFileLogger>(m_path + ".backup-log");
}

void BackupHandler::prep_logging()
{
    ensure_logger();
    // preformat time string for later logging
    std::time_t t = std::time(nullptr);
    auto tm = util::gmtime(t);
    std::strftime(m_time_buf, sizeof(m_time_buf), "%c", &tm);
}


void BackupHandler::backup_realm_if_needed(int current_file_format_version, int target_file_format_version)
{
    if (current_file_format_version == 0)
        return;
    if (current_file_format_version >= target_file_format_version)
        return;
    std::string backup_nm = backup_name(m_prefix, current_file_format_version);
    if (util::File::exists(backup_nm)) {
        return;
    }
    try {
        // ignore it, if attempt to get free space fails for any reason
        if (util::File::get_free_space(m_path) < util::File::get_size_static(m_path) * 2) {
            prep_logging();
            m_logger->error(util::LogCategory::storage, "%1 : Insufficient free space for backup: %2", m_time_buf,
                            backup_nm);
            return;
        }
    }
    catch (...) {
        // ignore error
    }
    {
        prep_logging();
        m_logger->info(util::LogCategory::storage, "%1 : Creating backup: %2", m_time_buf, backup_nm);
    }
    std::string part_name = backup_nm + ".part";
    // The backup file should be a 1-1 copy, so that we can get the original
    // contents including unchanged layout of data, freelists, etc
    // In doing so we forego the option of compacting the backup.
    // Silence any errors during the backup process, but should one occur
    // remove any backup files, since they cannot be trusted.
    try {
        util::File::copy(m_path, part_name);
        util::File::move(part_name, backup_nm);
        prep_logging();
        m_logger->info(util::LogCategory::storage, "%1 : Completed backup: %2", m_time_buf, backup_nm);
    }
    catch (...) {
        try {
            util::File::try_remove(part_name);
            util::File::try_remove(backup_nm);
        }
        catch (...) {
        }
    }
}

} // namespace realm

1	/*************************************************************************
2	*
3	* Copyright 2016 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#include <realm/backup_restore.hpp>
20	#include <realm/util/file.hpp>
21	#include <realm/util/time.hpp>
22
23	#include <vector>
24	#include <chrono>
25
26	namespace realm {
27
28	/*
29	* IMPORTANT: The following two arrays must be kept updated
30	* as new versions are released or if rollback is ever done.
31	*/
32
33	using VersionList = BackupHandler::VersionList;
34	using VersionTimeList = BackupHandler::VersionTimeList;
35
36	// Note: accepted versions should have new versions added at front
37	const VersionList BackupHandler::accepted_versions_ = {24, 23, 22, 21, 20, 11, 10};
38
39	// the pair is <version, age-in-seconds>
40	// we keep backup files in 3 months.
41	static constexpr int three_months = 3 * 31 * 24 * 60 * 60;
42	const VersionTimeList BackupHandler::delete_versions_{{23, three_months}, {22, three_months}, {21, three_months},
43	{20, three_months}, {11, three_months}, {10, three_months}};
44
45
46	// helper functions
47	static std::string backup_name(const std::string& prefix, int version)
48	{	829,737✔
49	return util::format("%1v%2.backup.realm", prefix, version);	829,737✔
50	}	829,737✔
51
52	static bool backup_exists(const std::string& prefix, int version)
53	{	828,591✔
54	std::string fname = backup_name(prefix, version);	828,591✔
55	return util::File::exists(fname);	828,591✔
56	}	828,591✔
57
58	std::string BackupHandler::get_prefix_from_path(const std::string& path)
59	{	138,156✔
60	auto size = path.size();	138,156✔
61		67,908✔
62	// remove a suffix ".realm" but add back the "."	67,908✔
63	if (size > 6 && path.substr(size - 6, 6) == ".realm") {	138,156✔
64	return path.substr(0, size - 5); // include '.'	110,034✔
65	}	110,034✔
66		13,932✔
67	// if no ".realm" suffix, at least ensure a terminating "."	13,932✔
68	if (path[size - 1] == '.') {	28,122✔
69	return path;	×
70	}	×
71	return path + ".";	28,122✔
72	}	28,122✔
73
74	BackupHandler::BackupHandler(const std::string& path, const VersionList& accepted,
75	const VersionTimeList& to_be_deleted)
76	{	138,144✔
77	m_path = path;	138,144✔
78	m_prefix = get_prefix_from_path(path);	138,144✔
79	m_accepted_versions = accepted;	138,144✔
80	m_delete_versions = to_be_deleted;	138,144✔
81	}	138,144✔
82
83	bool BackupHandler::must_restore_from_backup(int current_file_format_version) const
84	{	138,144✔
85	if (current_file_format_version == 0)	138,144✔
86	return false;	50,526✔
87	if (is_accepted_file_format(current_file_format_version))	87,618✔
88	return false;	87,594✔
89	for (auto v : m_accepted_versions) {	102✔
90	if (backup_exists(m_prefix, v))	102✔
91	return true;	12✔
92	}	102✔
93	return false;	18✔
94	}	24✔
95
96	bool BackupHandler::is_accepted_file_format(int version) const noexcept
97	{	175,224✔
98	for (auto v : m_accepted_versions) {	176,718✔
99	if (v == version)	176,718✔
100	return true;	175,188✔
101	}	176,718✔
102	return false;	86,610✔
103	}	175,224✔
104
105	void BackupHandler::restore_from_backup()
106	{	12✔
107	for (auto v : m_accepted_versions) {	18✔
108	if (backup_exists(m_prefix, v)) {	18✔
109	prep_logging();	12✔
110	auto backup_nm = backup_name(m_prefix, v);	12✔
111	m_logger->info(util::LogCategory::storage, "%1 : Restoring from backup: %2", m_time_buf, backup_nm);	12✔
112	util::File::move(backup_nm, m_path);	12✔
113	return;	12✔
114	}	12✔
115	}	18✔
116	}	12✔
117
118	void BackupHandler::cleanup_backups()
119	{	138,132✔
120	auto now = time(nullptr);	138,132✔
121	for (auto v : m_delete_versions) {	828,474✔
122	try {	828,474✔
123	if (backup_exists(m_prefix, v.first)) {	828,474✔
124	std::string fn = backup_name(m_prefix, v.first);	885✔
125	// Assuming time_t is in seconds (should be on posix, but...)	420✔
126	auto last_modified = util::File::last_write_time(fn);	885✔
127	double diff = difftime(now, last_modified);	885✔
128	if (diff > v.second) {	885✔
129	prep_logging();	6✔
130	m_logger->info(util::LogCategory::storage, "%1 : Removing old backup: %2 (age %3)", m_time_buf,	6✔
131	fn, diff);	6✔
132	util::File::remove(fn);	6✔
133	}	6✔
134	}	885✔
135	}	828,474✔
136	catch (...) // ignore any problems, just leave the files	828,474✔
137	{	407,211✔
138	}	×
139	}	828,474✔
140	}	138,132✔
141
142	void BackupHandler::ensure_logger()
143	{	306✔
144	m_logger = std::make_unique<util::AppendToFileLogger>(m_path + ".backup-log");	306✔
145	}	306✔
146
147	void BackupHandler::prep_logging()
148	{	306✔
149	ensure_logger();	306✔
150	// preformat time string for later logging	153✔
151	std::time_t t = std::time(nullptr);	306✔
152	auto tm = util::gmtime(t);	306✔
153	std::strftime(m_time_buf, sizeof(m_time_buf), "%c", &tm);	306✔
154	}	306✔
155
156
157	void BackupHandler::backup_realm_if_needed(int current_file_format_version, int target_file_format_version)
158	{	138,123✔
159	if (current_file_format_version == 0)	138,123✔
160	return;	50,526✔
161	if (current_file_format_version >= target_file_format_version)	87,597✔
162	return;	87,345✔
163	std::string backup_nm = backup_name(m_prefix, current_file_format_version);	252✔
164	if (util::File::exists(backup_nm)) {	252✔
165	return;	108✔
166	}	108✔
167	try {	144✔
168	// ignore it, if attempt to get free space fails for any reason	72✔
169	if (util::File::get_free_space(m_path) < util::File::get_size_static(m_path) * 2) {	144✔
170	prep_logging();	×
NEW 171	m_logger->error(util::LogCategory::storage, "%1 : Insufficient free space for backup: %2", m_time_buf,	×
NEW 172	backup_nm);	×
173	return;	×
174	}	×
175	}	×
176	catch (...) {	×
177	// ignore error
178	}	×
179	{	144✔
180	prep_logging();	144✔
181	m_logger->info(util::LogCategory::storage, "%1 : Creating backup: %2", m_time_buf, backup_nm);	144✔
182	}	144✔
183	std::string part_name = backup_nm + ".part";	144✔
184	// The backup file should be a 1-1 copy, so that we can get the original	72✔
185	// contents including unchanged layout of data, freelists, etc	72✔
186	// In doing so we forego the option of compacting the backup.	72✔
187	// Silence any errors during the backup process, but should one occur	72✔
188	// remove any backup files, since they cannot be trusted.	72✔
189	try {	144✔
190	util::File::copy(m_path, part_name);	144✔
191	util::File::move(part_name, backup_nm);	144✔
192	prep_logging();	144✔
193	m_logger->info(util::LogCategory::storage, "%1 : Completed backup: %2", m_time_buf, backup_nm);	144✔
194	}	144✔
195	catch (...) {	72✔
196	try {	×
197	util::File::try_remove(part_name);	×
198	util::File::try_remove(backup_nm);	×
199	}	×
200	catch (...) {	×
201	}	×
202	}	×
203	}	144✔
204
205	} // namespace realm

realm / realm-core / github_pull_request_281750

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous