jorgen.edelbo_338

Committed 03 Jul 2024 03:00PM UTC coverage: 90.856% (-0.008%) from 90.864%

Build # jorgen.edelbo_338

Build Type

Pull #7803

Evergreen

Committed by

nicola-cab

Commit Message

Merge branch 'next-major' into feature/string-compression

Pull Request Pull Request #7803: Feature/string compression

Run Details

103028 of 180606 branches covered (57.05%)

1144 of 1267 new or added lines in 33 files covered. (90.29%)

155 existing lines in 24 files now uncovered.

218583 of 240583 relevant lines covered (90.86%)

7959624.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.27

/src/realm/array_blobs_small.hpp

/*************************************************************************
 *
 * Copyright 2016 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#ifndef REALM_ARRAY_BLOBS_SMALL_HPP
#define REALM_ARRAY_BLOBS_SMALL_HPP

#include <realm/array_blob.hpp>
#include <realm/exceptions.hpp>

namespace realm {

/*
STORAGE FORMAT
---------------------------------------------------------------------------------------
ArraySmallBlobs stores binary elements using two ArrayInteger and one ArrayBlob. The ArrayBlob can only store one
single concecutive array of bytes (contrary to its 'Array' name that misleadingly indicates it could store multiple
elements).

Assume we have the strings "a", "", "abc", null, "ab". Then the three arrays will contain:

ArrayInteger    m_offsets   1, 1, 5, 5, 6
ArrayBlob       m_blob      aabcab
ArrayInteger    m_nulls     0, 0, 0, 1, 0 // 1 indicates null, 0 indicates non-null

So for each element the ArrayInteger, the ArrayInteger points into the ArrayBlob at the position of the first
byte of the next element.

m_nulls is always present (except for old database files; see below), so any ArraySmallBlobs is always nullable!
The nullable property (such as throwing exception upon set(null) on non-nullable column, etc) is handled on
column level only.

DATABASE FILE VERSION CHANGES
---------------------------------------------------------------------------------------
Old database files do not have any m_nulls array. To be backwardscompatible, many methods will have tests like
`if(Array::size() == 3)` and have a backwards compatible code paths for these (e.g. avoid writing to m_nulls
in set(), etc). This way no file format upgrade is needed to support nulls for BinaryData.
*/

class ArraySmallBlobs : public Array {
public:
    explicit ArraySmallBlobs(Allocator&) noexcept;
    ~ArraySmallBlobs() noexcept override {}

    // Disable copying, this is not allowed.
    ArraySmallBlobs& operator=(const ArraySmallBlobs&) = delete;
    ArraySmallBlobs(const ArraySmallBlobs&) = delete;

    /// Create a new empty binary array and attach this accessor to
    /// it. This does not modify the parent reference information of
    /// this accessor.
    ///
    /// Note that the caller assumes ownership of the allocated
    /// underlying node. It is not owned by the accessor.
    void create();

    //@{
    /// Overriding functions of Array
    void init_from_ref(ref_type) noexcept;
    void init_from_mem(MemRef) noexcept;
    void init_from_parent() noexcept;
    //@}

    bool is_empty() const noexcept;
    size_t size() const noexcept;

    BinaryData get(size_t ndx) const noexcept;
    StringData get_string(size_t ndx) const;
    bool is_null(size_t ndx) const;

    void add(BinaryData value, bool add_zero_term = false);
    void set(size_t ndx, BinaryData value, bool add_zero_term = false);
    void insert(size_t ndx, BinaryData value, bool add_zero_term = false);
    void add_string(StringData value);
    void set_string(size_t ndx, StringData value);
    void insert_string(size_t ndx, StringData value);
    void erase(size_t ndx);
    void truncate(size_t new_size);
    void clear();
    void destroy();

    size_t find_first(BinaryData value, bool is_string, size_t begin, size_t end) const noexcept;

    /// Get the specified element without the cost of constructing an
    /// array instance. If an array instance is already available, or
    /// you need to get multiple values, then this method will be
    /// slower.
    static BinaryData get(const char* header, size_t ndx, Allocator&) noexcept;
    static StringData get_string(const char* header, size_t ndx, Allocator& alloc) noexcept;

    static size_t get_size_from_header(const char*, Allocator&) noexcept;

    /// Construct a binary array of the specified size and return just
    /// the reference to the underlying memory. All elements will be
    /// initialized to the binary value `defaults`, which can be either
    /// null or zero-length non-null (value with size > 0 is not allowed as
    /// initialization value).
    static MemRef create_array(size_t size, Allocator&, BinaryData defaults);

    void update_from_parent() noexcept;

private:
    friend class ArrayString;
    Array m_offsets;
    ArrayBlob m_blob;
    Array m_nulls;

    StringData get_string_legacy(size_t ndx) const;
};


// Implementation:

inline ArraySmallBlobs::ArraySmallBlobs(Allocator& allocator) noexcept
    : Array(allocator)
    , m_offsets(allocator)
    , m_blob(allocator)
    , m_nulls(allocator)
{
    m_offsets.set_parent(this, 0);
    m_blob.set_parent(this, 1);
    m_nulls.set_parent(this, 2);
}

inline void ArraySmallBlobs::create()
{
    size_t init_size = 0;
    BinaryData defaults = BinaryData{};                          // This init value is ignored because size = 0
    MemRef mem = create_array(init_size, get_alloc(), defaults); // Throws
    init_from_mem(mem);
}

inline void ArraySmallBlobs::init_from_ref(ref_type ref) noexcept
{
    REALM_ASSERT(ref);
    char* header = get_alloc().translate(ref);
    init_from_mem(MemRef(header, ref, m_alloc));
}

inline void ArraySmallBlobs::init_from_parent() noexcept
{
    ref_type ref = get_ref_from_parent();
    init_from_ref(ref);
}

inline bool ArraySmallBlobs::is_empty() const noexcept
{
    return m_offsets.is_empty();
}

inline size_t ArraySmallBlobs::size() const noexcept
{
    return m_offsets.size();
}

inline BinaryData ArraySmallBlobs::get(size_t ndx) const noexcept
{
    REALM_ASSERT_3(ndx, <, m_offsets.size());

    if (m_nulls.get(ndx)) {
        return BinaryData();
    }
    else {
        size_t begin = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0;
        auto offset = m_offsets.get(ndx);
        size_t end = to_size_t(offset);

        BinaryData bd = BinaryData(m_blob.get(begin), end - begin);
        // Old database file (non-nullable column should never return null)
        REALM_ASSERT(!bd.is_null());
        return bd;
    }
}

inline bool ArraySmallBlobs::is_null(size_t ndx) const
{
    REALM_ASSERT_3(ndx, <, m_nulls.size());

    return m_nulls.get(ndx) != 0;
}

inline StringData ArraySmallBlobs::get_string(size_t ndx) const
{
    BinaryData bin = get(ndx);
    if (bin.is_null())
        return realm::null();
    else
        return StringData(bin.data(), bin.size() - 1); // Do not include terminating zero
}

inline StringData ArraySmallBlobs::get_string(const char* header, size_t ndx, Allocator& alloc) noexcept
{
    BinaryData bin = get(header, ndx, alloc);
    if (bin.is_null())
        return realm::null();
    else
        return StringData(bin.data(), bin.size() - 1); // Do not include terminating zero
}

inline void ArraySmallBlobs::add_string(StringData value)
{
    add(BinaryData(value.data(), value.size()), true);
}

inline void ArraySmallBlobs::set_string(size_t ndx, StringData value)
{
    set(ndx, BinaryData(value.data(), value.size()), true);
}

inline void ArraySmallBlobs::insert_string(size_t ndx, StringData value)
{
    insert(ndx, BinaryData(value.data(), value.size()), true);
}

inline void ArraySmallBlobs::truncate(size_t new_size)
{
    REALM_ASSERT(new_size == 0 || new_size < m_offsets.size());

    size_t sz = new_size ? to_size_t(m_offsets.get(new_size - 1)) : 0;

    m_offsets.truncate(new_size);
    m_blob.truncate(sz);
    m_nulls.truncate(new_size);
}

inline void ArraySmallBlobs::clear()
{
    m_blob.clear();
    m_offsets.clear();
    m_nulls.clear();
}

inline void ArraySmallBlobs::destroy()
{
    m_blob.destroy();
    m_offsets.destroy();
    m_nulls.destroy();
    Array::destroy();
}

inline size_t ArraySmallBlobs::get_size_from_header(const char* header, Allocator& alloc) noexcept
{
    ref_type offsets_ref = to_ref(Array::get(header, 0));
    const char* offsets_header = alloc.translate(offsets_ref);
    return Array::get_size_from_header(offsets_header);
}

inline void ArraySmallBlobs::update_from_parent() noexcept
{
    Array::update_from_parent();
    m_blob.update_from_parent();
    m_offsets.update_from_parent();
    m_nulls.update_from_parent();
}

} // namespace realm

#endif // REALM_ARRAY_BINARY_HPP

1	/*************************************************************************
2	*
3	* Copyright 2016 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#ifndef REALM_ARRAY_BLOBS_SMALL_HPP
20	#define REALM_ARRAY_BLOBS_SMALL_HPP
21
22	#include <realm/array_blob.hpp>
23	#include <realm/exceptions.hpp>
24
25	namespace realm {
26
27	/*
28	STORAGE FORMAT
29	---------------------------------------------------------------------------------------
30	ArraySmallBlobs stores binary elements using two ArrayInteger and one ArrayBlob. The ArrayBlob can only store one
31	single concecutive array of bytes (contrary to its 'Array' name that misleadingly indicates it could store multiple
32	elements).
33
34	Assume we have the strings "a", "", "abc", null, "ab". Then the three arrays will contain:
35
36	ArrayInteger m_offsets 1, 1, 5, 5, 6
37	ArrayBlob m_blob aabcab
38	ArrayInteger m_nulls 0, 0, 0, 1, 0 // 1 indicates null, 0 indicates non-null
39
40	So for each element the ArrayInteger, the ArrayInteger points into the ArrayBlob at the position of the first
41	byte of the next element.
42
43	m_nulls is always present (except for old database files; see below), so any ArraySmallBlobs is always nullable!
44	The nullable property (such as throwing exception upon set(null) on non-nullable column, etc) is handled on
45	column level only.
46
47	DATABASE FILE VERSION CHANGES
48	---------------------------------------------------------------------------------------
49	Old database files do not have any m_nulls array. To be backwardscompatible, many methods will have tests like
50	`if(Array::size() == 3)` and have a backwards compatible code paths for these (e.g. avoid writing to m_nulls
51	in set(), etc). This way no file format upgrade is needed to support nulls for BinaryData.
52	*/
53
54	class ArraySmallBlobs : public Array {
55	public:
56	explicit ArraySmallBlobs(Allocator&) noexcept;
57	~ArraySmallBlobs() noexcept override {}	16,638✔
58
59	// Disable copying, this is not allowed.
60	ArraySmallBlobs& operator=(const ArraySmallBlobs&) = delete;
61	ArraySmallBlobs(const ArraySmallBlobs&) = delete;
62
63	/// Create a new empty binary array and attach this accessor to
64	/// it. This does not modify the parent reference information of
65	/// this accessor.
66	///
67	/// Note that the caller assumes ownership of the allocated
68	/// underlying node. It is not owned by the accessor.
69	void create();
70
71	//@{
72	/// Overriding functions of Array
73	void init_from_ref(ref_type) noexcept;
74	void init_from_mem(MemRef) noexcept;
75	void init_from_parent() noexcept;
76	//@}
77
78	bool is_empty() const noexcept;
79	size_t size() const noexcept;
80
81	BinaryData get(size_t ndx) const noexcept;
82	StringData get_string(size_t ndx) const;
83	bool is_null(size_t ndx) const;
84
85	void add(BinaryData value, bool add_zero_term = false);
86	void set(size_t ndx, BinaryData value, bool add_zero_term = false);
87	void insert(size_t ndx, BinaryData value, bool add_zero_term = false);
88	void add_string(StringData value);
89	void set_string(size_t ndx, StringData value);
90	void insert_string(size_t ndx, StringData value);
91	void erase(size_t ndx);
92	void truncate(size_t new_size);
93	void clear();
94	void destroy();
95
96	size_t find_first(BinaryData value, bool is_string, size_t begin, size_t end) const noexcept;
97
98	/// Get the specified element without the cost of constructing an
99	/// array instance. If an array instance is already available, or
100	/// you need to get multiple values, then this method will be
101	/// slower.
102	static BinaryData get(const char* header, size_t ndx, Allocator&) noexcept;
103	static StringData get_string(const char* header, size_t ndx, Allocator& alloc) noexcept;
104
105	static size_t get_size_from_header(const char*, Allocator&) noexcept;
106
107	/// Construct a binary array of the specified size and return just
108	/// the reference to the underlying memory. All elements will be
109	/// initialized to the binary value `defaults`, which can be either
110	/// null or zero-length non-null (value with size > 0 is not allowed as
111	/// initialization value).
112	static MemRef create_array(size_t size, Allocator&, BinaryData defaults);
113
114	void update_from_parent() noexcept;
115
116	private:
117	friend class ArrayString;
118	Array m_offsets;
119	ArrayBlob m_blob;
120	Array m_nulls;
121
122	StringData get_string_legacy(size_t ndx) const;
123	};
124
125
126	// Implementation:
127
128	inline ArraySmallBlobs::ArraySmallBlobs(Allocator& allocator) noexcept
129	: Array(allocator)	5,383,761✔
130	, m_offsets(allocator)	5,383,761✔
131	, m_blob(allocator)	5,383,761✔
132	, m_nulls(allocator)	5,383,761✔
133	{	10,825,056✔
134	m_offsets.set_parent(this, 0);	10,825,056✔
135	m_blob.set_parent(this, 1);	10,825,056✔
136	m_nulls.set_parent(this, 2);	10,825,056✔
137	}	10,825,056✔
138
139	inline void ArraySmallBlobs::create()
140	{	150,534✔
141	size_t init_size = 0;	150,534✔
142	BinaryData defaults = BinaryData{}; // This init value is ignored because size = 0	150,534✔
143	MemRef mem = create_array(init_size, get_alloc(), defaults); // Throws	150,534✔
144	init_from_mem(mem);	150,534✔
145	}	150,534✔
146
147	inline void ArraySmallBlobs::init_from_ref(ref_type ref) noexcept
148	{	×
149	REALM_ASSERT(ref);	×
150	char* header = get_alloc().translate(ref);	×
151	init_from_mem(MemRef(header, ref, m_alloc));	×
152	}	×
153
154	inline void ArraySmallBlobs::init_from_parent() noexcept
155	{	×
156	ref_type ref = get_ref_from_parent();	×
157	init_from_ref(ref);	×
158	}	×
159
160	inline bool ArraySmallBlobs::is_empty() const noexcept
161	{	2✔
162	return m_offsets.is_empty();	2✔
163	}	2✔
164
165	inline size_t ArraySmallBlobs::size() const noexcept
166	{	5,772,321✔
167	return m_offsets.size();	5,772,321✔
168	}	5,772,321✔
169
170	inline BinaryData ArraySmallBlobs::get(size_t ndx) const noexcept
171	{	4,381,857✔
172	REALM_ASSERT_3(ndx, <, m_offsets.size());	4,381,857✔
173
174	if (m_nulls.get(ndx)) {	4,381,857✔
175	return BinaryData();	332,019✔
176	}	332,019✔
177	else {	4,049,838✔
178	size_t begin = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0;	4,049,838✔
179	auto offset = m_offsets.get(ndx);	4,049,838✔
180	size_t end = to_size_t(offset);	4,049,838✔
181
182	BinaryData bd = BinaryData(m_blob.get(begin), end - begin);	4,049,838✔
183	// Old database file (non-nullable column should never return null)
184	REALM_ASSERT(!bd.is_null());	4,049,838✔
185	return bd;	4,049,838✔
186	}	4,049,838✔
187	}	4,381,857✔
188
189	inline bool ArraySmallBlobs::is_null(size_t ndx) const
190	{	45,153✔
191	REALM_ASSERT_3(ndx, <, m_nulls.size());	45,153✔
192
193	return m_nulls.get(ndx) != 0;	45,153✔
194	}	45,153✔
195
196	inline StringData ArraySmallBlobs::get_string(size_t ndx) const
197	{	1,345,098✔
198	BinaryData bin = get(ndx);	1,345,098✔
199	if (bin.is_null())	1,345,098✔
200	return realm::null();	47,007✔
201	else	1,298,091✔
202	return StringData(bin.data(), bin.size() - 1); // Do not include terminating zero	1,298,091✔
203	}	1,345,098✔
204
205	inline StringData ArraySmallBlobs::get_string(const char* header, size_t ndx, Allocator& alloc) noexcept
UNCOV 206	{	×
UNCOV 207	BinaryData bin = get(header, ndx, alloc);	×
UNCOV 208	if (bin.is_null())	×
UNCOV 209	return realm::null();	×
UNCOV 210	else	×
UNCOV 211	return StringData(bin.data(), bin.size() - 1); // Do not include terminating zero	×
UNCOV 212	}	×
213
214	inline void ArraySmallBlobs::add_string(StringData value)
215	{	49,914✔
216	add(BinaryData(value.data(), value.size()), true);	49,914✔
217	}	49,914✔
218
219	inline void ArraySmallBlobs::set_string(size_t ndx, StringData value)
220	{	220,281✔
221	set(ndx, BinaryData(value.data(), value.size()), true);	220,281✔
222	}	220,281✔
223
224	inline void ArraySmallBlobs::insert_string(size_t ndx, StringData value)
225	{	269,802✔
226	insert(ndx, BinaryData(value.data(), value.size()), true);	269,802✔
227	}	269,802✔
228
229	inline void ArraySmallBlobs::truncate(size_t new_size)
230	{	120✔
231	REALM_ASSERT(new_size == 0 \|\| new_size < m_offsets.size());	120✔
232
233	size_t sz = new_size ? to_size_t(m_offsets.get(new_size - 1)) : 0;	120✔
234
235	m_offsets.truncate(new_size);	120✔
236	m_blob.truncate(sz);	120✔
237	m_nulls.truncate(new_size);	120✔
238	}	120✔
239
240	inline void ArraySmallBlobs::clear()
241	{	92,622✔
242	m_blob.clear();	92,622✔
243	m_offsets.clear();	92,622✔
244	m_nulls.clear();	92,622✔
245	}	92,622✔
246
247	inline void ArraySmallBlobs::destroy()
248	{	44,067✔
249	m_blob.destroy();	44,067✔
250	m_offsets.destroy();	44,067✔
251	m_nulls.destroy();	44,067✔
252	Array::destroy();	44,067✔
253	}	44,067✔
254
255	inline size_t ArraySmallBlobs::get_size_from_header(const char* header, Allocator& alloc) noexcept
256	{	×
257	ref_type offsets_ref = to_ref(Array::get(header, 0));	×
258	const char* offsets_header = alloc.translate(offsets_ref);	×
259	return Array::get_size_from_header(offsets_header);	×
260	}	×
261
262	inline void ArraySmallBlobs::update_from_parent() noexcept
263	{	×
264	Array::update_from_parent();	×
265	m_blob.update_from_parent();	×
266	m_offsets.update_from_parent();	×
267	m_nulls.update_from_parent();	×
268	}	×
269
270	} // namespace realm
271
272	#endif // REALM_ARRAY_BINARY_HPP

realm / realm-core / jorgen.edelbo_338

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous