nicola.cabiddu_1772

Committed 31 May 2024 12:58PM UTC coverage: 90.685%. First build

Build # nicola.cabiddu_1772

Build Type

Pull #7668

Evergreen

Committed by

jedelbo

Commit Message

Update tests

Pull Request Pull Request #7668: RCORE-2094 Compressing Integer Arrays

Run Details

102170 of 180138 branches covered (56.72%)

1749 of 1914 new or added lines in 36 files covered. (91.38%)

217024 of 239317 relevant lines covered (90.68%)

6624441.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.24

/src/realm/integer_compressor.hpp

/*************************************************************************
 *
 * Copyright 2023 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#ifndef REALM_INTEGER_COMPRESSOR_HPP
#define REALM_INTEGER_COMPRESSOR_HPP

#include <cstdint>
#include <cstddef>
#include <vector>
#include <realm/query_conditions.hpp>
#include <realm/array_direct.hpp>
#include <realm/node.hpp>

namespace realm {

class Array;
class QueryStateBase;
class IntegerCompressor {
public:
    // commit => encode, COW/insert => decode
    bool compress(const Array&, Array&) const;
    bool decompress(Array&) const;

    bool init(const char*);
    void set_vtable(Array&);

    // init from mem B
    inline uint64_t* data() const;
    inline size_t size() const;
    inline NodeHeader::Encoding get_encoding() const;
    inline uint8_t v_width() const;
    inline uint8_t ndx_width() const;
    inline size_t v_size() const;
    inline size_t ndx_size() const;

    inline uint64_t v_mask() const;
    inline uint64_t ndx_mask() const;
    inline uint64_t msb() const;
    inline uint64_t ndx_msb() const;
    inline uint64_t bitmask_v() const;
    inline uint64_t bitmask_ndx() const;

    int64_t get(size_t) const;

private:
    // getting and setting interface specifically for encoding formats
    inline void init_packed(const char*);
    inline void init_flex(const char*);

    static int64_t get_packed(const Array& arr, size_t ndx);
    static int64_t get_flex(const Array& arr, size_t ndx);

    static std::vector<int64_t> get_all_packed(const Array& arr, size_t begin, size_t end);
    static std::vector<int64_t> get_all_flex(const Array& arr, size_t begin, size_t end);

    static void get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]);
    static void get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]);
    static void set_packed(Array& arr, size_t ndx, int64_t val);
    static void set_flex(Array& arr, size_t ndx, int64_t val);
    // query interface
    template <class Cond>
    static bool find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index,
                            QueryStateBase* st);
    template <class Cond>
    static bool find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index,
                          QueryStateBase* st);

    // internal impl
    void compress_values(const Array&, std::vector<int64_t>&, std::vector<unsigned>&) const;
    inline bool is_packed() const;
    inline bool is_flex() const;

    // for testing
    bool always_compress(const Array&, Array&, Node::Encoding) const;

private:
    using Encoding = NodeHeader::Encoding;
    Encoding m_encoding{NodeHeader::Encoding::WTypBits};
    uint64_t* m_data;
    uint8_t m_v_width = 0, m_ndx_width = 0;
    size_t m_v_size = 0, m_ndx_size = 0;
};

inline void IntegerCompressor::init_packed(const char* h)
{
    m_data = (uint64_t*)NodeHeader::get_data_from_header(h);
    m_v_width = NodeHeader::get_element_size(h, Encoding::Packed);
    m_v_size = NodeHeader::get_num_elements(h, Encoding::Packed);
}

inline void IntegerCompressor::init_flex(const char* h)
{
    m_data = (uint64_t*)NodeHeader::get_data_from_header(h);
    m_v_width = NodeHeader::get_elementA_size(h);
    m_v_size = NodeHeader::get_arrayA_num_elements(h);
    m_ndx_width = NodeHeader::get_elementB_size(h);
    m_ndx_size = NodeHeader::get_arrayB_num_elements(h);
}

inline uint64_t* IntegerCompressor::data() const
{
    return m_data;
}

inline bool IntegerCompressor::is_packed() const
{
    return m_encoding == NodeHeader::Encoding::Packed;
}

inline bool IntegerCompressor::is_flex() const
{
    return m_encoding == NodeHeader::Encoding::Flex;
}

inline size_t IntegerCompressor::size() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return m_encoding == NodeHeader::Encoding::Packed ? v_size() : ndx_size();
}

inline size_t IntegerCompressor::v_size() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return m_v_size;
}

inline size_t IntegerCompressor::ndx_size() const
{
    REALM_ASSERT_DEBUG(is_flex());
    return m_ndx_size;
}

inline uint8_t IntegerCompressor::v_width() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return m_v_width;
}

inline uint8_t IntegerCompressor::ndx_width() const
{
    REALM_ASSERT_DEBUG(is_flex());
    return m_ndx_width;
}

inline NodeHeader::Encoding IntegerCompressor::get_encoding() const
{
    return m_encoding;
}

inline uint64_t IntegerCompressor::v_mask() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return 1ULL << (m_v_width - 1);
}

inline uint64_t IntegerCompressor::ndx_mask() const
{
    REALM_ASSERT_DEBUG(is_flex());
    return 1ULL << (m_ndx_width - 1);
}

inline uint64_t IntegerCompressor::msb() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return populate(m_v_width, v_mask());
}

inline uint64_t IntegerCompressor::ndx_msb() const
{
    REALM_ASSERT_DEBUG(is_flex());
    return populate(m_ndx_width, ndx_mask());
}

inline uint64_t IntegerCompressor::bitmask_v() const
{
    REALM_ASSERT_DEBUG(is_packed() || is_flex());
    return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_v_width);
}

inline uint64_t IntegerCompressor::bitmask_ndx() const
{
    REALM_ASSERT_DEBUG(is_flex());
    return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_ndx_width);
}

} // namespace realm
#endif // REALM_INTEGER_COMPRESSOR_HPP

1	/*************************************************************************
2	*
3	* Copyright 2023 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#ifndef REALM_INTEGER_COMPRESSOR_HPP
20	#define REALM_INTEGER_COMPRESSOR_HPP
21
22	#include <cstdint>
23	#include <cstddef>
24	#include <vector>
25	#include <realm/query_conditions.hpp>
26	#include <realm/array_direct.hpp>
27	#include <realm/node.hpp>
28
29	namespace realm {
30
31	class Array;
32	class QueryStateBase;
33	class IntegerCompressor {
34	public:
35	// commit => encode, COW/insert => decode
36	bool compress(const Array&, Array&) const;
37	bool decompress(Array&) const;
38
39	bool init(const char*);
40	void set_vtable(Array&);
41
42	// init from mem B
43	inline uint64_t* data() const;
44	inline size_t size() const;
45	inline NodeHeader::Encoding get_encoding() const;
46	inline uint8_t v_width() const;
47	inline uint8_t ndx_width() const;
48	inline size_t v_size() const;
49	inline size_t ndx_size() const;
50
51	inline uint64_t v_mask() const;
52	inline uint64_t ndx_mask() const;
53	inline uint64_t msb() const;
54	inline uint64_t ndx_msb() const;
55	inline uint64_t bitmask_v() const;
56	inline uint64_t bitmask_ndx() const;
57
58	int64_t get(size_t) const;
59
60	private:
61	// getting and setting interface specifically for encoding formats
62	inline void init_packed(const char*);
63	inline void init_flex(const char*);
64
65	static int64_t get_packed(const Array& arr, size_t ndx);
66	static int64_t get_flex(const Array& arr, size_t ndx);
67
68	static std::vector<int64_t> get_all_packed(const Array& arr, size_t begin, size_t end);
69	static std::vector<int64_t> get_all_flex(const Array& arr, size_t begin, size_t end);
70
71	static void get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]);
72	static void get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]);
73	static void set_packed(Array& arr, size_t ndx, int64_t val);
74	static void set_flex(Array& arr, size_t ndx, int64_t val);
75	// query interface
76	template <class Cond>
77	static bool find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index,
78	QueryStateBase* st);
79	template <class Cond>
80	static bool find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index,
81	QueryStateBase* st);
82
83	// internal impl
84	void compress_values(const Array&, std::vector<int64_t>&, std::vector<unsigned>&) const;
85	inline bool is_packed() const;
86	inline bool is_flex() const;
87
88	// for testing
89	bool always_compress(const Array&, Array&, Node::Encoding) const;
90
91	private:
92	using Encoding = NodeHeader::Encoding;
93	Encoding m_encoding{NodeHeader::Encoding::WTypBits};
94	uint64_t* m_data;
95	uint8_t m_v_width = 0, m_ndx_width = 0;
96	size_t m_v_size = 0, m_ndx_size = 0;
97	};
98
99	inline void IntegerCompressor::init_packed(const char* h)
100	{	44,688,594✔
101	m_data = (uint64_t*)NodeHeader::get_data_from_header(h);	44,688,594✔
102	m_v_width = NodeHeader::get_element_size(h, Encoding::Packed);	44,688,594✔
103	m_v_size = NodeHeader::get_num_elements(h, Encoding::Packed);	44,688,594✔
104	}	44,688,594✔
105
106	inline void IntegerCompressor::init_flex(const char* h)
107	{	93,357✔
108	m_data = (uint64_t*)NodeHeader::get_data_from_header(h);	93,357✔
109	m_v_width = NodeHeader::get_elementA_size(h);	93,357✔
110	m_v_size = NodeHeader::get_arrayA_num_elements(h);	93,357✔
111	m_ndx_width = NodeHeader::get_elementB_size(h);	93,357✔
112	m_ndx_size = NodeHeader::get_arrayB_num_elements(h);	93,357✔
113	}	93,357✔
114
115	inline uint64_t* IntegerCompressor::data() const
116	{	68,417,115✔
117	return m_data;	68,417,115✔
118	}	68,417,115✔
119
120	inline bool IntegerCompressor::is_packed() const
121	{	1,451,538,078✔
122	return m_encoding == NodeHeader::Encoding::Packed;	1,451,538,078✔
123	}	1,451,538,078✔
124
125	inline bool IntegerCompressor::is_flex() const
126	{	915,890,430✔
127	return m_encoding == NodeHeader::Encoding::Flex;	915,890,430✔
128	}	915,890,430✔
129
130	inline size_t IntegerCompressor::size() const
131	{	44,748,348✔
132	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	44,748,348✔
133	return m_encoding == NodeHeader::Encoding::Packed ? v_size() : ndx_size();	44,748,348✔
134	}	44,748,348✔
135
136	inline size_t IntegerCompressor::v_size() const
137	{	47,843,448✔
138	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	47,843,448✔
139	return m_v_size;	47,843,448✔
140	}	47,843,448✔
141
142	inline size_t IntegerCompressor::ndx_size() const
143	{	93,360✔
144	REALM_ASSERT_DEBUG(is_flex());	93,360✔
145	return m_ndx_size;	93,360✔
146	}	93,360✔
147
148	inline uint8_t IntegerCompressor::v_width() const
149	{	179,953,950✔
150	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	179,953,950✔
151	return m_v_width;	179,953,950✔
152	}	179,953,950✔
153
154	inline uint8_t IntegerCompressor::ndx_width() const
155	{	3,772,515✔
156	REALM_ASSERT_DEBUG(is_flex());	3,772,515✔
157	return m_ndx_width;	3,772,515✔
158	}	3,772,515✔
159
160	inline NodeHeader::Encoding IntegerCompressor::get_encoding() const
161	{	979,717,542✔
162	return m_encoding;	979,717,542✔
163	}	979,717,542✔
164
165	inline uint64_t IntegerCompressor::v_mask() const
166	{	160,916,961✔
167	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	160,916,961✔
168	return 1ULL << (m_v_width - 1);	160,916,961✔
169	}	160,916,961✔
170
171	inline uint64_t IntegerCompressor::ndx_mask() const
172	{	180✔
173	REALM_ASSERT_DEBUG(is_flex());	180✔
174	return 1ULL << (m_ndx_width - 1);	180✔
175	}	180✔
176
177	inline uint64_t IntegerCompressor::msb() const
178	{	3,933,717✔
179	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	3,933,717✔
180	return populate(m_v_width, v_mask());	3,933,717✔
181	}	3,933,717✔
182
183	inline uint64_t IntegerCompressor::ndx_msb() const
184	{	180✔
185	REALM_ASSERT_DEBUG(is_flex());	180✔
186	return populate(m_ndx_width, ndx_mask());	180✔
187	}	180✔
188
189	inline uint64_t IntegerCompressor::bitmask_v() const
NEW 190	{	×
NEW 191	REALM_ASSERT_DEBUG(is_packed() \|\| is_flex());	×
NEW 192	return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_v_width);	×
NEW 193	}	×
194
195	inline uint64_t IntegerCompressor::bitmask_ndx() const
NEW 196	{	×
NEW 197	REALM_ASSERT_DEBUG(is_flex());	×
NEW 198	return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_ndx_width);	×
NEW 199	}	×
200
201	} // namespace realm
202	#endif // REALM_INTEGER_COMPRESSOR_HPP

realm / realm-core / nicola.cabiddu_1772

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous