jorgen.edelbo_334

Committed 01 Jul 2024 07:22AM UTC coverage: 90.829% (-0.04%) from 90.865%

Build # jorgen.edelbo_334

Build Type

Pull #7803

Evergreen

Committed by

jedelbo

Commit Message

Merge branch 'next-major' into feature/string-compression

Pull Request Pull Request #7803: Feature/string compression

Run Details

102912 of 180568 branches covered (56.99%)

1141 of 1267 new or added lines in 33 files covered. (90.06%)

172 existing lines in 24 files now uncovered.

218291 of 240332 relevant lines covered (90.83%)

7818396.4 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/src/realm/string_compressor.hpp

/*************************************************************************
 *
 * Copyright 2016 Realm Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **************************************************************************/

#ifndef REALM_STRING_COMPRESSOR_HPP
#define REALM_STRING_COMPRESSOR_HPP

#include <realm/array_unsigned.hpp>
#include <realm/utilities.hpp>
#include <vector>

using CompressionSymbol = uint16_t;
using CompressedString = std::vector<CompressionSymbol>;

struct CompressedStringView {
    CompressionSymbol* data = 0;
    uint32_t size = 0;
    CompressedStringView() = default;
    CompressedStringView(CompressionSymbol* c_ptr, size_t s)
        : data(c_ptr)
        , size(uint32_t(s))
    {
    }
    explicit CompressedStringView(CompressedString& cs)
        : data(cs.data())
        , size(uint32_t(cs.size()))
    {
    }
    bool operator==(CompressedStringView& other)
    {
        if (size != other.size)
            return false;
        for (size_t i = 0; i < size; ++i) {
            if (data[i] != other.data[i])
                return false;
        }
        return true;
    }
};

namespace realm {
class StringCompressor {
public:
    StringCompressor(Allocator& alloc, Array& parent, size_t index, bool writable);
    void refresh(bool writable);
    ~StringCompressor();

    int compare(CompressedStringView& A, CompressedStringView& B);
    int compare(StringData sd, CompressedStringView& B);

    CompressedString compress(StringData, bool learn);
    std::string decompress(CompressedStringView& c_str);

private:
    struct SymbolDef {
        CompressionSymbol id = 0;
        CompressionSymbol expansion_a = 0;
        CompressionSymbol expansion_b = 0;
    };

    struct ExpandedSymbolDef {
        SymbolDef def;
        std::string_view expansion;
        // ^ points into storage managed by m_expansion_storage
        // we need the following 2 values to facilitate rollback of allocated storage
        uint32_t storage_index;  // index into m_expansion_storage
        uint32_t storage_offset; // offset into block.
    };

    void rebuild_internal();
    void expand_compression_map();
    void add_expansion(SymbolDef def);
    std::vector<ExpandedSymbolDef> m_symbols; // map from symbol -> symbolpair, 2 elements pr entry
    std::vector<SymbolDef> m_compression_map; // perfect hash from symbolpair to its symbol

    ArrayUnsigned m_data;
    constexpr static size_t storage_chunk_size = 4096;
    std::vector<std::string> m_expansion_storage;
};

} // namespace realm

#endif

1	/*************************************************************************
2	*
3	* Copyright 2016 Realm Inc.
4	*
5	* Licensed under the Apache License, Version 2.0 (the "License");
6	* you may not use this file except in compliance with the License.
7	* You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*
17	**************************************************************************/
18
19	#ifndef REALM_STRING_COMPRESSOR_HPP
20	#define REALM_STRING_COMPRESSOR_HPP
21
22	#include <realm/array_unsigned.hpp>
23	#include <realm/utilities.hpp>
24	#include <vector>
25
26	using CompressionSymbol = uint16_t;
27	using CompressedString = std::vector<CompressionSymbol>;
28
29	struct CompressedStringView {
30	CompressionSymbol* data = 0;
31	uint32_t size = 0;
NEW 32	CompressedStringView() = default;	×
33	CompressedStringView(CompressionSymbol* c_ptr, size_t s)
34	: data(c_ptr)	5,302,623✔
35	, size(uint32_t(s))	5,302,623✔
36	{	10,808,190✔
37	}	10,808,190✔
38	explicit CompressedStringView(CompressedString& cs)
39	: data(cs.data())	459,111✔
40	, size(uint32_t(cs.size()))	459,111✔
41	{	921,972✔
42	}	921,972✔
43	bool operator==(CompressedStringView& other)
44	{	921,963✔
45	if (size != other.size)	921,963✔
NEW 46	return false;	×
47	for (size_t i = 0; i < size; ++i) {	1,111,931,829✔
48	if (data[i] != other.data[i])	1,111,009,866✔
NEW 49	return false;	×
50	}	1,111,009,866✔
51	return true;	921,963✔
52	}	921,963✔
53	};
54
55	namespace realm {
56	class StringCompressor {
57	public:
58	StringCompressor(Allocator& alloc, Array& parent, size_t index, bool writable);
59	void refresh(bool writable);
60	~StringCompressor();
61
62	int compare(CompressedStringView& A, CompressedStringView& B);
63	int compare(StringData sd, CompressedStringView& B);
64
65	CompressedString compress(StringData, bool learn);
66	std::string decompress(CompressedStringView& c_str);
67
68	private:
69	struct SymbolDef {
70	CompressionSymbol id = 0;
71	CompressionSymbol expansion_a = 0;
72	CompressionSymbol expansion_b = 0;
73	};
74
75	struct ExpandedSymbolDef {
76	SymbolDef def;
77	std::string_view expansion;
78	// ^ points into storage managed by m_expansion_storage
79	// we need the following 2 values to facilitate rollback of allocated storage
80	uint32_t storage_index; // index into m_expansion_storage
81	uint32_t storage_offset; // offset into block.
82	};
83
84	void rebuild_internal();
85	void expand_compression_map();
86	void add_expansion(SymbolDef def);
87	std::vector<ExpandedSymbolDef> m_symbols; // map from symbol -> symbolpair, 2 elements pr entry
88	std::vector<SymbolDef> m_compression_map; // perfect hash from symbolpair to its symbol
89
90	ArrayUnsigned m_data;
91	constexpr static size_t storage_chunk_size = 4096;
92	std::vector<std::string> m_expansion_storage;
93	};
94
95	} // namespace realm
96
97	#endif

realm / realm-core / jorgen.edelbo_334

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous