• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

rieske / trans / 29684795945

19 Jul 2026 11:15AM UTC coverage: 90.771% (+0.2%) from 90.53%
29684795945

Pull #46

github

rieske
Experimental parser genrator optimization
Pull Request #46: Experimental parser generator optimization

467 of 475 new or added lines in 14 files covered. (98.32%)

16 existing lines in 6 files now uncovered.

5272 of 5808 relevant lines covered (90.77%)

233275.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.89
/src/parser/CanonicalCollection.cpp
1
#include "CanonicalCollection.h"
2

3
#include "util/LogManager.h"
4
#include "util/Logger.h"
5

6
#include <algorithm>
7
#include <cstdint>
8
#include <deque>
9
#include <unordered_map>
10
#include <utility>
11
#include <vector>
12

13
namespace {
14

15
Logger& logger = LogManager::getComponentLogger(Component::PARSER);
16

17
using namespace parser;
18

19

20
void closeItemSet(
113,346 ✔
21
        const FirstTable& first,
22
        const Grammar& grammar,
23
        GenerationBuffers& buffers,
24
        std::vector<LR1Item>& items) {
25
    auto& coreIndex = buffers.coreIndex;
113,346 ✔
26
    auto& worklist = buffers.worklist;
113,346 ✔
27
    buffers.prepareClosure(items.size());
113,346 ✔
28

29
    for (std::size_t i = 0; i < items.size(); ++i) {
304,860 ✔
30
        coreIndex.emplace(items[i].coreKey(), i);
191,514 ✔
31
        worklist.push_back(i);
191,514 ✔
32
    }
33
    items.reserve(std::max(items.capacity(), items.size() + 64));
113,346 ✔
34

35
    std::size_t head = 0;
113,346 ✔
36
    while (head < worklist.size()) {
3,039,418 ✔
37
        const std::size_t index = worklist[head++];
2,926,072 ✔
38
        if (!items[index].hasUnvisitedSymbols() || grammar.isTerminal(items[index].nextUnvisitedSymbol())) {
2,926,072 ✔
39
            continue;
896,562 ✔
40
        }
41

42
        const int B = items[index].nextUnvisitedSymbol();
2,029,510 ✔
43
        const LR1Item::LookaheadSet propagated = items[index].hasSymbolsAfterNext()
2,029,510 ✔
44
                ? first.firstBits(items[index].symbolAfterNext())
2,029,510 ✔
45
                : items[index].lookaheads();
568,156 ✔
46

47
        for (const auto& production : grammar.getProductionsOfSymbol(B)) {
12,826,582 ✔
48
            const std::uint64_t key =
49
                    (static_cast<std::uint64_t>(static_cast<std::uint32_t>(production.getId())) << 32);
10,797,072 ✔
50
            const auto existing = coreIndex.find(key);
10,797,072 ✔
51
            if (existing == coreIndex.end()) {
10,797,072 ✔
52
                items.emplace_back(production, propagated);
991,300 ✔
53
                const std::size_t newIndex = items.size() - 1;
991,300 ✔
54
                coreIndex.emplace(key, newIndex);
991,300 ✔
55
                worklist.push_back(newIndex);
991,300 ✔
56
            } else if (items[existing->second].mergeLookaheads(propagated)) {
9,805,772 ✔
57
                worklist.push_back(existing->second);
1,743,258 ✔
58
            }
59
        }
60
    }
61
}
113,346 ✔
62

63
std::unordered_map<int, std::vector<LR1Item>> gotoAllFrom(
12,004 ✔
64
        const std::vector<LR1Item>& I,
65
        const FirstTable& first,
66
        const Grammar& grammar,
67
        GenerationBuffers& buffers) {
68
    std::unordered_map<int, std::vector<LR1Item>> bySymbol;
12,004 ✔
69
    bySymbol.reserve(I.size());
12,004 ✔
70
    for (const auto& item : I) {
210,548 ✔
71
        if (item.hasUnvisitedSymbols()) {
198,544 ✔
72
            bySymbol[item.nextUnvisitedSymbol()].push_back(item.advance());
191,488 ✔
73
        }
74
    }
75
    for (auto& entry : bySymbol) {
125,324 ✔
76
        entry.second.reserve(entry.second.size() + 16);
113,320 ✔
77
        closeItemSet(first, grammar, buffers, entry.second);
113,320 ✔
78
    }
79
    return bySymbol;
12,004 ✔
NEW
UNCOV
80
}
×
81

82
// --- LALR(1) ---
83

84
std::vector<std::uint64_t> makeCoreSignature(const std::vector<LR1Item>& items, GenerationBuffers& buffers) {
54,632 ✔
85
    auto& signature = buffers.coreSignature;
54,632 ✔
86
    signature.clear();
54,632 ✔
87
    signature.reserve(items.size());
54,632 ✔
88
    for (const auto& item : items) {
646,946 ✔
89
        signature.push_back(item.coreKey());
592,314 ✔
90
    }
91
    std::sort(signature.begin(), signature.end());
54,632 ✔
92
    return signature;
54,632 ✔
93
}
94

95
struct CoreSignatureHash {
96
    std::size_t operator()(const std::vector<std::uint64_t>& signature) const noexcept {
74,586 ✔
97
        std::size_t hash = signature.size();
74,586 ✔
98
        for (const std::uint64_t key : signature) {
890,124 ✔
99
            hash ^= std::hash<std::uint64_t>{}(key) + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2);
815,538 ✔
100
        }
101
        return hash;
74,586 ✔
102
    }
103
};
104

105
bool mergeLookaheadsByCore(
51,370 ✔
106
        std::vector<LR1Item>& existing,
107
        const std::vector<LR1Item>& incoming,
108
        GenerationBuffers& buffers) {
109
    buffers.prepareMerge(existing.size());
51,370 ✔
110
    auto& index = buffers.mergeIndexByCore;
51,370 ✔
111
    for (std::size_t i = 0; i < existing.size(); ++i) {
592,620 ✔
112
        index.emplace(existing[i].coreKey(), i);
541,250 ✔
113
    }
114
    bool merged = false;
51,370 ✔
115
    for (const auto& item : incoming) {
592,620 ✔
116
        merged |= existing[index.at(item.coreKey())].mergeLookaheads(item.lookaheads());
541,250 ✔
117
    }
118
    return merged;
51,370 ✔
119
}
120

121
void computeLALR1(
16 ✔
122
        std::vector<std::vector<LR1Item>>& canonicalCollection,
123
        GotoMap& computedGotos,
124
        const std::vector<int>& grammarSymbols,
125
        const FirstTable& first,
126
        const Grammar& grammar,
127
        GenerationBuffers& buffers) {
128
    std::unordered_map<std::vector<std::uint64_t>, std::size_t, CoreSignatureHash> stateByCores;
16 ✔
129
    stateByCores.reserve(512);
16 ✔
130
    stateByCores.emplace(makeCoreSignature(canonicalCollection[0], buffers), 0);
16 ✔
131
    computedGotos.reserve(grammarSymbols.size() * 8);
16 ✔
132

133
    std::deque<std::size_t> worklist { 0 };
48 ✔
134
    std::vector<char> inQueue(1, 1);
16 ✔
135

136
    auto enqueue = [&](std::size_t state) {
6,470 ✔
137
        if (state >= inQueue.size()) {
6,470 ✔
138
            inQueue.resize(state + 1, 0);
3,246 ✔
139
        }
140
        if (!inQueue[state]) {
6,470 ✔
141
            inQueue[state] = 1;
5,550 ✔
142
            worklist.push_back(state);
5,550 ✔
143
        }
144
    };
6,470 ✔
145

146
    while (!worklist.empty()) {
5,582 ✔
147
        const std::size_t state = worklist.front();
5,566 ✔
148
        worklist.pop_front();
5,566 ✔
149
        inQueue[state] = 0;
5,566 ✔
150

151
        auto gotos = gotoAllFrom(canonicalCollection[state], first, grammar, buffers);
5,566 ✔
152
        for (const int X : grammarSymbols) {
783,300 ✔
153
            const auto git = gotos.find(X);
777,734 ✔
154
            if (git == gotos.end() || git->second.empty()) {
777,734 ✔
155
                continue;
723,118 ✔
156
            }
157
            auto& gotoSet = git->second;
54,616 ✔
158
            auto signature = makeCoreSignature(gotoSet, buffers);
54,616 ✔
159
            const auto existing = stateByCores.find(signature);
54,616 ✔
160
            if (existing == stateByCores.end()) {
54,616 ✔
161
                const std::size_t newState = canonicalCollection.size();
3,246 ✔
162
                canonicalCollection.push_back(std::move(gotoSet));
3,246 ✔
163
                stateByCores.emplace(std::move(signature), newState);
3,246 ✔
164
                computedGotos[{ state, X }] = newState;
3,246 ✔
165
                enqueue(newState);
3,246 ✔
166
            } else {
167
                const std::size_t target = existing->second;
51,370 ✔
168
                computedGotos[{ state, X }] = target;
51,370 ✔
169
                if (mergeLookaheadsByCore(canonicalCollection[target], gotoSet, buffers)) {
51,370 ✔
170
                    enqueue(target);
3,224 ✔
171
                }
172
            }
173
        }
54,616 ✔
174
    }
5,566 ✔
175
}
16 ✔
176

177
// --- LR(1) ---
178

179
using ItemSetSignature = std::vector<std::pair<std::uint64_t, LR1Item::LookaheadSet>>;
180

181
ItemSetSignature makeItemSetSignature(const std::vector<LR1Item>& items) {
58,714 ✔
182
    ItemSetSignature signature;
58,714 ✔
183
    signature.reserve(items.size());
58,714 ✔
184
    for (const auto& item : items) {
649,214 ✔
185
        signature.emplace_back(item.coreKey(), item.lookaheads());
590,500 ✔
186
    }
187
    // Cores are unique within a closed item set; sort by core only.
188
    std::sort(signature.begin(), signature.end(),
58,714 ✔
189
            [](const auto& left, const auto& right) {
3,577,638 ✔
190
                return left.first < right.first;
3,577,638 ✔
191
            });
192
    return signature;
58,714 ✔
NEW
UNCOV
193
}
×
194

195
struct ItemSetSignatureHash {
196
    std::size_t operator()(const ItemSetSignature& signature) const noexcept {
104,314 ✔
197
        std::size_t hash = signature.size();
104,314 ✔
198
        for (const auto& entry : signature) {
1,282,006 ✔
199
            hash ^= std::hash<std::uint64_t>{}(entry.first) + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2);
1,177,692 ✔
200
            for (std::size_t i = 0; i < entry.second.size(); i += 64) {
3,533,076 ✔
201
                unsigned long long word = 0;
2,355,384 ✔
202
                for (std::size_t b = 0; b < 64 && i + b < entry.second.size(); ++b) {
153,099,960 ✔
203
                    if (entry.second.test(i + b)) {
150,744,576 ✔
204
                        word |= 1ULL << b;
23,600,556 ✔
205
                    }
206
                }
207
                hash ^= std::hash<unsigned long long>{}(word) + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2);
2,355,384 ✔
208
            }
209
        }
210
        return hash;
104,314 ✔
211
    }
212
};
213

214
void computeLR1(
10 ✔
215
        std::vector<std::vector<LR1Item>>& canonicalCollection,
216
        GotoMap& computedGotos,
217
        const std::vector<int>& grammarSymbols,
218
        const FirstTable& first,
219
        const Grammar& grammar,
220
        GenerationBuffers& buffers) {
221
    std::unordered_map<ItemSetSignature, std::size_t, ItemSetSignatureHash> stateByItems;
10 ✔
222
    stateByItems.reserve(256);
10 ✔
223
    stateByItems.emplace(makeItemSetSignature(canonicalCollection[0]), 0);
10 ✔
224

225
    for (std::size_t i = 0; i < canonicalCollection.size(); ++i) {
6,448 ✔
226
        auto gotos = gotoAllFrom(canonicalCollection[i], first, grammar, buffers);
6,438 ✔
227
        for (const int X : grammarSymbols) {
839,604 ✔
228
            const auto git = gotos.find(X);
833,166 ✔
229
            if (git == gotos.end() || git->second.empty()) {
833,166 ✔
230
                continue;
774,462 ✔
231
            }
232
            auto& gotoSet = git->second;
58,704 ✔
233
            auto signature = makeItemSetSignature(gotoSet);
58,704 ✔
234
            const auto existing = stateByItems.find(signature);
58,704 ✔
235
            if (existing == stateByItems.end()) {
58,704 ✔
236
                const std::size_t newState = canonicalCollection.size();
6,428 ✔
237
                canonicalCollection.push_back(std::move(gotoSet));
6,428 ✔
238
                stateByItems.emplace(std::move(signature), newState);
6,428 ✔
239
                computedGotos[{ i, X }] = newState;
6,428 ✔
240
            } else {
241
                computedGotos[{ i, X }] = existing->second;
52,276 ✔
242
            }
243
        }
58,704 ✔
244
    }
6,438 ✔
245
}
10 ✔
246

247
} // namespace
248

249
namespace parser {
250

251
CanonicalCollection::CanonicalCollection(
26 ✔
252
        const FirstTable& firstTable,
253
        const Grammar& grammar,
254
        AutomatonKind kind) :
26 ✔
255
        firstTable { firstTable }
26 ✔
256
{
257
    std::vector<int> grammarSymbols = grammar.getNonterminalIDs();
26 ✔
258
    const auto& terminals = grammar.getTerminalIDs();
26 ✔
259
    grammarSymbols.insert(grammarSymbols.end(), terminals.begin(), terminals.end());
26 ✔
260

261
    std::vector<LR1Item> initialSet {
262
        LR1Item { grammar.getTopRule(), std::vector<int>{ grammar.getEndSymbol() }, grammar }
52 ✔
263
    };
104 ✔
264
    closeItemSet(firstTable, grammar, buffers, initialSet);
26 ✔
265
    canonicalCollection.push_back(std::move(initialSet));
26 ✔
266

267
    if (kind == AutomatonKind::LALR1) {
26 ✔
268
        computeLALR1(canonicalCollection, computedGotos, grammarSymbols, firstTable, grammar, buffers);
16 ✔
269
    } else {
270
        computeLR1(canonicalCollection, computedGotos, grammarSymbols, firstTable, grammar, buffers);
10 ✔
271
    }
272

273
    logCollection(grammar);
26 ✔
274
}
26 ✔
275

276
std::size_t CanonicalCollection::stateCount() const noexcept {
42 ✔
277
    return canonicalCollection.size();
42 ✔
278
}
279

280
const std::vector<LR1Item>& CanonicalCollection::setOfItemsAtState(size_t state) const {
3,492 ✔
281
    return canonicalCollection.at(state);
3,492 ✔
282
}
283

284
std::size_t CanonicalCollection::goTo(std::size_t stateFrom, int symbol) const {
21,276 ✔
285
    return computedGotos.at({ stateFrom, symbol });
21,276 ✔
286
}
287

288
std::optional<std::size_t> CanonicalCollection::findGoTo(std::size_t stateFrom, int symbol) const noexcept {
188,684 ✔
289
    const auto it = computedGotos.find({ stateFrom, symbol });
188,684 ✔
290
    return it != computedGotos.end() ? std::optional<std::size_t>{ it->second } : std::nullopt;
188,684 ✔
291
}
292

293
void CanonicalCollection::logCollection(const Grammar& grammar) const {
26 ✔
294
    if (logger.isNull()) {
26 ✔
295
        return;
24 ✔
296
    }
297
    logger << "\n*********************\nCanonical collection:\n*********************\n";
2 ✔
298
    int setNo = 0;
2 ✔
299
    for (const auto& setOfItems : canonicalCollection) {
16 ✔
300
        logger << "Set " << setNo++ << ":\n";
14 ✔
301
        for (const auto& item : setOfItems) {
42 ✔
302
            logger << item.str(grammar);
28 ✔
303
        }
304
        logger << "\n";
14 ✔
305
    }
306
    logger << "Computed GOTOs:\n";
2 ✔
307
    for (const auto& g : computedGotos) {
22 ✔
308
        logger << g.first.first << "\t" << g.first.second << "\t" << g.second << "\n";
20 ✔
309
    }
310
}
311

312
} // namespace parser
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc