• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

actioninja / refpack-rs / 20115958187

10 Dec 2025 10:56PM UTC coverage: 90.291% (-7.7%) from 97.965%
20115958187

push

github

actioninja
chore: Make longrunning CI not run in coverage!

1553 of 1720 relevant lines covered (90.29%)

359994.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.78
/src/data/compression/optimal.rs
1
////////////////////////////////////////////////////////////////////////////////
2
// This Source Code Form is subject to the terms of the Mozilla Public         /
3
// License, v. 2.0. If a copy of the MPL was not distributed with this         /
4
// file, You can obtain one at https://mozilla.org/MPL/2.0/.                   /
5
//                                                                             /
6
////////////////////////////////////////////////////////////////////////////////
7

8
use std::array;
9
use std::cmp::min;
10
use std::ops::Range;
11

12
use crate::data::compression::bytes_for_match;
13
use crate::data::compression::prefix_search::PrefixSearcher;
14
use crate::data::control::{COPY_LITERAL_MAX, Command, CommandKind, Control, LITERAL_MAX};
15

16
pub(crate) const HASH_CHAINING_LEVELS: usize = 4;
17

18
// state is packed into 32 bits for SIMD optimization purposes
19
// 31: literal/copy command flag
20
// when 0:
21
//   13-30: offset
22
//   11-12: literals
23
//   0-10: copy length
24
// when 1:
25
//   0-7 literals
26
#[derive(Copy, Clone, Default)]
27
struct CommandState(u32);
28

29
impl CommandState {
30
    fn literal(literal: u8) -> Self {
2,782,577✔
31
        Self((1 << 31) | literal as u32)
2,782,577✔
32
    }
2,782,577✔
33

34
    fn command(offset: u32, literal: u8, length: u16) -> Self {
648,952✔
35
        // it is assumed that none of the values ever exceed the maximum specified by refpack
36
        // doing these checks is possible but expensive since this function is in a hot part of the algorithm
37

38
        Self((offset << 13) | ((literal as u32) << 11) | (length as u32))
648,952✔
39
    }
648,952✔
40

41
    fn is_literal(self) -> bool {
3,052,296✔
42
        (self.0 & (1 << 31)) != 0
3,052,296✔
43
    }
3,052,296✔
44

45
    fn num_literals(self) -> u8 {
2,976,135✔
46
        if self.is_literal() { self.0 as u8 } else { 0 }
2,976,135✔
47
    }
2,976,135✔
48

49
    fn to_command(self) -> Command {
76,161✔
50
        if self.is_literal() {
76,161✔
51
            Command::new_literal((self.0 & 0xFF) as u8)
51,159✔
52
        } else {
53
            Command::new(
25,002✔
54
                (self.0 >> 13) & ((1 << 18) - 1),
25,002✔
55
                (self.0 & ((1 << 11) - 1)) as u16,
25,002✔
56
                ((self.0 >> 11) & 3) as u8,
25,002✔
57
            )
58
        }
59
    }
76,161✔
60
}
61

62
fn controls_from_state_slice(state: &[u32], input: &[u8]) -> Vec<Control> {
48,914✔
63
    let mut cur_pos = state.len() - 1;
48,914✔
64
    // add the output controls in reverse order in this list
65
    let mut controls = vec![];
48,914✔
66

67
    // special handling of the last literals: the last command must be a stop command
68
    // so we can take the number of literals at the end of the input and put them into the stop command
69
    let num_stop_literals = CommandState(state[cur_pos]).num_literals() % 4;
48,914✔
70

71
    // the current position includes the last byte of this literal, so subtract one
72
    let literal_pos = cur_pos + 1 - num_stop_literals as usize;
48,914✔
73
    controls.push(Control {
48,914✔
74
        command: Command::new_stop_unchecked(num_stop_literals),
48,914✔
75
        bytes: input[literal_pos..literal_pos + num_stop_literals as usize].to_vec(),
48,914✔
76
    });
48,914✔
77

78
    cur_pos -= num_stop_literals as usize;
48,914✔
79

80
    loop {
81
        // the bytes of the next command end at the current position
82
        let cur_command = CommandState(state[cur_pos]).to_command();
76,161✔
83

84
        if let CommandKind::Literal = cur_command.kind {
76,161✔
85
            assert_eq!(cur_command.literal % 4, 0);
51,159✔
86
        }
25,002✔
87

88
        let num_literal = cur_command.num_of_literal().unwrap_or(0);
76,161✔
89
        let num_copy = cur_command.offset_copy().unwrap_or((0, 0)).1;
76,161✔
90

91
        // total number of bytes in the input that the current command encodes
92
        let command_decompressed_bytes = num_literal + num_copy;
76,161✔
93

94
        // same as with the stop command
95
        let literal_pos = cur_pos + 1 - command_decompressed_bytes;
76,161✔
96
        controls.push(Control {
76,161✔
97
            command: cur_command,
76,161✔
98
            bytes: input[literal_pos..literal_pos + num_literal].to_vec(),
76,161✔
99
        });
76,161✔
100

101
        if command_decompressed_bytes > cur_pos {
76,161✔
102
            // the encoding should end at position -1, but unsigned integers cannot represent this
103
            debug_assert!(command_decompressed_bytes == cur_pos + 1);
48,914✔
104
            break;
48,914✔
105
        }
27,247✔
106
        cur_pos -= command_decompressed_bytes;
27,247✔
107
    }
108

109
    // we built the controls in reverse order, so reverse the vec
110
    controls.reverse();
48,914✔
111

112
    controls
48,914✔
113
}
48,914✔
114

115
fn update_state_simd(
648,952✔
116
    cost_state: &mut [u32],
648,952✔
117
    command_state: &mut [u32],
648,952✔
118
    new_cost: u32,
648,952✔
119
    range: Range<usize>,
648,952✔
120
    command_base: CommandState,
648,952✔
121
) {
648,952✔
122
    const CHUNK_SIZE: usize = 4;
123

124
    let mut cost_state_chunks = cost_state[range.clone()].chunks_exact_mut(CHUNK_SIZE);
648,952✔
125
    let mut command_state_chunks = command_state[range].chunks_exact_mut(CHUNK_SIZE);
648,952✔
126

127
    let chunk_bytes = cost_state_chunks.len() * CHUNK_SIZE;
648,952✔
128

129
    let new_commands_base = command_base.0;
648,952✔
130
    let new_commands_base_arr: [u32; CHUNK_SIZE] = array::from_fn(|i| new_commands_base + i as u32);
2,595,808✔
131

132
    for (n, (cost, command)) in (&mut cost_state_chunks)
648,952✔
133
        .zip(&mut command_state_chunks)
648,952✔
134
        .enumerate()
648,952✔
135
    {
136
        let cur_cost_arr: [u32; CHUNK_SIZE] = cost.try_into().unwrap();
72,588✔
137
        let new_cost_arr = cur_cost_arr.map(|c| c.min(new_cost));
290,352✔
138

139
        let none_changed = (0..CHUNK_SIZE).all(|i| cur_cost_arr[i] == new_cost_arr[i]);
258,624✔
140

141
        if none_changed {
72,588✔
142
            continue;
49,824✔
143
        }
22,764✔
144

145
        let new_commands_arr: [u32; CHUNK_SIZE] = array::from_fn(|i| {
91,056✔
146
            let new_command = new_commands_base_arr[i] + (n * CHUNK_SIZE) as u32;
91,056✔
147
            let old_command = command[i];
91,056✔
148

149
            if cur_cost_arr[i] > new_cost {
91,056✔
150
                new_command
54,492✔
151
            } else {
152
                old_command
36,564✔
153
            }
154
        });
91,056✔
155
        command.copy_from_slice(&new_commands_arr);
22,764✔
156

157
        cost.copy_from_slice(&new_cost_arr);
22,764✔
158
    }
159

160
    let new_commands_remainder = new_commands_base_arr.map(|c| c + chunk_bytes as u32);
2,595,808✔
161

162
    for ((cost, command), new_command) in cost_state_chunks
952,528✔
163
        .into_remainder()
648,952✔
164
        .iter_mut()
648,952✔
165
        .zip(command_state_chunks.into_remainder().iter_mut())
648,952✔
166
        .zip(new_commands_remainder)
648,952✔
167
    {
168
        if *cost > new_cost {
952,528✔
169
            *cost = new_cost;
164,108✔
170
            *command = new_command;
164,108✔
171
        }
788,420✔
172
    }
173
}
648,952✔
174

175
/// Search for the set of controls that can encode the input slice in the lowest amount of output bytes possible.
176
///
177
/// To understand this algorithm, first consider the case of using a hash chain prefix searcher
178
/// as it is equivalent to the case of using a multilevel hash chain.
179
///
180
/// This algorithm is in essence a variation of dijkstra's algorithm; for every node (byte position) that is opened
181
/// it is always known what the most cost-effective way to reach that point is.
182
///
183
/// For example, consider the case of an input consisting of all zeros:
184
/// it is known that to encode the first byte of the input the only option is a literal command of length 1.
185
/// Thus, the cost for encoding the first byte must be 2 (literal command + 1 literal).
186
/// Because copy commands can also include four literals the literal command cost
187
/// is not encoded until after four literals.
188
/// After the first byte has been encoded multiple other bytes can be reached via a copy command;
189
/// the short copy command can copy 3-10 bytes with a minimum offset of 1 and a cost of 2 bytes,
190
/// thus we know that positions 3-10 can be reached with a maximum cost of 3 (1 byte literal + 2 bytes short command),
191
/// and positions 1 and 2 can also be reached with a cost of 3 and 4 respectively (with 2 and 3 literal bytes).
192
///
193
/// Once all positions have been opened it is known that the last cost state is the minimum cost
194
/// for encoding all bytes in the input. It is then possible to encode all commands by tracing backwards
195
/// through the input while referencing the command state that is built in the search process.
196
pub(crate) fn encode_slice_hc<'a, PS: PrefixSearcher<'a>>(input: &'a [u8]) -> Vec<Control> {
50,458✔
197
    let input_length = input.len();
50,458✔
198

199
    // if the input is 3 bytes or fewer it is impossible to encode any copy commands
200
    // just return the stop commands with the input as literal bytes
201
    if input_length <= 3 {
50,458✔
202
        return vec![Control {
1,544✔
203
            command: Command::new_stop_unchecked(input_length as u8),
1,544✔
204
            bytes: Vec::from(input),
1,544✔
205
        }];
1,544✔
206
    }
48,914✔
207

208
    // build the prefix searcher
209
    // it will give us all previous occurrences of the current position along with their match length
210
    let mut prev = PS::build(input);
48,914✔
211

212
    // tracks the last command to encode all bytes in the input up to a certain point
213
    let mut command_state = vec![CommandState::default().0; input_length];
48,914✔
214
    // tracks the maximum cost to encode all bytes in the input up to a certain position
215
    let mut cost_state = vec![u32::MAX; input_length];
48,914✔
216
    // the state vecs could be combined into a single vec, but we store them separately for SIMD purposes
217

218
    // we know the first byte must be encoded as a literal, thus the cost is 1
219
    cost_state[0] = 1;
48,914✔
220
    // idem
221
    command_state[0] = CommandState::literal(1).0;
48,914✔
222

223
    // go through all the byte positions in the input
224
    for pos in 0..(input_length as u32 - 1) {
2,927,221✔
225
        // since this position has no unexplored predecessors
226
        // we know the cost to reach this byte is equivalent to the stored cost state
227
        let cur_cost = cost_state[pos as usize];
2,927,221✔
228
        // and the command to reach that state is the stored command
229
        let cur_command = command_state[pos as usize];
2,927,221✔
230
        // get the number of literals that are passed on into the next command
231
        // for copy commands this is always 0
232
        let cur_literals = CommandState(cur_command).num_literals();
2,927,221✔
233

234
        // there can't be any matches on the last 3 bytes since matches must always be at least 3 bytes
235
        if pos < (input_length - 3) as u32 {
2,927,221✔
236
            // we want to try encoding bytes for the next byte onward since the current position is already known
237
            // so the search position is the current position + 1
238
            let match_start_pos = pos + 1;
2,829,393✔
239

240
            // search for all matches with the search position
241
            prev.search(
2,829,393✔
242
                match_start_pos as usize,
2,829,393✔
243
                |match_pos, match_start, match_end| {
623,392✔
244
                    // for all bytes in this match, update the command and cost state
245
                    // for all positions that have a lower cost than the stored cost state
246

247
                    debug_assert!(match_start < match_end);
623,392✔
248

249
                    let offset = match_start_pos as usize - match_pos;
623,392✔
250

251
                    // loop through all ranges in the match that have an equal cost
252
                    // for SIMD optimization purposes
253
                    let mut i = match_start;
623,392✔
254
                    while i < match_end {
1,272,350✔
255
                        if let Some((command_bytes, interval_limit)) = bytes_for_match(i, offset) {
648,958✔
256
                            // get the cost to encode the current command (command_bytes)
257
                            if let Some(command_bytes) = command_bytes {
648,958✔
258
                                let match_length_start = i;
648,952✔
259
                                let match_length_end = min(interval_limit + 1, match_end);
648,952✔
260
                                let range = (pos as usize + match_length_start)
648,952✔
261
                                    ..(pos as usize + match_length_end);
648,952✔
262

648,952✔
263
                                // the cost for all encoded commands in this range
648,952✔
264
                                let new_cost = cur_cost + command_bytes as u32;
648,952✔
265

648,952✔
266
                                // now update the cost and command state in this range with the new cost
648,952✔
267
                                update_state_simd(
648,952✔
268
                                    &mut cost_state,
648,952✔
269
                                    &mut command_state,
648,952✔
270
                                    new_cost,
648,952✔
271
                                    range,
648,952✔
272
                                    CommandState::command(
648,952✔
273
                                        offset as u32,
648,952✔
274
                                        cur_literals % 4,
648,952✔
275
                                        match_length_start as u16,
648,952✔
276
                                    ),
648,952✔
277
                                );
648,952✔
278
                            }
648,952✔
279

280
                            i = interval_limit + 1;
648,958✔
281
                            continue;
648,958✔
282
                        }
×
283
                        break;
×
284
                    }
285
                },
623,392✔
286
            );
287
        }
97,828✔
288

289
        // update for the next literal
290
        let mut literal_cost = cur_cost;
2,927,221✔
291
        literal_cost += 1;
2,927,221✔
292
        let mut new_state_literal = cur_literals + 1;
2,927,221✔
293
        if new_state_literal > (LITERAL_MAX + COPY_LITERAL_MAX) {
2,927,221✔
294
            // literal + copy command cannot represent this amount, wrap back around to a new literal command
2,383✔
295
            new_state_literal = 4;
2,383✔
296
        }
2,924,838✔
297
        if new_state_literal == 4 {
2,927,221✔
298
            // a copy command cannot represent this, so we needed to add a new byte for the literal command
51,524✔
299
            literal_cost += 1;
51,524✔
300
        }
2,875,697✔
301

302
        if cost_state[pos as usize + 1] > literal_cost {
2,927,221✔
303
            cost_state[pos as usize + 1] = literal_cost;
2,733,663✔
304
            command_state[pos as usize + 1] = CommandState::literal(new_state_literal).0;
2,733,663✔
305
        }
2,733,663✔
306
    }
307

308
    // since we don't need the cost state for building the output command list
309
    // we can drop it early to save on peak memory usage
310
    drop(cost_state);
48,914✔
311

312
    // trace backwards through the command state to extract the output command list
313
    controls_from_state_slice(&command_state, input)
48,914✔
314
}
50,458✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc