16092050615

Committed 05 Jul 2025 09:06PM UTC coverage: 94.616% (+0.3%) from 94.304%

Build # 16092050615

Build Type

push

github

Committed by

web-flow

Commit Message

General refactor and cleanup (#38)

* General refactor and cleanup

* Rename direct_buffer->stream_buffer

Run Details

68 of 78 new or added lines in 2 files covered. (87.18%)

80 existing lines in 2 files now uncovered.

3743 of 3956 relevant lines covered (94.62%)

80.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.46

/picojson/src/stream_buffer.rs

// SPDX-License-Identifier: Apache-2.0

use crate::ParseError;

/// Error types for StreamBuffer operations
#[derive(Debug, PartialEq)]
pub enum StreamBufferError {
    /// Buffer is full and cannot accommodate more data
    BufferFull,
    /// Attempted to read beyond available data
    EndOfData,
    /// Invalid buffer state or operation
    InvalidState(&'static str),
}

impl From<StreamBufferError> for ParseError {
    fn from(err: StreamBufferError) -> Self {
        match err {
            StreamBufferError::BufferFull => ParseError::ScratchBufferFull,
            StreamBufferError::EndOfData => ParseError::EndOfData,
            StreamBufferError::InvalidState(msg) => ParseError::UnexpectedState(msg),
        }
    }
}

/// StreamBuffer manages a single buffer for both input and escape processing
///
/// Key design principles:
/// - Reader fills unused portions of buffer directly
/// - Unescaped content is copied to buffer start when needed
/// - Zero-copy string extraction when no escapes are present
/// - Guaranteed space for escape processing (unescaped ≤ escaped)
pub struct StreamBuffer<'a> {
    /// The entire buffer slice
    buffer: &'a mut [u8],
    /// Current position where tokenizer is reading
    tokenize_pos: usize,
    /// End of valid data from Reader (buffer[0..data_end] contains valid data)
    data_end: usize,
    /// Length of unescaped content at buffer start (0 if no unescaping active)
    unescaped_len: usize,
    /// Minimum space to reserve for escape processing
    escape_reserve: usize,
}

impl<'a> StreamBuffer<'a> {
    /// Create a new StreamBuffer with the given buffer slice
    pub fn new(buffer: &'a mut [u8]) -> Self {
        // Reserve ~12.5% of buffer for escape processing (>>3 instead of /10), minimum 64 bytes
        // Avoids expensive 32-bit division on 8-bit AVR targets
        let escape_reserve = (buffer.len() >> 3).max(64);

        Self {
            buffer,
            tokenize_pos: 0,
            data_end: 0,
            unescaped_len: 0,
            escape_reserve,
        }
    }

    /// Get the current byte at tokenize position
    pub fn current_byte(&self) -> Result<u8, StreamBufferError> {
        if self.tokenize_pos >= self.data_end {
            return Err(StreamBufferError::EndOfData);
        }
        self.buffer
            .get(self.tokenize_pos)
            .copied()
            .ok_or(StreamBufferError::EndOfData)
    }

    /// Advance the tokenize position by one byte
    pub fn advance(&mut self) -> Result<(), StreamBufferError> {
        if self.tokenize_pos >= self.data_end {
            return Err(StreamBufferError::EndOfData);
        }
        self.tokenize_pos = self.tokenize_pos.wrapping_add(1);
        Ok(())
    }

    /// Get remaining bytes available for reading
    pub fn remaining_bytes(&self) -> usize {
        self.data_end.saturating_sub(self.tokenize_pos)
    }

    /// Get slice for Reader to fill with new data
    /// Returns None if no space available
    pub fn get_fill_slice(&mut self) -> Option<&mut [u8]> {
        if self.data_end >= self.buffer.len() {
            return None;
        }
        self.buffer.get_mut(self.data_end..)
    }

    /// Mark that Reader filled `bytes_read` bytes
    pub fn mark_filled(&mut self, bytes_read: usize) -> Result<(), StreamBufferError> {
        let new_data_end = self.data_end.wrapping_add(bytes_read);
        if new_data_end > self.buffer.len() {
            return Err(StreamBufferError::InvalidState(
                "Attempted to mark more bytes than buffer space",
            ));
        }
        self.data_end = new_data_end;
        Ok(())
    }

    /// Start unescaping and copy existing content from a range in the buffer
    /// This handles the common case of starting escape processing partway through a string
    pub fn start_unescaping_with_copy(
        &mut self,
        max_escaped_len: usize,
        copy_start: usize,
        copy_end: usize,
    ) -> Result<(), StreamBufferError> {
        // Clear any previous unescaped content
        self.unescaped_len = 0;

        // Ensure we have space at the start for unescaping
        if max_escaped_len > self.buffer.len() {
            return Err(StreamBufferError::BufferFull);
        }

        // Copy existing content if there is any
        if copy_end > copy_start && copy_start < self.data_end {
            let span_len = copy_end.saturating_sub(copy_start);

            // Ensure the span fits in the buffer - return error instead of silent truncation
            if span_len > self.buffer.len() {
                return Err(StreamBufferError::BufferFull);
            }

            let src_range = copy_start..copy_start.wrapping_add(span_len);
            if src_range.end > self.buffer.len() {
                return Err(StreamBufferError::InvalidState(
                    "Source range out of bounds",
                ));
            }

            // Copy within the same buffer: move data from [copy_start..copy_end] to [0..span_len]
            // Use copy_within to handle overlapping ranges safely
            self.buffer.copy_within(src_range, 0);
            self.unescaped_len = span_len;
        }

        Ok(())
    }

    /// Get the unescaped content slice
    pub fn get_unescaped_slice(&self) -> Result<&[u8], StreamBufferError> {
        if self.unescaped_len == 0 {
            return Err(StreamBufferError::InvalidState(
                "No unescaped content available",
            ));
        }
        self.buffer
            .get(0..self.unescaped_len)
            .ok_or(StreamBufferError::InvalidState(
                "Unescaped length exceeds buffer size",
            ))
    }

    /// Clear unescaped content (call after yielding unescaped string)
    pub fn clear_unescaped(&mut self) {
        self.unescaped_len = 0;
    }

    /// Get current tokenize position (for string start tracking)
    pub fn current_position(&self) -> usize {
        self.tokenize_pos
    }

    /// Check if buffer is empty (no more data to process)
    pub fn is_empty(&self) -> bool {
        self.tokenize_pos >= self.data_end
    }

    /// Check if we have unescaped content ready
    pub fn has_unescaped_content(&self) -> bool {
        self.unescaped_len > 0
    }

    /// Append a single byte to the unescaped content
    pub fn append_unescaped_byte(&mut self, byte: u8) -> Result<(), StreamBufferError> {
        let available_space = self.buffer.len().saturating_sub(self.escape_reserve);
        if self.unescaped_len >= available_space {
            return Err(StreamBufferError::BufferFull);
        }

        if let Some(b) = self.buffer.get_mut(self.unescaped_len) {
            *b = byte;
            self.unescaped_len = self.unescaped_len.wrapping_add(1);
            Ok(())
        } else {
            Err(StreamBufferError::BufferFull)
        }
    }

    /// Get a string slice from the buffer (zero-copy)
    /// Used for strings without escapes
    pub fn get_string_slice(&self, start: usize, end: usize) -> Result<&[u8], StreamBufferError> {
        if start > end || end > self.data_end {
            return Err(StreamBufferError::InvalidState("Invalid slice bounds"));
        }
        self.buffer
            .get(start..end)
            .ok_or(StreamBufferError::InvalidState("Invalid slice bounds"))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_lifetime_expectations() {
        // This test demonstrates how StreamBuffer lifetimes should work
        let mut buffer = [0u8; 100];
        let mut stream_buffer = StreamBuffer::new(&mut buffer);

        // Simulate some data being in the buffer
        let test_data = b"hello world";
        stream_buffer.buffer[0..test_data.len()].copy_from_slice(test_data);
        stream_buffer.data_end = test_data.len();

        // Test that we can get buffer data

        // Test unescaped content - add some unescaped data
        stream_buffer.unescaped_len = 3;
        stream_buffer.buffer[0..3].copy_from_slice(b"abc");

        let unescaped_slice = stream_buffer.get_unescaped_slice().unwrap();
        assert_eq!(unescaped_slice, b"abc");

        // The key expectation: these slices should live as long as the original buffer
        // and be usable to create String::Borrowed(&'buffer str) and String::Unescaped(&'buffer str)
    }

    #[test]
    fn test_new_stream_buffer() {
        let mut buffer = [0u8; 100];
        let db = StreamBuffer::new(&mut buffer);

        assert_eq!(db.tokenize_pos, 0);
        assert_eq!(db.data_end, 0);
        assert_eq!(db.unescaped_len, 0);
        assert_eq!(db.escape_reserve, 64); // 12.5% of 100, minimum 64
        assert!(db.is_empty());
    }

    #[test]
    fn test_fill_and_advance() {
        let mut buffer = [0u8; 100];
        let mut db = StreamBuffer::new(&mut buffer);

        // Fill with some data
        {
            let fill_slice = db.get_fill_slice().unwrap();
            fill_slice[0..5].copy_from_slice(b"hello");
        }
        db.mark_filled(5).unwrap();

        assert_eq!(db.data_end, 5);
        assert_eq!(db.remaining_bytes(), 5);

        // Read bytes
        assert_eq!(db.current_byte().unwrap(), b'h');
        db.advance().unwrap();
        assert_eq!(db.current_byte().unwrap(), b'e');
        assert_eq!(db.remaining_bytes(), 4);
    }

    #[test]
    fn test_error_conditions() {
        let mut buffer = [0u8; 10];
        let mut db = StreamBuffer::new(&mut buffer);

        // EndOfData errors
        assert_eq!(db.current_byte().unwrap_err(), StreamBufferError::EndOfData);
        assert_eq!(db.advance().unwrap_err(), StreamBufferError::EndOfData);

        // No unescaped content
        assert!(db.get_unescaped_slice().is_err());
    }

    #[test]
    fn test_buffer_full_scenario() {
        // Test what happens when buffer gets completely full
        let mut buffer = [0u8; 10];
        let mut db = StreamBuffer::new(&mut buffer);

        // Fill buffer completely
        {
            let fill_slice = db.get_fill_slice().unwrap();
            fill_slice.copy_from_slice(b"0123456789");
        }
        db.mark_filled(10).unwrap();

        // No more space for filling
        assert!(db.get_fill_slice().is_none());

        // We can still read from buffer
        assert_eq!(db.current_byte().unwrap(), b'0');
        assert_eq!(db.remaining_bytes(), 10);
    }

    #[test]
    fn test_minimal_buffer_with_long_token() {
        // Test very small buffer with a token that doesn't fit
        let mut buffer = [0u8; 8]; // Very small buffer
        let mut db = StreamBuffer::new(&mut buffer);

        // Try to put a string that's almost as big as the buffer
        {
            let fill_slice = db.get_fill_slice().unwrap();
            fill_slice[0..6].copy_from_slice(b"\"hello"); // Start of a long string, no closing quote
        }
        db.mark_filled(6).unwrap();

        // Advance through the data
        for _ in 0..6 {
            db.advance().unwrap();
        }

        // Now buffer is exhausted but we don't have a complete token
        assert!(db.is_empty());
        assert_eq!(db.remaining_bytes(), 0);

        // This simulates the scenario where we need more data but can't fit it
        // The parser would need to handle this by buffering the incomplete token
    }

    #[test]
    fn test_reader_returns_zero_bytes() {
        let mut buffer = [0u8; 20];
        let mut db = StreamBuffer::new(&mut buffer);

        // Simulate Reader returning 0 bytes (EOF)
        {
            let fill_slice = db.get_fill_slice().unwrap();
            assert_eq!(fill_slice.len(), 20);
            // Reader returns 0 bytes - simulating EOF or no data available
        }
        db.mark_filled(0).unwrap(); // Reader returned 0

        assert!(db.is_empty());
        assert_eq!(db.data_end, 0);
        assert_eq!(db.remaining_bytes(), 0);

        // Should still be able to get fill slice for next attempt
        let fill_slice = db.get_fill_slice().unwrap();
        assert_eq!(fill_slice.len(), 20);
    }

    #[test]
    fn test_maximum_escape_reserve_scenario() {
        let mut buffer = [0u8; 100];
        let db = StreamBuffer::new(&mut buffer);

        // Check escape reserve calculation
        assert_eq!(db.escape_reserve, 64); // max(100>>3, 64) = 64

        // Test with smaller buffer
        let mut small_buffer = [0u8; 50];
        let small_db = StreamBuffer::new(&mut small_buffer);
        assert_eq!(small_db.escape_reserve, 64); // Still 64 (minimum)

        // Test with larger buffer
        let mut large_buffer = [0u8; 1000];
        let large_db = StreamBuffer::new(&mut large_buffer);
        assert_eq!(large_db.escape_reserve, 125); // 1000 >> 3 = 125
    }

    #[test]
    fn test_boundary_conditions() {
        let mut buffer = [0u8; 3]; // Absolute minimum
        let mut db = StreamBuffer::new(&mut buffer);

        // Can't even hold a proper JSON token, but should not crash
        {
            let fill_slice = db.get_fill_slice().unwrap();
            fill_slice.copy_from_slice(b"\"a\"");
        }
        db.mark_filled(3).unwrap();

        // Should be able to read through it
        assert_eq!(db.current_byte().unwrap(), b'"');
        db.advance().unwrap();
        assert_eq!(db.current_byte().unwrap(), b'a');
        db.advance().unwrap();
        assert_eq!(db.current_byte().unwrap(), b'"');
        db.advance().unwrap();

        assert!(db.is_empty());
    }

    #[test]
    fn test_start_unescaping_with_copy_span_too_large() {
        let mut buffer = [0u8; 10]; // Small buffer
        let mut db = StreamBuffer::new(&mut buffer);

        // Fill buffer with some data
        {
            let fill_slice = db.get_fill_slice().unwrap();
            fill_slice.copy_from_slice(b"0123456789");
        }
        db.mark_filled(10).unwrap();

        // Try to copy a span that's larger than the entire buffer
        let copy_start = 0;
        let copy_end = 15; // This span (15 bytes) is larger than buffer (10 bytes)
        let max_escaped_len = 5; // This is fine

        // Should return BufferFull error instead of silently truncating
        let result = db.start_unescaping_with_copy(max_escaped_len, copy_start, copy_end);
        assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);

        // Test boundary case: span exactly equals buffer size should work
        let copy_end_exact = 10; // Span of exactly 10 bytes (buffer size)
        let result = db.start_unescaping_with_copy(max_escaped_len, 0, copy_end_exact);
        assert!(result.is_ok());
        assert_eq!(db.unescaped_len, 10);

        // Test valid smaller span should work
        db.clear_unescaped();
        let result = db.start_unescaping_with_copy(max_escaped_len, 2, 6); // 4 byte span
        assert!(result.is_ok());
        assert_eq!(db.unescaped_len, 4);
        assert_eq!(db.get_unescaped_slice().unwrap(), b"2345");
    }

    #[test]
    fn test_append_unescaped_byte_respects_escape_reserve() {
        let mut buffer = [0u8; 100]; // 100 byte buffer
        let mut db = StreamBuffer::new(&mut buffer);

        // Check escape reserve was set correctly (12.5% of 100, minimum 64)
        assert_eq!(db.escape_reserve, 64);

        // Should be able to append up to (buffer_len - escape_reserve) bytes
        let max_unescaped = 100 - db.escape_reserve; // 100 - 64 = 36

        // Fill up to the limit - should succeed
        for i in 0..max_unescaped {
            let result = db.append_unescaped_byte(b'A');
            assert!(result.is_ok(), "Failed at byte {}", i);
        }

        assert_eq!(db.unescaped_len, max_unescaped);

        // One more byte should fail due to escape reserve constraint
        let result = db.append_unescaped_byte(b'B');
        assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);

        // Verify we didn't exceed the escape reserve boundary
        assert_eq!(db.unescaped_len, max_unescaped);
    }

    #[test]
    fn test_append_unescaped_byte_escape_reserve_larger_than_buffer() {
        let mut buffer = [0u8; 10]; // Very small buffer
        let mut db = StreamBuffer::new(&mut buffer);

        // Even small buffers get minimum 64 byte escape reserve, but that's larger than buffer
        assert_eq!(db.escape_reserve, 64); // minimum

        // Since escape_reserve (64) > buffer.len() (10), no bytes should be appendable
        // This should not panic with underflow, but return BufferFull error
        let result = db.append_unescaped_byte(b'A');
        assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);

        // Test with even smaller buffer to ensure we handle underflow correctly
        let mut tiny_buffer = [0u8; 3];
        let mut tiny_db = StreamBuffer::new(&mut tiny_buffer);
        assert_eq!(tiny_db.escape_reserve, 64); // Still minimum 64

        // Should handle this gracefully without panic
        let result = tiny_db.append_unescaped_byte(b'B');
        assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);
    }
}

impl crate::number_parser::NumberExtractor for StreamBuffer<'_> {
    fn get_number_slice(
        &self,
        start: usize,
        end: usize,
    ) -> Result<&[u8], crate::shared::ParseError> {
        self.get_string_slice(start, end)
            .map_err(|_| crate::shared::ParseError::UnexpectedState("Invalid number slice bounds"))
    }

    fn current_position(&self) -> usize {
        self.tokenize_pos
    }

    fn is_empty(&self) -> bool {
        self.tokenize_pos >= self.data_end
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2
3	use crate::ParseError;
4
5	/// Error types for StreamBuffer operations
6	#[derive(Debug, PartialEq)]
7	pub enum StreamBufferError {
8	/// Buffer is full and cannot accommodate more data
9	BufferFull,
10	/// Attempted to read beyond available data
11	EndOfData,
12	/// Invalid buffer state or operation
13	InvalidState(&'static str),
14	}
15
16	impl From<StreamBufferError> for ParseError {
NEW 17	fn from(err: StreamBufferError) -> Self {	×
18	match err {	×
NEW 19	StreamBufferError::BufferFull => ParseError::ScratchBufferFull,	×
NEW 20	StreamBufferError::EndOfData => ParseError::EndOfData,	×
NEW 21	StreamBufferError::InvalidState(msg) => ParseError::UnexpectedState(msg),	×
22	}
23	}	×
24	}
25
26	/// StreamBuffer manages a single buffer for both input and escape processing
27	///
28	/// Key design principles:
29	/// - Reader fills unused portions of buffer directly
30	/// - Unescaped content is copied to buffer start when needed
31	/// - Zero-copy string extraction when no escapes are present
32	/// - Guaranteed space for escape processing (unescaped ≤ escaped)
33	pub struct StreamBuffer<'a> {
34	/// The entire buffer slice
35	buffer: &'a mut [u8],
36	/// Current position where tokenizer is reading
37	tokenize_pos: usize,
38	/// End of valid data from Reader (buffer[0..data_end] contains valid data)
39	data_end: usize,
40	/// Length of unescaped content at buffer start (0 if no unescaping active)
41	unescaped_len: usize,
42	/// Minimum space to reserve for escape processing
43	escape_reserve: usize,
44	}
45
46	impl<'a> StreamBuffer<'a> {
47	/// Create a new StreamBuffer with the given buffer slice
48	pub fn new(buffer: &'a mut [u8]) -> Self {	53✔
49	// Reserve ~12.5% of buffer for escape processing (>>3 instead of /10), minimum 64 bytes
50	// Avoids expensive 32-bit division on 8-bit AVR targets
51	let escape_reserve = (buffer.len() >> 3).max(64);	53✔
52
53	Self {	53✔
54	buffer,	53✔
55	tokenize_pos: 0,	53✔
56	data_end: 0,	53✔
57	unescaped_len: 0,	53✔
58	escape_reserve,	53✔
59	}	53✔
60	}	53✔
61
62	/// Get the current byte at tokenize position
63	pub fn current_byte(&self) -> Result<u8, StreamBufferError> {	597✔
64	if self.tokenize_pos >= self.data_end {	597✔
65	return Err(StreamBufferError::EndOfData);	1✔
66	}	596✔
67	self.buffer	596✔
68	.get(self.tokenize_pos)	596✔
69	.copied()	596✔
70	.ok_or(StreamBufferError::EndOfData)	596✔
71	}	597✔
72
73	/// Advance the tokenize position by one byte
74	pub fn advance(&mut self) -> Result<(), StreamBufferError> {	601✔
75	if self.tokenize_pos >= self.data_end {	601✔
76	return Err(StreamBufferError::EndOfData);	1✔
77	}	600✔
78	self.tokenize_pos = self.tokenize_pos.wrapping_add(1);	600✔
79	Ok(())	600✔
80	}	601✔
81
82	/// Get remaining bytes available for reading
83	pub fn remaining_bytes(&self) -> usize {	10✔
84	self.data_end.saturating_sub(self.tokenize_pos)	10✔
85	}	10✔
86
87	/// Get slice for Reader to fill with new data
88	/// Returns None if no space available
89	pub fn get_fill_slice(&mut self) -> Option<&mut [u8]> {	629✔
90	if self.data_end >= self.buffer.len() {	629✔
91	return None;	1✔
92	}	628✔
93	self.buffer.get_mut(self.data_end..)	628✔
94	}	629✔
95
96	/// Mark that Reader filled `bytes_read` bytes
97	pub fn mark_filled(&mut self, bytes_read: usize) -> Result<(), StreamBufferError> {	627✔
98	let new_data_end = self.data_end.wrapping_add(bytes_read);	627✔
99	if new_data_end > self.buffer.len() {	627✔
NEW 100	return Err(StreamBufferError::InvalidState(	×
101	"Attempted to mark more bytes than buffer space",	×
102	));	×
103	}	627✔
104	self.data_end = new_data_end;	627✔
105	Ok(())	627✔
106	}	627✔
107
108	/// Start unescaping and copy existing content from a range in the buffer
109	/// This handles the common case of starting escape processing partway through a string
110	pub fn start_unescaping_with_copy(	8✔
111	&mut self,	8✔
112	max_escaped_len: usize,	8✔
113	copy_start: usize,	8✔
114	copy_end: usize,	8✔
115	) -> Result<(), StreamBufferError> {	8✔
116	// Clear any previous unescaped content
117	self.unescaped_len = 0;	8✔
118
119	// Ensure we have space at the start for unescaping
120	if max_escaped_len > self.buffer.len() {	8✔
NEW 121	return Err(StreamBufferError::BufferFull);	×
122	}	8✔
123
124	// Copy existing content if there is any
125	if copy_end > copy_start && copy_start < self.data_end {	8✔
126	let span_len = copy_end.saturating_sub(copy_start);	8✔
127
128	// Ensure the span fits in the buffer - return error instead of silent truncation
129	if span_len > self.buffer.len() {	8✔
130	return Err(StreamBufferError::BufferFull);	1✔
131	}	7✔
132
133	let src_range = copy_start..copy_start.wrapping_add(span_len);	7✔
134	if src_range.end > self.buffer.len() {	7✔
NEW 135	return Err(StreamBufferError::InvalidState(	×
136	"Source range out of bounds",	×
137	));	×
138	}	7✔
139
140	// Copy within the same buffer: move data from [copy_start..copy_end] to [0..span_len]
141	// Use copy_within to handle overlapping ranges safely
142	self.buffer.copy_within(src_range, 0);	7✔
143	self.unescaped_len = span_len;	7✔
144	}	×
145
146	Ok(())	7✔
147	}	8✔
148
149	/// Get the unescaped content slice
150	pub fn get_unescaped_slice(&self) -> Result<&[u8], StreamBufferError> {	8✔
151	if self.unescaped_len == 0 {	8✔
152	return Err(StreamBufferError::InvalidState(	1✔
153	"No unescaped content available",	1✔
154	));	1✔
155	}	7✔
156	self.buffer	7✔
157	.get(0..self.unescaped_len)	7✔
158	.ok_or(StreamBufferError::InvalidState(	7✔
159	"Unescaped length exceeds buffer size",	7✔
160	))	7✔
161	}	8✔
162
163	/// Clear unescaped content (call after yielding unescaped string)
164	pub fn clear_unescaped(&mut self) {	3✔
165	self.unescaped_len = 0;	3✔
166	}	3✔
167
168	/// Get current tokenize position (for string start tracking)
169	pub fn current_position(&self) -> usize {	106✔
170	self.tokenize_pos	106✔
171	}	106✔
172
173	/// Check if buffer is empty (no more data to process)
174	pub fn is_empty(&self) -> bool {	625✔
175	self.tokenize_pos >= self.data_end	625✔
176	}	625✔
177
178	/// Check if we have unescaped content ready
179	pub fn has_unescaped_content(&self) -> bool {	212✔
180	self.unescaped_len > 0	212✔
181	}	212✔
182
183	/// Append a single byte to the unescaped content
184	pub fn append_unescaped_byte(&mut self, byte: u8) -> Result<(), StreamBufferError> {	83✔
185	let available_space = self.buffer.len().saturating_sub(self.escape_reserve);	83✔
186	if self.unescaped_len >= available_space {	83✔
187	return Err(StreamBufferError::BufferFull);	3✔
188	}	80✔
189
190	if let Some(b) = self.buffer.get_mut(self.unescaped_len) {	80✔
191	*b = byte;	80✔
192	self.unescaped_len = self.unescaped_len.wrapping_add(1);	80✔
193	Ok(())	80✔
194	} else {
NEW 195	Err(StreamBufferError::BufferFull)	×
196	}
197	}	83✔
198
199	/// Get a string slice from the buffer (zero-copy)
200	/// Used for strings without escapes
201	pub fn get_string_slice(&self, start: usize, end: usize) -> Result<&[u8], StreamBufferError> {	68✔
202	if start > end \|\| end > self.data_end {	68✔
NEW 203	return Err(StreamBufferError::InvalidState("Invalid slice bounds"));	×
204	}	68✔
205	self.buffer	68✔
206	.get(start..end)	68✔
207	.ok_or(StreamBufferError::InvalidState("Invalid slice bounds"))	68✔
208	}	68✔
209	}
210
211	#[cfg(test)]
212	mod tests {
213	use super::*;
214
215	#[test]
216	fn test_lifetime_expectations() {	1✔
217	// This test demonstrates how StreamBuffer lifetimes should work
218	let mut buffer = [0u8; 100];	1✔
219	let mut stream_buffer = StreamBuffer::new(&mut buffer);	1✔
220
221	// Simulate some data being in the buffer
222	let test_data = b"hello world";	1✔
223	stream_buffer.buffer[0..test_data.len()].copy_from_slice(test_data);	1✔
224	stream_buffer.data_end = test_data.len();	1✔
225
226	// Test that we can get buffer data
227
228	// Test unescaped content - add some unescaped data
229	stream_buffer.unescaped_len = 3;	1✔
230	stream_buffer.buffer[0..3].copy_from_slice(b"abc");	1✔
231
232	let unescaped_slice = stream_buffer.get_unescaped_slice().unwrap();	1✔
233	assert_eq!(unescaped_slice, b"abc");	1✔
234
235	// The key expectation: these slices should live as long as the original buffer
236	// and be usable to create String::Borrowed(&'buffer str) and String::Unescaped(&'buffer str)
237	}	1✔
238
239	#[test]
240	fn test_new_stream_buffer() {	1✔
241	let mut buffer = [0u8; 100];	1✔
242	let db = StreamBuffer::new(&mut buffer);	1✔
243
244	assert_eq!(db.tokenize_pos, 0);	1✔
245	assert_eq!(db.data_end, 0);	1✔
246	assert_eq!(db.unescaped_len, 0);	1✔
247	assert_eq!(db.escape_reserve, 64); // 12.5% of 100, minimum 64	1✔
248	assert!(db.is_empty());	1✔
249	}	1✔
250
251	#[test]
252	fn test_fill_and_advance() {	1✔
253	let mut buffer = [0u8; 100];	1✔
254	let mut db = StreamBuffer::new(&mut buffer);	1✔
255
256	// Fill with some data
257	{	1✔
258	let fill_slice = db.get_fill_slice().unwrap();	1✔
259	fill_slice[0..5].copy_from_slice(b"hello");	1✔
260	}	1✔
261	db.mark_filled(5).unwrap();	1✔
262
263	assert_eq!(db.data_end, 5);	1✔
264	assert_eq!(db.remaining_bytes(), 5);	1✔
265
266	// Read bytes
267	assert_eq!(db.current_byte().unwrap(), b'h');	1✔
268	db.advance().unwrap();	1✔
269	assert_eq!(db.current_byte().unwrap(), b'e');	1✔
270	assert_eq!(db.remaining_bytes(), 4);	1✔
271	}	1✔
272
273	#[test]
274	fn test_error_conditions() {	1✔
275	let mut buffer = [0u8; 10];	1✔
276	let mut db = StreamBuffer::new(&mut buffer);	1✔
277
278	// EndOfData errors
279	assert_eq!(db.current_byte().unwrap_err(), StreamBufferError::EndOfData);	1✔
280	assert_eq!(db.advance().unwrap_err(), StreamBufferError::EndOfData);	1✔
281
282	// No unescaped content
283	assert!(db.get_unescaped_slice().is_err());	1✔
284	}	1✔
285
286	#[test]
287	fn test_buffer_full_scenario() {	1✔
288	// Test what happens when buffer gets completely full
289	let mut buffer = [0u8; 10];	1✔
290	let mut db = StreamBuffer::new(&mut buffer);	1✔
291
292	// Fill buffer completely
293	{	1✔
294	let fill_slice = db.get_fill_slice().unwrap();	1✔
295	fill_slice.copy_from_slice(b"0123456789");	1✔
296	}	1✔
297	db.mark_filled(10).unwrap();	1✔
298
299	// No more space for filling
300	assert!(db.get_fill_slice().is_none());	1✔
301
302	// We can still read from buffer
303	assert_eq!(db.current_byte().unwrap(), b'0');	1✔
304	assert_eq!(db.remaining_bytes(), 10);	1✔
305	}	1✔
306
307	#[test]
308	fn test_minimal_buffer_with_long_token() {	1✔
309	// Test very small buffer with a token that doesn't fit
310	let mut buffer = [0u8; 8]; // Very small buffer	1✔
311	let mut db = StreamBuffer::new(&mut buffer);	1✔
312
313	// Try to put a string that's almost as big as the buffer
314	{	1✔
315	let fill_slice = db.get_fill_slice().unwrap();	1✔
316	fill_slice[0..6].copy_from_slice(b"\"hello"); // Start of a long string, no closing quote	1✔
317	}	1✔
318	db.mark_filled(6).unwrap();	1✔
319
320	// Advance through the data
321	for _ in 0..6 {	7✔
322	db.advance().unwrap();	6✔
323	}	6✔
324
325	// Now buffer is exhausted but we don't have a complete token
326	assert!(db.is_empty());	1✔
327	assert_eq!(db.remaining_bytes(), 0);	1✔
328
329	// This simulates the scenario where we need more data but can't fit it
330	// The parser would need to handle this by buffering the incomplete token
331	}	1✔
332
333	#[test]
334	fn test_reader_returns_zero_bytes() {	1✔
335	let mut buffer = [0u8; 20];	1✔
336	let mut db = StreamBuffer::new(&mut buffer);	1✔
337
338	// Simulate Reader returning 0 bytes (EOF)
339	{
340	let fill_slice = db.get_fill_slice().unwrap();	1✔
341	assert_eq!(fill_slice.len(), 20);	1✔
342	// Reader returns 0 bytes - simulating EOF or no data available
343	}
344	db.mark_filled(0).unwrap(); // Reader returned 0	1✔
345
346	assert!(db.is_empty());	1✔
347	assert_eq!(db.data_end, 0);	1✔
348	assert_eq!(db.remaining_bytes(), 0);	1✔
349
350	// Should still be able to get fill slice for next attempt
351	let fill_slice = db.get_fill_slice().unwrap();	1✔
352	assert_eq!(fill_slice.len(), 20);	1✔
353	}	1✔
354
355	#[test]
356	fn test_maximum_escape_reserve_scenario() {	1✔
357	let mut buffer = [0u8; 100];	1✔
358	let db = StreamBuffer::new(&mut buffer);	1✔
359
360	// Check escape reserve calculation
361	assert_eq!(db.escape_reserve, 64); // max(100>>3, 64) = 64	1✔
362
363	// Test with smaller buffer
364	let mut small_buffer = [0u8; 50];	1✔
365	let small_db = StreamBuffer::new(&mut small_buffer);	1✔
366	assert_eq!(small_db.escape_reserve, 64); // Still 64 (minimum)	1✔
367
368	// Test with larger buffer
369	let mut large_buffer = [0u8; 1000];	1✔
370	let large_db = StreamBuffer::new(&mut large_buffer);	1✔
371	assert_eq!(large_db.escape_reserve, 125); // 1000 >> 3 = 125	1✔
372	}	1✔
373
374	#[test]
375	fn test_boundary_conditions() {	1✔
376	let mut buffer = [0u8; 3]; // Absolute minimum	1✔
377	let mut db = StreamBuffer::new(&mut buffer);	1✔
378
379	// Can't even hold a proper JSON token, but should not crash
380	{	1✔
381	let fill_slice = db.get_fill_slice().unwrap();	1✔
382	fill_slice.copy_from_slice(b"\"a\"");	1✔
383	}	1✔
384	db.mark_filled(3).unwrap();	1✔
385
386	// Should be able to read through it
387	assert_eq!(db.current_byte().unwrap(), b'"');	1✔
388	db.advance().unwrap();	1✔
389	assert_eq!(db.current_byte().unwrap(), b'a');	1✔
390	db.advance().unwrap();	1✔
391	assert_eq!(db.current_byte().unwrap(), b'"');	1✔
392	db.advance().unwrap();	1✔
393
394	assert!(db.is_empty());	1✔
395	}	1✔
396
397	#[test]
398	fn test_start_unescaping_with_copy_span_too_large() {	1✔
399	let mut buffer = [0u8; 10]; // Small buffer	1✔
400	let mut db = StreamBuffer::new(&mut buffer);	1✔
401
402	// Fill buffer with some data
403	{	1✔
404	let fill_slice = db.get_fill_slice().unwrap();	1✔
405	fill_slice.copy_from_slice(b"0123456789");	1✔
406	}	1✔
407	db.mark_filled(10).unwrap();	1✔
408
409	// Try to copy a span that's larger than the entire buffer
410	let copy_start = 0;	1✔
411	let copy_end = 15; // This span (15 bytes) is larger than buffer (10 bytes)	1✔
412	let max_escaped_len = 5; // This is fine	1✔
413
414	// Should return BufferFull error instead of silently truncating
415	let result = db.start_unescaping_with_copy(max_escaped_len, copy_start, copy_end);	1✔
416	assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);	1✔
417
418	// Test boundary case: span exactly equals buffer size should work
419	let copy_end_exact = 10; // Span of exactly 10 bytes (buffer size)	1✔
420	let result = db.start_unescaping_with_copy(max_escaped_len, 0, copy_end_exact);	1✔
421	assert!(result.is_ok());	1✔
422	assert_eq!(db.unescaped_len, 10);	1✔
423
424	// Test valid smaller span should work
425	db.clear_unescaped();	1✔
426	let result = db.start_unescaping_with_copy(max_escaped_len, 2, 6); // 4 byte span	1✔
427	assert!(result.is_ok());	1✔
428	assert_eq!(db.unescaped_len, 4);	1✔
429	assert_eq!(db.get_unescaped_slice().unwrap(), b"2345");	1✔
430	}	1✔
431
432	#[test]
433	fn test_append_unescaped_byte_respects_escape_reserve() {	1✔
434	let mut buffer = [0u8; 100]; // 100 byte buffer	1✔
435	let mut db = StreamBuffer::new(&mut buffer);	1✔
436
437	// Check escape reserve was set correctly (12.5% of 100, minimum 64)
438	assert_eq!(db.escape_reserve, 64);	1✔
439
440	// Should be able to append up to (buffer_len - escape_reserve) bytes
441	let max_unescaped = 100 - db.escape_reserve; // 100 - 64 = 36	1✔
442
443	// Fill up to the limit - should succeed
444	for i in 0..max_unescaped {	36✔
445	let result = db.append_unescaped_byte(b'A');	36✔
446	assert!(result.is_ok(), "Failed at byte {}", i);	36✔
447	}
448
449	assert_eq!(db.unescaped_len, max_unescaped);	1✔
450
451	// One more byte should fail due to escape reserve constraint
452	let result = db.append_unescaped_byte(b'B');	1✔
453	assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);	1✔
454
455	// Verify we didn't exceed the escape reserve boundary
456	assert_eq!(db.unescaped_len, max_unescaped);	1✔
457	}	1✔
458
459	#[test]
460	fn test_append_unescaped_byte_escape_reserve_larger_than_buffer() {	1✔
461	let mut buffer = [0u8; 10]; // Very small buffer	1✔
462	let mut db = StreamBuffer::new(&mut buffer);	1✔
463
464	// Even small buffers get minimum 64 byte escape reserve, but that's larger than buffer
465	assert_eq!(db.escape_reserve, 64); // minimum	1✔
466
467	// Since escape_reserve (64) > buffer.len() (10), no bytes should be appendable
468	// This should not panic with underflow, but return BufferFull error
469	let result = db.append_unescaped_byte(b'A');	1✔
470	assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);	1✔
471
472	// Test with even smaller buffer to ensure we handle underflow correctly
473	let mut tiny_buffer = [0u8; 3];	1✔
474	let mut tiny_db = StreamBuffer::new(&mut tiny_buffer);	1✔
475	assert_eq!(tiny_db.escape_reserve, 64); // Still minimum 64	1✔
476
477	// Should handle this gracefully without panic
478	let result = tiny_db.append_unescaped_byte(b'B');	1✔
479	assert_eq!(result.unwrap_err(), StreamBufferError::BufferFull);	1✔
480	}	1✔
481	}
482
483	impl crate::number_parser::NumberExtractor for StreamBuffer<'_> {
484	fn get_number_slice(	38✔
485	&self,	38✔
486	start: usize,	38✔
487	end: usize,	38✔
488	) -> Result<&[u8], crate::shared::ParseError> {	38✔
489	self.get_string_slice(start, end)	38✔
490	.map_err(\|_\| crate::shared::ParseError::UnexpectedState("Invalid number slice bounds"))	38✔
491	}	38✔
492
493	fn current_position(&self) -> usize {	38✔
494	self.tokenize_pos	38✔
495	}	38✔
496
497	fn is_empty(&self) -> bool {	21✔
498	self.tokenize_pos >= self.data_end	21✔
499	}	21✔
500	}

kaidokert / picojson-rs / 16092050615

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous