• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

facet-rs / facet / 19946635754

04 Dec 2025 10:56PM UTC coverage: 58.306% (+0.5%) from 57.787%
19946635754

Pull #1078

github

web-flow
Merge 286a539ef into cb87d63aa
Pull Request #1078: feat(facet-json): streaming JSON deserializer

1253 of 1541 new or added lines in 6 files covered. (81.31%)

8 existing lines in 2 files now uncovered.

23561 of 40409 relevant lines covered (58.31%)

499.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.88
/facet-json/src/reader.rs
1
//! High-level streaming JSON reader.
2
//!
3
//! `JsonReader` wraps `ScanBuffer` and `Scanner` to provide a convenient
4
//! streaming token iterator that handles buffer management automatically.
5
//!
6
//! # Design: No Compaction
7
//!
8
//! This reader uses a simple grow-or-reset strategy:
9
//! - When scanner returns a complete token: materialize it, continue
10
//! - When scanner returns Eof (end of buffer): reset buffer, refill, continue
11
//! - When scanner returns NeedMore (mid-token): grow buffer if full, refill, continue
12
//!
13
//! We never compact (shift data left) because:
14
//! - Reset handles the "all data processed" case
15
//! - Grow handles the "mid-token" case
16
//! - Scanner indices remain stable (no adjustment needed)
17

18
use alloc::string::String;
19

20
use crate::scan_buffer::ScanBuffer;
21
use crate::scanner::{
22
    ParsedNumber, ScanError, ScanErrorKind, Scanner, SpannedToken, Token as ScanToken,
23
    decode_string_owned, parse_number,
24
};
25
use facet_reflect::Span;
26

27
/// Error from JSON reader operations
28
#[derive(Debug)]
29
pub enum ReaderError {
30
    /// IO error during refill
31
    Io(std::io::Error),
32
    /// Scanner error
33
    Scan(ScanError),
34
}
35

36
impl From<std::io::Error> for ReaderError {
NEW
37
    fn from(err: std::io::Error) -> Self {
×
NEW
38
        ReaderError::Io(err)
×
NEW
39
    }
×
40
}
41

42
impl From<ScanError> for ReaderError {
NEW
43
    fn from(err: ScanError) -> Self {
×
NEW
44
        ReaderError::Scan(err)
×
NEW
45
    }
×
46
}
47

48
impl core::fmt::Display for ReaderError {
NEW
49
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
×
NEW
50
        match self {
×
NEW
51
            ReaderError::Io(e) => write!(f, "IO error: {e}"),
×
NEW
52
            ReaderError::Scan(e) => write!(f, "scan error: {:?}", e.kind),
×
53
        }
NEW
54
    }
×
55
}
56

57
impl std::error::Error for ReaderError {}
58

59
/// A materialized JSON token with its value decoded.
60
#[derive(Debug, Clone, PartialEq)]
61
pub enum JsonToken {
62
    /// `{`
63
    ObjectStart,
64
    /// `}`
65
    ObjectEnd,
66
    /// `[`
67
    ArrayStart,
68
    /// `]`
69
    ArrayEnd,
70
    /// `:`
71
    Colon,
72
    /// `,`
73
    Comma,
74
    /// `null`
75
    Null,
76
    /// `true`
77
    True,
78
    /// `false`
79
    False,
80
    /// String value (decoded, with escapes processed)
81
    String(String),
82
    /// Unsigned integer
83
    U64(u64),
84
    /// Signed integer
85
    I64(i64),
86
    /// Unsigned 128-bit integer
87
    U128(u128),
88
    /// Signed 128-bit integer
89
    I128(i128),
90
    /// Floating point number
91
    F64(f64),
92
    /// End of input
93
    Eof,
94
}
95

96
/// Spanned JSON token with location information
97
#[derive(Debug, Clone)]
98
pub struct SpannedJsonToken {
99
    /// The token
100
    pub token: JsonToken,
101
    /// Source span (byte offset from start of input)
102
    pub span: Span,
103
}
104

105
/// Streaming JSON reader that handles buffer management automatically.
106
#[cfg(feature = "std")]
107
pub struct JsonReader<R> {
108
    reader: R,
109
    buffer: ScanBuffer,
110
    scanner: Scanner,
111
    /// Total bytes processed (for span calculation across refills)
112
    bytes_processed: usize,
113
}
114

115
#[cfg(feature = "std")]
116
impl<R: std::io::Read> JsonReader<R> {
117
    /// Create a new streaming JSON reader.
118
    pub fn new(reader: R) -> Self {
3✔
119
        Self {
3✔
120
            reader,
3✔
121
            buffer: ScanBuffer::new(),
3✔
122
            scanner: Scanner::new(),
3✔
123
            bytes_processed: 0,
3✔
124
        }
3✔
125
    }
3✔
126

127
    /// Create a new streaming JSON reader with custom buffer capacity.
128
    pub fn with_capacity(reader: R, capacity: usize) -> Self {
4✔
129
        Self {
4✔
130
            reader,
4✔
131
            buffer: ScanBuffer::with_capacity(capacity),
4✔
132
            scanner: Scanner::new(),
4✔
133
            bytes_processed: 0,
4✔
134
        }
4✔
135
    }
4✔
136

137
    /// Read the next token from the stream.
138
    pub fn next_token(&mut self) -> Option<Result<SpannedJsonToken, ReaderError>> {
52✔
139
        loop {
140
            // Ensure we have data to scan
141
            if self.buffer.filled() == 0 && !self.buffer.is_eof() {
77✔
142
                match self.buffer.refill(&mut self.reader) {
7✔
143
                    Ok(0) => {
NEW
144
                        return Some(Ok(SpannedJsonToken {
×
NEW
145
                            token: JsonToken::Eof,
×
NEW
146
                            span: Span::new(self.bytes_processed, 0),
×
NEW
147
                        }));
×
148
                    }
149
                    Ok(_) => {}
7✔
NEW
150
                    Err(e) => return Some(Err(ReaderError::Io(e))),
×
151
                }
152
            }
70✔
153

154
            let result = self.scanner.next_token(self.buffer.data());
77✔
155

156
            match result {
77✔
157
                Ok(spanned) => {
77✔
158
                    match &spanned.token {
77✔
159
                        ScanToken::NeedMore { .. } => {
160
                            // Mid-token, need more data
161
                            // Grow buffer if it's full, then refill
162
                            if self.buffer.filled() == self.buffer.capacity() {
16✔
163
                                self.buffer.grow();
5✔
164
                            }
11✔
165

166
                            match self.buffer.refill(&mut self.reader) {
16✔
NEW
167
                                Ok(0) if self.buffer.is_eof() => {
×
168
                                    // True EOF while mid-token = error
NEW
169
                                    return Some(Err(ReaderError::Scan(ScanError {
×
NEW
170
                                        kind: ScanErrorKind::UnexpectedEof("incomplete token"),
×
NEW
171
                                        span: Span::new(self.bytes_processed, 0),
×
NEW
172
                                    })));
×
173
                                }
174
                                Ok(_) => continue,
16✔
NEW
175
                                Err(e) => return Some(Err(ReaderError::Io(e))),
×
176
                            }
177
                        }
178
                        ScanToken::Eof => {
179
                            // Scanner reached end of buffer
180
                            if !self.buffer.is_eof() {
16✔
181
                                // Not true EOF - all tokens processed, reset and refill
182
                                self.bytes_processed += self.scanner.pos();
16✔
183
                                self.buffer.reset();
16✔
184
                                self.scanner.set_pos(0);
16✔
185

186
                                match self.buffer.refill(&mut self.reader) {
16✔
187
                                    Ok(0) => {
188
                                        return Some(Ok(SpannedJsonToken {
7✔
189
                                            token: JsonToken::Eof,
7✔
190
                                            span: Span::new(self.bytes_processed, 0),
7✔
191
                                        }));
7✔
192
                                    }
193
                                    Ok(_) => continue,
9✔
NEW
194
                                    Err(e) => return Some(Err(ReaderError::Io(e))),
×
195
                                }
NEW
196
                            }
×
197
                            // True EOF
NEW
198
                            return Some(Ok(SpannedJsonToken {
×
NEW
199
                                token: JsonToken::Eof,
×
NEW
200
                                span: Span::new(self.bytes_processed + spanned.span.offset, 0),
×
NEW
201
                            }));
×
202
                        }
203
                        _ => {
204
                            // Complete token - materialize it
205
                            return Some(self.materialize_token(&spanned));
45✔
206
                        }
207
                    }
208
                }
NEW
209
                Err(e) => {
×
NEW
210
                    return Some(Err(ReaderError::Scan(ScanError {
×
NEW
211
                        kind: e.kind,
×
NEW
212
                        span: Span::new(self.bytes_processed + e.span.offset, e.span.len),
×
NEW
213
                    })));
×
214
                }
215
            }
216
        }
217
    }
52✔
218

219
    fn materialize_token(&self, spanned: &SpannedToken) -> Result<SpannedJsonToken, ReaderError> {
45✔
220
        let buf = self.buffer.data();
45✔
221
        let span = Span::new(self.bytes_processed + spanned.span.offset, spanned.span.len);
45✔
222

223
        let token = match &spanned.token {
45✔
224
            ScanToken::ObjectStart => JsonToken::ObjectStart,
5✔
225
            ScanToken::ObjectEnd => JsonToken::ObjectEnd,
5✔
226
            ScanToken::ArrayStart => JsonToken::ArrayStart,
2✔
227
            ScanToken::ArrayEnd => JsonToken::ArrayEnd,
2✔
228
            ScanToken::Colon => JsonToken::Colon,
6✔
229
            ScanToken::Comma => JsonToken::Comma,
6✔
NEW
230
            ScanToken::Null => JsonToken::Null,
×
NEW
231
            ScanToken::True => JsonToken::True,
×
NEW
232
            ScanToken::False => JsonToken::False,
×
233
            ScanToken::String { start, end, .. } => {
11✔
234
                let s = decode_string_owned(buf, *start, *end).map_err(ReaderError::Scan)?;
11✔
235
                JsonToken::String(s)
11✔
236
            }
237
            ScanToken::Number { start, end, hint } => {
8✔
238
                let parsed = parse_number(buf, *start, *end, *hint).map_err(ReaderError::Scan)?;
8✔
239
                match parsed {
8✔
240
                    ParsedNumber::U64(n) => JsonToken::U64(n),
5✔
241
                    ParsedNumber::I64(n) => JsonToken::I64(n),
1✔
NEW
242
                    ParsedNumber::U128(n) => JsonToken::U128(n),
×
NEW
243
                    ParsedNumber::I128(n) => JsonToken::I128(n),
×
244
                    ParsedNumber::F64(n) => JsonToken::F64(n),
2✔
245
                }
246
            }
NEW
247
            ScanToken::Eof | ScanToken::NeedMore { .. } => unreachable!(),
×
248
        };
249

250
        Ok(SpannedJsonToken { token, span })
45✔
251
    }
45✔
252
}
253

254
/// Async streaming JSON reader for tokio.
255
#[cfg(feature = "tokio")]
256
pub struct AsyncJsonReader<R> {
257
    reader: R,
258
    buffer: ScanBuffer,
259
    scanner: Scanner,
260
    bytes_processed: usize,
261
}
262

263
#[cfg(feature = "tokio")]
264
impl<R: tokio::io::AsyncRead + Unpin> AsyncJsonReader<R> {
265
    /// Create a new async streaming JSON reader.
266
    pub fn new(reader: R) -> Self {
267
        Self {
268
            reader,
269
            buffer: ScanBuffer::new(),
270
            scanner: Scanner::new(),
271
            bytes_processed: 0,
272
        }
273
    }
274

275
    /// Create with custom buffer capacity.
276
    pub fn with_capacity(reader: R, capacity: usize) -> Self {
277
        Self {
278
            reader,
279
            buffer: ScanBuffer::with_capacity(capacity),
280
            scanner: Scanner::new(),
281
            bytes_processed: 0,
282
        }
283
    }
284

285
    /// Read the next token asynchronously.
286
    pub async fn next_token(&mut self) -> Option<Result<SpannedJsonToken, ReaderError>> {
287
        loop {
288
            if self.buffer.filled() == 0 && !self.buffer.is_eof() {
289
                match self.buffer.refill_tokio(&mut self.reader).await {
290
                    Ok(0) => {
291
                        return Some(Ok(SpannedJsonToken {
292
                            token: JsonToken::Eof,
293
                            span: Span::new(self.bytes_processed, 0),
294
                        }));
295
                    }
296
                    Ok(_) => {}
297
                    Err(e) => return Some(Err(ReaderError::Io(e))),
298
                }
299
            }
300

301
            let result = self.scanner.next_token(self.buffer.data());
302

303
            match result {
304
                Ok(spanned) => match &spanned.token {
305
                    ScanToken::NeedMore { .. } => {
306
                        if self.buffer.filled() == self.buffer.capacity() {
307
                            self.buffer.grow();
308
                        }
309
                        match self.buffer.refill_tokio(&mut self.reader).await {
310
                            Ok(0) if self.buffer.is_eof() => {
311
                                return Some(Err(ReaderError::Scan(ScanError {
312
                                    kind: ScanErrorKind::UnexpectedEof("incomplete token"),
313
                                    span: Span::new(self.bytes_processed, 0),
314
                                })));
315
                            }
316
                            Ok(_) => continue,
317
                            Err(e) => return Some(Err(ReaderError::Io(e))),
318
                        }
319
                    }
320
                    ScanToken::Eof => {
321
                        if !self.buffer.is_eof() {
322
                            self.bytes_processed += self.scanner.pos();
323
                            self.buffer.reset();
324
                            self.scanner.set_pos(0);
325
                            match self.buffer.refill_tokio(&mut self.reader).await {
326
                                Ok(0) => {
327
                                    return Some(Ok(SpannedJsonToken {
328
                                        token: JsonToken::Eof,
329
                                        span: Span::new(self.bytes_processed, 0),
330
                                    }));
331
                                }
332
                                Ok(_) => continue,
333
                                Err(e) => return Some(Err(ReaderError::Io(e))),
334
                            }
335
                        }
336
                        return Some(Ok(SpannedJsonToken {
337
                            token: JsonToken::Eof,
338
                            span: Span::new(self.bytes_processed + spanned.span.offset, 0),
339
                        }));
340
                    }
341
                    _ => {
342
                        return Some(self.materialize_token(&spanned));
343
                    }
344
                },
345
                Err(e) => {
346
                    return Some(Err(ReaderError::Scan(ScanError {
347
                        kind: e.kind,
348
                        span: Span::new(self.bytes_processed + e.span.offset, e.span.len),
349
                    })));
350
                }
351
            }
352
        }
353
    }
354

355
    fn materialize_token(&self, spanned: &SpannedToken) -> Result<SpannedJsonToken, ReaderError> {
356
        let buf = self.buffer.data();
357
        let span = Span::new(self.bytes_processed + spanned.span.offset, spanned.span.len);
358

359
        let token = match &spanned.token {
360
            ScanToken::ObjectStart => JsonToken::ObjectStart,
361
            ScanToken::ObjectEnd => JsonToken::ObjectEnd,
362
            ScanToken::ArrayStart => JsonToken::ArrayStart,
363
            ScanToken::ArrayEnd => JsonToken::ArrayEnd,
364
            ScanToken::Colon => JsonToken::Colon,
365
            ScanToken::Comma => JsonToken::Comma,
366
            ScanToken::Null => JsonToken::Null,
367
            ScanToken::True => JsonToken::True,
368
            ScanToken::False => JsonToken::False,
369
            ScanToken::String { start, end, .. } => {
370
                let s = decode_string_owned(buf, *start, *end).map_err(ReaderError::Scan)?;
371
                JsonToken::String(s)
372
            }
373
            ScanToken::Number { start, end, hint } => {
374
                let parsed = parse_number(buf, *start, *end, *hint).map_err(ReaderError::Scan)?;
375
                match parsed {
376
                    ParsedNumber::U64(n) => JsonToken::U64(n),
377
                    ParsedNumber::I64(n) => JsonToken::I64(n),
378
                    ParsedNumber::U128(n) => JsonToken::U128(n),
379
                    ParsedNumber::I128(n) => JsonToken::I128(n),
380
                    ParsedNumber::F64(n) => JsonToken::F64(n),
381
                }
382
            }
383
            ScanToken::Eof | ScanToken::NeedMore { .. } => unreachable!(),
384
        };
385

386
        Ok(SpannedJsonToken { token, span })
387
    }
388
}
389

390
/// Async streaming JSON reader for futures-io (smol, async-std).
391
#[cfg(feature = "futures-io")]
392
pub struct FuturesJsonReader<R> {
393
    reader: R,
394
    buffer: ScanBuffer,
395
    scanner: Scanner,
396
    bytes_processed: usize,
397
}
398

399
#[cfg(feature = "futures-io")]
400
impl<R: futures_io::AsyncRead + Unpin> FuturesJsonReader<R> {
401
    /// Create a new async streaming JSON reader.
402
    pub fn new(reader: R) -> Self {
403
        Self {
404
            reader,
405
            buffer: ScanBuffer::new(),
406
            scanner: Scanner::new(),
407
            bytes_processed: 0,
408
        }
409
    }
410

411
    /// Create with custom buffer capacity.
412
    pub fn with_capacity(reader: R, capacity: usize) -> Self {
413
        Self {
414
            reader,
415
            buffer: ScanBuffer::with_capacity(capacity),
416
            scanner: Scanner::new(),
417
            bytes_processed: 0,
418
        }
419
    }
420

421
    /// Read the next token asynchronously.
422
    pub async fn next_token(&mut self) -> Option<Result<SpannedJsonToken, ReaderError>> {
423
        loop {
424
            if self.buffer.filled() == 0 && !self.buffer.is_eof() {
425
                match self.buffer.refill_futures(&mut self.reader).await {
426
                    Ok(0) => {
427
                        return Some(Ok(SpannedJsonToken {
428
                            token: JsonToken::Eof,
429
                            span: Span::new(self.bytes_processed, 0),
430
                        }));
431
                    }
432
                    Ok(_) => {}
433
                    Err(e) => return Some(Err(ReaderError::Io(e))),
434
                }
435
            }
436

437
            let result = self.scanner.next_token(self.buffer.data());
438

439
            match result {
440
                Ok(spanned) => match &spanned.token {
441
                    ScanToken::NeedMore { .. } => {
442
                        if self.buffer.filled() == self.buffer.capacity() {
443
                            self.buffer.grow();
444
                        }
445
                        match self.buffer.refill_futures(&mut self.reader).await {
446
                            Ok(0) if self.buffer.is_eof() => {
447
                                return Some(Err(ReaderError::Scan(ScanError {
448
                                    kind: ScanErrorKind::UnexpectedEof("incomplete token"),
449
                                    span: Span::new(self.bytes_processed, 0),
450
                                })));
451
                            }
452
                            Ok(_) => continue,
453
                            Err(e) => return Some(Err(ReaderError::Io(e))),
454
                        }
455
                    }
456
                    ScanToken::Eof => {
457
                        if !self.buffer.is_eof() {
458
                            self.bytes_processed += self.scanner.pos();
459
                            self.buffer.reset();
460
                            self.scanner.set_pos(0);
461
                            match self.buffer.refill_futures(&mut self.reader).await {
462
                                Ok(0) => {
463
                                    return Some(Ok(SpannedJsonToken {
464
                                        token: JsonToken::Eof,
465
                                        span: Span::new(self.bytes_processed, 0),
466
                                    }));
467
                                }
468
                                Ok(_) => continue,
469
                                Err(e) => return Some(Err(ReaderError::Io(e))),
470
                            }
471
                        }
472
                        return Some(Ok(SpannedJsonToken {
473
                            token: JsonToken::Eof,
474
                            span: Span::new(self.bytes_processed + spanned.span.offset, 0),
475
                        }));
476
                    }
477
                    _ => {
478
                        return Some(self.materialize_token(&spanned));
479
                    }
480
                },
481
                Err(e) => {
482
                    return Some(Err(ReaderError::Scan(ScanError {
483
                        kind: e.kind,
484
                        span: Span::new(self.bytes_processed + e.span.offset, e.span.len),
485
                    })));
486
                }
487
            }
488
        }
489
    }
490

491
    fn materialize_token(&self, spanned: &SpannedToken) -> Result<SpannedJsonToken, ReaderError> {
492
        let buf = self.buffer.data();
493
        let span = Span::new(self.bytes_processed + spanned.span.offset, spanned.span.len);
494

495
        let token = match &spanned.token {
496
            ScanToken::ObjectStart => JsonToken::ObjectStart,
497
            ScanToken::ObjectEnd => JsonToken::ObjectEnd,
498
            ScanToken::ArrayStart => JsonToken::ArrayStart,
499
            ScanToken::ArrayEnd => JsonToken::ArrayEnd,
500
            ScanToken::Colon => JsonToken::Colon,
501
            ScanToken::Comma => JsonToken::Comma,
502
            ScanToken::Null => JsonToken::Null,
503
            ScanToken::True => JsonToken::True,
504
            ScanToken::False => JsonToken::False,
505
            ScanToken::String { start, end, .. } => {
506
                let s = decode_string_owned(buf, *start, *end).map_err(ReaderError::Scan)?;
507
                JsonToken::String(s)
508
            }
509
            ScanToken::Number { start, end, hint } => {
510
                let parsed = parse_number(buf, *start, *end, *hint).map_err(ReaderError::Scan)?;
511
                match parsed {
512
                    ParsedNumber::U64(n) => JsonToken::U64(n),
513
                    ParsedNumber::I64(n) => JsonToken::I64(n),
514
                    ParsedNumber::U128(n) => JsonToken::U128(n),
515
                    ParsedNumber::I128(n) => JsonToken::I128(n),
516
                    ParsedNumber::F64(n) => JsonToken::F64(n),
517
                }
518
            }
519
            ScanToken::Eof | ScanToken::NeedMore { .. } => unreachable!(),
520
        };
521

522
        Ok(SpannedJsonToken { token, span })
523
    }
524
}
525

526
// ============================================================================
527
// Tests
528
// ============================================================================
529

530
#[cfg(all(test, feature = "std"))]
531
mod tests {
532
    use super::*;
533
    use std::io::{Cursor, Read};
534

535
    /// A reader wrapper that simulates short reads by returning at most N bytes per read.
536
    struct ShortReadAdapter<R> {
537
        inner: R,
538
        max_bytes_per_read: usize,
539
    }
540

541
    impl<R> ShortReadAdapter<R> {
542
        fn new(inner: R, max_bytes_per_read: usize) -> Self {
3✔
543
            Self {
3✔
544
                inner,
3✔
545
                max_bytes_per_read,
3✔
546
            }
3✔
547
        }
3✔
548
    }
549

550
    impl<R: Read> Read for ShortReadAdapter<R> {
551
        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
28✔
552
            let len = buf.len().min(self.max_bytes_per_read);
28✔
553
            self.inner.read(&mut buf[..len])
28✔
554
        }
28✔
555
    }
556

557
    fn collect_tokens<R: Read>(reader: &mut JsonReader<R>) -> Vec<JsonToken> {
7✔
558
        let mut tokens = Vec::new();
7✔
559
        loop {
560
            let result = reader.next_token().unwrap().unwrap();
52✔
561
            let is_eof = matches!(result.token, JsonToken::Eof);
52✔
562
            tokens.push(result.token);
52✔
563
            if is_eof {
52✔
564
                break;
7✔
565
            }
45✔
566
        }
567
        tokens
7✔
568
    }
7✔
569

570
    #[test]
571
    fn test_simple() {
1✔
572
        let json = r#"{"name": "test", "value": 42}"#;
1✔
573
        let mut reader = JsonReader::new(Cursor::new(json));
1✔
574
        let tokens = collect_tokens(&mut reader);
1✔
575

576
        assert_eq!(
1✔
577
            tokens,
578
            vec![
1✔
579
                JsonToken::ObjectStart,
1✔
580
                JsonToken::String("name".to_string()),
1✔
581
                JsonToken::Colon,
1✔
582
                JsonToken::String("test".to_string()),
1✔
583
                JsonToken::Comma,
1✔
584
                JsonToken::String("value".to_string()),
1✔
585
                JsonToken::Colon,
1✔
586
                JsonToken::U64(42),
1✔
587
                JsonToken::ObjectEnd,
1✔
588
                JsonToken::Eof,
1✔
589
            ]
590
        );
591
    }
1✔
592

593
    #[test]
594
    fn test_small_buffer() {
1✔
595
        // Use a tiny 4-byte buffer - forces buffer growth for strings
596
        let json = r#"{"hello": "world"}"#;
1✔
597
        let mut reader = JsonReader::with_capacity(Cursor::new(json), 4);
1✔
598
        let tokens = collect_tokens(&mut reader);
1✔
599

600
        assert_eq!(
1✔
601
            tokens,
602
            vec![
1✔
603
                JsonToken::ObjectStart,
1✔
604
                JsonToken::String("hello".to_string()),
1✔
605
                JsonToken::Colon,
1✔
606
                JsonToken::String("world".to_string()),
1✔
607
                JsonToken::ObjectEnd,
1✔
608
                JsonToken::Eof,
1✔
609
            ]
610
        );
611
    }
1✔
612

613
    #[test]
614
    fn test_short_reads() {
1✔
615
        // Simulate network-like conditions: only 1-2 bytes per read
616
        let json = r#"{"hello": "world"}"#;
1✔
617
        let adapter = ShortReadAdapter::new(Cursor::new(json), 2);
1✔
618
        let mut reader = JsonReader::with_capacity(adapter, 4);
1✔
619
        let tokens = collect_tokens(&mut reader);
1✔
620

621
        assert_eq!(
1✔
622
            tokens,
623
            vec![
1✔
624
                JsonToken::ObjectStart,
1✔
625
                JsonToken::String("hello".to_string()),
1✔
626
                JsonToken::Colon,
1✔
627
                JsonToken::String("world".to_string()),
1✔
628
                JsonToken::ObjectEnd,
1✔
629
                JsonToken::Eof,
1✔
630
            ]
631
        );
632
    }
1✔
633

634
    #[test]
635
    fn test_single_byte_reads() {
1✔
636
        // Extreme case: 1 byte at a time
637
        let json = r#"[1, 2, 3]"#;
1✔
638
        let adapter = ShortReadAdapter::new(Cursor::new(json), 1);
1✔
639
        let mut reader = JsonReader::with_capacity(adapter, 2);
1✔
640
        let tokens = collect_tokens(&mut reader);
1✔
641

642
        assert_eq!(
1✔
643
            tokens,
644
            vec![
1✔
645
                JsonToken::ArrayStart,
1✔
646
                JsonToken::U64(1),
1✔
647
                JsonToken::Comma,
1✔
648
                JsonToken::U64(2),
1✔
649
                JsonToken::Comma,
1✔
650
                JsonToken::U64(3),
1✔
651
                JsonToken::ArrayEnd,
1✔
652
                JsonToken::Eof,
1✔
653
            ]
654
        );
655
    }
1✔
656

657
    #[test]
658
    fn test_numbers() {
1✔
659
        let json = r#"[1, -5, 3.14, 1e10]"#;
1✔
660
        let mut reader = JsonReader::new(Cursor::new(json));
1✔
661
        let tokens = collect_tokens(&mut reader);
1✔
662

663
        assert!(matches!(tokens[1], JsonToken::U64(1)));
1✔
664
        assert!(matches!(tokens[3], JsonToken::I64(-5)));
1✔
665
        assert!(matches!(tokens[5], JsonToken::F64(_)));
1✔
666
        assert!(matches!(tokens[7], JsonToken::F64(_)));
1✔
667
    }
1✔
668

669
    #[test]
670
    fn test_escapes() {
1✔
671
        let json = r#"{"msg": "hello\nworld"}"#;
1✔
672
        let mut reader = JsonReader::new(Cursor::new(json));
1✔
673
        let tokens = collect_tokens(&mut reader);
1✔
674

675
        assert_eq!(tokens[3], JsonToken::String("hello\nworld".to_string()));
1✔
676
    }
1✔
677

678
    #[test]
679
    fn test_escapes_with_short_reads() {
1✔
680
        // Escapes spanning read boundaries
681
        let json = r#"{"msg": "a\nb\tc"}"#;
1✔
682
        let adapter = ShortReadAdapter::new(Cursor::new(json), 3);
1✔
683
        let mut reader = JsonReader::with_capacity(adapter, 4);
1✔
684
        let tokens = collect_tokens(&mut reader);
1✔
685

686
        assert_eq!(tokens[3], JsonToken::String("a\nb\tc".to_string()));
1✔
687
    }
1✔
688
}
689

690
#[cfg(all(test, feature = "tokio"))]
691
mod tokio_tests {
692
    use super::*;
693
    use std::io::Cursor;
694

695
    #[tokio::test]
696
    async fn test_async_simple() {
697
        let json = r#"{"name": "test", "value": 42}"#;
698
        let cursor = Cursor::new(json.as_bytes().to_vec());
699
        let mut reader = AsyncJsonReader::new(cursor);
700

701
        let mut tokens = Vec::new();
702
        loop {
703
            let result = reader.next_token().await.unwrap().unwrap();
704
            let is_eof = matches!(result.token, JsonToken::Eof);
705
            tokens.push(result.token);
706
            if is_eof {
707
                break;
708
            }
709
        }
710

711
        assert_eq!(
712
            tokens,
713
            vec![
714
                JsonToken::ObjectStart,
715
                JsonToken::String("name".to_string()),
716
                JsonToken::Colon,
717
                JsonToken::String("test".to_string()),
718
                JsonToken::Comma,
719
                JsonToken::String("value".to_string()),
720
                JsonToken::Colon,
721
                JsonToken::U64(42),
722
                JsonToken::ObjectEnd,
723
                JsonToken::Eof,
724
            ]
725
        );
726
    }
727

728
    #[tokio::test]
729
    async fn test_async_small_buffer() {
730
        let json = r#"{"hello": "world"}"#;
731
        let cursor = Cursor::new(json.as_bytes().to_vec());
732
        let mut reader = AsyncJsonReader::with_capacity(cursor, 4);
733

734
        let mut tokens = Vec::new();
735
        loop {
736
            let result = reader.next_token().await.unwrap().unwrap();
737
            let is_eof = matches!(result.token, JsonToken::Eof);
738
            tokens.push(result.token);
739
            if is_eof {
740
                break;
741
            }
742
        }
743

744
        assert_eq!(
745
            tokens,
746
            vec![
747
                JsonToken::ObjectStart,
748
                JsonToken::String("hello".to_string()),
749
                JsonToken::Colon,
750
                JsonToken::String("world".to_string()),
751
                JsonToken::ObjectEnd,
752
                JsonToken::Eof,
753
            ]
754
        );
755
    }
756
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc