23177332371

Committed 17 Mar 2026 03:41AM UTC coverage: 94.165% (-0.5%) from 94.685%

Build # 23177332371

Build Type

push

github

Committed by

jzombie

Commit Message

Prepare for 0.5.0-alpha

Coverage Stats

18899 of 20070 relevant lines covered (94.17%)

2323.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.41

/src/utils.rs

//! Text normalization helpers shared by source implementations.

use chrono::{DateTime, Utc};
use std::fs;
use std::path::Path;

use crate::data::{RecordSection, SectionRole};
use crate::types::Sentence;

/// Collapse repeated whitespace in-place while preserving single spaces.
/// Collapse runs of whitespace into single spaces and trim.
pub fn normalize_inline_whitespace<T: AsRef<str>>(text: T) -> String {
    let mut normalized = String::new();
    let mut seen_space = false;
    for ch in text.as_ref().chars() {
        if ch.is_whitespace() {
            if !seen_space {
                normalized.push(' ');
                seen_space = true;
            }
        } else {
            normalized.push(ch);
            seen_space = false;
        }
    }
    normalized.trim().to_string()
}

/// Split a block of text into sentences, falling back to the whole string when needed.
/// Heuristic sentence splitter with tokenizer-friendly rules.
pub fn sentences(text: &str) -> Vec<Sentence> {
    let mut results = Vec::new();

    for block in text.split("\n\n") {
        if block.trim().is_empty() {
            continue;
        }
        let normalized = normalize_inline_whitespace(block);
        if normalized.is_empty() {
            continue;
        }
        push_block_sentences(&normalized, &mut results);
    }

    results
}

/// Convenience helper to construct a `RecordSection` with normalized text metadata.
/// Convenience helper to build a `RecordSection` with precomputed sentences.
pub fn make_section(role: SectionRole, heading: Option<&str>, text: &str) -> RecordSection {
    RecordSection {
        role,
        heading: heading.map(|h| h.to_string()),
        text: text.to_string(),
        sentences: sentences(text),
    }
}

fn push_block_sentences(block: &str, results: &mut Vec<Sentence>) {
    let chars: Vec<char> = block.chars().collect();
    let mut buffer = String::new();

    for (idx, ch) in chars.iter().enumerate() {
        buffer.push(*ch);
        if is_sentence_boundary(&chars, idx) {
            let trimmed = buffer.trim();
            if !trimmed.is_empty() {
                results.push(trimmed.to_string());
            }
            buffer.clear();
        }
    }

    let trailing = buffer.trim();
    if !trailing.is_empty() {
        results.push(trailing.to_string());
    }
}

fn is_sentence_boundary(chars: &[char], idx: usize) -> bool {
    match chars[idx] {
        '.' => is_dot_boundary(chars, idx),
        '!' | '?' => true,
        _ => false,
    }
}

fn is_dot_boundary(chars: &[char], idx: usize) -> bool {
    if is_decimal_middle(chars, idx) || is_ticker_middle(chars, idx) {
        return false;
    }
    if idx + 1 < chars.len() && chars[idx + 1] == '.' {
        return false;
    }
    true
}

fn is_decimal_middle(chars: &[char], idx: usize) -> bool {
    idx > 0
        && idx + 1 < chars.len()
        && chars[idx - 1].is_ascii_digit()
        && chars[idx + 1].is_ascii_digit()
}

fn is_ticker_middle(chars: &[char], idx: usize) -> bool {
    idx > 0
        && idx + 1 < chars.len()
        && is_ticker_char(chars[idx - 1])
        && is_ticker_char(chars[idx + 1])
}

fn is_ticker_char(ch: char) -> bool {
    ch.is_ascii_uppercase() || ch.is_ascii_digit()
}

// ---------------------------------------------------------------------------
// Filesystem helpers
// ---------------------------------------------------------------------------

/// True if the path has a `.txt` extension (case-insensitive).
pub fn is_text_file(path: &Path) -> bool {
    path.extension()
        .and_then(|ext| ext.to_str())
        .map(|ext| ext.eq_ignore_ascii_case("txt"))
        .unwrap_or(false)
}

/// Best-effort file modified time.
pub fn file_mtime(path: &Path) -> Option<DateTime<Utc>> {
    let metadata = fs::metadata(path).ok()?;
    let modified = metadata.modified().ok()?;
    Some(system_time_to_utc(modified))
}

/// Best-effort (created_at, updated_at) pair for a file.
pub fn file_times(path: &Path) -> (DateTime<Utc>, DateTime<Utc>) {
    let metadata = fs::metadata(path).ok();
    let updated_at = metadata
        .as_ref()
        .and_then(|meta| meta.modified().ok())
        .map(system_time_to_utc)
        .unwrap_or_else(Utc::now);
    let created_at = metadata
        .and_then(|meta| meta.created().ok())
        .map(system_time_to_utc)
        .unwrap_or(updated_at);
    (created_at, updated_at)
}

fn system_time_to_utc(time: std::time::SystemTime) -> DateTime<Utc> {
    DateTime::<Utc>::from(time)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn normalize_inline_whitespace_collapses_runs() {
        let input = "Alpha\n\n  Beta\tGamma";
        assert_eq!(normalize_inline_whitespace(input), "Alpha Beta Gamma");
    }

    #[test]
    fn sentences_falls_back_to_full_text_when_needed() {
        let text = "   \n";
        let result = sentences(text);
        assert!(result.is_empty());

        let text2 = "Single block without punctuation";
        let result2 = sentences(text2);
        assert_eq!(
            result2,
            vec![String::from("Single block without punctuation")]
        );
    }

    #[test]
    fn make_section_populates_sentences() {
        let section = make_section(SectionRole::Context, Some("Summary"), "Line one. Line two!");
        assert_eq!(section.heading.as_deref(), Some("Summary"));
        assert_eq!(section.sentences.len(), 2);
        assert_eq!(section.role, SectionRole::Context);
    }

    #[test]
    fn sentences_keep_decimal_values_together() {
        let text = "Price closed at 3.14. Outlook improved.";
        let result = sentences(text);
        assert_eq!(result, vec!["Price closed at 3.14.", "Outlook improved."]);
    }

    #[test]
    fn sentences_keep_dot_tickers_together() {
        let text = "BRK.B rallied while RDS.A lagged.";
        let result = sentences(text);
        assert_eq!(result, vec!["BRK.B rallied while RDS.A lagged."]);
    }

    #[test]
    fn file_time_helpers_handle_existing_and_missing_paths() {
        use tempfile::tempdir;
        let temp = tempdir().unwrap();
        let existing = temp.path().join("exists.txt");
        std::fs::write(&existing, "hello").unwrap();

        assert!(file_mtime(&existing).is_some());
        let (created_at, updated_at) = file_times(&existing);
        assert!(updated_at >= created_at);

        let missing = temp.path().join("missing.txt");
        assert!(file_mtime(&missing).is_none());
        let (missing_created, missing_updated) = file_times(&missing);
        assert!(missing_updated >= missing_created);
    }

    #[test]
    fn sentences_treat_blank_line_as_boundary() {
        let text = "First line without punctuation\n\nSecond line with more context.";
        let result = sentences(text);
        assert_eq!(
            result,
            vec![
                "First line without punctuation".to_string(),
                "Second line with more context.".to_string()
            ]
        );
    }

    #[test]
    fn sentences_keep_ellipsis_together() {
        let text = "Wait... really? Yes.";
        let result = sentences(text);
        assert_eq!(result, vec!["Wait...", "really?", "Yes."]);
    }

    #[test]
    fn is_text_file_matches_txt_case_insensitively() {
        use std::path::PathBuf;
        assert!(is_text_file(&PathBuf::from("hello.txt")));
        assert!(is_text_file(&PathBuf::from("hello.TXT")));
        assert!(is_text_file(&PathBuf::from("hello.Txt")));
        assert!(!is_text_file(&PathBuf::from("hello.md")));
        assert!(!is_text_file(&PathBuf::from("hello")));
    }
}

1	//! Text normalization helpers shared by source implementations.
2
3	use chrono::{DateTime, Utc};
4	use std::fs;
5	use std::path::Path;
6
7	use crate::data::{RecordSection, SectionRole};
8	use crate::types::Sentence;
9
10	/// Collapse repeated whitespace in-place while preserving single spaces.
11	/// Collapse runs of whitespace into single spaces and trim.
12	pub fn normalize_inline_whitespace<T: AsRef<str>>(text: T) -> String {	13,905✔
13	let mut normalized = String::new();	13,905✔
14	let mut seen_space = false;	13,905✔
15	for ch in text.as_ref().chars() {	262,399✔
16	if ch.is_whitespace() {	262,399✔
17	if !seen_space {	37,845✔
18	normalized.push(' ');	37,839✔
19	seen_space = true;	37,839✔
20	}	37,839✔
21	} else {	224,554✔
22	normalized.push(ch);	224,554✔
23	seen_space = false;	224,554✔
24	}	224,554✔
25	}
26	normalized.trim().to_string()	13,905✔
27	}	13,905✔
28
29	/// Split a block of text into sentences, falling back to the whole string when needed.
30	/// Heuristic sentence splitter with tokenizer-friendly rules.
31	pub fn sentences(text: &str) -> Vec<Sentence> {	13,823✔
32	let mut results = Vec::new();	13,823✔
33
34	for block in text.split("\n\n") {	13,824✔
35	if block.trim().is_empty() {	13,824✔
36	continue;	1✔
37	}	13,823✔
38	let normalized = normalize_inline_whitespace(block);	13,823✔
39	if normalized.is_empty() {	13,823✔
40	continue;	×
41	}	13,823✔
42	push_block_sentences(&normalized, &mut results);	13,823✔
43	}
44
45	results	13,823✔
46	}	13,823✔
47
48	/// Convenience helper to construct a `RecordSection` with normalized text metadata.
49	/// Convenience helper to build a `RecordSection` with precomputed sentences.
50	pub fn make_section(role: SectionRole, heading: Option<&str>, text: &str) -> RecordSection {	13,817✔
51	RecordSection {
52	role,	13,817✔
53	heading: heading.map(\|h\| h.to_string()),	13,817✔
54	text: text.to_string(),	13,817✔
55	sentences: sentences(text),	13,817✔
56	}
57	}	13,817✔
58
59	fn push_block_sentences(block: &str, results: &mut Vec<Sentence>) {	13,823✔
60	let chars: Vec<char> = block.chars().collect();	13,823✔
61	let mut buffer = String::new();	13,823✔
62
63	for (idx, ch) in chars.iter().enumerate() {	261,366✔
64	buffer.push(*ch);	261,366✔
65	if is_sentence_boundary(&chars, idx) {	261,366✔
66	let trimmed = buffer.trim();	315✔
67	if !trimmed.is_empty() {	315✔
68	results.push(trimmed.to_string());	315✔
69	}	315✔
70	buffer.clear();	315✔
71	}	261,051✔
72	}
73
74	let trailing = buffer.trim();	13,823✔
75	if !trailing.is_empty() {	13,823✔
76	results.push(trailing.to_string());	13,654✔
77	}	13,654✔
78	}	13,823✔
79
80	fn is_sentence_boundary(chars: &[char], idx: usize) -> bool {	261,366✔
81	match chars[idx] {	261,366✔
82	'.' => is_dot_boundary(chars, idx),	318✔
83	'!' \| '?' => true,	2✔
84	_ => false,	261,046✔
85	}
86	}	261,366✔
87
88	fn is_dot_boundary(chars: &[char], idx: usize) -> bool {	318✔
89	if is_decimal_middle(chars, idx) \|\| is_ticker_middle(chars, idx) {	318✔
90	return false;	3✔
91	}	315✔
92	if idx + 1 < chars.len() && chars[idx + 1] == '.' {	315✔
93	return false;	2✔
94	}	313✔
95	true	313✔
96	}	318✔
97
98	fn is_decimal_middle(chars: &[char], idx: usize) -> bool {	318✔
99	idx > 0	318✔
100	&& idx + 1 < chars.len()	318✔
101	&& chars[idx - 1].is_ascii_digit()	150✔
102	&& chars[idx + 1].is_ascii_digit()	144✔
103	}	318✔
104
105	fn is_ticker_middle(chars: &[char], idx: usize) -> bool {	317✔
106	idx > 0	317✔
107	&& idx + 1 < chars.len()	317✔
108	&& is_ticker_char(chars[idx - 1])	149✔
109	&& is_ticker_char(chars[idx + 1])	145✔
110	}	317✔
111
112	fn is_ticker_char(ch: char) -> bool {	294✔
113	ch.is_ascii_uppercase() \|\| ch.is_ascii_digit()	294✔
114	}	294✔
115
116	// ---------------------------------------------------------------------------
117	// Filesystem helpers
118	// ---------------------------------------------------------------------------
119
120	/// True if the path has a `.txt` extension (case-insensitive).
121	pub fn is_text_file(path: &Path) -> bool {	669✔
122	path.extension()	669✔
123	.and_then(\|ext\| ext.to_str())	669✔
124	.map(\|ext\| ext.eq_ignore_ascii_case("txt"))	669✔
125	.unwrap_or(false)	669✔
126	}	669✔
127
128	/// Best-effort file modified time.
129	pub fn file_mtime(path: &Path) -> Option<DateTime<Utc>> {	2✔
130	let metadata = fs::metadata(path).ok()?;	2✔
131	let modified = metadata.modified().ok()?;	1✔
132	Some(system_time_to_utc(modified))	1✔
133	}	2✔
134
135	/// Best-effort (created_at, updated_at) pair for a file.
136	pub fn file_times(path: &Path) -> (DateTime<Utc>, DateTime<Utc>) {	34✔
137	let metadata = fs::metadata(path).ok();	34✔
138	let updated_at = metadata	34✔
139	.as_ref()	34✔
140	.and_then(\|meta\| meta.modified().ok())	34✔
141	.map(system_time_to_utc)	34✔
142	.unwrap_or_else(Utc::now);	34✔
143	let created_at = metadata	34✔
144	.and_then(\|meta\| meta.created().ok())	34✔
145	.map(system_time_to_utc)	34✔
146	.unwrap_or(updated_at);	34✔
147	(created_at, updated_at)	34✔
148	}	34✔
149
150	fn system_time_to_utc(time: std::time::SystemTime) -> DateTime<Utc> {	67✔
151	DateTime::<Utc>::from(time)	67✔
152	}	67✔
153
154	#[cfg(test)]
155	mod tests {
156	use super::*;
157
158	#[test]
159	fn normalize_inline_whitespace_collapses_runs() {	1✔
160	let input = "Alpha\n\n Beta\tGamma";	1✔
161	assert_eq!(normalize_inline_whitespace(input), "Alpha Beta Gamma");	1✔
162	}	1✔
163
164	#[test]
165	fn sentences_falls_back_to_full_text_when_needed() {	1✔
166	let text = " \n";	1✔
167	let result = sentences(text);	1✔
168	assert!(result.is_empty());	1✔
169
170	let text2 = "Single block without punctuation";	1✔
171	let result2 = sentences(text2);	1✔
172	assert_eq!(	1✔
173	result2,
174	vec![String::from("Single block without punctuation")]	1✔
175	);
176	}	1✔
177
178	#[test]
179	fn make_section_populates_sentences() {	1✔
180	let section = make_section(SectionRole::Context, Some("Summary"), "Line one. Line two!");	1✔
181	assert_eq!(section.heading.as_deref(), Some("Summary"));	1✔
182	assert_eq!(section.sentences.len(), 2);	1✔
183	assert_eq!(section.role, SectionRole::Context);	1✔
184	}	1✔
185
186	#[test]
187	fn sentences_keep_decimal_values_together() {	1✔
188	let text = "Price closed at 3.14. Outlook improved.";	1✔
189	let result = sentences(text);	1✔
190	assert_eq!(result, vec!["Price closed at 3.14.", "Outlook improved."]);	1✔
191	}	1✔
192
193	#[test]
194	fn sentences_keep_dot_tickers_together() {	1✔
195	let text = "BRK.B rallied while RDS.A lagged.";	1✔
196	let result = sentences(text);	1✔
197	assert_eq!(result, vec!["BRK.B rallied while RDS.A lagged."]);	1✔
198	}	1✔
199
200	#[test]
201	fn file_time_helpers_handle_existing_and_missing_paths() {	1✔
202	use tempfile::tempdir;
203	let temp = tempdir().unwrap();	1✔
204	let existing = temp.path().join("exists.txt");	1✔
205	std::fs::write(&existing, "hello").unwrap();	1✔
206
207	assert!(file_mtime(&existing).is_some());	1✔
208	let (created_at, updated_at) = file_times(&existing);	1✔
209	assert!(updated_at >= created_at);	1✔
210
211	let missing = temp.path().join("missing.txt");	1✔
212	assert!(file_mtime(&missing).is_none());	1✔
213	let (missing_created, missing_updated) = file_times(&missing);	1✔
214	assert!(missing_updated >= missing_created);	1✔
215	}	1✔
216
217	#[test]
218	fn sentences_treat_blank_line_as_boundary() {	1✔
219	let text = "First line without punctuation\n\nSecond line with more context.";	1✔
220	let result = sentences(text);	1✔
221	assert_eq!(	1✔
222	result,
223	vec![	1✔
224	"First line without punctuation".to_string(),	1✔
225	"Second line with more context.".to_string()	1✔
226	]
227	);
228	}	1✔
229
230	#[test]
231	fn sentences_keep_ellipsis_together() {	1✔
232	let text = "Wait... really? Yes.";	1✔
233	let result = sentences(text);	1✔
234	assert_eq!(result, vec!["Wait...", "really?", "Yes."]);	1✔
235	}	1✔
236
237	#[test]
238	fn is_text_file_matches_txt_case_insensitively() {	1✔
239	use std::path::PathBuf;
240	assert!(is_text_file(&PathBuf::from("hello.txt")));	1✔
241	assert!(is_text_file(&PathBuf::from("hello.TXT")));	1✔
242	assert!(is_text_file(&PathBuf::from("hello.Txt")));	1✔
243	assert!(!is_text_file(&PathBuf::from("hello.md")));	1✔
244	assert!(!is_text_file(&PathBuf::from("hello")));	1✔
245	}	1✔
246	}

jzombie / rust-triplets / 23177332371

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous