• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

dcdpr / jp / 20372941205

19 Dec 2025 02:26PM UTC coverage: 52.025% (+4.3%) from 47.733%
20372941205

push

github

web-flow
test(llm, conversation): Improve test determinism and infrastructure (#322)

This commit enhances test reliability and reproducibility by introducing
fixed timestamps and consistent ordering throughout the test suite.
Tests now use deterministic timestamps (`2020-01-01 00:00:00.0`) instead
of dynamic ones, ensuring snapshot tests remain stable across runs.

The `ConversationStream` now tracks its creation timestamp explicitly,
which is used consistently in serialization and test fixtures. This
change required updating the `add_config_delta()` method to accept a
`ConfigDelta` with an explicit timestamp rather than inferring it at
call time.

JSON serialization now preserves key ordering across all crates by
enabling the `preserve_order` feature on `serde_json`. This ensures
deterministic output in fixtures and snapshots, particularly important
for provider responses and configuration serialization.

Model-related types (`ModelIdConfig`, `ProviderId`, `Name`) now
implement ordering traits, enabling the openrouter provider to sort and
deduplicate model lists. This prevents duplicate entries in model
listings and provides consistent ordering for better test reliability.

Test fixture handling has been improved with better JSON body formatting
and more careful whitespace handling, particularly for server-sent
events that require specific newline patterns. The mock infrastructure
now better preserves the structure of HTTP responses in YAML fixtures.

Additional improvements include updating the `saphyr` dependency to fix
multi-newline ending issues, refining array type inference in OpenAI's
parameter sanitization, and removing large blocks of commented-out test
code that are no longer needed.

---------

Signed-off-by: Jean Mertz <git@jeanmertz.com>

103 of 134 new or added lines in 10 files covered. (76.87%)

1237 existing lines in 38 files now uncovered.

8774 of 16865 relevant lines covered (52.02%)

135.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.66
/crates/jp_cli/src/cmd/init.rs
1
use std::{env, fs, path::PathBuf, str::FromStr as _};
2

3
use crossterm::style::Stylize as _;
4
use duct::cmd;
5
use jp_config::{
6
    PartialAppConfig,
7
    conversation::tool::RunMode,
8
    model::id::{ModelIdConfig, Name, PartialModelIdConfig, ProviderId},
9
};
10
use jp_workspace::Workspace;
11
use path_clean::PathClean as _;
12

13
use crate::{DEFAULT_STORAGE_DIR, Output, ctx::IntoPartialAppConfig};
14

15
#[derive(Debug, clap::Args)]
16
pub(crate) struct Init {
17
    /// Path to initialize the workspace at. Defaults to the current directory.
18
    path: Option<PathBuf>,
19
}
20

21
impl Init {
22
    pub(crate) fn run(&self) -> Output {
×
23
        let cwd = std::env::current_dir()?;
×
24
        let mut root = self
×
25
            .path
×
26
            .clone()
×
27
            .unwrap_or_else(|| PathBuf::from("."))
×
28
            .clean();
×
29

30
        if !root.is_absolute() {
×
31
            root = cwd.join(root);
×
32
        }
×
33

34
        fs::create_dir_all(&root)?;
×
35

36
        let storage = root.join(DEFAULT_STORAGE_DIR);
×
37
        let id = jp_workspace::Id::new();
×
38
        jp_id::global::set(id.to_string());
×
39

40
        let mut workspace =
×
41
            Workspace::new_with_id(root.clone(), id.clone()).persisted_at(&storage)?;
×
42

43
        id.store(&storage)?;
×
44

45
        workspace = workspace.with_local_storage()?;
×
46

47
        let mut config = default_config();
×
48
        if let Some(id) = default_model() {
×
49
            print!("Using model {}", id.to_string().bold().blue());
×
50
            let note = "  (to use a different model, update `.jp/config.toml`)".to_owned();
×
51
            println!("{}\n", note.grey().italic());
×
52

×
53
            config.assistant.model.id = PartialModelIdConfig {
×
54
                provider: Some(id.provider),
×
55
                name: Some(id.name),
×
56
            }
×
57
            .into();
×
58
        }
×
59

60
        let data = toml::to_string_pretty(&config)?;
×
61
        fs::write(storage.join("config.toml"), data)?;
×
62
        fs::create_dir_all(storage.join("config.d"))?;
×
63

64
        workspace.persist()?;
×
65

66
        let loc = if root == cwd {
×
67
            "current directory".to_owned()
×
68
        } else {
69
            root.to_string_lossy().bold().to_string()
×
70
        };
71

72
        Ok(format!("Initialized workspace at {loc}").into())
×
73
    }
×
74
}
75

76
#[expect(clippy::too_many_lines)]
77
fn default_config() -> jp_config::PartialAppConfig {
1✔
78
    let mut cfg = jp_config::PartialAppConfig::default();
1✔
79
    cfg.extends
1✔
80
        .get_or_insert_default()
1✔
81
        .push("config.d/**/*".into());
1✔
82

83
    // This is a required field without a default value (that is, the
84
    // `ToolsDefaultsConfig` type does not set a default value for `run`).
85
    //
86
    // By setting it explicitly, we ensure that the default generated config
87
    // file has this value set, which exposes it to the user. This is desired,
88
    // as this is an important security feature, which we don't want users to
89
    // have to rely on a default value that might change in the future.
90
    cfg.conversation.tools.defaults.run = Some(RunMode::Ask);
1✔
91

92
    if has_anthropic() {
1✔
93
        cfg.providers.llm.aliases.extend([
1✔
94
            ("anthropic".to_owned(), PartialModelIdConfig {
1✔
95
                provider: Some(ProviderId::Anthropic),
1✔
96
                name: Some(Name("claude-sonnet-4-5".into())),
1✔
97
            }),
1✔
98
            ("claude".to_owned(), PartialModelIdConfig {
1✔
99
                provider: Some(ProviderId::Anthropic),
1✔
100
                name: Some(Name("claude-sonnet-4-5".into())),
1✔
101
            }),
1✔
102
            ("sonnet".to_owned(), PartialModelIdConfig {
1✔
103
                provider: Some(ProviderId::Anthropic),
1✔
104
                name: Some(Name("claude-sonnet-4-5".into())),
1✔
105
            }),
1✔
106
            ("opus".to_owned(), PartialModelIdConfig {
1✔
107
                provider: Some(ProviderId::Anthropic),
1✔
108
                name: Some(Name("claude-opus-4-5".into())),
1✔
109
            }),
1✔
110
            ("haiku".to_owned(), PartialModelIdConfig {
1✔
111
                provider: Some(ProviderId::Anthropic),
1✔
112
                name: Some(Name("claude-haiku-4-5".into())),
1✔
113
            }),
1✔
114
        ]);
1✔
115
    }
1✔
116

117
    if has_openai() {
1✔
118
        cfg.providers.llm.aliases.extend([
1✔
119
            ("openai".to_owned(), PartialModelIdConfig {
1✔
120
                provider: Some(ProviderId::Openai),
1✔
121
                name: Some(Name("gpt-5.2".into())),
1✔
122
            }),
1✔
123
            ("chatgpt".to_owned(), PartialModelIdConfig {
1✔
124
                provider: Some(ProviderId::Openai),
1✔
125
                name: Some(Name("gpt-5.2".into())),
1✔
126
            }),
1✔
127
            ("gpt".to_owned(), PartialModelIdConfig {
1✔
128
                provider: Some(ProviderId::Openai),
1✔
129
                name: Some(Name("gpt-5.2".into())),
1✔
130
            }),
1✔
131
            ("gpt5".to_owned(), PartialModelIdConfig {
1✔
132
                provider: Some(ProviderId::Openai),
1✔
133
                name: Some(Name("gpt-5.2".into())),
1✔
134
            }),
1✔
135
            ("gpt5-mini".to_owned(), PartialModelIdConfig {
1✔
136
                provider: Some(ProviderId::Openai),
1✔
137
                name: Some(Name("gpt-5-mini".into())),
1✔
138
            }),
1✔
139
            ("gpt-mini".to_owned(), PartialModelIdConfig {
1✔
140
                provider: Some(ProviderId::Openai),
1✔
141
                name: Some(Name("gpt-5-mini".into())),
1✔
142
            }),
1✔
143
            ("gpt5-nano".to_owned(), PartialModelIdConfig {
1✔
144
                provider: Some(ProviderId::Openai),
1✔
145
                name: Some(Name("gpt-5-nano".into())),
1✔
146
            }),
1✔
147
            ("gpt-nano".to_owned(), PartialModelIdConfig {
1✔
148
                provider: Some(ProviderId::Openai),
1✔
149
                name: Some(Name("gpt-5-nano".into())),
1✔
150
            }),
1✔
151
            ("o3-research".to_owned(), PartialModelIdConfig {
1✔
152
                provider: Some(ProviderId::Openai),
1✔
153
                name: Some(Name("o3-deep-research".into())),
1✔
154
            }),
1✔
155
            ("o4-mini-research".to_owned(), PartialModelIdConfig {
1✔
156
                provider: Some(ProviderId::Openai),
1✔
157
                name: Some(Name("o4-mini-deep-research".into())),
1✔
158
            }),
1✔
159
            ("codex".to_owned(), PartialModelIdConfig {
1✔
160
                provider: Some(ProviderId::Openai),
1✔
161
                name: Some(Name("gpt-5.1-codex".into())),
1✔
162
            }),
1✔
163
            ("codex-max".to_owned(), PartialModelIdConfig {
1✔
164
                provider: Some(ProviderId::Openai),
1✔
165
                name: Some(Name("gpt-5.1-codex-max".into())),
1✔
166
            }),
1✔
167
            ("gpt-5-codex".to_owned(), PartialModelIdConfig {
1✔
168
                provider: Some(ProviderId::Openai),
1✔
169
                name: Some(Name("gpt-5.1-codex".into())),
1✔
170
            }),
1✔
171
            ("codex-mini".to_owned(), PartialModelIdConfig {
1✔
172
                provider: Some(ProviderId::Openai),
1✔
173
                name: Some(Name("gpt-5.1-codex-mini".into())),
1✔
174
            }),
1✔
175
        ]);
1✔
176
    }
1✔
177

178
    if has_google() {
1✔
179
        cfg.providers.llm.aliases.extend([
1✔
180
            ("google".to_owned(), PartialModelIdConfig {
1✔
181
                provider: Some(ProviderId::Google),
1✔
182
                name: Some(Name("gemini-pro-latest".into())),
1✔
183
            }),
1✔
184
            ("gemini".to_owned(), PartialModelIdConfig {
1✔
185
                provider: Some(ProviderId::Google),
1✔
186
                name: Some(Name("gemini-pro-latest".into())),
1✔
187
            }),
1✔
188
            ("gemini-pro".to_owned(), PartialModelIdConfig {
1✔
189
                provider: Some(ProviderId::Google),
1✔
190
                name: Some(Name("gemini-pro-latest".into())),
1✔
191
            }),
1✔
192
            ("gemini-flash".to_owned(), PartialModelIdConfig {
1✔
193
                provider: Some(ProviderId::Google),
1✔
194
                name: Some(Name("gemini-flash-latest".into())),
1✔
195
            }),
1✔
196
            ("gemini-lite".to_owned(), PartialModelIdConfig {
1✔
197
                provider: Some(ProviderId::Google),
1✔
198
                name: Some(Name("gemini-flash-lite-latest".into())),
1✔
199
            }),
1✔
200
        ]);
1✔
201
    }
1✔
202

203
    cfg
1✔
204
}
1✔
205

206
fn has_anthropic() -> bool {
1✔
207
    env::var("ANTHROPIC_API_KEY").is_ok()
1✔
208
}
1✔
209

210
fn has_openai() -> bool {
1✔
211
    env::var("OPENAI_API_KEY").is_ok()
1✔
212
}
1✔
213

214
fn has_google() -> bool {
1✔
215
    env::var("GOOGLE_API_KEY").is_ok()
1✔
216
}
1✔
217

218
fn default_model() -> Option<ModelIdConfig> {
×
219
    env::var("JP_CFG_ASSISTANT_MODEL_ID")
×
220
        .ok()
×
221
        .and_then(|v| ModelIdConfig::from_str(&v).ok())
×
222
        .or_else(|| {
×
223
            let models = cmd!("ollama", "list")
×
UNCOV
224
                .pipe(cmd!("cut", "-d", " ", "-f1"))
×
225
                .pipe(cmd!("tail", "-n+2"))
×
226
                .read()
×
227
                .unwrap_or_default();
×
228

229
            let models = models.lines().map(str::trim).collect::<Vec<_>>();
×
230
            let model = if let Some(model) = models.iter().find(|m| m.starts_with("llama")) {
×
231
                model
×
UNCOV
232
            } else if let Some(model) = models.iter().find(|m| m.starts_with("gemma")) {
×
233
                model
×
UNCOV
234
            } else if let Some(model) = models.iter().find(|m| m.starts_with("qwen")) {
×
UNCOV
235
                model
×
236
            } else {
237
                return None;
×
238
            };
239

240
            format!("ollama/{model}").parse().ok()
×
241
        })
×
242
        // TODO: Use `Config` env vars here.
243
        .or_else(|| {
×
244
            env::var("ANTHROPIC_API_KEY")
×
245
                .is_ok()
×
246
                .then(|| "anthropic/claude-sonnet-4-5".parse().ok())
×
247
                .flatten()
×
248
        })
×
249
        .or_else(|| {
×
250
            env::var("OPENAI_API_KEY")
×
251
                .is_ok()
×
252
                .then(|| "openai/gpt-5.1".parse().ok())
×
253
                .flatten()
×
254
        })
×
255
        .or_else(|| {
×
256
            env::var("GEMINI_API_KEY")
×
257
                .is_ok()
×
UNCOV
258
                .then(|| "google/gemini-3-pro-preview".parse().ok())
×
UNCOV
259
                .flatten()
×
260
        })
×
261
}
×
262

263
impl IntoPartialAppConfig for Init {
264
    fn apply_cli_config(
×
265
        &self,
×
266
        _workspace: Option<&Workspace>,
×
267
        partial: PartialAppConfig,
×
UNCOV
268
        _: Option<&PartialAppConfig>,
×
UNCOV
269
    ) -> std::result::Result<PartialAppConfig, Box<dyn std::error::Error + Send + Sync>> {
×
UNCOV
270
        Ok(partial)
×
UNCOV
271
    }
×
272
}
273

274
#[cfg(test)]
275
mod tests {
276
    use serial_test::serial;
277
    use test_log::test;
278

279
    use super::*;
280

281
    pub(crate) struct EnvVarGuard {
282
        name: String,
283
        original_value: Option<String>,
284
    }
285

286
    impl EnvVarGuard {
287
        pub fn set(name: &str, value: &str) -> Self {
3✔
288
            let name = name.to_string();
3✔
289
            let original_value = std::env::var(&name).ok();
3✔
290
            unsafe { std::env::set_var(&name, value) };
3✔
291
            Self {
3✔
292
                name,
3✔
293
                original_value,
3✔
294
            }
3✔
295
        }
3✔
296
    }
297

298
    impl Drop for EnvVarGuard {
299
        fn drop(&mut self) {
3✔
300
            if let Some(ref original) = self.original_value {
3✔
UNCOV
301
                unsafe { std::env::set_var(&self.name, original) };
×
302
            } else {
3✔
303
                unsafe { std::env::remove_var(&self.name) };
3✔
304
            }
3✔
305
        }
3✔
306
    }
307

308
    #[serial(env_vars)]
309
    #[test]
310
    fn test_default_config() {
1✔
311
        let _env1 = EnvVarGuard::set("ANTHROPIC_API_KEY", "foo");
1✔
312
        let _env2 = EnvVarGuard::set("OPENAI_API_KEY", "bar");
1✔
313
        let _env3 = EnvVarGuard::set("GOOGLE_API_KEY", "baz");
1✔
314

315
        let config = default_config();
1✔
316

317
        insta::assert_toml_snapshot!(config);
1✔
318
    }
319
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc