• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

ImmediatePlatform / Immediate.Jobs / 30304771671

27 Jul 2026 08:56PM UTC coverage: 65.995%. First build
30304771671

Pull #33

github

web-flow
Merge 46c3047f1 into e9a147742
Pull Request #33: Improve Storage API

1521 of 2481 new or added lines in 28 files covered. (61.31%)

4326 of 6555 relevant lines covered (66.0%)

2.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.67
/src/Immediate.Jobs.Shared/InMemoryJobStorage.cs
1
namespace Immediate.Jobs.Shared;
2

3
#pragma warning disable IDE0391 // Keep synchronous storage methods async for .NET 11 runtime-async state machines.
4

5
/// <summary>
6
/// A best-effort, non-durable, single-node provider intended for development and tests.
7
/// </summary>
8
public sealed class InMemoryJobStorage(TimeProvider timeProvider) :
4✔
9
        IRecurringJobStorage,
10
        IJobGraphStorage,
11
        IJobStorageReplica
12
{
13
        private readonly Lock _gate = new();
4✔
14
        private readonly Dictionary<string, JobRecord> _jobs = new(StringComparer.Ordinal);
4✔
15
        private readonly Dictionary<string, JobBatchRecord> _batches = new(StringComparer.Ordinal);
4✔
16
        private readonly List<JobContinuationEdge> _edges = [];
4✔
17
        private readonly HashSet<JobContinuationEdge> _settledEdges = [];
4✔
18
        private readonly Dictionary<string, RecurringJobSchedule> _recurring = new(StringComparer.Ordinal);
4✔
19
        private readonly Dictionary<string, JobServerSnapshot> _servers = new(StringComparer.Ordinal);
4✔
20
        private readonly HashSet<string> _recurringKeys = new(StringComparer.Ordinal);
4✔
21

22
        /// <inheritdoc />
23
        public ValueTask InitializeAsync(CancellationToken cancellationToken = default)
24
        {
25
                cancellationToken.ThrowIfCancellationRequested();
4✔
26
                return ValueTask.CompletedTask;
4✔
27
        }
28

29
        /// <inheritdoc />
30
        public ValueTask EnqueueAsync(JobRecord job, CancellationToken cancellationToken = default)
31
        {
32
                ArgumentNullException.ThrowIfNull(job);
4✔
33
                cancellationToken.ThrowIfCancellationRequested();
4✔
34
                lock (_gate)
35
                {
36
                        if (!_jobs.TryAdd(job.Id, job))
4✔
NEW
37
                                throw new ImmediateJobException($"Job '{job.Id}' already exists.");
×
38
                }
4✔
39

40
                return ValueTask.CompletedTask;
4✔
41
        }
42

43
        /// <inheritdoc />
44
        public ValueTask EnqueueContinuationAsync(
45
                JobRecord job,
46
                IReadOnlyList<JobContinuationEdge> edges,
47
                CancellationToken cancellationToken = default
48
        )
49
        {
50
                ArgumentNullException.ThrowIfNull(job);
4✔
51
                ArgumentNullException.ThrowIfNull(edges);
4✔
52
                cancellationToken.ThrowIfCancellationRequested();
4✔
53
                lock (_gate)
54
                {
55
                        ValidateNewJob(job);
4✔
56
                        ValidateEdges([job], edges, batchId: null);
4✔
57

58
                        var restoreExistingState = HasTerminalParent(edges) &&
4✔
59
                                (job.State != JobState.AwaitingContinuation || job.RemainingDependencies < edges.Count);
4✔
60
                        _jobs.Add(job.Id, restoreExistingState ? job : NormalizeWaitingJob(job, edges.Count));
4✔
61
                        _edges.AddRange(edges);
4✔
62
                        if (restoreExistingState)
4✔
63
                                MarkTerminalParentEdgesSettled(edges);
×
64
                        else
65
                                EvaluateAlreadyTerminalParents(edges);
4✔
66
                }
4✔
67

68
                return ValueTask.CompletedTask;
4✔
69
        }
70

71
        /// <inheritdoc />
72
        public ValueTask EnqueueBatchAsync(
73
                JobBatchRecord batch,
74
                IReadOnlyList<JobRecord> jobs,
75
                IReadOnlyList<JobContinuationEdge> edges,
76
                CancellationToken cancellationToken = default
77
        )
78
        {
79
                ArgumentNullException.ThrowIfNull(batch);
4✔
80
                ArgumentNullException.ThrowIfNull(jobs);
4✔
81
                ArgumentNullException.ThrowIfNull(edges);
4✔
82
                cancellationToken.ThrowIfCancellationRequested();
4✔
83
                lock (_gate)
84
                {
85
                        ValidateBatch(batch, jobs, edges);
4✔
86

87
                        var restoreExistingState = IsRecoveredBatch(batch, jobs, edges);
4✔
88
                        _batches.Add(batch.Id, batch);
4✔
89
                        var incomingCounts = edges
4✔
90
                                .GroupBy(static edge => edge.ChildJobId, StringComparer.Ordinal)
4✔
91
                                .ToDictionary(static group => group.Key, static group => group.Count(), StringComparer.Ordinal);
4✔
92
                        foreach (var job in jobs)
4✔
93
                        {
94
                                _jobs.Add(
4✔
95
                                        job.Id,
4✔
96
                                        restoreExistingState
4✔
97
                                                ? job
4✔
98
                                                : incomingCounts.TryGetValue(job.Id, out var dependencyCount)
4✔
99
                                                ? NormalizeWaitingJob(job, dependencyCount)
4✔
100
                                                : job with { BatchId = batch.Id, RemainingDependencies = 0, FailedDependencies = 0 }
4✔
101
                                );
4✔
102
                        }
103

104
                        _edges.AddRange(edges);
4✔
105
                        if (restoreExistingState)
4✔
106
                                MarkTerminalParentEdgesSettled(edges);
4✔
107
                        else
108
                                EvaluateAlreadyTerminalParents(edges);
4✔
109
                }
4✔
110

111
                return ValueTask.CompletedTask;
4✔
112
        }
113

114
        /// <inheritdoc />
115
        public async ValueTask<IReadOnlyList<JobRecord>> AcquireDueJobsAsync(
116
                JobAcquisitionRequest request,
117
                CancellationToken cancellationToken = default
118
        )
119
        {
120
                ArgumentNullException.ThrowIfNull(request);
4✔
121
                ArgumentException.ThrowIfNullOrWhiteSpace(request.WorkerId);
4✔
122
                ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(request.Lease, TimeSpan.Zero);
4✔
123
                ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(request.BatchSize, 0);
4✔
124
                cancellationToken.ThrowIfCancellationRequested();
4✔
125
                var now = timeProvider.GetUtcNow();
4✔
126
                lock (_gate)
127
                {
128
                        foreach (var expired in _jobs.Values.Where(x => x.State == JobState.Active && x.LeaseExpiresAt <= now).ToArray())
4✔
129
                        {
130
                                _jobs[expired.Id] = expired with
4✔
131
                                {
4✔
132
                                        State = JobState.Pending,
4✔
133
                                        WorkerId = null,
4✔
134
                                        LeaseExpiresAt = null,
4✔
135
                                };
4✔
136
                        }
137

138
                        var acquired = new List<JobRecord>(request.BatchSize);
4✔
139
                        foreach (var queue in request.Queues)
4✔
140
                        {
141
                                var queueCapacity = Math.Min(queue.Capacity, request.BatchSize - acquired.Count);
4✔
142
                                if (queueCapacity <= 0)
4✔
143
                                        continue;
144

145
                                var jobCapacities = queue.JobCapacities.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal);
4✔
146
                                foreach (var candidate in _jobs.Values
4✔
147
                                        .Where(job => job.QueueName == queue.QueueName &&
4✔
148
                                                jobCapacities.ContainsKey(job.JobName) &&
4✔
149
                                                job.State is JobState.Pending or JobState.Scheduled &&
4✔
150
                                                job.DueAt <= now)
4✔
151
                                        .OrderBy(job => job.DueAt)
4✔
152
                                        .ThenBy(job => job.CreatedAt)
4✔
153
                                        .ThenBy(job => job.Id))
4✔
154
                                {
155
                                        if (queueCapacity == 0)
4✔
156
                                                break;
1✔
157
                                        if (jobCapacities[candidate.JobName] <= 0)
4✔
158
                                                continue;
159

160
                                        var job = candidate with
4✔
161
                                        {
4✔
162
                                                State = JobState.Active,
4✔
163
                                                Attempt = candidate.Attempt + 1,
4✔
164
                                                WorkerId = request.WorkerId,
4✔
165
                                                LeaseExpiresAt = now + request.Lease,
4✔
166
                                                ExecutionTraceId = null,
4✔
167
                                                ExecutionSpanId = null,
4✔
168
                                                ExecutionStartedAt = null,
4✔
169
                                        };
4✔
170
                                        _jobs[job.Id] = job;
4✔
171
                                        MarkBatchStarted(job.BatchId, now);
4✔
172
                                        acquired.Add(job);
4✔
173
                                        jobCapacities[job.JobName]--;
4✔
174
                                        queueCapacity--;
4✔
175
                                }
176
                        }
177

178
                        return acquired;
4✔
179
                }
180
        }
4✔
181

182
        /// <inheritdoc />
183
        public async ValueTask<IReadOnlyList<JobRecord>> AcquireJobsAsync(
184
                IReadOnlyCollection<string> jobIds,
185
                string workerId,
186
                TimeSpan lease,
187
                CancellationToken cancellationToken = default
188
        )
189
        {
190
                ArgumentNullException.ThrowIfNull(jobIds);
4✔
191
                ArgumentException.ThrowIfNullOrWhiteSpace(workerId);
4✔
192
                ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(lease, TimeSpan.Zero);
4✔
193
                cancellationToken.ThrowIfCancellationRequested();
4✔
194
                var now = timeProvider.GetUtcNow();
4✔
195
                lock (_gate)
196
                {
197
                        var acquired = new List<JobRecord>(jobIds.Count);
4✔
198
                        foreach (var id in jobIds)
4✔
199
                        {
200
                                if (!_jobs.TryGetValue(id, out var job))
4✔
201
                                        continue;
202
                                if (job.State == JobState.Active && job.LeaseExpiresAt <= now)
4✔
203
                                {
204
                                        job = job with { State = JobState.Pending, WorkerId = null, LeaseExpiresAt = null };
4✔
205
                                        _jobs[id] = job;
4✔
206
                                }
207

208
                                if (job.State is not (JobState.Pending or JobState.Scheduled) || job.DueAt > now)
4✔
209
                                        continue;
210

211
                                job = job with
4✔
212
                                {
4✔
213
                                        State = JobState.Active,
4✔
214
                                        Attempt = job.Attempt + 1,
4✔
215
                                        WorkerId = workerId,
4✔
216
                                        LeaseExpiresAt = now + lease,
4✔
217
                                        ExecutionTraceId = null,
4✔
218
                                        ExecutionSpanId = null,
4✔
219
                                        ExecutionStartedAt = null,
4✔
220
                                };
4✔
221
                                _jobs[id] = job;
4✔
222
                                MarkBatchStarted(job.BatchId, now);
4✔
223
                                acquired.Add(job);
4✔
224
                        }
225

226
                        return acquired;
4✔
227
                }
228
        }
4✔
229

230
        /// <inheritdoc />
231
        public ValueTask SetExecutionTelemetryAsync(
232
                string jobId,
233
                string workerId,
234
                string? traceId,
235
                string? spanId,
236
                DateTimeOffset startedAt,
237
                CancellationToken cancellationToken = default
238
        )
239
        {
240
                cancellationToken.ThrowIfCancellationRequested();
4✔
241
                lock (_gate)
242
                {
243
                        var job = GetOwnedActive(jobId, workerId);
4✔
244
                        _jobs[jobId] = job with
4✔
245
                        {
4✔
246
                                ExecutionTraceId = traceId,
4✔
247
                                ExecutionSpanId = spanId,
4✔
248
                                ExecutionStartedAt = startedAt,
4✔
249
                        };
4✔
250
                }
4✔
251

252
                return ValueTask.CompletedTask;
4✔
253
        }
254

255
        /// <inheritdoc />
256
        public ValueTask RenewLeaseAsync(string jobId, string workerId, TimeSpan lease, CancellationToken cancellationToken = default)
257
        {
258
                cancellationToken.ThrowIfCancellationRequested();
×
259
                lock (_gate)
260
                {
261
                        var job = GetOwnedActive(jobId, workerId);
×
262
                        _jobs[jobId] = job with { LeaseExpiresAt = timeProvider.GetUtcNow() + lease };
×
263
                }
×
264

265
                return ValueTask.CompletedTask;
×
266
        }
267

268
        /// <inheritdoc />
269
        public ValueTask CompleteAsync(string jobId, string workerId, CancellationToken cancellationToken = default)
270
                => CompleteWithContinuationsAsync(jobId, workerId, [], cancellationToken);
4✔
271

272
        /// <inheritdoc />
273
        public ValueTask CompleteWithContinuationsAsync(
274
                string jobId,
275
                string workerId,
276
                IReadOnlyList<JobContinuationAddition> additions,
277
                CancellationToken cancellationToken = default
278
        )
279
        {
280
                ArgumentNullException.ThrowIfNull(additions);
4✔
281
                cancellationToken.ThrowIfCancellationRequested();
4✔
282
                lock (_gate)
283
                {
284
                        var current = GetOwnedActive(jobId, workerId);
4✔
285
                        var existingWaiters = GetUnsettledWaiters(jobId);
4✔
286
                        var newJobIds = new HashSet<string>(StringComparer.Ordinal);
4✔
287
                        var dependencyEdges = new List<JobContinuationEdge>(additions.Count);
4✔
288
                        var trackedAdditions = 0;
4✔
289
                        foreach (var addition in additions)
4✔
290
                        {
291
                                ArgumentNullException.ThrowIfNull(addition);
4✔
292
                                ArgumentNullException.ThrowIfNull(addition.Job);
4✔
293
                                ValidateNewJob(addition.Job);
4✔
294
                                if (!newJobIds.Add(addition.Job.Id))
4✔
NEW
295
                                        throw new ImmediateJobException($"Job '{addition.Job.Id}' occurs more than once in the completion buffer.");
×
296
                                if (addition.Job.State is not (JobState.Pending or JobState.Scheduled))
4✔
NEW
297
                                        throw new ImmediateJobException($"Dynamic continuation '{addition.Job.Id}' has invalid state '{addition.Job.State}'.");
×
298
                                if (!Enum.IsDefined(addition.Trigger))
4✔
299
                                        throw new ArgumentOutOfRangeException(nameof(additions), "Unknown continuation trigger.");
×
300

301
                                if (addition.Options == ContinuationOptions.Detached)
4✔
302
                                {
303
                                        if (addition.Job.BatchId is not null)
×
NEW
304
                                                throw new ImmediateJobException("A detached continuation cannot belong to a batch.");
×
305
                                }
306
                                else if (addition.Options is ContinuationOptions.BesideContinuations or ContinuationOptions.BeforeContinuations)
4✔
307
                                {
308
                                        if (current.BatchId is null || addition.Job.BatchId != current.BatchId)
4✔
NEW
309
                                                throw new ImmediateJobException("A batch-tracked continuation must belong to the current job's batch.");
×
310
                                        trackedAdditions++;
4✔
311
                                }
312
                                else
313
                                {
314
                                        throw new ArgumentOutOfRangeException(nameof(additions), "Unknown continuation option.");
×
315
                                }
316

317
                                dependencyEdges.Add(new()
4✔
318
                                {
4✔
319
                                        ChildJobId = addition.Job.Id,
4✔
320
                                        ParentJobId = jobId,
4✔
321
                                        Trigger = addition.Trigger,
4✔
322
                                });
4✔
323
                        }
324

325
                        if (dependencyEdges.Count != 0)
4✔
326
                                ValidateEdges([.. additions.Select(static addition => addition.Job)], dependencyEdges, current.BatchId);
4✔
327
                        ValidateSplice(existingWaiters, additions.Count(static addition => addition.Options == ContinuationOptions.BeforeContinuations));
4✔
328

329
                        IncrementBatchMembers(current.BatchId, trackedAdditions);
4✔
330
                        foreach (var addition in additions)
4✔
331
                        {
332
                                _jobs.Add(addition.Job.Id, NormalizeWaitingJob(addition.Job, dependencyCount: 1));
4✔
333
                                if (addition.Options == ContinuationOptions.BeforeContinuations)
4✔
334
                                        SpliceBeforeWaiters(addition.Job.Id, existingWaiters);
4✔
335
                        }
336

337
                        _edges.AddRange(dependencyEdges);
4✔
338
                        TransitionToTerminal(jobId, JobState.Succeeded, error: null, timeProvider.GetUtcNow());
4✔
339
                }
4✔
340

341
                return ValueTask.CompletedTask;
4✔
342
        }
343

344
        /// <inheritdoc />
345
        public ValueTask AddBatchJobAsync(
346
                string currentJobId,
347
                JobRecord job,
348
                ContinuationOptions options,
349
                CancellationToken cancellationToken = default
350
        )
351
        {
352
                ArgumentNullException.ThrowIfNull(job);
×
353
                cancellationToken.ThrowIfCancellationRequested();
×
354
                lock (_gate)
355
                {
356
                        if (!_jobs.TryGetValue(currentJobId, out var current) || current.State != JobState.Active)
×
NEW
357
                                throw new ImmediateJobException($"Job '{currentJobId}' is not currently active.");
×
358
                        if (current.BatchId is null || !_batches.ContainsKey(current.BatchId))
×
NEW
359
                                throw new ImmediateJobException("The current job does not belong to a batch.");
×
360
                        if (options == ContinuationOptions.Detached)
×
NEW
361
                                throw new ImmediateJobException("IJOB020: AddToBatchAsync(JobDetails, ...) cannot create detached work.");
×
362
                        if (options is not (ContinuationOptions.BesideContinuations or ContinuationOptions.BeforeContinuations))
×
363
                                throw new ArgumentOutOfRangeException(nameof(options));
×
364
                        ValidateNewJob(job);
×
365
                        if (job.BatchId != current.BatchId)
×
NEW
366
                                throw new ImmediateJobException("The new job must belong to the current job's batch.");
×
367
                        if (job.State is JobState.Active or JobState.AwaitingContinuation || IsTerminal(job.State))
×
NEW
368
                                throw new ImmediateJobException($"Concurrent batch member '{job.Id}' has invalid state '{job.State}'.");
×
369

370
                        var existingWaiters = options == ContinuationOptions.BeforeContinuations
×
371
                                ? GetUnsettledWaiters(currentJobId)
×
372
                                : [];
×
373
                        ValidateSplice(existingWaiters, options == ContinuationOptions.BeforeContinuations ? 1 : 0);
×
374
                        IncrementBatchMembers(current.BatchId, 1);
×
375
                        _jobs.Add(job.Id, job with { RemainingDependencies = 0 });
×
376
                        if (options == ContinuationOptions.BeforeContinuations)
×
377
                                SpliceBeforeWaiters(job.Id, existingWaiters);
×
378
                }
×
379

380
                return ValueTask.CompletedTask;
×
381
        }
382

383
        /// <inheritdoc />
384
        public ValueTask FailAsync(
385
                string jobId,
386
                string workerId,
387
                string error,
388
                DateTimeOffset? nextRetryAt,
389
                CancellationToken cancellationToken = default
390
        )
391
        {
392
                cancellationToken.ThrowIfCancellationRequested();
4✔
393
                lock (_gate)
394
                {
395
                        var job = GetOwnedActive(jobId, workerId);
4✔
396
                        if (nextRetryAt.HasValue)
4✔
397
                        {
398
                                var now = timeProvider.GetUtcNow();
4✔
399
                                _jobs[jobId] = job with
4✔
400
                                {
4✔
401
                                        State = nextRetryAt <= now ? JobState.Pending : JobState.Scheduled,
4✔
402
                                        DueAt = nextRetryAt.Value,
4✔
403
                                        WorkerId = null,
4✔
404
                                        LeaseExpiresAt = null,
4✔
405
                                        LastError = error,
4✔
406
                                        CompletedAt = null,
4✔
407
                                };
4✔
408
                        }
409
                        else
410
                        {
411
                                TransitionToTerminal(jobId, JobState.Failed, error, timeProvider.GetUtcNow());
4✔
412
                        }
413
                }
4✔
414

415
                return ValueTask.CompletedTask;
4✔
416
        }
417

418
        /// <inheritdoc />
419
        public ValueTask UpsertRecurringAsync(RecurringJobSchedule schedule, CancellationToken cancellationToken = default)
420
        {
421
                ArgumentNullException.ThrowIfNull(schedule);
4✔
422
                cancellationToken.ThrowIfCancellationRequested();
4✔
423
                lock (_gate)
424
                {
425
                        if (_recurring.TryGetValue(schedule.Name, out var current))
4✔
426
                        {
427
                                if (current.IsCodeDefined && !schedule.IsCodeDefined)
4✔
428
                                        throw new ImmediateJobException("Code-defined recurring schedules cannot be replaced by dynamic schedules.");
4✔
429

430
                                schedule = schedule with { IsPaused = current.IsPaused, LastRunAt = current.LastRunAt };
4✔
431
                        }
432

433
                        _recurring[schedule.Name] = schedule;
4✔
434
                }
4✔
435

436
                return ValueTask.CompletedTask;
4✔
437
        }
438

439
        /// <inheritdoc />
440
        public ValueTask RemoveObsoleteCodeDefinedRecurringAsync(
441
                IReadOnlyCollection<string> activeScheduleNames,
442
                CancellationToken cancellationToken = default
443
        )
444
        {
445
                ArgumentNullException.ThrowIfNull(activeScheduleNames);
4✔
446
                cancellationToken.ThrowIfCancellationRequested();
4✔
447
                var activeNames = activeScheduleNames.ToHashSet(StringComparer.Ordinal);
4✔
448
                lock (_gate)
449
                {
450
                        var obsoleteNames = _recurring
4✔
451
                                .Where(schedule => schedule.Value.IsCodeDefined && !activeNames.Contains(schedule.Key))
4✔
452
                                .Select(static schedule => schedule.Key)
4✔
453
                                .ToArray();
4✔
454
                        foreach (var name in obsoleteNames)
4✔
455
                                _ = _recurring.Remove(name);
4✔
456
                }
4✔
457

458
                return ValueTask.CompletedTask;
4✔
459
        }
460

461
        /// <inheritdoc />
462
        public ValueTask RemoveRecurringAsync(string name, CancellationToken cancellationToken = default)
463
        {
464
                cancellationToken.ThrowIfCancellationRequested();
×
465
                lock (_gate)
466
                {
467
                        if (_recurring.TryGetValue(name, out var schedule) && schedule.IsCodeDefined)
×
NEW
468
                                throw new ImmediateJobException("Code-defined recurring schedules cannot be deleted.");
×
469

470
                        _ = _recurring.Remove(name);
×
471
                }
×
472

473
                return ValueTask.CompletedTask;
×
474
        }
475

476
        /// <inheritdoc />
477
        public ValueTask PauseRecurringAsync(string name, CancellationToken cancellationToken = default) =>
NEW
478
                SetRecurringPausedAsync(name, true, cancellationToken);
×
479

480
        /// <inheritdoc />
481
        public ValueTask ResumeRecurringAsync(string name, CancellationToken cancellationToken = default) =>
NEW
482
                SetRecurringPausedAsync(name, false, cancellationToken);
×
483

484
        /// <inheritdoc />
485
        public async ValueTask<IReadOnlyList<RecurringJobSchedule>> GetDueRecurringAsync(
486
                DateTimeOffset now,
487
                int batchSize,
488
                CancellationToken cancellationToken = default
489
        )
490
        {
491
                cancellationToken.ThrowIfCancellationRequested();
4✔
492
                lock (_gate)
493
                {
494
                        return
4✔
495
                        [
4✔
496
                                .. _recurring.Values.Where(x => !x.IsPaused && x.NextRunAt <= now).OrderBy(x => x.NextRunAt).Take(batchSize),
4✔
497
                        ];
4✔
498
                }
499
        }
4✔
500

501
        /// <inheritdoc />
502
        public async ValueTask<bool> MaterializeRecurringAsync(
503
                RecurringJobSchedule schedule,
504
                JobRecord job,
505
                DateTimeOffset nextRunAt,
506
                CancellationToken cancellationToken = default
507
        )
508
        {
509
                ArgumentNullException.ThrowIfNull(schedule);
4✔
510
                ArgumentNullException.ThrowIfNull(job);
4✔
511
                cancellationToken.ThrowIfCancellationRequested();
4✔
512
                lock (_gate)
513
                {
514
                        if (!_recurring.TryGetValue(schedule.Name, out var current) || current.NextRunAt != schedule.NextRunAt)
4✔
515
                                return false;
×
516

517
                        var inserted = job.RecurringKey is null || _recurringKeys.Add(job.RecurringKey);
4✔
518
                        if (inserted)
4✔
519
                                _jobs[job.Id] = job;
4✔
520
                        _recurring[schedule.Name] = current with { LastRunAt = schedule.NextRunAt, NextRunAt = nextRunAt };
4✔
521
                        return inserted;
4✔
522
                }
523
        }
4✔
524

525
        /// <inheritdoc />
526
        public async ValueTask<JobMonitoringSnapshot> GetMonitoringSnapshotAsync(CancellationToken cancellationToken = default)
527
        {
528
                cancellationToken.ThrowIfCancellationRequested();
4✔
529
                lock (_gate)
530
                {
531
                        var counts = Enum.GetValues<JobState>().ToDictionary(state => state, state => _jobs.Values.LongCount(x => x.State == state));
4✔
532
                        var cutoff = timeProvider.GetUtcNow() - TimeSpan.FromMinutes(2);
4✔
533
                        IReadOnlyList<JobServerSnapshot> servers = [.. _servers.Values.Where(x => x.LastHeartbeat >= cutoff)];
4✔
534
                        return new(timeProvider.GetUtcNow(), counts, [.. _recurring.Values], servers)
4✔
535
                        {
4✔
536
                                Capabilities = this.GetCapabilities(),
4✔
537
                        };
4✔
538
                }
539
        }
4✔
540

541
        /// <inheritdoc />
542
        public async ValueTask<IReadOnlyList<JobRecord>> QueryJobsAsync(JobQuery query, CancellationToken cancellationToken = default)
543
        {
544
                ArgumentNullException.ThrowIfNull(query);
4✔
545
                cancellationToken.ThrowIfCancellationRequested();
4✔
546
                lock (_gate)
547
                {
548
                        var jobs = _jobs.Values.AsEnumerable();
4✔
549
                        if (query.Id is { } id)
4✔
550
                                jobs = jobs.Where(x => x.Id == id);
4✔
551
                        if (query.State is { } state)
4✔
552
                                jobs = jobs.Where(x => x.State == state);
4✔
553
                        if (!string.IsNullOrWhiteSpace(query.QueueName))
4✔
554
                                jobs = jobs.Where(x => x.QueueName == query.QueueName);
4✔
555

556
                        if (!string.IsNullOrWhiteSpace(query.Search))
4✔
557
                                jobs = jobs.Where(x => x.JobName.Contains(query.Search, StringComparison.OrdinalIgnoreCase));
3✔
558

559
                        return
4✔
560
                        [
4✔
561
                                .. jobs.OrderByDescending(x => x.CreatedAt).Skip(Math.Max(0, query.Skip)).Take(Math.Clamp(query.Take, 1, 1000)),
4✔
562
                        ];
4✔
563
                }
564
        }
4✔
565

566
        /// <inheritdoc />
567
        public async ValueTask<BatchStatus?> GetBatchStatusAsync(
568
                string batchId,
569
                CancellationToken cancellationToken = default
570
        )
571
        {
572
                ArgumentException.ThrowIfNullOrWhiteSpace(batchId);
4✔
573
                cancellationToken.ThrowIfCancellationRequested();
4✔
574
                lock (_gate)
575
                {
576
                        if (!_batches.TryGetValue(batchId, out var batch))
4✔
577
                                return null;
4✔
578

579
                        return ToStatus(batch);
4✔
580
                }
581
        }
4✔
582

583
        /// <inheritdoc />
584
        public async ValueTask<IReadOnlyList<BatchStatus>> QueryBatchesAsync(
585
                JobBatchQuery query,
586
                CancellationToken cancellationToken = default
587
        )
588
        {
589
                ArgumentNullException.ThrowIfNull(query);
4✔
590
                ArgumentOutOfRangeException.ThrowIfNegative(query.Skip);
4✔
591
                ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(query.Take, 0);
4✔
592
                cancellationToken.ThrowIfCancellationRequested();
4✔
593
                lock (_gate)
594
                {
595
                        var batches = _batches.Values.AsEnumerable();
4✔
596
                        if (query.State is { } state)
4✔
597
                                batches = batches.Where(batch => batch.State == state);
×
598
                        return
4✔
599
                        [
4✔
600
                                .. batches.OrderByDescending(static batch => batch.CreatedAt)
4✔
601
                                        .ThenBy(static batch => batch.Id, StringComparer.Ordinal)
4✔
602
                                        .Skip(query.Skip)
4✔
603
                                        .Take(query.Take)
4✔
604
                                        .Select(ToStatus),
4✔
605
                        ];
4✔
606
                }
607
        }
4✔
608

609
        /// <inheritdoc />
610
        public async ValueTask<IReadOnlyList<BatchMemberStatus>> QueryBatchMembersAsync(
611
                string batchId,
612
                BatchMemberQuery query,
613
                CancellationToken cancellationToken = default
614
        )
615
        {
616
                ArgumentException.ThrowIfNullOrWhiteSpace(batchId);
4✔
617
                ArgumentNullException.ThrowIfNull(query);
4✔
618
                ArgumentOutOfRangeException.ThrowIfNegative(query.Skip);
4✔
619
                ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(query.Take, 0);
4✔
620
                cancellationToken.ThrowIfCancellationRequested();
4✔
621
                lock (_gate)
622
                {
623
                        if (!_batches.ContainsKey(batchId))
4✔
624
                                return [];
×
625

626
                        var members = _jobs.Values.Where(job => job.BatchId == batchId);
4✔
627
                        if (query.State is { } state)
4✔
628
                                members = members.Where(job => job.State == state);
4✔
629

630
                        return
4✔
631
                        [
4✔
632
                                .. members
4✔
633
                                        .OrderBy(job => job.CreatedAt)
4✔
634
                                        .ThenBy(job => job.Id, StringComparer.Ordinal)
4✔
635
                                        .Skip(query.Skip)
4✔
636
                                        .Take(query.Take)
4✔
637
                                        .Select(static job => new BatchMemberStatus(
4✔
638
                                                job.Id,
4✔
639
                                                job.JobName,
4✔
640
                                                job.QueueName,
4✔
641
                                                job.State,
4✔
642
                                                job.Attempt,
4✔
643
                                                job.CreatedAt,
4✔
644
                                                job.CompletedAt,
4✔
645
                                                job.LastError
4✔
646
                                        )),
4✔
647
                        ];
4✔
648
                }
649
        }
4✔
650

651
        /// <inheritdoc />
652
        public async ValueTask<BatchGraph?> GetBatchGraphAsync(
653
                string batchId,
654
                CancellationToken cancellationToken = default
655
        )
656
        {
657
                ArgumentException.ThrowIfNullOrWhiteSpace(batchId);
4✔
658
                cancellationToken.ThrowIfCancellationRequested();
4✔
659
                lock (_gate)
660
                {
661
                        if (!_batches.ContainsKey(batchId))
4✔
662
                                return null;
×
663

664
                        var members = _jobs.Values
4✔
665
                                .Where(job => job.BatchId == batchId)
4✔
666
                                .OrderBy(job => job.CreatedAt)
4✔
667
                                .ThenBy(job => job.Id, StringComparer.Ordinal)
4✔
668
                                .ToArray();
4✔
669
                        var memberIds = members.Select(static job => job.Id).ToHashSet(StringComparer.Ordinal);
4✔
670
                        return new(
4✔
671
                                batchId,
4✔
672
                                [.. members.Select(static job => new BatchGraphNode(job.Id, job.JobName, job.State))],
4✔
673
                                [.. _edges.Where(edge => memberIds.Contains(edge.ChildJobId)).Select(ToGraphEdge)]
4✔
674
                        );
4✔
675
                }
676
        }
4✔
677

678
        /// <inheritdoc />
679
        public async ValueTask<JobStatus?> GetJobStatusAsync(
680
                string jobId,
681
                CancellationToken cancellationToken = default
682
        )
683
        {
684
                ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
4✔
685
                cancellationToken.ThrowIfCancellationRequested();
4✔
686
                lock (_gate)
687
                {
688
                        if (!_jobs.TryGetValue(jobId, out var job))
4✔
689
                                return null;
×
690

691
                        return new(
4✔
692
                                job.Id,
4✔
693
                                job.JobName,
4✔
694
                                job.QueueName,
4✔
695
                                job.State,
4✔
696
                                job.Attempt,
4✔
697
                                MaxAttempts: 0,
4✔
698
                                job.CreatedAt,
4✔
699
                                job.DueAt,
4✔
700
                                job.CompletedAt,
4✔
701
                                job.LastError,
4✔
702
                                job.BatchId,
4✔
703
                                [.. _edges.Where(edge => edge.ChildJobId == jobId).Select(ToGraphEdge)]
4✔
704
                        );
4✔
705
                }
706
        }
4✔
707

708
        /// <inheritdoc />
709
        public ValueTask CancelBatchAsync(string batchId, CancellationToken cancellationToken = default)
710
        {
711
                ArgumentException.ThrowIfNullOrWhiteSpace(batchId);
4✔
712
                cancellationToken.ThrowIfCancellationRequested();
4✔
713
                lock (_gate)
714
                {
715
                        if (!_batches.TryGetValue(batchId, out var batch))
4✔
716
                                throw new KeyNotFoundException($"Batch '{batchId}' was not found.");
×
717
                        if (batch.State != BatchState.Executing)
4✔
NEW
718
                                throw new ImmediateJobException("Only an executing batch can be cancelled.");
×
719

720
                        foreach (var jobId in _jobs.Values
4✔
721
                                .Where(job => job.BatchId == batchId && !IsTerminal(job.State))
4✔
722
                                .Select(static job => job.Id)
4✔
723
                                .ToArray())
4✔
724
                        {
725
                                TransitionToTerminal(jobId, JobState.Cancelled, error: null, timeProvider.GetUtcNow());
4✔
726
                        }
727
                }
4✔
728

729
                return ValueTask.CompletedTask;
4✔
730
        }
731

732
        /// <inheritdoc />
733
        public ValueTask DeleteBatchAsync(string batchId, CancellationToken cancellationToken = default)
734
        {
735
                ArgumentException.ThrowIfNullOrWhiteSpace(batchId);
4✔
736
                cancellationToken.ThrowIfCancellationRequested();
4✔
737
                lock (_gate)
738
                {
739
                        if (!_batches.TryGetValue(batchId, out var batch))
4✔
740
                                throw new KeyNotFoundException($"Batch '{batchId}' was not found.");
×
741
                        if (batch.State == BatchState.Executing)
4✔
NEW
742
                                throw new ImmediateJobException("Only a terminal batch can be deleted.");
×
743

744
                        var jobIds = _jobs.Values
4✔
745
                                .Where(job => job.BatchId == batchId)
4✔
746
                                .Select(static job => job.Id)
4✔
747
                                .ToHashSet(StringComparer.Ordinal);
4✔
748
                        foreach (var jobId in jobIds)
4✔
749
                                _ = _jobs.Remove(jobId);
4✔
750
                        _ = _batches.Remove(batchId);
4✔
751
                        RemoveEdgesForJobs(jobIds, [batchId]);
4✔
752
                }
4✔
753

754
                return ValueTask.CompletedTask;
4✔
755
        }
756

757
        /// <inheritdoc />
758
        public ValueTask RetryAsync(string jobId, CancellationToken cancellationToken = default)
759
        {
760
                cancellationToken.ThrowIfCancellationRequested();
×
761
                lock (_gate)
762
                {
763
                        if (!_jobs.TryGetValue(jobId, out var job) || job.State != JobState.Failed)
×
NEW
764
                                throw new ImmediateJobException("Only failed jobs can be retried.");
×
765

766
                        _jobs[jobId] = job with
×
767
                        {
×
768
                                State = JobState.Pending,
×
769
                                DueAt = timeProvider.GetUtcNow(),
×
770
                                CompletedAt = null,
×
771
                                LastError = null,
×
772
                        };
×
773
                        if (job.BatchId is { } batchId && _batches.TryGetValue(batchId, out var batch))
×
774
                        {
775
                                _batches[batchId] = batch with
×
776
                                {
×
777
                                        State = BatchState.Executing,
×
778
                                        PendingCount = batch.PendingCount + 1,
×
779
                                        FailedCount = Math.Max(0, batch.FailedCount - 1),
×
780
                                        CompletedAt = null,
×
781
                                };
×
782
                        }
783
                }
×
784

785
                return ValueTask.CompletedTask;
×
786
        }
787

788
        /// <inheritdoc />
789
        public ValueTask DeleteAsync(string jobId, CancellationToken cancellationToken = default)
790
        {
791
                cancellationToken.ThrowIfCancellationRequested();
×
792
                lock (_gate)
793
                {
794
                        if (!_jobs.TryGetValue(jobId, out var job))
×
795
                                return ValueTask.CompletedTask;
×
796
                        if (!IsTerminal(job.State))
×
NEW
797
                                throw new ImmediateJobException("Only terminal jobs can be deleted.");
×
798
                        if (job.BatchId is not null)
×
NEW
799
                                throw new ImmediateJobException("Batch members cannot be deleted individually.");
×
800

801
                        _ = _jobs.Remove(jobId);
×
802
                        RemoveEdgesForJobs(new(StringComparer.Ordinal) { jobId });
×
803
                }
×
804

805
                return ValueTask.CompletedTask;
×
806
        }
×
807

808
        /// <inheritdoc />
809
        public ValueTask PurgeJobsAsync(
810
                TimeSpan succeededRetention,
811
                TimeSpan failedRetention,
812
                CancellationToken cancellationToken = default
813
        )
814
        {
815
                cancellationToken.ThrowIfCancellationRequested();
4✔
816
                var now = timeProvider.GetUtcNow();
4✔
817
                lock (_gate)
818
                {
819
                        var standaloneJobIds = _jobs.Values
4✔
820
                                .Where(static job => job.BatchId is null)
4✔
821
                                .Where(x => x.CompletedAt is { } completed &&
4✔
822
                                        (x.State == JobState.Succeeded && completed < now - succeededRetention ||
4✔
823
                                         x.State is JobState.Failed or JobState.Cancelled && completed < now - failedRetention))
4✔
NEW
824
                                .Select(static job => job.Id)
×
825
                                .ToHashSet(StringComparer.Ordinal);
4✔
826
                        foreach (var id in standaloneJobIds)
4✔
NEW
827
                                _ = _jobs.Remove(id);
×
828
                        RemoveEdgesForJobs(standaloneJobIds);
4✔
829
                }
4✔
830

831
                return ValueTask.CompletedTask;
4✔
832
        }
833

834
        /// <inheritdoc />
835
        public ValueTask PurgeBatchesAsync(
836
                TimeSpan batchSucceededRetention,
837
                TimeSpan batchFailedRetention,
838
                CancellationToken cancellationToken = default
839
        )
840
        {
841
                cancellationToken.ThrowIfCancellationRequested();
4✔
842
                var now = timeProvider.GetUtcNow();
4✔
843
                lock (_gate)
844
                {
845
                        var batchIds = _batches.Values
4✔
846
                                .Where(batch => batch.CompletedAt is { } completed &&
×
847
                                        (batch.State == BatchState.Succeeded && completed < now - batchSucceededRetention ||
×
848
                                         batch.State is BatchState.Failed or BatchState.Cancelled && completed < now - batchFailedRetention))
×
849
                                .Select(static batch => batch.Id)
×
850
                                .ToHashSet(StringComparer.Ordinal);
4✔
851
                        var batchJobIds = _jobs.Values
4✔
852
                                .Where(job => job.BatchId is { } batchId && batchIds.Contains(batchId))
4✔
853
                                .Select(static job => job.Id)
×
854
                                .ToHashSet(StringComparer.Ordinal);
4✔
855
                        foreach (var id in batchJobIds)
4✔
856
                                _ = _jobs.Remove(id);
×
857
                        foreach (var batchId in batchIds)
4✔
858
                                _ = _batches.Remove(batchId);
×
859
                        RemoveEdgesForJobs(batchJobIds, batchIds);
4✔
860
                }
4✔
861

862
                return ValueTask.CompletedTask;
4✔
863
        }
864

865
        /// <inheritdoc />
866
        public ValueTask HeartbeatAsync(JobServerSnapshot server, CancellationToken cancellationToken = default)
867
        {
868
                ArgumentNullException.ThrowIfNull(server);
4✔
869
                cancellationToken.ThrowIfCancellationRequested();
4✔
870
                lock (_gate)
871
                        _servers[server.WorkerId] = server;
4✔
872
                return ValueTask.CompletedTask;
4✔
873
        }
874

875
        /// <inheritdoc />
876
        public async ValueTask<bool> IsHealthyAsync(CancellationToken cancellationToken = default)
877
        {
878
                cancellationToken.ThrowIfCancellationRequested();
×
879
                return true;
×
880
        }
881

882
        private void ValidateNewJob(JobRecord job)
883
        {
884
                ArgumentException.ThrowIfNullOrWhiteSpace(job.Id);
4✔
885
                ArgumentException.ThrowIfNullOrWhiteSpace(job.JobName);
4✔
886
                if (_jobs.ContainsKey(job.Id))
4✔
NEW
887
                        throw new ImmediateJobException($"Job '{job.Id}' already exists.");
×
888
        }
4✔
889

890
        private void ValidateBatch(
891
                JobBatchRecord batch,
892
                IReadOnlyList<JobRecord> jobs,
893
                IReadOnlyList<JobContinuationEdge> edges
894
        )
895
        {
896
                ArgumentException.ThrowIfNullOrWhiteSpace(batch.Id);
4✔
897
                if (_batches.ContainsKey(batch.Id))
4✔
NEW
898
                        throw new ImmediateJobException($"Batch '{batch.Id}' already exists.");
×
899
                if (jobs.Count == 0)
4✔
NEW
900
                        throw new ImmediateJobException("An atomic batch cannot be empty.");
×
901
                var succeeded = jobs.Count(static job => job.State == JobState.Succeeded);
4✔
902
                var failed = jobs.Count(static job => job.State == JobState.Failed);
4✔
903
                var cancelled = jobs.Count(static job => job.State == JobState.Cancelled);
4✔
904
                var pending = jobs.Count - succeeded - failed - cancelled;
4✔
905
                var expectedState = pending != 0
4✔
906
                        ? BatchState.Executing
4✔
907
                        : failed != 0 ? BatchState.Failed : cancelled != 0 ? BatchState.Cancelled : BatchState.Succeeded;
4✔
908
                if (batch.TotalJobs != jobs.Count ||
4✔
909
                        batch.PendingCount != pending ||
4✔
910
                        batch.SucceededCount != succeeded ||
4✔
911
                        batch.FailedCount != failed ||
4✔
912
                        batch.CancelledCount != cancelled ||
4✔
913
                        batch.State != expectedState ||
4✔
914
                        (pending == 0) != (batch.CompletedAt is not null))
4✔
915
                {
NEW
916
                        throw new ImmediateJobException("A batch header does not match its members or aggregate state.");
×
917
                }
918

919
                var jobIds = new HashSet<string>(StringComparer.Ordinal);
4✔
920
                foreach (var job in jobs)
4✔
921
                {
922
                        ValidateNewJob(job);
4✔
923
                        if (!jobIds.Add(job.Id))
4✔
NEW
924
                                throw new ImmediateJobException($"Job '{job.Id}' occurs more than once in the batch.");
×
925
                        if (job.BatchId != batch.Id)
4✔
NEW
926
                                throw new ImmediateJobException($"Job '{job.Id}' does not belong to batch '{batch.Id}'.");
×
927
                }
928

929
                ValidateEdges(jobs, edges, batch.Id);
4✔
930
        }
4✔
931

932
        private void ValidateEdges(
933
                IReadOnlyList<JobRecord> newJobs,
934
                IReadOnlyList<JobContinuationEdge> edges,
935
                string? batchId
936
        )
937
        {
938
                if (batchId is null && edges.Count == 0)
4✔
NEW
939
                        throw new ImmediateJobException("A continuation must have at least one parent.");
×
940

941
                var newJobIds = newJobs.Select(static job => job.Id).ToHashSet(StringComparer.Ordinal);
4✔
942
                var logicalEdges = new HashSet<(string Child, string ParentKind, string Parent)>(
4✔
943
                        EqualityComparer<(string Child, string ParentKind, string Parent)>.Default
4✔
944
                );
4✔
945
                var outgoing = newJobIds.ToDictionary(static id => id, static _ => new List<string>(), StringComparer.Ordinal);
4✔
946
                var incoming = newJobIds.ToDictionary(static id => id, static _ => 0, StringComparer.Ordinal);
4✔
947
                foreach (var edge in edges)
4✔
948
                {
949
                        ArgumentNullException.ThrowIfNull(edge);
4✔
950
                        ArgumentException.ThrowIfNullOrWhiteSpace(edge.ChildJobId);
4✔
951
                        if (!Enum.IsDefined(edge.Trigger))
4✔
952
                                throw new ArgumentOutOfRangeException(nameof(edges), "Unknown continuation trigger.");
×
953
                        if (!newJobIds.Contains(edge.ChildJobId))
4✔
NEW
954
                                throw new ImmediateJobException($"Continuation child '{edge.ChildJobId}' is not part of the atomic insert.");
×
955

956
                        var hasJobParent = !string.IsNullOrWhiteSpace(edge.ParentJobId);
4✔
957
                        var hasBatchParent = !string.IsNullOrWhiteSpace(edge.ParentBatchId);
4✔
958
                        if (hasJobParent == hasBatchParent)
4✔
NEW
959
                                throw new ImmediateJobException("A continuation edge must have exactly one job or batch parent.");
×
960

961
                        if (hasJobParent)
4✔
962
                        {
963
                                var parentId = edge.ParentJobId!;
4✔
964
                                if (parentId == edge.ChildJobId)
4✔
NEW
965
                                        throw new ImmediateJobException($"Continuation job '{edge.ChildJobId}' cannot depend on itself.");
×
966
                                if (!newJobIds.Contains(parentId) && !_jobs.ContainsKey(parentId))
4✔
967
                                        throw new KeyNotFoundException($"Continuation parent job '{parentId}' was not found.");
4✔
968
                                if (!logicalEdges.Add((edge.ChildJobId, "job", parentId)))
4✔
NEW
969
                                        throw new ImmediateJobException($"Duplicate continuation edge '{parentId}' -> '{edge.ChildJobId}'.");
×
970

971
                                if (newJobIds.Contains(parentId))
4✔
972
                                {
973
                                        outgoing[parentId].Add(edge.ChildJobId);
4✔
974
                                        incoming[edge.ChildJobId]++;
4✔
975
                                }
976
                        }
977
                        else
978
                        {
979
                                var parentId = edge.ParentBatchId!;
4✔
980
                                if (!_batches.ContainsKey(parentId))
4✔
981
                                        throw new KeyNotFoundException($"Continuation parent batch '{parentId}' was not found.");
×
982
                                if (!logicalEdges.Add((edge.ChildJobId, "batch", parentId)))
4✔
NEW
983
                                        throw new ImmediateJobException($"Duplicate continuation edge from batch '{parentId}' to '{edge.ChildJobId}'.");
×
984
                        }
985
                }
986

987
                var ready = new Queue<string>(incoming.Where(static pair => pair.Value == 0).Select(static pair => pair.Key));
4✔
988
                var visited = 0;
4✔
989
                while (ready.TryDequeue(out var parentId))
4✔
990
                {
991
                        visited++;
4✔
992
                        foreach (var childId in outgoing[parentId])
4✔
993
                        {
994
                                if (--incoming[childId] == 0)
4✔
995
                                        ready.Enqueue(childId);
4✔
996
                        }
997
                }
998

999
                if (visited != newJobIds.Count)
4✔
NEW
1000
                        throw new ImmediateJobException("IJOB018: The continuation graph contains a dependency cycle.");
×
1001
        }
4✔
1002

1003
        private static JobRecord NormalizeWaitingJob(JobRecord job, int dependencyCount) => job with
4✔
1004
        {
4✔
1005
                State = dependencyCount == 0 ? job.State : JobState.AwaitingContinuation,
4✔
1006
                RemainingDependencies = dependencyCount,
4✔
1007
                FailedDependencies = 0,
4✔
1008
                WorkerId = null,
4✔
1009
                LeaseExpiresAt = null,
4✔
1010
                CompletedAt = null,
4✔
1011
        };
4✔
1012

1013
        private bool IsRecoveredBatch(
1014
                JobBatchRecord batch,
1015
                IReadOnlyList<JobRecord> jobs,
1016
                IReadOnlyList<JobContinuationEdge> edges
1017
        )
1018
        {
1019
                if (batch.StartedAt is not null ||
4✔
1020
                        batch.CompletedAt is not null ||
4✔
1021
                        batch.SucceededCount != 0 ||
4✔
1022
                        batch.FailedCount != 0 ||
4✔
1023
                        batch.CancelledCount != 0 ||
4✔
1024
                        jobs.Any(static job => job.State is JobState.Active or JobState.Succeeded or JobState.Failed or JobState.Cancelled))
4✔
1025
                {
1026
                        return true;
4✔
1027
                }
1028

1029
                if (!HasTerminalParent(edges))
4✔
1030
                        return false;
4✔
1031

1032
                var incomingCounts = edges
4✔
1033
                        .GroupBy(static edge => edge.ChildJobId, StringComparer.Ordinal)
4✔
1034
                        .ToDictionary(static group => group.Key, static group => group.Count(), StringComparer.Ordinal);
4✔
1035
                return jobs.Any(job => incomingCounts.TryGetValue(job.Id, out var incoming) &&
4✔
1036
                        (job.State != JobState.AwaitingContinuation || job.RemainingDependencies < incoming));
4✔
1037
        }
1038

1039
        private bool HasTerminalParent(IEnumerable<JobContinuationEdge> edges) => edges.Any(edge =>
4✔
1040
                edge.ParentJobId is { } parentJobId &&
4✔
1041
                        _jobs.TryGetValue(parentJobId, out var parentJob) &&
4✔
1042
                        IsTerminal(parentJob.State) ||
4✔
1043
                edge.ParentBatchId is { } parentBatchId &&
4✔
1044
                        _batches.TryGetValue(parentBatchId, out var parentBatch) &&
4✔
1045
                        IsTerminal(parentBatch.State));
4✔
1046

1047
        private void MarkTerminalParentEdgesSettled(IEnumerable<JobContinuationEdge> edges)
1048
        {
1049
                foreach (var edge in edges)
4✔
1050
                {
1051
                        if (edge.ParentJobId is { } parentJobId &&
4✔
1052
                                _jobs.TryGetValue(parentJobId, out var parentJob) &&
4✔
1053
                                IsTerminal(parentJob.State) ||
4✔
1054
                                edge.ParentBatchId is { } parentBatchId &&
4✔
1055
                                _batches.TryGetValue(parentBatchId, out var parentBatch) &&
4✔
1056
                                IsTerminal(parentBatch.State))
4✔
1057
                        {
1058
                                _ = _settledEdges.Add(edge);
4✔
1059
                        }
1060
                }
1061
        }
4✔
1062

1063
        private void EvaluateAlreadyTerminalParents(IReadOnlyList<JobContinuationEdge> edges)
1064
        {
1065
                foreach (var parentId in edges
4✔
1066
                        .Where(static edge => edge.ParentJobId is not null)
4✔
1067
                        .Select(static edge => edge.ParentJobId!)
4✔
1068
                        .Distinct(StringComparer.Ordinal)
4✔
1069
                        .Where(parentId => _jobs.TryGetValue(parentId, out var parent) && IsTerminal(parent.State))
4✔
1070
                        .ToArray())
4✔
1071
                {
1072
                        ProcessTerminalJob(parentId);
4✔
1073
                }
1074

1075
                foreach (var parentId in edges
4✔
1076
                        .Where(static edge => edge.ParentBatchId is not null)
4✔
1077
                        .Select(static edge => edge.ParentBatchId!)
4✔
1078
                        .Distinct(StringComparer.Ordinal)
4✔
1079
                        .Where(parentId => _batches.TryGetValue(parentId, out var parent) && IsTerminal(parent.State))
4✔
1080
                        .ToArray())
4✔
1081
                {
1082
                        ProcessTerminalBatch(parentId);
4✔
1083
                }
1084
        }
4✔
1085

1086
        private void TransitionToTerminal(
1087
                string jobId,
1088
                JobState terminalState,
1089
                string? error,
1090
                DateTimeOffset completedAt
1091
        )
1092
        {
1093
                if (!IsTerminal(terminalState))
4✔
1094
                        throw new ArgumentOutOfRangeException(nameof(terminalState));
×
1095
                if (!_jobs.TryGetValue(jobId, out var job) || IsTerminal(job.State))
4✔
1096
                        return;
4✔
1097

1098
                _jobs[jobId] = job with
4✔
1099
                {
4✔
1100
                        State = terminalState,
4✔
1101
                        WorkerId = null,
4✔
1102
                        LeaseExpiresAt = null,
4✔
1103
                        LastError = error,
4✔
1104
                        CompletedAt = completedAt,
4✔
1105
                };
4✔
1106
                UpdateBatchAfterTerminal(job.BatchId, terminalState, completedAt);
4✔
1107
                ProcessTerminalJob(jobId);
4✔
1108
        }
4✔
1109

1110
        private void UpdateBatchAfterTerminal(string? batchId, JobState state, DateTimeOffset completedAt)
1111
        {
1112
                if (batchId is null || !_batches.TryGetValue(batchId, out var batch))
4✔
1113
                        return;
4✔
1114

1115
                var pending = Math.Max(0, batch.PendingCount - 1);
4✔
1116
                batch = batch with
4✔
1117
                {
4✔
1118
                        PendingCount = pending,
4✔
1119
                        SucceededCount = batch.SucceededCount + (state == JobState.Succeeded ? 1 : 0),
4✔
1120
                        FailedCount = batch.FailedCount + (state == JobState.Failed ? 1 : 0),
4✔
1121
                        CancelledCount = batch.CancelledCount + (state == JobState.Cancelled ? 1 : 0),
4✔
1122
                };
4✔
1123
                if (pending == 0)
4✔
1124
                {
1125
                        batch = batch with
4✔
1126
                        {
4✔
1127
                                State = batch.FailedCount != 0
4✔
1128
                                        ? BatchState.Failed
4✔
1129
                                        : batch.CancelledCount != 0 ? BatchState.Cancelled : BatchState.Succeeded,
4✔
1130
                                CompletedAt = completedAt,
4✔
1131
                        };
4✔
1132
                }
1133

1134
                _batches[batchId] = batch;
4✔
1135
                if (pending == 0)
4✔
1136
                        ProcessTerminalBatch(batchId);
4✔
1137
        }
4✔
1138

1139
        private void ProcessTerminalJob(string parentJobId)
1140
        {
1141
                if (!_jobs.TryGetValue(parentJobId, out var parent) || !IsTerminal(parent.State))
4✔
1142
                        return;
×
1143

1144
                foreach (var edge in _edges
4✔
1145
                        .Where(edge => edge.ParentJobId == parentJobId && !_settledEdges.Contains(edge))
4✔
1146
                        .ToArray())
4✔
1147
                {
1148
                        _ = _settledEdges.Add(edge);
4✔
1149
                        SettleEdge(
4✔
1150
                                edge,
4✔
1151
                                parentSucceeded: parent.State == JobState.Succeeded,
4✔
1152
                                parentFailed: parent.State == JobState.Failed
4✔
1153
                        );
4✔
1154
                }
1155
        }
4✔
1156

1157
        private void ProcessTerminalBatch(string parentBatchId)
1158
        {
1159
                if (!_batches.TryGetValue(parentBatchId, out var parent) || !IsTerminal(parent.State))
4✔
1160
                        return;
×
1161

1162
                foreach (var edge in _edges
4✔
1163
                        .Where(edge => edge.ParentBatchId == parentBatchId && !_settledEdges.Contains(edge))
4✔
1164
                        .ToArray())
4✔
1165
                {
1166
                        _ = _settledEdges.Add(edge);
4✔
1167
                        SettleEdge(
4✔
1168
                                edge,
4✔
1169
                                parentSucceeded: parent.State == BatchState.Succeeded,
4✔
1170
                                parentFailed: parent.State == BatchState.Failed
4✔
1171
                        );
4✔
1172
                }
1173
        }
4✔
1174

1175
        private void SettleEdge(JobContinuationEdge edge, bool parentSucceeded, bool parentFailed)
1176
        {
1177
                if (!_jobs.TryGetValue(edge.ChildJobId, out var child) || IsTerminal(child.State))
4✔
1178
                        return;
×
1179
                if (edge.Trigger == ContinuationTrigger.Success && !parentSucceeded)
4✔
1180
                {
1181
                        _jobs[child.Id] = child with { RemainingDependencies = 0 };
4✔
1182
                        TransitionToTerminal(child.Id, JobState.Cancelled, error: null, timeProvider.GetUtcNow());
4✔
1183
                        return;
4✔
1184
                }
1185

1186
                var remaining = Math.Max(0, child.RemainingDependencies - 1);
4✔
1187
                var failed = child.FailedDependencies + (parentFailed ? 1 : 0);
4✔
1188
                if (remaining == 0 && edge.Trigger == ContinuationTrigger.Failure && failed == 0)
4✔
1189
                {
1190
                        _jobs[child.Id] = child with { RemainingDependencies = 0, FailedDependencies = failed };
4✔
1191
                        TransitionToTerminal(child.Id, JobState.Cancelled, error: null, timeProvider.GetUtcNow());
4✔
1192
                        return;
4✔
1193
                }
1194

1195
                _jobs[child.Id] = child with
4✔
1196
                {
4✔
1197
                        State = remaining == 0 && child.State == JobState.AwaitingContinuation
4✔
1198
                                ? GetReadyState(child)
4✔
1199
                                : child.State,
4✔
1200
                        RemainingDependencies = remaining,
4✔
1201
                        FailedDependencies = failed,
4✔
1202
                };
4✔
1203
        }
4✔
1204

1205
        private JobState GetReadyState(JobRecord job) =>
1206
                job.DueAt <= timeProvider.GetUtcNow() ? JobState.Pending : JobState.Scheduled;
4✔
1207

1208
        private JobContinuationEdge[] GetUnsettledWaiters(string parentJobId) =>
1209
        [
4✔
1210
                .. _edges.Where(edge => edge.ParentJobId == parentJobId &&
4✔
1211
                        !_settledEdges.Contains(edge) &&
4✔
1212
                        _jobs.TryGetValue(edge.ChildJobId, out var child) &&
4✔
1213
                        !IsTerminal(child.State)),
4✔
1214
        ];
4✔
1215

1216
        private void SpliceBeforeWaiters(
1217
                string newParentJobId,
1218
                IReadOnlyList<JobContinuationEdge> existingWaiters
1219
        )
1220
        {
1221
                foreach (var existingEdge in existingWaiters)
4✔
1222
                {
1223
                        if (!_jobs.TryGetValue(existingEdge.ChildJobId, out var child) || IsTerminal(child.State))
4✔
1224
                                continue;
1225

1226
                        _jobs[child.Id] = child with
4✔
1227
                        {
4✔
1228
                                State = JobState.AwaitingContinuation,
4✔
1229
                                RemainingDependencies = child.RemainingDependencies + 1,
4✔
1230
                        };
4✔
1231
                        _edges.Add(new()
4✔
1232
                        {
4✔
1233
                                ChildJobId = child.Id,
4✔
1234
                                ParentJobId = newParentJobId,
4✔
1235
                                Trigger = existingEdge.Trigger,
4✔
1236
                        });
4✔
1237
                }
1238
        }
4✔
1239

1240
        private void ValidateSplice(IReadOnlyList<JobContinuationEdge> existingWaiters, int additions)
1241
        {
1242
                if (additions == 0)
4✔
1243
                        return;
4✔
1244
                foreach (var existingEdge in existingWaiters)
4✔
1245
                {
1246
                        if (_jobs.TryGetValue(existingEdge.ChildJobId, out var child) &&
4✔
1247
                                child.RemainingDependencies > int.MaxValue - additions)
4✔
1248
                        {
NEW
1249
                                throw new ImmediateJobException($"Continuation dependency count overflow for job '{child.Id}'.");
×
1250
                        }
1251
                }
1252
        }
4✔
1253

1254
        private void IncrementBatchMembers(string? batchId, int count)
1255
        {
1256
                if (count == 0)
4✔
1257
                        return;
4✔
1258
                if (batchId is null || !_batches.TryGetValue(batchId, out var batch))
4✔
NEW
1259
                        throw new ImmediateJobException("The current job's batch was not found.");
×
1260
                if (batch.TotalJobs > int.MaxValue - count || batch.PendingCount > int.MaxValue - count)
4✔
NEW
1261
                        throw new ImmediateJobException($"Batch '{batchId}' member count overflow.");
×
1262

1263
                _batches[batchId] = batch with
4✔
1264
                {
4✔
1265
                        TotalJobs = batch.TotalJobs + count,
4✔
1266
                        PendingCount = batch.PendingCount + count,
4✔
1267
                };
4✔
1268
        }
4✔
1269

1270
        private void MarkBatchStarted(string? batchId, DateTimeOffset startedAt)
1271
        {
1272
                if (batchId is not null && _batches.TryGetValue(batchId, out var batch) && batch.StartedAt is null)
4✔
1273
                        _batches[batchId] = batch with { StartedAt = startedAt };
4✔
1274
        }
4✔
1275

1276
        private static bool IsTerminal(JobState state) =>
1277
                state is JobState.Succeeded or JobState.Failed or JobState.Cancelled;
4✔
1278

1279
        private static bool IsTerminal(BatchState state) => state is not BatchState.Executing;
4✔
1280

1281
        private static BatchStatus ToStatus(JobBatchRecord batch) => new(
4✔
1282
                batch.Id,
4✔
1283
                batch.State,
4✔
1284
                batch.TotalJobs,
4✔
1285
                batch.SucceededCount,
4✔
1286
                batch.FailedCount,
4✔
1287
                batch.CancelledCount,
4✔
1288
                batch.PendingCount,
4✔
1289
                batch.CreatedAt,
4✔
1290
                batch.StartedAt,
4✔
1291
                batch.CompletedAt,
4✔
1292
                batch.TotalJobs == 0 ? 0 : (double)(batch.TotalJobs - batch.PendingCount) / batch.TotalJobs
4✔
1293
        );
4✔
1294

1295
        private static BatchGraphEdge ToGraphEdge(JobContinuationEdge edge) => new(
4✔
1296
                edge.ChildJobId,
4✔
1297
                edge.ParentJobId,
4✔
1298
                edge.ParentBatchId,
4✔
1299
                edge.Trigger
4✔
1300
        );
4✔
1301

1302
        private void RemoveEdgesForJobs(
1303
                HashSet<string> jobIds,
1304
                HashSet<string>? batchIds = null
1305
        )
1306
        {
1307
                if (jobIds.Count == 0 && (batchIds is null || batchIds.Count == 0))
4✔
1308
                        return;
4✔
1309

1310
                var batches = batchIds ?? [];
4✔
1311
                for (var index = _edges.Count - 1; index >= 0; index--)
4✔
1312
                {
1313
                        var edge = _edges[index];
4✔
1314
                        if (!jobIds.Contains(edge.ChildJobId) &&
4✔
1315
                                (edge.ParentJobId is null || !jobIds.Contains(edge.ParentJobId)) &&
4✔
1316
                                (edge.ParentBatchId is null || !batches.Contains(edge.ParentBatchId)))
4✔
1317
                        {
1318
                                continue;
1319
                        }
1320

1321
                        _edges.RemoveAt(index);
4✔
1322
                        _ = _settledEdges.Remove(edge);
4✔
1323
                }
1324
        }
4✔
1325

1326
        private JobRecord GetOwnedActive(string jobId, string workerId)
1327
        {
1328
                if (!_jobs.TryGetValue(jobId, out var job) || job.State != JobState.Active || job.WorkerId != workerId)
4✔
NEW
1329
                        throw new ImmediateJobException($"Worker '{workerId}' does not own active job '{jobId}'.");
×
1330
                return job;
4✔
1331
        }
1332

1333
        private ValueTask SetRecurringPausedAsync(string name, bool isPaused, CancellationToken cancellationToken)
1334
        {
1335
                cancellationToken.ThrowIfCancellationRequested();
×
1336
                lock (_gate)
1337
                {
1338
                        if (!_recurring.TryGetValue(name, out var schedule))
×
1339
                                throw new KeyNotFoundException($"Recurring schedule '{name}' was not found.");
×
1340

1341
                        _recurring[name] = schedule with { IsPaused = isPaused };
×
1342
                }
×
1343

1344
                return ValueTask.CompletedTask;
×
1345
        }
1346
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc