• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

medplum / medplum / 28812604563

06 Jul 2026 06:03PM UTC coverage: 91.899% (-0.02%) from 91.919%
28812604563

Pull #9491

github

web-flow
Merge face95eb6 into 08fcd9cb9
Pull Request #9491: feat(agent): auto-retry of messages already committed but failed

21816 of 24788 branches covered (88.01%)

Branch coverage included in aggregate %.

201 of 223 new or added lines in 7 files covered. (90.13%)

51 existing lines in 8 files now uncovered.

38774 of 41143 relevant lines covered (94.24%)

12495.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.04
/packages/agent/src/queue/durable-queue.ts
1
// SPDX-FileCopyrightText: Copyright Orangebot, Inc. and Medplum contributors
2
// SPDX-License-Identifier: Apache-2.0
3

4
import type { ILogger } from '@medplum/core';
5
import { normalizeErrorString } from '@medplum/core';
6
import { randomUUID } from 'node:crypto';
7
import { chmodSync, existsSync } from 'node:fs';
8
import type { DatabaseSync, SQLInputValue, StatementSync } from 'node:sqlite';
9
import {
10
  CHANNEL_DEPTH,
11
  CHECKPOINT_WAL,
12
  CLAIM_NEXT,
13
  COMMIT_SEQ_NO,
14
  COUNT_BY_STATE,
15
  DB_SIZE_BYTES,
16
  ENQUEUE,
17
  ENQUEUE_REJECTED,
18
  FIND_BY_CALLBACK,
19
  FIND_BY_ID,
20
  FIND_SEEN_BY_CONTROL_ID,
21
  GET_LEASE,
22
  HEARTBEAT_LEASE,
23
  LIST_QUEUED_IDS_FOR_CHANNEL,
24
  MARK_BOT_FAILED,
25
  MARK_PROCESSED,
26
  MARK_SENT,
27
  PEEK_LAST_SEQ_NO,
28
  RECORD_SERVER_RESPONSE,
29
  RECOVER_CLAIMED,
30
  RECOVER_INFLIGHT,
31
  RECOVER_INFLIGHT_GUARANTEED,
32
  RELEASE_LEASE,
33
  REQUEUE,
34
  SCHEDULE_RETRY,
35
  SET_ACK_OUTCOME,
36
  TRY_ACQUIRE_LEASE,
37
} from './queries';
38
import { runMigrations } from './schema';
39
import type {
40
  AckOutcome,
41
  ClaimedRow,
42
  EnqueueInput,
43
  EnqueueRejectedInput,
44
  EnqueueResult,
45
  InboundRow,
46
  MessageState,
47
  NackedRow,
48
  QueueErrorCode,
49
} from './types';
50
import {
51
  AckOutcome as AckOutcomeValues,
52
  assertRowState,
53
  MessageState as MessageStateValues,
54
  QueueLeaseError,
55
} from './types';
56

57
/**
58
 * Default lease lifetime. Picked to be:
59
 * - Long enough that ordinary GC pauses or slow disks don't cost us the lease
60
 *   under load.
61
 * - Short enough that, when an agent dies un-gracefully, a peer can take over
62
 *   within tens of seconds rather than minutes.
63
 */
64
export const DEFAULT_LEASE_TTL_MS = 30_000;
16✔
65

66
/**
67
 * How often the leaseholder re-extends its lease. Must be comfortably less than
68
 * {@link DEFAULT_LEASE_TTL_MS} so transient delays don't let the lease expire.
69
 */
70
export const DEFAULT_LEASE_HEARTBEAT_MS = 10_000;
16✔
71

72
/**
73
 * How often a non-leader retries acquisition. Shorter is fine — the underlying
74
 * SQL is a single conditional UPSERT and is cheap.
75
 */
76
export const DEFAULT_LEASE_ACQUIRE_RETRY_MS = 2_000;
16✔
77

78
/**
79
 * How often the queue folds its WAL back into the main DB file. SQLite only
80
 * checkpoints piggybacked on commits, so once traffic stops nothing else would
81
 * drain the WAL until the next retention sweep or close(); this loop bounds how
82
 * far the main DB file can lag behind the last write. Matches the agent
83
 * heartbeat cadence — the loop's former home, before the queue took ownership.
84
 */
85
export const DEFAULT_CHECKPOINT_INTERVAL_MS = 10_000;
16✔
86

87
export interface DurableQueueOptions {
88
  /** Filesystem path to the SQLite DB file. */
89
  path: string;
90
  /** Logger for migration / lifecycle messages. */
91
  log: ILogger;
92
  /** Override the per-process dispatch-lease holder ID. Defaults to a fresh UUID. */
93
  leaseHolder?: string;
94
  /** Override the dispatch-lease TTL in ms. */
95
  leaseTtlMs?: number;
96
  /** Override the dispatch-lease heartbeat interval in ms. */
97
  leaseHeartbeatMs?: number;
98
  /** Override the dispatch-lease acquire retry interval in ms. */
99
  leaseAcquireRetryMs?: number;
100
  /** Override the WAL-checkpoint loop interval in ms. */
101
  checkpointIntervalMs?: number;
102
}
103

104
/** Counts by lifecycle state — used by stats and the retention sweeper. */
105
export type StateCounts = Record<MessageState, number>;
106

107
/** Per-channel queue depth snapshot returned by {@link DurableQueue.getChannelDepths}. */
108
export interface ChannelDepth {
109
  queued: number;
110
  /** Claimed by a worker but not yet written to the socket. */
111
  claimed: number;
112
  /** Written to the socket, awaiting the server response. */
113
  inflight: number;
114
  oldestQueuedAgeMs: number | null;
115
}
116

117
/**
118
 * SQLite-backed durable FIFO for inbound HL7 messages.
119
 *
120
 * One instance owns one synchronous `node:sqlite` connection plus a bag of
121
 * prepared statements covering the hot path. All channel intake and worker
122
 * dispatch goes through this object. See DURABLE_QUEUE_ARCHITECTURE.md §3, §5.
123
 *
124
 * The class is designed to be opened once at agent startup, used for the
125
 * lifetime of the process, and closed exactly once during {@link DurableQueue.close}.
126
 */
127
export class DurableQueue {
128
  private readonly db: DatabaseSync;
129
  private readonly log: ILogger;
130
  private readonly path: string;
131
  private closed = false;
161✔
132
  // The dispatch-lease holder ID of THIS process, set when startDispatchLease()
133
  // begins the acquire/heartbeat loop (or bound directly via setLeaseHolder in
134
  // tests). Dispatch-class mutations are gated on the live lease still belonging
135
  // to this holder; when a peer takes over, the gate throws QueueLeaseError so the
136
  // demoted worker stops driving the queue. Undefined when the dispatch lease was
137
  // never started (single-process use and most unit tests), in which case the gate
138
  // is inert because no lease row exists.
139
  private leaseHolderId: string | undefined;
140

141
  // ── Dispatch-lease orchestration (formerly DispatchLeaseManager) ──
142
  // Lightweight leader election for the DISPATCH path. The lease gates *dispatch
143
  // only* — claiming rows and driving them to the server (the worker +
144
  // recoverOnStartup). It deliberately does NOT gate intake (followers still
145
  // persist inbound messages via enqueue), maintenance (WAL checkpoint, retention
146
  // sweep), or diagnostics (stats): those are valid on any process with the queue
147
  // file open. Exactly one process may dispatch at a time, which is the
148
  // load-bearing primitive that makes zero-downtime upgrades safe: during the
149
  // overlap window where the old and new agent processes both have the SQLite file
150
  // open, only the leaseholder claims/sends rows; the non-leader keeps accepting
151
  // and persisting inbound traffic but does not dispatch.
152
  //
153
  // There is deliberately NO lost-leadership callback: loss is enforced at the
154
  // data layer instead — the dispatch ops throw QueueLeaseError once a peer holds
155
  // the lease (see assertNotDemoted), so the worker self-detects and drains
156
  // without pushing an event down a chain of callbacks.
157
  private readonly leaseTtlMs: number;
158
  private readonly leaseHeartbeatMs: number;
159
  private readonly leaseAcquireRetryMs: number;
160
  private leaseLeader = false;
161✔
161
  private leaseLoopActive = false;
161✔
162
  private leaseAcquireTimer: NodeJS.Timeout | undefined;
163
  private leaseHeartbeatTimer: NodeJS.Timeout | undefined;
164
  private onBecameLeader: (() => void) | undefined;
165
  // True when the WAL may contain frames not yet checkpointed into the main DB
166
  // file. SQLite only attempts checkpoints piggybacked on commits, so once
167
  // traffic stops nothing would ever drain the WAL — the queue's own checkpoint
168
  // loop (startCheckpointLoop) polls this via checkpointWalIfDirty(). Starts true
169
  // because open() itself writes (pragmas, migrations, lease).
170
  private walDirty = true;
161✔
171

172
  // WAL-checkpoint loop. Runs while the queue is open, INDEPENDENT of the
173
  // dispatch lease: a follower still accepts intake writes (enqueue runs on any
174
  // process with the file open), so WAL draining can't be gated on leadership the
175
  // way the lease timers are. Started in open(), cleared in close().
176
  private readonly checkpointIntervalMs: number;
177
  private checkpointTimer: NodeJS.Timeout | undefined;
178

179
  // Prepared statements — created once at open(), reused for every call.
180
  // Names mirror the public methods that use them.
181
  private readonly enqueueStmt: StatementSync;
182
  private readonly enqueueRejectedStmt: StatementSync;
183
  private readonly peekLastSeqNoStmt: StatementSync;
184
  private readonly commitSeqNoStmt: StatementSync;
185
  private readonly findSeenByControlIdStmt: StatementSync;
186
  private readonly claimNextStmt: StatementSync;
187
  private readonly markSentStmt: StatementSync;
188
  private readonly findByCallbackStmt: StatementSync;
189
  private readonly findByIdStmt: StatementSync;
190
  private readonly recordServerResponseStmt: StatementSync;
191
  private readonly markProcessedStmt: StatementSync;
192
  private readonly markBotFailedStmt: StatementSync;
193
  private readonly setAckOutcomeStmt: StatementSync;
194
  private readonly scheduleRetryStmt: StatementSync;
195
  private readonly requeueStmt: StatementSync;
196
  private readonly recoverInflightStmt: StatementSync;
197
  private readonly recoverInflightGuaranteedStmt: StatementSync;
198
  private readonly recoverClaimedStmt: StatementSync;
199
  private readonly listQueuedIdsForChannelStmt: StatementSync;
200
  private readonly countByStateStmt: StatementSync;
201
  private readonly channelDepthStmt: StatementSync;
202
  private readonly tryAcquireLeaseStmt: StatementSync;
203
  private readonly heartbeatLeaseStmt: StatementSync;
204
  private readonly releaseLeaseStmt: StatementSync;
205
  private readonly getLeaseStmt: StatementSync;
206
  private readonly checkpointStmt: StatementSync;
207
  private readonly dbSizeBytesStmt: StatementSync;
208

209
  constructor(db: DatabaseSync, options: DurableQueueOptions) {
210
    this.db = db;
161✔
211
    this.log = options.log;
161✔
212
    this.path = options.path;
161✔
213
    this.leaseHolderId = options.leaseHolder;
161✔
214
    this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
161✔
215
    this.leaseHeartbeatMs = options.leaseHeartbeatMs ?? DEFAULT_LEASE_HEARTBEAT_MS;
161✔
216
    this.leaseAcquireRetryMs = options.leaseAcquireRetryMs ?? DEFAULT_LEASE_ACQUIRE_RETRY_MS;
161✔
217
    this.checkpointIntervalMs = options.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
161✔
218

219
    // Single-writer, durable-across-crash, WAL-friendly settings (§5).
220
    // Picked for HL7 inbound rates and the durability contract we owe the sender:
221
    //   synchronous=NORMAL gives crash safety without per-write fsync;
222
    //   WAL lets workers read while intake writes;
223
    //   mmap_size keeps the read path off the kernel I/O hot path.
224
    this.db.exec(`
161✔
225
      PRAGMA journal_mode = WAL;
226
      PRAGMA synchronous  = NORMAL;
227
      PRAGMA temp_store   = MEMORY;
228
      PRAGMA cache_size   = -65536;
229
      PRAGMA mmap_size    = 268435456;
230
      PRAGMA wal_autocheckpoint = 1000;
231
      PRAGMA busy_timeout = 5000;
232
      PRAGMA foreign_keys = ON;
233
    `);
234

235
    runMigrations(this.db);
161✔
236

237
    this.enqueueStmt = this.db.prepare(ENQUEUE);
161✔
238

239
    this.enqueueRejectedStmt = this.db.prepare(ENQUEUE_REJECTED);
161✔
240

241
    // Per-channel monotonic sequence counter (assignSeqNo). Single-process +
242
    // synchronous, so no locking is needed between peek and commit.
243
    this.peekLastSeqNoStmt = this.db.prepare(PEEK_LAST_SEQ_NO);
161✔
244
    this.commitSeqNoStmt = this.db.prepare(COMMIT_SEQ_NO);
161✔
245

246
    // Canonical prior row for a (channel, control_id): any state except `nacked`
247
    // (nacked rows are rejected-intake audit records and intentionally reuse
248
    // control IDs, so they're not a "real" prior delivery to dedupe against).
249
    // id DESC returns the most recent in the unlikely event legacy duplicates
250
    // predate the dedupe-on-intake logic.
251
    this.findSeenByControlIdStmt = this.db.prepare(FIND_SEEN_BY_CONTROL_ID);
161✔
252

253
    // FIFO claim: take the lowest-id queued row for this channel, flip it to
254
    // `claimed` in the same statement so concurrent workers can't double-claim.
255
    // node:sqlite is synchronous and the agent is single-process, so RETURNING is
256
    // enough — no advisory locking needed.
257
    this.claimNextStmt = this.db.prepare(CLAIM_NEXT);
161✔
258

259
    // Phase A → B: the App's send path calls this the moment the transmit request
260
    // is written to the socket, flipping `claimed` → `inflight` and stamping sent_at.
261
    this.markSentStmt = this.db.prepare(MARK_SENT);
161✔
262

263
    this.findByCallbackStmt = this.db.prepare(FIND_BY_CALLBACK);
161✔
264

265
    this.findByIdStmt = this.db.prepare(FIND_BY_ID);
161✔
266

267
    this.recordServerResponseStmt = this.db.prepare(RECORD_SERVER_RESPONSE);
161✔
268

269
    this.markProcessedStmt = this.db.prepare(MARK_PROCESSED);
161✔
270

271
    this.markBotFailedStmt = this.db.prepare(MARK_BOT_FAILED);
161✔
272

273
    this.setAckOutcomeStmt = this.db.prepare(SET_ACK_OUTCOME);
161✔
274

275
    // Auto-retry transition: a claimed/inflight row goes back to `queued` with a
276
    // future next_attempt_at so the channel's FIFO head blocks (and the row is
277
    // not re-claimed) until the backoff elapses. See SCHEDULE_RETRY.
278
    this.scheduleRetryStmt = this.db.prepare(SCHEDULE_RETRY);
161✔
279

280
    this.requeueStmt = this.db.prepare(REQUEUE);
161✔
281

282
    // Crash recovery splits three ways on whether the request reached the wire
283
    // and the channel's guaranteed-delivery setting:
284
    //   recoverClaimed — rows left `claimed` provably never reached the server,
285
    //     so they always return to `queued` for a clean re-dispatch with no
286
    //     duplicate risk (regardless of guaranteed delivery).
287
    //   recoverInflight — rows left `inflight` on a NORMAL channel are ambiguous
288
    //     (the server may or may not have processed them), so they land in
289
    //     `failed` for operator review, never silently retried. The source leg's
290
    //     ack_outcome is left as-is (`pending`): we don't know whether an ACK was owed.
291
    //   recoverInflightGuaranteed — rows left `inflight` on a GUARANTEED channel
292
    //     return to `queued` (duplication risk accepted) so the channel keeps
293
    //     trying until upstream gives a definitive answer (§4.1).
294
    this.recoverInflightStmt = this.db.prepare(RECOVER_INFLIGHT);
161✔
295
    this.recoverInflightGuaranteedStmt = this.db.prepare(RECOVER_INFLIGHT_GUARANTEED);
161✔
296
    this.recoverClaimedStmt = this.db.prepare(RECOVER_CLAIMED);
161✔
297

298
    this.listQueuedIdsForChannelStmt = this.db.prepare(LIST_QUEUED_IDS_FOR_CHANNEL);
161✔
299

300
    this.countByStateStmt = this.db.prepare(COUNT_BY_STATE);
161✔
301

302
    this.channelDepthStmt = this.db.prepare(CHANNEL_DEPTH);
161✔
303

304
    this.tryAcquireLeaseStmt = this.db.prepare(TRY_ACQUIRE_LEASE);
161✔
305

306
    this.heartbeatLeaseStmt = this.db.prepare(HEARTBEAT_LEASE);
161✔
307

308
    this.releaseLeaseStmt = this.db.prepare(RELEASE_LEASE);
161✔
309

310
    this.getLeaseStmt = this.db.prepare(GET_LEASE);
161✔
311

312
    // Prepared once like every other statement — checkpointWal() runs on every
313
    // checkpoint-loop tick and every retention sweep, so re-compiling it per call
314
    // would leak GC-finalised statement objects proportional to that frequency.
315
    this.checkpointStmt = this.db.prepare(CHECKPOINT_WAL);
161✔
316

317
    // Prepared once like the rest — the retention sweeper calls getDbSizeBytes()
318
    // in a tight loop while purging under size pressure, so re-compiling it per
319
    // call would leak GC-finalised statement objects proportional to sweep work.
320
    this.dbSizeBytesStmt = this.db.prepare(DB_SIZE_BYTES);
161✔
321
  }
322

323
  /**
324
   * Opens (or creates) the DB file at `options.path`, runs migrations, and
325
   * returns a ready DurableQueue.
326
   *
327
   * Constructs synchronously — `node:sqlite` is itself synchronous so there's
328
   * no benefit to making this `async`.
329
   *
330
   * @param options - Path + logger.
331
   * @returns The opened DurableQueue.
332
   */
333
  static open(options: DurableQueueOptions): DurableQueue {
334
    // Lazy require so this module can be imported in environments where
335
    // node:sqlite isn't available (e.g. type-only tooling).
336
    // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/consistent-type-imports
337
    const { DatabaseSync } = require('node:sqlite') as typeof import('node:sqlite');
160✔
338
    const db = new DatabaseSync(options.path);
160✔
339
    const queue = new DurableQueue(db, options);
160✔
340
    // Lock the DB file down to the agent's user. Best-effort — fails silently on
341
    // platforms (Windows) where chmod is a no-op.
342
    if (existsSync(options.path)) {
160!
343
      try {
160✔
344
        chmodSync(options.path, 0o600);
160✔
345
      } catch {
346
        // ignore
347
      }
348
    }
349
    // Start the WAL-checkpoint loop here (not in the constructor) so a bare
350
    // `new DurableQueue(db, opts)` — used by unit tests — stays timer-free; the
351
    // production lifecycle is open() … close(), and close() clears it.
352
    queue.startCheckpointLoop();
160✔
353
    return queue;
160✔
354
  }
355

356
  /** Closes the underlying SQLite handle. Idempotent. */
357
  close(): void {
358
    if (this.closed) {
163✔
359
      return;
2✔
360
    }
361
    this.closed = true;
161✔
362
    // Stop the WAL-checkpoint loop — the final flush below covers shutdown, and a
363
    // checkpointWalIfDirty() tick is a no-op once `closed` is set anyway.
364
    this.stopCheckpointLoop();
161✔
365
    // Release the lease BEFORE closing the DB so a waiting peer can take over
366
    // immediately rather than waiting for our TTL to expire. Idempotent and a
367
    // no-op when the dispatch lease was never started.
368
    this.stopDispatchLease();
161✔
369
    // SQLite only checkpoints + deletes the WAL on close when this is the last
370
    // connection to the file. An upgrade-overlap peer or an operator's sqlite3
371
    // shell defeats that, so flush explicitly — a clean shutdown should always
372
    // leave a self-contained main DB file (e.g. for file-level backups).
373
    this.checkpointWal();
161✔
374
    try {
161✔
375
      this.db.close();
161✔
376
    } catch (err) {
377
      this.log.warn(`Error while closing durable queue DB: ${normalizeErrorString(err)}`);
×
378
    }
379
  }
380

381
  /**
382
   * Unconditionally runs a TRUNCATE checkpoint, folding the WAL into the main
383
   * DB file and truncating the WAL to zero bytes. Best-effort: failures are
384
   * logged, and an incomplete checkpoint (a peer connection pinning part of the
385
   * WAL) leaves the dirty flag set so the next attempt retries.
386
   * @returns True when the checkpoint fully completed.
387
   */
388
  checkpointWal(): boolean {
389
    try {
216✔
390
      const row = this.checkpointStmt.get() as { busy: number } | undefined;
216✔
391
      if (row?.busy) {
216!
392
        return false;
×
393
      }
394
      this.walDirty = false;
216✔
395
      return true;
216✔
396
    } catch (err) {
397
      this.log.warn(`wal_checkpoint failed: ${normalizeErrorString(err)}`);
×
398
      return false;
×
399
    }
400
  }
401

402
  /**
403
   * Runs {@link DurableQueue.checkpointWal} only when writes have landed since
404
   * the last completed checkpoint. Driven by the queue's own checkpoint loop
405
   * (see {@link DurableQueue.startCheckpointLoop}) every `checkpointIntervalMs`,
406
   * so the WAL drains shortly after traffic stops instead of waiting for the next
407
   * retention sweep or for close(). A no-op on an idle queue.
408
   * @returns True when a checkpoint ran and fully completed.
409
   */
410
  checkpointWalIfDirty(): boolean {
411
    if (this.closed || !this.walDirty) {
6✔
412
      return false;
1✔
413
    }
414
    return this.checkpointWal();
5✔
415
  }
416

417
  /**
418
   * Starts the periodic WAL-checkpoint loop. Idempotent. Called once by
419
   * {@link DurableQueue.open}; runs regardless of dispatch leadership (see the
420
   * `checkpointTimer` field) and is torn down by {@link DurableQueue.close}.
421
   */
422
  private startCheckpointLoop(): void {
423
    if (this.checkpointTimer) {
160!
424
      return;
×
425
    }
426
    this.checkpointTimer = setInterval(() => this.checkpointWalIfDirty(), this.checkpointIntervalMs);
160✔
427
    // Don't keep the event loop alive solely for housekeeping — same as the
428
    // dispatch-lease timers.
429
    if (typeof this.checkpointTimer.unref === 'function') {
160!
430
      this.checkpointTimer.unref();
160✔
431
    }
432
  }
433

434
  /** Stops the WAL-checkpoint loop. Idempotent and a no-op when never started. */
435
  private stopCheckpointLoop(): void {
436
    if (this.checkpointTimer) {
161✔
437
      clearInterval(this.checkpointTimer);
160✔
438
      this.checkpointTimer = undefined;
160✔
439
    }
440
  }
441

442
  /** @returns Filesystem path of the underlying SQLite file. */
443
  getPath(): string {
444
    return this.path;
×
445
  }
446

447
  /** @returns Direct access to the underlying handle. Used by retention sweeper for PRAGMA queries. */
448
  getDb(): DatabaseSync {
449
    return this.db;
125✔
450
  }
451

452
  /**
453
   * Runs `fn` inside a single SQLite transaction, committing if it returns and
454
   * rolling back if it throws. Used to make a row state transition atomic with an
455
   * external side effect — e.g. flipping a row `claimed` → `inflight` only if the
456
   * socket write that puts it on the wire also succeeds (see `App.sendToWebSocket`):
457
   * if the write throws, the marker rolls back and the row stays `claimed`
458
   * (provably unsent, safe to requeue) rather than a phantom `inflight`.
459
   *
460
   * `node:sqlite` is synchronous, so `fn` must be synchronous too — do any `await`
461
   * (e.g. token refresh) before calling, not inside.
462
   * @param fn - The work to run transactionally.
463
   * @returns Whatever `fn` returns.
464
   */
465
  runInTransaction<T>(fn: () => T): T {
466
    this.db.exec('BEGIN');
34✔
467
    try {
34✔
468
      const result = fn();
34✔
469
      this.db.exec('COMMIT');
34✔
470
      return result;
34✔
471
    } catch (err) {
472
      this.db.exec('ROLLBACK');
1✔
473
      throw err;
1✔
474
    }
475
  }
476

477
  /**
478
   * Inserts a new `queued` row.
479
   *
480
   * If any prior non-`nacked` row already owns this `(channel_name,
481
   * msg_control_id)` — in `queued`, `claimed`, `inflight`, `processed`,
482
   * `rejected`, or `failed` — we don't insert; we surface a `duplicate` result
483
   * carrying that row so the caller can compare bodies and decide between
484
   * replaying the prior ACK and rejecting the collision (§8). This SELECT is the
485
   * single dedup authority on the intake path: the partial unique index only
486
   * covers the live `queued`/`claimed`/`inflight` window, so it can't recognize a
487
   * retransmit of an already-`processed`/`rejected`/`failed` row —
488
   * it remains only as a last-resort guard against an active-window race (not
489
   * expected in the single-process agent).
490
   * @param input - Fields to persist on the new row.
491
   * @param options - Optional behavior.
492
   * @param options.assignSeqNo - When provided, enqueue assigns the channel's next
493
   *   sequence number — but only after the duplicate check passes, so a retransmit
494
   *   never consumes one. It peeks the persisted counter (a non-consuming read),
495
   *   calls this callback with the candidate so the caller can stamp it into MSH.13
496
   *   and return the finalized bytes to persist, then advances the counter in the
497
   *   SAME transaction as the insert (so a crash can't store the row while leaving
498
   *   the counter behind, which would reuse the number on restart). On a duplicate
499
   *   the callback is never invoked.
500
   * @returns Either the inserted row, or the prior row that owns the MSH.10.
501
   */
502
  enqueue(input: EnqueueInput, options?: { assignSeqNo?: (candidate: number) => Buffer }): EnqueueResult {
503
    if (input.msgControlId) {
769!
504
      const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
769✔
505
      if (existing) {
769✔
506
        return { kind: 'duplicate', existing };
19✔
507
      }
508
    }
509

510
    // Not a duplicate — only now assign the sequence number (if configured). Peek
511
    // is non-consuming; the counter is advanced (commitSeqNo) inside the insert
512
    // transaction below, so a failed insert burns no number and a duplicate (above)
513
    // never reaches here.
514
    let finalizedMessage = input.finalizedMessage;
750✔
515
    let seqNo = input.seqNo;
750✔
516
    let seqNoToCommit: number | undefined;
517
    if (options?.assignSeqNo) {
750✔
518
      const candidate = this.peekNextSeqNo(input.channelName);
9✔
519
      finalizedMessage = options.assignSeqNo(candidate);
9✔
520
      seqNo = candidate;
9✔
521
      seqNoToCommit = candidate;
9✔
522
    }
523

524
    try {
750✔
525
      const runInsert = (): number => {
750✔
526
        const info = this.enqueueStmt.run(
750✔
527
          input.channelName,
528
          input.remote,
529
          input.msgControlId,
530
          input.msgType,
531
          toBlob(input.originalMessage),
532
          toBlob(finalizedMessage),
533
          input.encoding,
534
          input.enhancedMode,
535
          input.callbackId,
536
          seqNo,
537
          input.receivedAt,
538
          input.guaranteedDelivery ? 1 : 0
750✔
539
        );
540
        return Number(info.lastInsertRowid);
750✔
541
      };
542

543
      let id: number;
544
      if (seqNoToCommit !== undefined) {
750✔
545
        // Atomically insert the row and advance the sequence counter.
546
        this.db.exec('BEGIN');
9✔
547
        try {
9✔
548
          id = runInsert();
9✔
549
          this.commitSeqNo(input.channelName, seqNoToCommit);
9✔
550
          this.db.exec('COMMIT');
9✔
551
        } catch (txErr) {
552
          this.db.exec('ROLLBACK');
1✔
553
          throw txErr;
1✔
554
        }
555
      } else {
556
        id = runInsert();
741✔
557
      }
558
      this.walDirty = true;
749✔
559
      const row = this.getById(id);
749✔
560
      if (!row) {
749!
561
        throw new Error(`enqueue: inserted row id=${id} could not be re-read`);
×
562
      }
563
      // The insert (just committed above) is what durably accepted the message —
564
      // this re-read only recovers the row for the caller. Don't assert it's
565
      // still 'queued': a peer process sharing this DB file may have already
566
      // claimed (or, in principle, settled) it between the commit and this
567
      // read. That's not a failed intake, so we return whatever state it's in
568
      // rather than throwing and having the caller NACK an already-committed message.
569
      return { kind: 'inserted', row };
749✔
570
    } catch (err) {
571
      if (isUniqueConstraintError(err) && input.msgControlId) {
1!
572
        const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
×
573
        if (existing) {
×
574
          return { kind: 'duplicate', existing };
×
575
        }
576
      }
577
      throw err;
1✔
578
    }
579
  }
580

581
  /**
582
   * Inserts an audit row in the `nacked` terminal state — used when intake
583
   * rejected the message (duplicate, malformed) but we still want a forensics
584
   * record. Failure to write this is non-fatal; the caller is expected to log.
585
   * @param input - Fields to persist plus the `lastError`/`errorCode` describing why we rejected.
586
   * @returns The newly inserted audit row, or null if even the audit insert failed.
587
   */
588
  enqueueRejected(input: EnqueueRejectedInput): NackedRow | null {
589
    try {
12✔
590
      const info = this.enqueueRejectedStmt.run(
12✔
591
        input.channelName,
592
        input.remote,
593
        input.msgControlId,
594
        input.msgType,
595
        toBlob(input.originalMessage),
596
        toBlob(input.finalizedMessage),
597
        input.encoding,
598
        input.enhancedMode,
599
        input.callbackId,
600
        input.lastError,
601
        input.errorCode,
602
        input.seqNo,
603
        input.receivedAt
604
      );
605
      this.walDirty = true;
12✔
606
      // ENQUEUE_REJECTED always inserts in the 'nacked' state.
607
      const row = this.getById(Number(info.lastInsertRowid));
12✔
608
      assertRowState(row, MessageStateValues.NACKED);
12✔
609
      return row;
12✔
610
    } catch (err) {
611
      this.log.warn(`enqueueRejected failed: ${normalizeErrorString(err)}`);
×
612
      return null;
×
613
    }
614
  }
615

616
  /**
617
   * Atomically claims the next `queued` row for `channelName`, flipping it to
618
   * `claimed` and bumping `attempt_count`. The row stays `claimed` until
619
   * {@link markSent} flips it to `inflight` once the request hits the socket.
620
   * Honors a retry backoff: a `queued` row whose `next_attempt_at` is still in
621
   * the future is not handed out, and because the FIFO head-of-line is preserved
622
   * (the predicate is on the outer update, see {@link CLAIM_NEXT}) the whole
623
   * channel waits behind it rather than skipping ahead.
624
   * @param channelName - The channel to claim from.
625
   * @param now - Override the timestamp written to `processing_started_at` and
626
   *   compared against `next_attempt_at` (for tests).
627
   * @returns The claimed row, or `null` if the channel queue is empty or its head is backing off.
628
   */
629
  claimNext(channelName: string, now: number = Date.now()): ClaimedRow | null {
210✔
630
    this.assertNotDemoted();
210✔
631
    // Bind `now` twice: once for processing_started_at, once for the
632
    // next_attempt_at backoff predicate on the outer update.
633
    const raw = this.claimNextStmt.get(now, channelName, now) as Record<string, SQLInputValue> | undefined;
210✔
634
    if (raw) {
210✔
635
      this.walDirty = true;
113✔
636
    }
637
    // CLAIM_NEXT sets state='claimed' in the same statement (RETURNING *), so the
638
    // decoded row is always a ClaimedRow.
639
    return raw ? (rowFromSql(raw) as ClaimedRow) : null;
209✔
640
  }
641

642
  /**
643
   * Phase A → B transition: records that the transmit request for `callbackId`
644
   * was written to the WebSocket, flipping the row from `claimed` to `inflight`
645
   * and stamping `sent_at`. Called from the App's send path the instant the
646
   * request leaves the process — the durable marker that distinguishes a
647
   * provably-unsent row (safe to requeue on crash) from an ambiguous in-flight
648
   * one (failed for review). Guarded on `state = 'claimed'`, so it's a no-op for
649
   * legacy (non-durable) sends and for any row already past `claimed`.
650
   * @param callbackId - The callback ID of the transmit request just sent.
651
   * @param now - Override for `sent_at` (for tests).
652
   * @returns True if a `claimed` row was flipped to `inflight`.
653
   */
654
  markSent(callbackId: string, now: number = Date.now()): boolean {
44✔
655
    const info = this.markSentStmt.run(now, callbackId);
44✔
656
    if (Number(info.changes) > 0) {
44✔
657
      this.walDirty = true;
41✔
658
      return true;
41✔
659
    }
660
    return false;
3✔
661
  }
662

663
  /**
664
   * Looks up a row by its server-callback ID. Returns `null` if not found.
665
   * @param callbackId - The callback ID echoed back by the Medplum server.
666
   * @returns The matching row, or null.
667
   */
668
  findByCallback(callbackId: string): InboundRow | null {
669
    const raw = this.findByCallbackStmt.get(callbackId) as Record<string, SQLInputValue> | undefined;
8✔
670
    return raw ? rowFromSql(raw) : null;
8✔
671
  }
672

673
  /**
674
   * Looks up a row by its primary key.
675
   * @param id - The row's primary key.
676
   * @returns The matching row, or null.
677
   */
678
  getById(id: number): InboundRow | null {
679
    const raw = this.findByIdStmt.get(id) as Record<string, SQLInputValue> | undefined;
995✔
680
    return raw ? rowFromSql(raw) : null;
995!
681
  }
682

683
  /**
684
   * Records the server's `agent:transmit:response` body + status against the row.
685
   * @param id - Row primary key.
686
   * @param statusCode - HTTP-style status code returned by the server.
687
   * @param body - Response payload, or null when the server omitted one.
688
   */
689
  recordServerResponse(id: number, statusCode: number | null, body: Buffer | string | null): void {
690
    this.assertNotDemoted();
73✔
691
    this.recordServerResponseStmt.run(statusCode, body === null ? null : toBlob(body), id);
73!
692
    this.walDirty = true;
73✔
693
  }
694

695
  /**
696
   * Terminal Bot-leg transition: the server accepted the message (2xx). The
697
   * source leg is recorded independently via `ackOutcome` — `delivered` when the
698
   * app-level ACK reached the source, `undelivered` when it couldn't (e.g. the
699
   * source connection had closed). An `undelivered` row is fully processed
700
   * upstream and must never be re-dispatched; it recovers when the source
701
   * retransmits and the stored ACK is replayed (see {@link setAckOutcome}).
702
   *
703
   * Attempt-scoped (see {@link MARK_PROCESSED}): a no-op if `attemptCount`
704
   * doesn't match the row's current attempt.
705
   * @param id - Row primary key.
706
   * @param attemptCount - The attempt this response answers; must match the
707
   *   row's current `attempt_count` for the write to apply.
708
   * @param ackOutcome - Source-leg result: `delivered` or `undelivered`.
709
   * @param now - Override for `processed_at` (for tests).
710
   * @returns True if the row was settled; false if this attempt was superseded.
711
   */
712
  markProcessed(id: number, attemptCount: number, ackOutcome: AckOutcome, now: number = Date.now()): boolean {
56✔
713
    this.assertNotDemoted();
56✔
714
    const info = this.markProcessedStmt.run(ackOutcome, now, id, attemptCount);
56✔
715
    if (Number(info.changes) > 0) {
56✔
716
      this.walDirty = true;
54✔
717
      return true;
54✔
718
    }
719
    return false;
2✔
720
  }
721

722
  /**
723
   * Terminal Bot-leg transition: the server **rejected** the message itself
724
   * (permanent 4xx). Retrying can never help — the content must be triaged.
725
   *
726
   * Attempt-scoped (see {@link MARK_PROCESSED}): a no-op if `attemptCount`
727
   * doesn't match the row's current attempt.
728
   * @param id - Row primary key.
729
   * @param attemptCount - The attempt this failure answers; must match the
730
   *   row's current `attempt_count` for the write to apply.
731
   * @param error - Human-readable error string, written to `last_error`.
732
   * @param errorCode - Machine-readable classification, written to `error_code`.
733
   * @param now - Override for `errored_at` (for tests).
734
   * @returns True if the row was settled; false if this attempt was superseded.
735
   */
736
  markRejected(
737
    id: number,
738
    attemptCount: number,
739
    error: string,
740
    errorCode: QueueErrorCode,
741
    now: number = Date.now()
7✔
742
  ): boolean {
743
    this.assertNotDemoted();
7✔
744
    const info = this.markBotFailedStmt.run(MessageStateValues.REJECTED, now, error, errorCode, id, attemptCount);
7✔
745
    if (Number(info.changes) > 0) {
7!
746
      this.walDirty = true;
7✔
747
      return true;
7✔
748
    }
NEW
749
    return false;
×
750
  }
751

752
  /**
753
   * Terminal-for-now Bot-leg transition: a **transient/ambiguous** failure
754
   * (5xx, 429, response timeout, dispatch error, interrupted). The retry/
755
   * operator-review candidate — distinct from a `rejected` message so a future
756
   * retry policy can re-dispatch `failed` rows without ever touching `rejected`
757
   * ones (or `processed` + `undelivered` ones, whose Bot leg already succeeded).
758
   *
759
   * Attempt-scoped (see {@link MARK_PROCESSED}): a no-op if `attemptCount`
760
   * doesn't match the row's current attempt.
761
   * @param id - Row primary key.
762
   * @param attemptCount - The attempt this failure answers; must match the
763
   *   row's current `attempt_count` for the write to apply.
764
   * @param error - Human-readable error string, written to `last_error`.
765
   * @param errorCode - Machine-readable classification, written to `error_code`.
766
   * @param now - Override for `errored_at` (for tests).
767
   * @returns True if the row was settled; false if this attempt was superseded.
768
   */
769
  markFailed(
770
    id: number,
771
    attemptCount: number,
772
    error: string,
773
    errorCode: QueueErrorCode,
774
    now: number = Date.now()
16✔
775
  ): boolean {
776
    this.assertNotDemoted();
16✔
777
    const info = this.markBotFailedStmt.run(MessageStateValues.FAILED, now, error, errorCode, id, attemptCount);
16✔
778
    if (Number(info.changes) > 0) {
16!
779
      this.walDirty = true;
16✔
780
      return true;
16✔
781
    }
NEW
782
    return false;
×
783
  }
784

785
  /**
786
   * Updates only the source-leg {@link AckOutcome}, leaving the Bot-leg `state`
787
   * untouched. Used when a duplicate retransmit replays a previously
788
   * `undelivered` ACK and it lands — flipping the row to `delivered` so it no
789
   * longer reads as awaiting source delivery.
790
   * @param id - Row primary key.
791
   * @param ackOutcome - The new source-leg outcome.
792
   */
793
  setAckOutcome(id: number, ackOutcome: AckOutcome): void {
794
    this.setAckOutcomeStmt.run(ackOutcome, id);
4✔
795
    this.walDirty = true;
4✔
796
  }
797

798
  /**
799
   * Auto-retry transition: returns a `claimed`/`inflight`/`queued` row to
800
   * `queued`, scheduled to become claimable at `nextAttemptAt`. Because the row
801
   * keeps its id and claims are ordered by id, it sits at the head of its
802
   * channel's FIFO and blocks younger rows until it either succeeds or exhausts
803
   * its attempts — preserving per-channel ordering across retries.
804
   *
805
   * Distinct from {@link markFailed}: a retry stays `queued` (still in flight),
806
   * whereas `markFailed` is the terminal landing for a failure the policy is not
807
   * retrying. The worker decides which to call by gating the row's
808
   * {@link QueueErrorCode} against the retry policy (see worker.ts). Gated by the
809
   * dispatch lease like the other settle ops: a demoted process must not
810
   * reschedule. Attempt-scoped (see {@link MARK_PROCESSED}): also a no-op if
811
   * `attemptCount` doesn't match the row's current attempt — this is what lets
812
   * the late-response settle path (`onServerResponse` in worker.ts) call this on
813
   * an already-`queued` row without risking clobbering a newer attempt.
814
   * @param id - Row primary key.
815
   * @param attemptCount - The attempt this decision was made for; must match the
816
   *   row's current `attempt_count` for the write to apply.
817
   * @param error - Human-readable error string, written to `last_error`.
818
   * @param errorCode - Machine-readable classification, written to `error_code`.
819
   * @param nextAttemptAt - Earliest timestamp (ms) at which the row may be claimed again.
820
   * @returns True if the row was rescheduled; false if this attempt was superseded.
821
   */
822
  scheduleRetry(
823
    id: number,
824
    attemptCount: number,
825
    error: string,
826
    errorCode: QueueErrorCode,
827
    nextAttemptAt: number
828
  ): boolean {
829
    this.assertNotDemoted();
20✔
830
    const info = this.scheduleRetryStmt.run(error, errorCode, nextAttemptAt, id, attemptCount);
20✔
831
    if (Number(info.changes) > 0) {
20✔
832
      this.walDirty = true;
19✔
833
      return true;
19✔
834
    }
835
    return false;
1✔
836
  }
837

838
  /**
839
   * Returns a `claimed` row to `queued` — used when the WS connection drops
840
   * before the transmit request was ever written to the socket, so retrying on
841
   * reconnect carries no duplicate-delivery risk. (A row that already reached the
842
   * socket is `inflight`, not `claimed`, and this is a no-op for it — its outcome
843
   * is ambiguous and owned by the response timeout.) Because the row keeps its
844
   * original id and claims are ordered by id, it goes back to the front of its
845
   * channel's FIFO.
846
   * @param id - Row primary key.
847
   * @returns True if the row was requeued; false if it was not in `claimed`.
848
   */
849
  requeue(id: number): boolean {
850
    this.assertNotDemoted();
3✔
851
    const info = this.requeueStmt.run(id);
3✔
852
    if (Number(info.changes) > 0) {
3✔
853
      this.walDirty = true;
2✔
854
      return true;
2✔
855
    }
856
    return false;
1✔
857
  }
858

859
  /**
860
   * Recovers rows left mid-flight by a previous process, splitting on whether the
861
   * request reached the wire and the channel's guaranteed-delivery setting. Runs
862
   * once at startup, after acquiring the dispatch lease (§10):
863
   * - `claimed` rows provably never left the process (`sent_at` is NULL) → always
864
   *   returned to `queued` for a clean re-dispatch with no duplicate-delivery risk.
865
   * - `inflight` rows on a NORMAL channel are ambiguous (the server may have
866
   *   processed them) → `failed` with error code `interrupted`, surfaced for
867
   *   operator review, never silently retried.
868
   * - `inflight` rows on a GUARANTEED-delivery channel → returned to `queued`
869
   *   (duplication risk accepted) so the channel keeps trying until upstream gives
870
   *   a definitive answer (§4.1).
871
   * @param now - Override for `errored_at` (for tests).
872
   * @returns Counts of rows promoted to `failed` and returned to `queued`, respectively.
873
   */
874
  recoverOnStartup(now: number = Date.now()): { failed: number; requeued: number } {
38✔
875
    const failed = Number(this.recoverInflightStmt.run(now).changes);
38✔
876
    // Both legs return a row to `queued`: provably-unsent `claimed` rows (always
877
    // safe) and guaranteed-delivery `inflight` rows (duplication risk accepted).
878
    const requeued =
879
      Number(this.recoverClaimedStmt.run().changes) + Number(this.recoverInflightGuaranteedStmt.run().changes);
38✔
880
    if (failed > 0 || requeued > 0) {
38✔
881
      this.walDirty = true;
4✔
882
    }
883
    return { failed, requeued };
38✔
884
  }
885

886
  /**
887
   * Lists all `queued` row IDs for a channel, in FIFO order. Used by the
888
   * worker on startup to repopulate its in-memory wake signal.
889
   * @param channelName - The channel to query.
890
   * @returns Row IDs in FIFO order.
891
   */
892
  listQueuedIdsForChannel(channelName: string): number[] {
893
    const rows = this.listQueuedIdsForChannelStmt.all(channelName) as { id: number }[];
1✔
894
    return rows.map((r) => r.id);
2✔
895
  }
896

897
  /** @returns Counts of rows by state. Missing states are reported as 0. */
898
  countByState(): StateCounts {
899
    const counts: StateCounts = {
268✔
900
      queued: 0,
901
      claimed: 0,
902
      inflight: 0,
903
      processed: 0,
904
      rejected: 0,
905
      failed: 0,
906
      nacked: 0,
907
    };
908
    const rows = this.countByStateStmt.all() as { state: MessageState; n: number }[];
268✔
909
    for (const r of rows) {
268✔
910
      counts[r.state] = r.n;
311✔
911
    }
912
    return counts;
267✔
913
  }
914

915
  /**
916
   * @param channelName - The channel to query.
917
   * @param now - Override for the "now" timestamp used in `oldestQueuedAgeMs`.
918
   * @returns Depth snapshot for `channelName` (queued/claimed/inflight counts + oldest queued age).
919
   */
920
  getChannelDepth(channelName: string, now: number = Date.now()): ChannelDepth {
16✔
921
    const row = this.channelDepthStmt.get(channelName) as
16✔
922
      | { queued: number | null; claimed: number | null; inflight: number | null; oldest_queued_at: number | null }
923
      | undefined;
924
    return {
16✔
925
      queued: row?.queued ?? 0,
16!
926
      claimed: row?.claimed ?? 0,
16!
927
      inflight: row?.inflight ?? 0,
16!
928
      oldestQueuedAgeMs: row?.oldest_queued_at ? now - row.oldest_queued_at : null,
16!
929
    };
930
  }
931

932
  /**
933
   * Attempts to acquire (or re-acquire) the queue lease as `holder`.
934
   *
935
   * Succeeds when there is no current lease, the current lease is already held
936
   * by us (refresh case), or the prior holder's lease has expired. Fails when a
937
   * different holder still has a valid lease — caller should wait and retry.
938
   *
939
   * The lease is the cross-process coordination primitive that makes zero-downtime
940
   * upgrades safe: only the holder runs workers and `recoverOnStartup`, so two
941
   * processes sharing the DB during the upgrade overlap don't fight over rows.
942
   * @param holder - Stable identifier for this process (a per-process UUID).
943
   * @param ttlMs - How long the new lease should remain valid before another
944
   *                process can take over (also drives the heartbeat cadence).
945
   * @param now - Override the "now" timestamp (for tests).
946
   * @returns True if the lease is now held by `holder`; false if a foreign holder still owns it.
947
   */
948
  tryAcquireLease(holder: string, ttlMs: number, now: number = Date.now()): boolean {
99✔
949
    const expiresAt = now + ttlMs;
99✔
950
    const info = this.tryAcquireLeaseStmt.run(holder, now, expiresAt, holder, now);
99✔
951
    if (Number(info.changes) > 0) {
99✔
952
      this.walDirty = true;
78✔
953
    }
954
    return Number(info.changes) > 0;
99✔
955
  }
956

957
  /**
958
   * Extends the existing lease held by `holder`. Fails (returns false) when the
959
   * lease is no longer ours — most commonly because a peer took over after our
960
   * TTL elapsed. A leader that sees `false` here must stop driving the queue.
961
   * @param holder - The same identifier passed to {@link DurableQueue.tryAcquireLease}.
962
   * @param ttlMs - New TTL for the lease (added to `now`).
963
   * @param now - Override the "now" timestamp (for tests).
964
   * @returns True if the heartbeat extended our lease; false if we lost it.
965
   */
966
  heartbeatLease(holder: string, ttlMs: number, now: number = Date.now()): boolean {
15✔
967
    const info = this.heartbeatLeaseStmt.run(now + ttlMs, holder);
15✔
968
    if (Number(info.changes) > 0) {
15✔
969
      this.walDirty = true;
9✔
970
    }
971
    return Number(info.changes) > 0;
15✔
972
  }
973

974
  /**
975
   * Releases the lease if (and only if) `holder` still owns it. Idempotent. The
976
   * holder check means a process that already lost its lease won't accidentally
977
   * delete a newer leader's row.
978
   * @param holder - The identifier of the lease to release.
979
   */
980
  releaseLease(holder: string): void {
981
    const info = this.releaseLeaseStmt.run(holder);
64✔
982
    if (Number(info.changes) > 0) {
64✔
983
      this.walDirty = true;
61✔
984
    }
985
  }
986

987
  /**
988
   * @returns The current lease, or null if no row exists. Used for diagnostics
989
   *          and stats; the acquire/heartbeat methods do the actual coordination.
990
   */
991
  getCurrentLease(): { holder: string; expiresAt: number } | null {
992
    const row = this.getLeaseStmt.get() as { holder: string; expires_at: number } | undefined;
418✔
993
    return row ? { holder: row.holder, expiresAt: row.expires_at } : null;
418✔
994
  }
995

996
  /**
997
   * Binds this queue to the lease holder ID of the local process, so the
998
   * dispatch gate ({@link assertNotDemoted}) can tell "us" from a peer.
999
   * {@link startDispatchLease} sets this automatically; call it directly only in
1000
   * tests that drive {@link tryAcquireLease}/{@link heartbeatLease} by hand.
1001
   * @param holderId - This process's lease holder ID.
1002
   */
1003
  setLeaseHolder(holderId: string): void {
1004
    this.leaseHolderId = holderId;
3✔
1005
  }
1006

1007
  /**
1008
   * Begins the dispatch-lease acquire-and-heartbeat loop for this process.
1009
   *
1010
   * Tries to acquire the lease immediately. On success it fires `onBecameLeader`
1011
   * and begins heartbeating every `leaseHeartbeatMs`; on failure it retries every
1012
   * `leaseAcquireRetryMs` until the lease frees up. `onBecameLeader` fires every
1013
   * time this process transitions from follower to leader (typically once, but
1014
   * again if it loses the lease mid-run and reclaims it). There is no
1015
   * lost-leadership callback by design — loss surfaces as a {@link QueueLeaseError}
1016
   * from the dispatch ops, which the worker catches to drain itself.
1017
   *
1018
   * Idempotent: calling it again while the loop is already active is a no-op,
1019
   * regardless of whether this process is currently a retrying follower or the
1020
   * leader. A repeat call does not re-acquire, re-heartbeat, re-fire
1021
   * `onBecameLeader`, or replace the callback — so the App can call it on every
1022
   * heartbeat tick (and when already the leader) without restarting the loop.
1023
   * @param onBecameLeader - Callback invoked when this process takes the lease.
1024
   */
1025
  startDispatchLease(onBecameLeader: () => void): void {
1026
    if (this.leaseLoopActive) {
56✔
1027
      return;
1✔
1028
    }
1029
    this.onBecameLeader = onBecameLeader;
55✔
1030
    this.leaseLoopActive = true;
55✔
1031
    if (!this.leaseHolderId) {
55!
1032
      this.leaseHolderId = randomUUID();
55✔
1033
    }
1034
    this.tryAcquireDispatchLease();
55✔
1035
  }
1036

1037
  /**
1038
   * Stops the dispatch-lease loop and releases the lease if we hold it.
1039
   * Idempotent and a no-op when the loop was never started.
1040
   * @returns True if we were the leader when stopping (and released the lease).
1041
   */
1042
  stopDispatchLease(): boolean {
1043
    this.leaseLoopActive = false;
186✔
1044
    this.onBecameLeader = undefined;
186✔
1045
    this.clearLeaseAcquireTimer();
186✔
1046
    this.clearLeaseHeartbeatTimer();
186✔
1047
    const wasLeader = this.leaseLeader;
186✔
1048
    if (this.leaseLeader && this.leaseHolderId) {
186✔
1049
      try {
52✔
1050
        this.releaseLease(this.leaseHolderId);
52✔
1051
      } catch (err) {
1052
        this.log.warn(`Failed to release queue lease: ${normalizeErrorString(err)}`);
×
1053
      }
1054
      this.leaseLeader = false;
52✔
1055
    }
1056
    return wasLeader;
186✔
1057
  }
1058

1059
  /** @returns True if this process currently holds the dispatch lease. */
1060
  isLeader(): boolean {
1061
    return this.leaseLeader;
1,242✔
1062
  }
1063

1064
  /** @returns The dispatch-lease holder ID for this process (for diagnostics), or undefined if never started. */
1065
  getLeaseHolderId(): string | undefined {
1066
    return this.leaseHolderId;
14✔
1067
  }
1068

1069
  private tryAcquireDispatchLease(): void {
1070
    if (!this.leaseLoopActive || this.closed || !this.leaseHolderId) {
77!
1071
      return;
×
1072
    }
1073
    let acquired = false;
77✔
1074
    try {
77✔
1075
      acquired = this.tryAcquireLease(this.leaseHolderId, this.leaseTtlMs);
77✔
1076
    } catch (err) {
1077
      this.log.warn(`Queue lease acquire threw: ${normalizeErrorString(err)}`);
×
1078
    }
1079

1080
    if (acquired) {
77✔
1081
      this.leaseLeader = true;
57✔
1082
      this.clearLeaseAcquireTimer();
57✔
1083
      this.log.info(`Acquired queue lease (holder=${this.leaseHolderId}).`);
57✔
1084
      this.scheduleLeaseHeartbeat();
57✔
1085
      // Fire callback last so any exception from it doesn't leave timers stopped.
1086
      try {
57✔
1087
        this.onBecameLeader?.();
57✔
1088
      } catch (err) {
1089
        this.log.error(`onBecameLeader callback threw: ${normalizeErrorString(err)}`);
×
1090
      }
1091
      return;
57✔
1092
    }
1093

1094
    // Someone else holds the lease — try again in a bit.
1095
    this.scheduleLeaseAcquireRetry();
20✔
1096
  }
1097

1098
  private scheduleLeaseAcquireRetry(): void {
1099
    if (!this.leaseLoopActive || this.leaseAcquireTimer) {
25!
1100
      return;
×
1101
    }
1102
    this.leaseAcquireTimer = setTimeout(() => {
25✔
1103
      this.leaseAcquireTimer = undefined;
22✔
1104
      this.tryAcquireDispatchLease();
22✔
1105
    }, this.leaseAcquireRetryMs);
1106
    if (typeof this.leaseAcquireTimer.unref === 'function') {
25!
1107
      this.leaseAcquireTimer.unref();
25✔
1108
    }
1109
  }
1110

1111
  private scheduleLeaseHeartbeat(): void {
1112
    if (!this.leaseLoopActive || this.leaseHeartbeatTimer) {
57!
1113
      return;
×
1114
    }
1115
    this.leaseHeartbeatTimer = setInterval(() => this.leaseHeartbeat(), this.leaseHeartbeatMs);
57✔
1116
    if (typeof this.leaseHeartbeatTimer.unref === 'function') {
57!
1117
      this.leaseHeartbeatTimer.unref();
57✔
1118
    }
1119
  }
1120

1121
  private leaseHeartbeat(): void {
1122
    if (!this.leaseLoopActive || !this.leaseLeader || !this.leaseHolderId) {
13!
1123
      return;
×
1124
    }
1125
    let extended = false;
13✔
1126
    try {
13✔
1127
      extended = this.heartbeatLease(this.leaseHolderId, this.leaseTtlMs);
13✔
1128
    } catch (err) {
1129
      this.log.warn(`Queue lease heartbeat threw: ${normalizeErrorString(err)}`);
×
1130
      return;
×
1131
    }
1132
    if (!extended) {
13✔
1133
      // We lost the lease — a peer took over after our TTL elapsed. Drop back to
1134
      // follower mode and start trying to reclaim it. We do NOT push a drain
1135
      // event: the dispatch ops are bound to our holder and now throw
1136
      // QueueLeaseError (a peer owns the lease), so the worker self-detects and
1137
      // drains on its next claim/in-flight check — typically well before this
1138
      // heartbeat even runs. (A small overlap window is inherent to TTL leases:
1139
      // the peer could only acquire after our TTL expired.)
1140
      this.log.error(`Lost queue lease (holder=${this.leaseHolderId}); peer took over.`);
5✔
1141
      this.leaseLeader = false;
5✔
1142
      this.clearLeaseHeartbeatTimer();
5✔
1143
      this.scheduleLeaseAcquireRetry();
5✔
1144
    }
1145
  }
1146

1147
  private clearLeaseAcquireTimer(): void {
1148
    if (this.leaseAcquireTimer) {
243✔
1149
      clearTimeout(this.leaseAcquireTimer);
3✔
1150
      this.leaseAcquireTimer = undefined;
3✔
1151
    }
1152
  }
1153

1154
  private clearLeaseHeartbeatTimer(): void {
1155
    if (this.leaseHeartbeatTimer) {
191✔
1156
      clearInterval(this.leaseHeartbeatTimer);
57✔
1157
      this.leaseHeartbeatTimer = undefined;
57✔
1158
    }
1159
  }
1160

1161
  /**
1162
   * @returns True when a lease row exists and a DIFFERENT holder owns it — i.e. a
1163
   * peer has taken over and this process has been demoted. False when no lease
1164
   * exists (no coordination in play) or the lease is ours. Note this intentionally
1165
   * does NOT consider expiry: a merely-expired-but-still-ours lease will be
1166
   * re-extended by our next heartbeat, so only a foreign holder means "demoted",
1167
   * which keeps this exactly aligned with {@link DurableQueue.isLeader}.
1168
   */
1169
  isLeaseHeldByPeer(): boolean {
1170
    const lease = this.getCurrentLease();
387✔
1171
    return lease !== null && lease.holder !== this.leaseHolderId;
387✔
1172
  }
1173

1174
  /**
1175
   * Throws {@link QueueLeaseError} when a peer holds the lease. Called at the top
1176
   * of every dispatch-class mutation so a demoted process can't claim, dispatch,
1177
   * or settle rows the new leader now owns — the authoritative, data-layer half of
1178
   * leadership enforcement (the {@link DurableQueue.isLeader} flag is the
1179
   * cheap optimistic half). Intake, maintenance, diagnostics, and the physical
1180
   * `markSent` marker are deliberately NOT gated.
1181
   */
1182
  private assertNotDemoted(): void {
1183
    if (this.isLeaseHeldByPeer()) {
385✔
1184
      throw new QueueLeaseError(this.leaseHolderId, this.getCurrentLease()?.holder);
2✔
1185
    }
1186
  }
1187

1188
  /**
1189
   * @returns Size of the underlying database file in bytes, computed from
1190
   * `page_count * page_size`. Used by the retention sweeper.
1191
   */
1192
  getDbSizeBytes(): number {
1193
    const row = this.dbSizeBytesStmt.get() as { bytes: number } | undefined;
144✔
1194
    return row?.bytes ?? 0;
144!
1195
  }
1196

1197
  /**
1198
   * @param channelName - The channel to search.
1199
   * @param msgControlId - MSH.10 to look up.
1200
   * @returns The most recent non-`nacked` row owning this `(channel, control_id)`, or null.
1201
   */
1202
  findSeenByControlId(channelName: string, msgControlId: string): InboundRow | null {
1203
    const raw = this.findSeenByControlIdStmt.get(channelName, msgControlId) as
1,297✔
1204
      | Record<string, SQLInputValue>
1205
      | undefined;
1206
    return raw ? rowFromSql(raw) : null;
1,297✔
1207
  }
1208

1209
  /**
1210
   * Returns the sequence number that {@link commitSeqNo} would next persist for
1211
   * a channel, WITHOUT advancing the counter. The first value for a channel is
1212
   * 0; thereafter it is the last committed value + 1. Read-only: callers stamp
1213
   * this candidate into MSH.13 and only {@link commitSeqNo} it once the row is
1214
   * durably enqueued, so a failed insert never burns a sequence number.
1215
   * @param channelName - The channel whose counter to peek.
1216
   * @returns The next sequence number to assign.
1217
   */
1218
  peekNextSeqNo(channelName: string): number {
1219
    const row = this.peekLastSeqNoStmt.get(channelName) as { last_seq_no: number } | undefined;
24✔
1220
    return row === undefined ? 0 : row.last_seq_no + 1;
24✔
1221
  }
1222

1223
  /**
1224
   * Persists `seqNo` as the channel's last assigned sequence number. Production
1225
   * intake commits this inside the insert transaction via
1226
   * {@link enqueue}'s `commitSeqNo` option, so the row and the counter advance
1227
   * atomically; call it directly only when not pairing it with an insert. The
1228
   * counter survives restarts, keeping MSH.13 sequence numbers monotonic.
1229
   * @param channelName - The channel whose counter to advance.
1230
   * @param seqNo - The sequence number that was assigned and successfully enqueued.
1231
   */
1232
  commitSeqNo(channelName: string, seqNo: number): void {
1233
    this.commitSeqNoStmt.run(channelName, seqNo);
13✔
1234
    this.walDirty = true;
13✔
1235
  }
1236
}
1237

1238
/**
1239
 * Decodes a raw SQL row into the {@link InboundRow} union member matching its
1240
 * `state`. Centralized so every read path produces the same shape — adding a
1241
 * column means touching one place.
1242
 *
1243
 * The state-bound lifecycle columns are read with a non-null cast on the members
1244
 * whose state provably sets them (e.g. `processed_at` on a `processed` row): the
1245
 * transition statements in queries.ts guarantee those writes, so we trust the
1246
 * schema rather than re-checking at every read.
1247
 * @param raw - The raw row object returned by `node:sqlite`.
1248
 * @returns The decoded row.
1249
 */
1250
function rowFromSql(raw: Record<string, SQLInputValue>): InboundRow {
1251
  const base = {
1,661✔
1252
    id: raw.id as number,
1253
    channelName: raw.channel_name as string,
1254
    remote: raw.remote as string,
1255
    msgControlId: (raw.msg_control_id as string | null) ?? null,
1,661!
1256
    msgType: (raw.msg_type as string | null) ?? null,
1,661!
1257
    originalMessage: toBuffer(raw.original_message),
1258
    finalizedMessage: toBuffer(raw.finalized_message),
1259
    encoding: (raw.encoding as string | null) ?? null,
1,661!
1260
    enhancedMode: (raw.enhanced_mode as 'standard' | 'aaMode' | null) ?? null,
1,661!
1261
    attemptCount: raw.attempt_count as number,
1262
    guaranteedDelivery: Boolean(raw.guaranteed_delivery),
1263
    callbackId: raw.callback_id as string,
1264
    seqNo: (raw.seq_no as number | null) ?? null,
3,302✔
1265
    receivedAt: raw.received_at as number,
1266
    ackOutcome: (raw.ack_outcome as AckOutcome | null) ?? AckOutcomeValues.PENDING,
1,661!
1267
    serverStatusCode: (raw.server_status_code as number | null) ?? null,
3,221✔
1268
    serverResponseBody:
1269
      raw.server_response_body === null || raw.server_response_body === undefined
3,423✔
1270
        ? null
1271
        : toBuffer(raw.server_response_body),
1272
    lastError: (raw.last_error as string | null) ?? null,
3,227✔
1273
    errorCode: (raw.error_code as QueueErrorCode | null) ?? null,
3,227✔
1274
  };
1275

1276
  const state = raw.state as MessageState;
1,661✔
1277
  const processingStartedAt = raw.processing_started_at as number;
1,661✔
1278
  const sentAt = (raw.sent_at as number | null) ?? null;
1,661✔
1279

1280
  switch (state) {
1,661!
1281
    case MessageStateValues.QUEUED:
1282
      return { ...base, state, nextAttemptAt: (raw.next_attempt_at as number | null) ?? null };
1,289✔
1283
    case MessageStateValues.CLAIMED:
1284
      return { ...base, state, processingStartedAt };
205✔
1285
    case MessageStateValues.INFLIGHT:
1286
      return { ...base, state, processingStartedAt, sentAt: sentAt as number };
9✔
1287
    case MessageStateValues.PROCESSED:
1288
      return { ...base, state, processingStartedAt, sentAt: sentAt as number, processedAt: raw.processed_at as number };
86✔
1289
    case MessageStateValues.REJECTED:
1290
      return {
18✔
1291
        ...base,
1292
        state,
1293
        processingStartedAt,
1294
        sentAt: sentAt as number,
1295
        erroredAt: raw.errored_at as number,
1296
        lastError: base.lastError as string,
1297
        errorCode: base.errorCode as QueueErrorCode,
1298
      };
1299
    case MessageStateValues.FAILED:
1300
      return {
42✔
1301
        ...base,
1302
        state,
1303
        processingStartedAt,
1304
        sentAt,
1305
        erroredAt: raw.errored_at as number,
1306
        lastError: base.lastError as string,
1307
        errorCode: base.errorCode as QueueErrorCode,
1308
      };
1309
    case MessageStateValues.NACKED:
1310
      return { ...base, state, lastError: base.lastError as string, errorCode: base.errorCode as QueueErrorCode };
12✔
1311
    default:
NEW
1312
      state satisfies never;
×
NEW
1313
      throw new Error(`rowFromSql: unknown row state '${String(state)}' for row id=${base.id}`);
×
1314
  }
1315
}
1316

1317
function toBlob(value: Buffer | string): Buffer {
1318
  return typeof value === 'string' ? Buffer.from(value) : value;
1,596✔
1319
}
1320

1321
function toBuffer(value: SQLInputValue): Buffer {
1322
  if (Buffer.isBuffer(value)) {
3,423!
1323
    return value;
×
1324
  }
1325
  if (value instanceof Uint8Array) {
3,423!
1326
    return Buffer.from(value);
3,423✔
1327
  }
1328
  if (typeof value === 'string') {
×
1329
    return Buffer.from(value);
×
1330
  }
1331
  // SQLite NULL or unexpected — return empty to keep the shape stable; callers
1332
  // that needed a value should have checked for null upstream.
1333
  return Buffer.alloc(0);
×
1334
}
1335

1336
/**
1337
 * `node:sqlite` raises constraint failures as an Error whose message starts
1338
 * with "UNIQUE constraint failed". On recent Node versions a `code` property
1339
 * carries `ERR_SQLITE_CONSTRAINT_UNIQUE` (or `errcode = 2067`). We check all
1340
 * three so we keep working when the property shape evolves and across test
1341
 * runtimes (babel-jest can defeat `instanceof Error` checks).
1342
 * @param err - Value caught from a `node:sqlite` call.
1343
 * @returns True when the error represents a UNIQUE constraint failure.
1344
 */
1345
export function isUniqueConstraintError(err: unknown): boolean {
1346
  if (err === null || typeof err !== 'object') {
5✔
1347
    return false;
1✔
1348
  }
1349
  const asAny = err as { code?: string; errcode?: number; message?: string };
4✔
1350
  if (typeof asAny.code === 'string' && asAny.code.includes('CONSTRAINT_UNIQUE')) {
4✔
1351
    return true;
1✔
1352
  }
1353
  // SQLITE_CONSTRAINT_UNIQUE extended result code.
1354
  if (asAny.errcode === 2067) {
3!
1355
    return true;
×
1356
  }
1357
  if (typeof asAny.message === 'string' && /UNIQUE constraint failed/i.test(asAny.message)) {
3✔
1358
    return true;
1✔
1359
  }
1360
  return false;
2✔
1361
}
1362

1363
// Re-export so callers don't need to import from two modules.
1364
export { MessageStateValues as MessageState };
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc