• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

medplum / medplum / 28132730675

24 Jun 2026 09:55PM UTC coverage: 91.855% (-0.1%) from 91.978%
28132730675

push

github

web-flow
feat(agent): add durable queue for HL7 messages (#9346)

* feat(agent): add durable queue

Signed-off-by: Derrick Farris <derrick@medplum.com>

* test(hl7): update new tests to use `vitest`

Signed-off-by: Derrick Farris <derrick@medplum.com>

* [autofix.ci] apply automated fixes

* tmp: swap to local mock server url

Signed-off-by: Derrick Farris <derrick@medplum.com>

* test(agent): convert new durable queue tests to `vitest`

Signed-off-by: Derrick Farris <derrick@medplum.com>

* test(agent): fix type of `process.exit` mock

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): fix regression after merge

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): flush WAL on queue teardown

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): separate retention/checkpoints, sweep at startup

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): requeue when message has not been sent yet

Signed-off-by: Derrick Farris <derrick@medplum.com>

* [autofix.ci] apply automated fixes

* feat(agent): add `errorCode` and error classification

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): fix dedupe logic, and store seq nos atomically

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): ensure storage error (CE/AE) can be retried

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): split channel -> queue from queue -> Bot status

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): don't discard late ACKs

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): suppress app-level ACK in AA-mode

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent,hl7): move deferred commit ACK logic to agent

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent,hl7): revert all changes, all ACK logic in agent

Signed-off-by: Derrick Farris <derrick@medplum.com>

* fix(agent): fix duplicate dedup logic

Signed-off-by: Derrick Far... (continued)

20975 of 23888 branches covered (87.81%)

Branch coverage included in aggregate %.

765 of 842 new or added lines in 10 files covered. (90.86%)

37847 of 40150 relevant lines covered (94.26%)

12574.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.1
/packages/agent/src/queue/durable-queue.ts
1
// SPDX-FileCopyrightText: Copyright Orangebot, Inc. and Medplum contributors
2
// SPDX-License-Identifier: Apache-2.0
3

4
import type { ILogger } from '@medplum/core';
5
import { normalizeErrorString } from '@medplum/core';
6
import { randomUUID } from 'node:crypto';
7
import { chmodSync, existsSync } from 'node:fs';
8
import type { DatabaseSync, SQLInputValue, StatementSync } from 'node:sqlite';
9
import {
10
  CHANNEL_DEPTH,
11
  CHECKPOINT_WAL,
12
  CLAIM_NEXT,
13
  COMMIT_SEQ_NO,
14
  COUNT_BY_STATE,
15
  DB_SIZE_BYTES,
16
  ENQUEUE,
17
  ENQUEUE_REJECTED,
18
  FIND_BY_CALLBACK,
19
  FIND_BY_ID,
20
  FIND_SEEN_BY_CONTROL_ID,
21
  GET_LEASE,
22
  HEARTBEAT_LEASE,
23
  LIST_QUEUED_IDS_FOR_CHANNEL,
24
  MARK_BOT_FAILED,
25
  MARK_PROCESSED,
26
  MARK_SENT,
27
  PEEK_LAST_SEQ_NO,
28
  RECORD_SERVER_RESPONSE,
29
  RECOVER_CLAIMED,
30
  RECOVER_INFLIGHT,
31
  RELEASE_LEASE,
32
  REQUEUE,
33
  SET_ACK_OUTCOME,
34
  TRY_ACQUIRE_LEASE,
35
} from './queries';
36
import { runMigrations } from './schema';
37
import type {
38
  AckOutcome,
39
  EnqueueInput,
40
  EnqueueRejectedInput,
41
  EnqueueResult,
42
  InboundRow,
43
  MessageState,
44
  QueueErrorCode,
45
} from './types';
46
import { AckOutcome as AckOutcomeValues, MessageState as MessageStateValues, QueueLeaseError } from './types';
47

48
/**
49
 * Default lease lifetime. Picked to be:
50
 * - Long enough that ordinary GC pauses or slow disks don't cost us the lease
51
 *   under load.
52
 * - Short enough that, when an agent dies un-gracefully, a peer can take over
53
 *   within tens of seconds rather than minutes.
54
 */
55
export const DEFAULT_LEASE_TTL_MS = 30_000;
16✔
56

57
/**
58
 * How often the leaseholder re-extends its lease. Must be comfortably less than
59
 * {@link DEFAULT_LEASE_TTL_MS} so transient delays don't let the lease expire.
60
 */
61
export const DEFAULT_LEASE_HEARTBEAT_MS = 10_000;
16✔
62

63
/**
64
 * How often a non-leader retries acquisition. Shorter is fine — the underlying
65
 * SQL is a single conditional UPSERT and is cheap.
66
 */
67
export const DEFAULT_LEASE_ACQUIRE_RETRY_MS = 2_000;
16✔
68

69
/**
70
 * How often the queue folds its WAL back into the main DB file. SQLite only
71
 * checkpoints piggybacked on commits, so once traffic stops nothing else would
72
 * drain the WAL until the next retention sweep or close(); this loop bounds how
73
 * far the main DB file can lag behind the last write. Matches the agent
74
 * heartbeat cadence — the loop's former home, before the queue took ownership.
75
 */
76
export const DEFAULT_CHECKPOINT_INTERVAL_MS = 10_000;
16✔
77

78
export interface DurableQueueOptions {
79
  /** Filesystem path to the SQLite DB file. */
80
  path: string;
81
  /** Logger for migration / lifecycle messages. */
82
  log: ILogger;
83
  /** Override the per-process dispatch-lease holder ID. Defaults to a fresh UUID. */
84
  leaseHolder?: string;
85
  /** Override the dispatch-lease TTL in ms. */
86
  leaseTtlMs?: number;
87
  /** Override the dispatch-lease heartbeat interval in ms. */
88
  leaseHeartbeatMs?: number;
89
  /** Override the dispatch-lease acquire retry interval in ms. */
90
  leaseAcquireRetryMs?: number;
91
  /** Override the WAL-checkpoint loop interval in ms. */
92
  checkpointIntervalMs?: number;
93
}
94

95
/** Counts by lifecycle state — used by stats and the retention sweeper. */
96
export type StateCounts = Record<MessageState, number>;
97

98
/** Per-channel queue depth snapshot returned by {@link DurableQueue.getChannelDepths}. */
99
export interface ChannelDepth {
100
  queued: number;
101
  /** Claimed by a worker but not yet written to the socket. */
102
  claimed: number;
103
  /** Written to the socket, awaiting the server response. */
104
  inflight: number;
105
  oldestQueuedAgeMs: number | null;
106
}
107

108
/**
109
 * SQLite-backed durable FIFO for inbound HL7 messages.
110
 *
111
 * One instance owns one synchronous `node:sqlite` connection plus a bag of
112
 * prepared statements covering the hot path. All channel intake and worker
113
 * dispatch goes through this object. See DURABLE_QUEUE_ARCHITECTURE.md §3, §5.
114
 *
115
 * The class is designed to be opened once at agent startup, used for the
116
 * lifetime of the process, and closed exactly once during {@link DurableQueue.close}.
117
 */
118
export class DurableQueue {
119
  private readonly db: DatabaseSync;
120
  private readonly log: ILogger;
121
  private readonly path: string;
122
  private closed = false;
131✔
123
  // The dispatch-lease holder ID of THIS process, set when startDispatchLease()
124
  // begins the acquire/heartbeat loop (or bound directly via setLeaseHolder in
125
  // tests). Dispatch-class mutations are gated on the live lease still belonging
126
  // to this holder; when a peer takes over, the gate throws QueueLeaseError so the
127
  // demoted worker stops driving the queue. Undefined when the dispatch lease was
128
  // never started (single-process use and most unit tests), in which case the gate
129
  // is inert because no lease row exists.
130
  private leaseHolderId: string | undefined;
131

132
  // ── Dispatch-lease orchestration (formerly DispatchLeaseManager) ──
133
  // Lightweight leader election for the DISPATCH path. The lease gates *dispatch
134
  // only* — claiming rows and driving them to the server (the worker +
135
  // recoverOnStartup). It deliberately does NOT gate intake (followers still
136
  // persist inbound messages via enqueue), maintenance (WAL checkpoint, retention
137
  // sweep), or diagnostics (stats): those are valid on any process with the queue
138
  // file open. Exactly one process may dispatch at a time, which is the
139
  // load-bearing primitive that makes zero-downtime upgrades safe: during the
140
  // overlap window where the old and new agent processes both have the SQLite file
141
  // open, only the leaseholder claims/sends rows; the non-leader keeps accepting
142
  // and persisting inbound traffic but does not dispatch.
143
  //
144
  // There is deliberately NO lost-leadership callback: loss is enforced at the
145
  // data layer instead — the dispatch ops throw QueueLeaseError once a peer holds
146
  // the lease (see assertNotDemoted), so the worker self-detects and drains
147
  // without pushing an event down a chain of callbacks.
148
  private readonly leaseTtlMs: number;
149
  private readonly leaseHeartbeatMs: number;
150
  private readonly leaseAcquireRetryMs: number;
151
  private leaseLeader = false;
131✔
152
  private leaseLoopActive = false;
131✔
153
  private leaseAcquireTimer: NodeJS.Timeout | undefined;
154
  private leaseHeartbeatTimer: NodeJS.Timeout | undefined;
155
  private onBecameLeader: (() => void) | undefined;
156
  // True when the WAL may contain frames not yet checkpointed into the main DB
157
  // file. SQLite only attempts checkpoints piggybacked on commits, so once
158
  // traffic stops nothing would ever drain the WAL — the queue's own checkpoint
159
  // loop (startCheckpointLoop) polls this via checkpointWalIfDirty(). Starts true
160
  // because open() itself writes (pragmas, migrations, lease).
161
  private walDirty = true;
131✔
162

163
  // WAL-checkpoint loop. Runs while the queue is open, INDEPENDENT of the
164
  // dispatch lease: a follower still accepts intake writes (enqueue runs on any
165
  // process with the file open), so WAL draining can't be gated on leadership the
166
  // way the lease timers are. Started in open(), cleared in close().
167
  private readonly checkpointIntervalMs: number;
168
  private checkpointTimer: NodeJS.Timeout | undefined;
169

170
  // Prepared statements — created once at open(), reused for every call.
171
  // Names mirror the public methods that use them.
172
  private readonly enqueueStmt: StatementSync;
173
  private readonly enqueueRejectedStmt: StatementSync;
174
  private readonly peekLastSeqNoStmt: StatementSync;
175
  private readonly commitSeqNoStmt: StatementSync;
176
  private readonly findSeenByControlIdStmt: StatementSync;
177
  private readonly claimNextStmt: StatementSync;
178
  private readonly markSentStmt: StatementSync;
179
  private readonly findByCallbackStmt: StatementSync;
180
  private readonly findByIdStmt: StatementSync;
181
  private readonly recordServerResponseStmt: StatementSync;
182
  private readonly markProcessedStmt: StatementSync;
183
  private readonly markBotFailedStmt: StatementSync;
184
  private readonly setAckOutcomeStmt: StatementSync;
185
  private readonly requeueStmt: StatementSync;
186
  private readonly recoverInflightStmt: StatementSync;
187
  private readonly recoverClaimedStmt: StatementSync;
188
  private readonly listQueuedIdsForChannelStmt: StatementSync;
189
  private readonly countByStateStmt: StatementSync;
190
  private readonly channelDepthStmt: StatementSync;
191
  private readonly tryAcquireLeaseStmt: StatementSync;
192
  private readonly heartbeatLeaseStmt: StatementSync;
193
  private readonly releaseLeaseStmt: StatementSync;
194
  private readonly getLeaseStmt: StatementSync;
195
  private readonly checkpointStmt: StatementSync;
196
  private readonly dbSizeBytesStmt: StatementSync;
197

198
  constructor(db: DatabaseSync, options: DurableQueueOptions) {
199
    this.db = db;
131✔
200
    this.log = options.log;
131✔
201
    this.path = options.path;
131✔
202
    this.leaseHolderId = options.leaseHolder;
131✔
203
    this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
131✔
204
    this.leaseHeartbeatMs = options.leaseHeartbeatMs ?? DEFAULT_LEASE_HEARTBEAT_MS;
131✔
205
    this.leaseAcquireRetryMs = options.leaseAcquireRetryMs ?? DEFAULT_LEASE_ACQUIRE_RETRY_MS;
131✔
206
    this.checkpointIntervalMs = options.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
131✔
207

208
    // Single-writer, durable-across-crash, WAL-friendly settings (§5).
209
    // Picked for HL7 inbound rates and the durability contract we owe the sender:
210
    //   synchronous=NORMAL gives crash safety without per-write fsync;
211
    //   WAL lets workers read while intake writes;
212
    //   mmap_size keeps the read path off the kernel I/O hot path.
213
    this.db.exec(`
131✔
214
      PRAGMA journal_mode = WAL;
215
      PRAGMA synchronous  = NORMAL;
216
      PRAGMA temp_store   = MEMORY;
217
      PRAGMA cache_size   = -65536;
218
      PRAGMA mmap_size    = 268435456;
219
      PRAGMA wal_autocheckpoint = 1000;
220
      PRAGMA busy_timeout = 5000;
221
      PRAGMA foreign_keys = ON;
222
    `);
223

224
    runMigrations(this.db);
131✔
225

226
    this.enqueueStmt = this.db.prepare(ENQUEUE);
131✔
227

228
    this.enqueueRejectedStmt = this.db.prepare(ENQUEUE_REJECTED);
131✔
229

230
    // Per-channel monotonic sequence counter (assignSeqNo). Single-process +
231
    // synchronous, so no locking is needed between peek and commit.
232
    this.peekLastSeqNoStmt = this.db.prepare(PEEK_LAST_SEQ_NO);
131✔
233
    this.commitSeqNoStmt = this.db.prepare(COMMIT_SEQ_NO);
131✔
234

235
    // Canonical prior row for a (channel, control_id): any state except `nacked`
236
    // (nacked rows are rejected-intake audit records and intentionally reuse
237
    // control IDs, so they're not a "real" prior delivery to dedupe against).
238
    // id DESC returns the most recent in the unlikely event legacy duplicates
239
    // predate the dedupe-on-intake logic.
240
    this.findSeenByControlIdStmt = this.db.prepare(FIND_SEEN_BY_CONTROL_ID);
131✔
241

242
    // FIFO claim: take the lowest-id queued row for this channel, flip it to
243
    // `claimed` in the same statement so concurrent workers can't double-claim.
244
    // node:sqlite is synchronous and the agent is single-process, so RETURNING is
245
    // enough — no advisory locking needed.
246
    this.claimNextStmt = this.db.prepare(CLAIM_NEXT);
131✔
247

248
    // Phase A → B: the App's send path calls this the moment the transmit request
249
    // is written to the socket, flipping `claimed` → `inflight` and stamping sent_at.
250
    this.markSentStmt = this.db.prepare(MARK_SENT);
131✔
251

252
    this.findByCallbackStmt = this.db.prepare(FIND_BY_CALLBACK);
131✔
253

254
    this.findByIdStmt = this.db.prepare(FIND_BY_ID);
131✔
255

256
    this.recordServerResponseStmt = this.db.prepare(RECORD_SERVER_RESPONSE);
131✔
257

258
    this.markProcessedStmt = this.db.prepare(MARK_PROCESSED);
131✔
259

260
    this.markBotFailedStmt = this.db.prepare(MARK_BOT_FAILED);
131✔
261

262
    this.setAckOutcomeStmt = this.db.prepare(SET_ACK_OUTCOME);
131✔
263

264
    this.requeueStmt = this.db.prepare(REQUEUE);
131✔
265

266
    // Crash recovery splits on whether the request reached the wire:
267
    //   recoverInflight — rows left `inflight` are ambiguous (the server may or
268
    //     may not have processed them), so they land in `failed` for operator
269
    //     review, never `rejected`. The source leg's ack_outcome is left as-is
270
    //     (`pending`): we genuinely don't know whether an ACK was owed.
271
    //   recoverClaimed — rows left `claimed` provably never reached the server,
272
    //     so they return to `queued` for a clean re-dispatch with no duplicate risk.
273
    this.recoverInflightStmt = this.db.prepare(RECOVER_INFLIGHT);
131✔
274
    this.recoverClaimedStmt = this.db.prepare(RECOVER_CLAIMED);
131✔
275

276
    this.listQueuedIdsForChannelStmt = this.db.prepare(LIST_QUEUED_IDS_FOR_CHANNEL);
131✔
277

278
    this.countByStateStmt = this.db.prepare(COUNT_BY_STATE);
131✔
279

280
    this.channelDepthStmt = this.db.prepare(CHANNEL_DEPTH);
131✔
281

282
    this.tryAcquireLeaseStmt = this.db.prepare(TRY_ACQUIRE_LEASE);
131✔
283

284
    this.heartbeatLeaseStmt = this.db.prepare(HEARTBEAT_LEASE);
131✔
285

286
    this.releaseLeaseStmt = this.db.prepare(RELEASE_LEASE);
131✔
287

288
    this.getLeaseStmt = this.db.prepare(GET_LEASE);
131✔
289

290
    // Prepared once like every other statement — checkpointWal() runs on every
291
    // checkpoint-loop tick and every retention sweep, so re-compiling it per call
292
    // would leak GC-finalised statement objects proportional to that frequency.
293
    this.checkpointStmt = this.db.prepare(CHECKPOINT_WAL);
131✔
294

295
    // Prepared once like the rest — the retention sweeper calls getDbSizeBytes()
296
    // in a tight loop while purging under size pressure, so re-compiling it per
297
    // call would leak GC-finalised statement objects proportional to sweep work.
298
    this.dbSizeBytesStmt = this.db.prepare(DB_SIZE_BYTES);
131✔
299
  }
300

301
  /**
302
   * Opens (or creates) the DB file at `options.path`, runs migrations, and
303
   * returns a ready DurableQueue.
304
   *
305
   * Constructs synchronously — `node:sqlite` is itself synchronous so there's
306
   * no benefit to making this `async`.
307
   *
308
   * @param options - Path + logger.
309
   * @returns The opened DurableQueue.
310
   */
311
  static open(options: DurableQueueOptions): DurableQueue {
312
    // Lazy require so this module can be imported in environments where
313
    // node:sqlite isn't available (e.g. type-only tooling).
314
    // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/consistent-type-imports
315
    const { DatabaseSync } = require('node:sqlite') as typeof import('node:sqlite');
131✔
316
    const db = new DatabaseSync(options.path);
131✔
317
    const queue = new DurableQueue(db, options);
131✔
318
    // Lock the DB file down to the agent's user. Best-effort — fails silently on
319
    // platforms (Windows) where chmod is a no-op.
320
    if (existsSync(options.path)) {
131!
321
      try {
131✔
322
        chmodSync(options.path, 0o600);
131✔
323
      } catch {
324
        // ignore
325
      }
326
    }
327
    // Start the WAL-checkpoint loop here (not in the constructor) so a bare
328
    // `new DurableQueue(db, opts)` — used by unit tests — stays timer-free; the
329
    // production lifecycle is open() … close(), and close() clears it.
330
    queue.startCheckpointLoop();
131✔
331
    return queue;
131✔
332
  }
333

334
  /** Closes the underlying SQLite handle. Idempotent. */
335
  close(): void {
336
    if (this.closed) {
133✔
337
      return;
2✔
338
    }
339
    this.closed = true;
131✔
340
    // Stop the WAL-checkpoint loop — the final flush below covers shutdown, and a
341
    // checkpointWalIfDirty() tick is a no-op once `closed` is set anyway.
342
    this.stopCheckpointLoop();
131✔
343
    // Release the lease BEFORE closing the DB so a waiting peer can take over
344
    // immediately rather than waiting for our TTL to expire. Idempotent and a
345
    // no-op when the dispatch lease was never started.
346
    this.stopDispatchLease();
131✔
347
    // SQLite only checkpoints + deletes the WAL on close when this is the last
348
    // connection to the file. An upgrade-overlap peer or an operator's sqlite3
349
    // shell defeats that, so flush explicitly — a clean shutdown should always
350
    // leave a self-contained main DB file (e.g. for file-level backups).
351
    this.checkpointWal();
131✔
352
    try {
131✔
353
      this.db.close();
131✔
354
    } catch (err) {
NEW
355
      this.log.warn(`Error while closing durable queue DB: ${normalizeErrorString(err)}`);
×
356
    }
357
  }
358

359
  /**
360
   * Unconditionally runs a TRUNCATE checkpoint, folding the WAL into the main
361
   * DB file and truncating the WAL to zero bytes. Best-effort: failures are
362
   * logged, and an incomplete checkpoint (a peer connection pinning part of the
363
   * WAL) leaves the dirty flag set so the next attempt retries.
364
   * @returns True when the checkpoint fully completed.
365
   */
366
  checkpointWal(): boolean {
367
    try {
184✔
368
      const row = this.checkpointStmt.get() as { busy: number } | undefined;
184✔
369
      if (row?.busy) {
184!
NEW
370
        return false;
×
371
      }
372
      this.walDirty = false;
184✔
373
      return true;
184✔
374
    } catch (err) {
NEW
375
      this.log.warn(`wal_checkpoint failed: ${normalizeErrorString(err)}`);
×
NEW
376
      return false;
×
377
    }
378
  }
379

380
  /**
381
   * Runs {@link DurableQueue.checkpointWal} only when writes have landed since
382
   * the last completed checkpoint. Driven by the queue's own checkpoint loop
383
   * (see {@link DurableQueue.startCheckpointLoop}) every `checkpointIntervalMs`,
384
   * so the WAL drains shortly after traffic stops instead of waiting for the next
385
   * retention sweep or for close(). A no-op on an idle queue.
386
   * @returns True when a checkpoint ran and fully completed.
387
   */
388
  checkpointWalIfDirty(): boolean {
389
    if (this.closed || !this.walDirty) {
6✔
390
      return false;
1✔
391
    }
392
    return this.checkpointWal();
5✔
393
  }
394

395
  /**
396
   * Starts the periodic WAL-checkpoint loop. Idempotent. Called once by
397
   * {@link DurableQueue.open}; runs regardless of dispatch leadership (see the
398
   * `checkpointTimer` field) and is torn down by {@link DurableQueue.close}.
399
   */
400
  private startCheckpointLoop(): void {
401
    if (this.checkpointTimer) {
131!
NEW
402
      return;
×
403
    }
404
    this.checkpointTimer = setInterval(() => this.checkpointWalIfDirty(), this.checkpointIntervalMs);
131✔
405
    // Don't keep the event loop alive solely for housekeeping — same as the
406
    // dispatch-lease timers.
407
    if (typeof this.checkpointTimer.unref === 'function') {
131!
408
      this.checkpointTimer.unref();
131✔
409
    }
410
  }
411

412
  /** Stops the WAL-checkpoint loop. Idempotent and a no-op when never started. */
413
  private stopCheckpointLoop(): void {
414
    if (this.checkpointTimer) {
131!
415
      clearInterval(this.checkpointTimer);
131✔
416
      this.checkpointTimer = undefined;
131✔
417
    }
418
  }
419

420
  /** @returns Filesystem path of the underlying SQLite file. */
421
  getPath(): string {
NEW
422
    return this.path;
×
423
  }
424

425
  /** @returns Direct access to the underlying handle. Used by retention sweeper for PRAGMA queries. */
426
  getDb(): DatabaseSync {
427
    return this.db;
114✔
428
  }
429

430
  /**
431
   * Runs `fn` inside a single SQLite transaction, committing if it returns and
432
   * rolling back if it throws. Used to make a row state transition atomic with an
433
   * external side effect — e.g. flipping a row `claimed` → `inflight` only if the
434
   * socket write that puts it on the wire also succeeds (see `App.sendToWebSocket`):
435
   * if the write throws, the marker rolls back and the row stays `claimed`
436
   * (provably unsent, safe to requeue) rather than a phantom `inflight`.
437
   *
438
   * `node:sqlite` is synchronous, so `fn` must be synchronous too — do any `await`
439
   * (e.g. token refresh) before calling, not inside.
440
   * @param fn - The work to run transactionally.
441
   * @returns Whatever `fn` returns.
442
   */
443
  runInTransaction<T>(fn: () => T): T {
444
    this.db.exec('BEGIN');
30✔
445
    try {
30✔
446
      const result = fn();
30✔
447
      this.db.exec('COMMIT');
30✔
448
      return result;
30✔
449
    } catch (err) {
450
      this.db.exec('ROLLBACK');
1✔
451
      throw err;
1✔
452
    }
453
  }
454

455
  /**
456
   * Inserts a new `queued` row.
457
   *
458
   * If any prior non-`nacked` row already owns this `(channel_name,
459
   * msg_control_id)` — in `queued`, `claimed`, `inflight`, `processed`,
460
   * `rejected`, or `failed` — we don't insert; we surface a `duplicate` result
461
   * carrying that row so the caller can compare bodies and decide between
462
   * replaying the prior ACK and rejecting the collision (§8). This SELECT is the
463
   * single dedup authority on the intake path: the partial unique index only
464
   * covers the live `queued`/`claimed`/`inflight` window, so it can't recognize a
465
   * retransmit of an already-`processed`/`rejected`/`failed` row —
466
   * it remains only as a last-resort guard against an active-window race (not
467
   * expected in the single-process agent).
468
   * @param input - Fields to persist on the new row.
469
   * @param options - Optional behavior.
470
   * @param options.assignSeqNo - When provided, enqueue assigns the channel's next
471
   *   sequence number — but only after the duplicate check passes, so a retransmit
472
   *   never consumes one. It peeks the persisted counter (a non-consuming read),
473
   *   calls this callback with the candidate so the caller can stamp it into MSH.13
474
   *   and return the finalized bytes to persist, then advances the counter in the
475
   *   SAME transaction as the insert (so a crash can't store the row while leaving
476
   *   the counter behind, which would reuse the number on restart). On a duplicate
477
   *   the callback is never invoked.
478
   * @returns Either the inserted row, or the prior row that owns the MSH.10.
479
   */
480
  enqueue(input: EnqueueInput, options?: { assignSeqNo?: (candidate: number) => Buffer }): EnqueueResult {
481
    if (input.msgControlId) {
768!
482
      const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
768✔
483
      if (existing) {
768✔
484
        return { kind: 'duplicate', existing };
19✔
485
      }
486
    }
487

488
    // Not a duplicate — only now assign the sequence number (if configured). Peek
489
    // is non-consuming; the counter is advanced (commitSeqNo) inside the insert
490
    // transaction below, so a failed insert burns no number and a duplicate (above)
491
    // never reaches here.
492
    let finalizedMessage = input.finalizedMessage;
749✔
493
    let seqNo = input.seqNo;
749✔
494
    let seqNoToCommit: number | undefined;
495
    if (options?.assignSeqNo) {
749✔
496
      const candidate = this.peekNextSeqNo(input.channelName);
9✔
497
      finalizedMessage = options.assignSeqNo(candidate);
9✔
498
      seqNo = candidate;
9✔
499
      seqNoToCommit = candidate;
9✔
500
    }
501

502
    try {
749✔
503
      const runInsert = (): number => {
749✔
504
        const info = this.enqueueStmt.run(
749✔
505
          input.channelName,
506
          input.remote,
507
          input.msgControlId,
508
          input.msgType,
509
          toBlob(input.originalMessage),
510
          toBlob(finalizedMessage),
511
          input.encoding,
512
          input.enhancedMode,
513
          input.callbackId,
514
          seqNo,
515
          input.receivedAt
516
        );
517
        return Number(info.lastInsertRowid);
749✔
518
      };
519

520
      let id: number;
521
      if (seqNoToCommit !== undefined) {
749✔
522
        // Atomically insert the row and advance the sequence counter.
523
        this.db.exec('BEGIN');
9✔
524
        try {
9✔
525
          id = runInsert();
9✔
526
          this.commitSeqNo(input.channelName, seqNoToCommit);
9✔
527
          this.db.exec('COMMIT');
9✔
528
        } catch (txErr) {
529
          this.db.exec('ROLLBACK');
1✔
530
          throw txErr;
1✔
531
        }
532
      } else {
533
        id = runInsert();
740✔
534
      }
535
      this.walDirty = true;
748✔
536
      const row = this.getById(id);
748✔
537
      if (!row) {
748!
NEW
538
        throw new Error(`enqueue: inserted row id=${id} could not be re-read`);
×
539
      }
540
      return { kind: 'inserted', row };
748✔
541
    } catch (err) {
542
      if (isUniqueConstraintError(err) && input.msgControlId) {
1!
NEW
543
        const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
×
NEW
544
        if (existing) {
×
NEW
545
          return { kind: 'duplicate', existing };
×
546
        }
547
      }
548
      throw err;
1✔
549
    }
550
  }
551

552
  /**
553
   * Inserts an audit row in the `nacked` terminal state — used when intake
554
   * rejected the message (duplicate, malformed) but we still want a forensics
555
   * record. Failure to write this is non-fatal; the caller is expected to log.
556
   * @param input - Fields to persist plus the `lastError`/`errorCode` describing why we rejected.
557
   * @returns The newly inserted audit row, or null if even the audit insert failed.
558
   */
559
  enqueueRejected(input: EnqueueRejectedInput): InboundRow | null {
560
    try {
12✔
561
      const info = this.enqueueRejectedStmt.run(
12✔
562
        input.channelName,
563
        input.remote,
564
        input.msgControlId,
565
        input.msgType,
566
        toBlob(input.originalMessage),
567
        toBlob(input.finalizedMessage),
568
        input.encoding,
569
        input.enhancedMode,
570
        input.callbackId,
571
        input.lastError,
572
        input.errorCode,
573
        input.seqNo,
574
        input.receivedAt
575
      );
576
      this.walDirty = true;
12✔
577
      return this.getById(Number(info.lastInsertRowid));
12✔
578
    } catch (err) {
NEW
579
      this.log.warn(`enqueueRejected failed: ${normalizeErrorString(err)}`);
×
NEW
580
      return null;
×
581
    }
582
  }
583

584
  /**
585
   * Atomically claims the next `queued` row for `channelName`, flipping it to
586
   * `claimed` and bumping `attempt_count`. The row stays `claimed` until
587
   * {@link markSent} flips it to `inflight` once the request hits the socket.
588
   * @param channelName - The channel to claim from.
589
   * @param now - Override the timestamp written to `processing_started_at` (for tests).
590
   * @returns The claimed row, or `null` if the channel queue is empty.
591
   */
592
  claimNext(channelName: string, now: number = Date.now()): InboundRow | null {
125✔
593
    this.assertNotDemoted();
125✔
594
    const raw = this.claimNextStmt.get(now, channelName) as Record<string, SQLInputValue> | undefined;
125✔
595
    if (raw) {
125✔
596
      this.walDirty = true;
71✔
597
    }
598
    return raw ? rowFromSql(raw) : null;
124✔
599
  }
600

601
  /**
602
   * Phase A → B transition: records that the transmit request for `callbackId`
603
   * was written to the WebSocket, flipping the row from `claimed` to `inflight`
604
   * and stamping `sent_at`. Called from the App's send path the instant the
605
   * request leaves the process — the durable marker that distinguishes a
606
   * provably-unsent row (safe to requeue on crash) from an ambiguous in-flight
607
   * one (failed for review). Guarded on `state = 'claimed'`, so it's a no-op for
608
   * legacy (non-durable) sends and for any row already past `claimed`.
609
   * @param callbackId - The callback ID of the transmit request just sent.
610
   * @param now - Override for `sent_at` (for tests).
611
   * @returns True if a `claimed` row was flipped to `inflight`.
612
   */
613
  markSent(callbackId: string, now: number = Date.now()): boolean {
37✔
614
    const info = this.markSentStmt.run(now, callbackId);
37✔
615
    if (Number(info.changes) > 0) {
37✔
616
      this.walDirty = true;
34✔
617
      return true;
34✔
618
    }
619
    return false;
3✔
620
  }
621

622
  /**
623
   * Looks up a row by its server-callback ID. Returns `null` if not found.
624
   * @param callbackId - The callback ID echoed back by the Medplum server.
625
   * @returns The matching row, or null.
626
   */
627
  findByCallback(callbackId: string): InboundRow | null {
628
    const raw = this.findByCallbackStmt.get(callbackId) as Record<string, SQLInputValue> | undefined;
7✔
629
    return raw ? rowFromSql(raw) : null;
7✔
630
  }
631

632
  /**
633
   * Looks up a row by its primary key.
634
   * @param id - The row's primary key.
635
   * @returns The matching row, or null.
636
   */
637
  getById(id: number): InboundRow | null {
638
    const raw = this.findByIdStmt.get(id) as Record<string, SQLInputValue> | undefined;
899✔
639
    return raw ? rowFromSql(raw) : null;
899!
640
  }
641

642
  /**
643
   * Records the server's `agent:transmit:response` body + status against the row.
644
   * @param id - Row primary key.
645
   * @param statusCode - HTTP-style status code returned by the server.
646
   * @param body - Response payload, or null when the server omitted one.
647
   */
648
  recordServerResponse(id: number, statusCode: number | null, body: Buffer | string | null): void {
649
    this.assertNotDemoted();
45✔
650
    this.recordServerResponseStmt.run(statusCode, body === null ? null : toBlob(body), id);
45!
651
    this.walDirty = true;
45✔
652
  }
653

654
  /**
655
   * Terminal Bot-leg transition: the server accepted the message (2xx). The
656
   * source leg is recorded independently via `ackOutcome` — `delivered` when the
657
   * app-level ACK reached the source, `undelivered` when it couldn't (e.g. the
658
   * source connection had closed). An `undelivered` row is fully processed
659
   * upstream and must never be re-dispatched; it recovers when the source
660
   * retransmits and the stored ACK is replayed (see {@link setAckOutcome}).
661
   * @param id - Row primary key.
662
   * @param ackOutcome - Source-leg result: `delivered` or `undelivered`.
663
   * @param now - Override for `processed_at` (for tests).
664
   */
665
  markProcessed(id: number, ackOutcome: AckOutcome, now: number = Date.now()): void {
41✔
666
    this.assertNotDemoted();
41✔
667
    this.markProcessedStmt.run(ackOutcome, now, id);
41✔
668
    this.walDirty = true;
41✔
669
  }
670

671
  /**
672
   * Terminal Bot-leg transition: the server **rejected** the message itself
673
   * (permanent 4xx). Retrying can never help — the content must be triaged.
674
   * @param id - Row primary key.
675
   * @param error - Human-readable error string, written to `last_error`.
676
   * @param errorCode - Machine-readable classification, written to `error_code`.
677
   * @param now - Override for `errored_at` (for tests).
678
   */
679
  markRejected(id: number, error: string, errorCode: QueueErrorCode, now: number = Date.now()): void {
4✔
680
    this.assertNotDemoted();
4✔
681
    this.markBotFailedStmt.run(MessageStateValues.REJECTED, now, error, errorCode, id);
4✔
682
    this.walDirty = true;
4✔
683
  }
684

685
  /**
686
   * Terminal-for-now Bot-leg transition: a **transient/ambiguous** failure
687
   * (5xx, 429, response timeout, dispatch error, interrupted). The retry/
688
   * operator-review candidate — distinct from a `rejected` message so a future
689
   * retry policy can re-dispatch `failed` rows without ever touching `rejected`
690
   * ones (or `processed` + `undelivered` ones, whose Bot leg already succeeded).
691
   * @param id - Row primary key.
692
   * @param error - Human-readable error string, written to `last_error`.
693
   * @param errorCode - Machine-readable classification, written to `error_code`.
694
   * @param now - Override for `errored_at` (for tests).
695
   */
696
  markFailed(id: number, error: string, errorCode: QueueErrorCode, now: number = Date.now()): void {
15✔
697
    this.assertNotDemoted();
15✔
698
    this.markBotFailedStmt.run(MessageStateValues.FAILED, now, error, errorCode, id);
15✔
699
    this.walDirty = true;
15✔
700
  }
701

702
  /**
703
   * Updates only the source-leg {@link AckOutcome}, leaving the Bot-leg `state`
704
   * untouched. Used when a duplicate retransmit replays a previously
705
   * `undelivered` ACK and it lands — flipping the row to `delivered` so it no
706
   * longer reads as awaiting source delivery.
707
   * @param id - Row primary key.
708
   * @param ackOutcome - The new source-leg outcome.
709
   */
710
  setAckOutcome(id: number, ackOutcome: AckOutcome): void {
711
    this.setAckOutcomeStmt.run(ackOutcome, id);
2✔
712
    this.walDirty = true;
2✔
713
  }
714

715
  /**
716
   * Returns a `claimed` row to `queued` — used when the WS connection drops
717
   * before the transmit request was ever written to the socket, so retrying on
718
   * reconnect carries no duplicate-delivery risk. (A row that already reached the
719
   * socket is `inflight`, not `claimed`, and this is a no-op for it — its outcome
720
   * is ambiguous and owned by the response timeout.) Because the row keeps its
721
   * original id and claims are ordered by id, it goes back to the front of its
722
   * channel's FIFO.
723
   * @param id - Row primary key.
724
   * @returns True if the row was requeued; false if it was not in `claimed`.
725
   */
726
  requeue(id: number): boolean {
727
    this.assertNotDemoted();
3✔
728
    const info = this.requeueStmt.run(id);
3✔
729
    if (Number(info.changes) > 0) {
3✔
730
      this.walDirty = true;
2✔
731
      return true;
2✔
732
    }
733
    return false;
1✔
734
  }
735

736
  /**
737
   * Recovers rows left mid-flight by a previous process, splitting on whether the
738
   * request reached the wire. Runs once at startup (§10):
739
   * - `inflight` rows are ambiguous (the server may have processed them) → `failed`
740
   *   with error code `interrupted`, surfaced for operator review, never silently retried.
741
   * - `claimed` rows provably never left the process (`sent_at` is NULL) → returned
742
   *   to `queued` for a clean re-dispatch with no duplicate-delivery risk.
743
   * @param now - Override for `errored_at` (for tests).
744
   * @returns Counts of `claimed` rows requeued and `inflight` rows failed.
745
   */
746
  recoverOnStartup(now: number = Date.now()): { requeued: number; failed: number } {
35✔
747
    const failedInfo = this.recoverInflightStmt.run(now);
35✔
748
    const requeuedInfo = this.recoverClaimedStmt.run();
35✔
749
    const failed = Number(failedInfo.changes);
35✔
750
    const requeued = Number(requeuedInfo.changes);
35✔
751
    if (failed > 0 || requeued > 0) {
35✔
752
      this.walDirty = true;
3✔
753
    }
754
    return { requeued, failed };
35✔
755
  }
756

757
  /**
758
   * Lists all `queued` row IDs for a channel, in FIFO order. Used by the
759
   * worker on startup to repopulate its in-memory wake signal.
760
   * @param channelName - The channel to query.
761
   * @returns Row IDs in FIFO order.
762
   */
763
  listQueuedIdsForChannel(channelName: string): number[] {
764
    const rows = this.listQueuedIdsForChannelStmt.all(channelName) as { id: number }[];
1✔
765
    return rows.map((r) => r.id);
2✔
766
  }
767

768
  /** @returns Counts of rows by state. Missing states are reported as 0. */
769
  countByState(): StateCounts {
770
    const counts: StateCounts = {
226✔
771
      queued: 0,
772
      claimed: 0,
773
      inflight: 0,
774
      processed: 0,
775
      rejected: 0,
776
      failed: 0,
777
      nacked: 0,
778
    };
779
    const rows = this.countByStateStmt.all() as { state: MessageState; n: number }[];
226✔
780
    for (const r of rows) {
226✔
781
      counts[r.state] = r.n;
267✔
782
    }
783
    return counts;
225✔
784
  }
785

786
  /**
787
   * @param channelName - The channel to query.
788
   * @param now - Override for the "now" timestamp used in `oldestQueuedAgeMs`.
789
   * @returns Depth snapshot for `channelName` (queued/claimed/inflight counts + oldest queued age).
790
   */
791
  getChannelDepth(channelName: string, now: number = Date.now()): ChannelDepth {
16✔
792
    const row = this.channelDepthStmt.get(channelName) as
16✔
793
      | { queued: number | null; claimed: number | null; inflight: number | null; oldest_queued_at: number | null }
794
      | undefined;
795
    return {
16✔
796
      queued: row?.queued ?? 0,
16!
797
      claimed: row?.claimed ?? 0,
16!
798
      inflight: row?.inflight ?? 0,
16!
799
      oldestQueuedAgeMs: row?.oldest_queued_at ? now - row.oldest_queued_at : null,
16!
800
    };
801
  }
802

803
  /**
804
   * Attempts to acquire (or re-acquire) the queue lease as `holder`.
805
   *
806
   * Succeeds when there is no current lease, the current lease is already held
807
   * by us (refresh case), or the prior holder's lease has expired. Fails when a
808
   * different holder still has a valid lease — caller should wait and retry.
809
   *
810
   * The lease is the cross-process coordination primitive that makes zero-downtime
811
   * upgrades safe: only the holder runs workers and `recoverOnStartup`, so two
812
   * processes sharing the DB during the upgrade overlap don't fight over rows.
813
   * @param holder - Stable identifier for this process (a per-process UUID).
814
   * @param ttlMs - How long the new lease should remain valid before another
815
   *                process can take over (also drives the heartbeat cadence).
816
   * @param now - Override the "now" timestamp (for tests).
817
   * @returns True if the lease is now held by `holder`; false if a foreign holder still owns it.
818
   */
819
  tryAcquireLease(holder: string, ttlMs: number, now: number = Date.now()): boolean {
97✔
820
    const expiresAt = now + ttlMs;
97✔
821
    const info = this.tryAcquireLeaseStmt.run(holder, now, expiresAt, holder, now);
97✔
822
    if (Number(info.changes) > 0) {
97✔
823
      this.walDirty = true;
76✔
824
    }
825
    return Number(info.changes) > 0;
97✔
826
  }
827

828
  /**
829
   * Extends the existing lease held by `holder`. Fails (returns false) when the
830
   * lease is no longer ours — most commonly because a peer took over after our
831
   * TTL elapsed. A leader that sees `false` here must stop driving the queue.
832
   * @param holder - The same identifier passed to {@link DurableQueue.tryAcquireLease}.
833
   * @param ttlMs - New TTL for the lease (added to `now`).
834
   * @param now - Override the "now" timestamp (for tests).
835
   * @returns True if the heartbeat extended our lease; false if we lost it.
836
   */
837
  heartbeatLease(holder: string, ttlMs: number, now: number = Date.now()): boolean {
15✔
838
    const info = this.heartbeatLeaseStmt.run(now + ttlMs, holder);
15✔
839
    if (Number(info.changes) > 0) {
15✔
840
      this.walDirty = true;
9✔
841
    }
842
    return Number(info.changes) > 0;
15✔
843
  }
844

845
  /**
846
   * Releases the lease if (and only if) `holder` still owns it. Idempotent. The
847
   * holder check means a process that already lost its lease won't accidentally
848
   * delete a newer leader's row.
849
   * @param holder - The identifier of the lease to release.
850
   */
851
  releaseLease(holder: string): void {
852
    const info = this.releaseLeaseStmt.run(holder);
62✔
853
    if (Number(info.changes) > 0) {
62✔
854
      this.walDirty = true;
59✔
855
    }
856
  }
857

858
  /**
859
   * @returns The current lease, or null if no row exists. Used for diagnostics
860
   *          and stats; the acquire/heartbeat methods do the actual coordination.
861
   */
862
  getCurrentLease(): { holder: string; expiresAt: number } | null {
863
    const row = this.getLeaseStmt.get() as { holder: string; expires_at: number } | undefined;
266✔
864
    return row ? { holder: row.holder, expiresAt: row.expires_at } : null;
266✔
865
  }
866

867
  /**
868
   * Binds this queue to the lease holder ID of the local process, so the
869
   * dispatch gate ({@link assertNotDemoted}) can tell "us" from a peer.
870
   * {@link startDispatchLease} sets this automatically; call it directly only in
871
   * tests that drive {@link tryAcquireLease}/{@link heartbeatLease} by hand.
872
   * @param holderId - This process's lease holder ID.
873
   */
874
  setLeaseHolder(holderId: string): void {
875
    this.leaseHolderId = holderId;
3✔
876
  }
877

878
  /**
879
   * Begins the dispatch-lease acquire-and-heartbeat loop for this process.
880
   *
881
   * Tries to acquire the lease immediately. On success it fires `onBecameLeader`
882
   * and begins heartbeating every `leaseHeartbeatMs`; on failure it retries every
883
   * `leaseAcquireRetryMs` until the lease frees up. `onBecameLeader` fires every
884
   * time this process transitions from follower to leader (typically once, but
885
   * again if it loses the lease mid-run and reclaims it). There is no
886
   * lost-leadership callback by design — loss surfaces as a {@link QueueLeaseError}
887
   * from the dispatch ops, which the worker catches to drain itself.
888
   *
889
   * Idempotent: calling it again while the loop is already active is a no-op,
890
   * regardless of whether this process is currently a retrying follower or the
891
   * leader. A repeat call does not re-acquire, re-heartbeat, re-fire
892
   * `onBecameLeader`, or replace the callback — so the App can call it on every
893
   * heartbeat tick (and when already the leader) without restarting the loop.
894
   * @param onBecameLeader - Callback invoked when this process takes the lease.
895
   */
896
  startDispatchLease(onBecameLeader: () => void): void {
897
    if (this.leaseLoopActive) {
54✔
898
      return;
1✔
899
    }
900
    this.onBecameLeader = onBecameLeader;
53✔
901
    this.leaseLoopActive = true;
53✔
902
    if (!this.leaseHolderId) {
53!
903
      this.leaseHolderId = randomUUID();
53✔
904
    }
905
    this.tryAcquireDispatchLease();
53✔
906
  }
907

908
  /**
909
   * Stops the dispatch-lease loop and releases the lease if we hold it.
910
   * Idempotent and a no-op when the loop was never started.
911
   * @returns True if we were the leader when stopping (and released the lease).
912
   */
913
  stopDispatchLease(): boolean {
914
    this.leaseLoopActive = false;
156✔
915
    this.onBecameLeader = undefined;
156✔
916
    this.clearLeaseAcquireTimer();
156✔
917
    this.clearLeaseHeartbeatTimer();
156✔
918
    const wasLeader = this.leaseLeader;
156✔
919
    if (this.leaseLeader && this.leaseHolderId) {
156✔
920
      try {
50✔
921
        this.releaseLease(this.leaseHolderId);
50✔
922
      } catch (err) {
NEW
923
        this.log.warn(`Failed to release queue lease: ${normalizeErrorString(err)}`);
×
924
      }
925
      this.leaseLeader = false;
50✔
926
    }
927
    return wasLeader;
156✔
928
  }
929

930
  /** @returns True if this process currently holds the dispatch lease. */
931
  isLeader(): boolean {
932
    return this.leaseLeader;
1,239✔
933
  }
934

935
  /** @returns The dispatch-lease holder ID for this process (for diagnostics), or undefined if never started. */
936
  getLeaseHolderId(): string | undefined {
937
    return this.leaseHolderId;
14✔
938
  }
939

940
  private tryAcquireDispatchLease(): void {
941
    if (!this.leaseLoopActive || this.closed || !this.leaseHolderId) {
75!
NEW
942
      return;
×
943
    }
944
    let acquired = false;
75✔
945
    try {
75✔
946
      acquired = this.tryAcquireLease(this.leaseHolderId, this.leaseTtlMs);
75✔
947
    } catch (err) {
NEW
948
      this.log.warn(`Queue lease acquire threw: ${normalizeErrorString(err)}`);
×
949
    }
950

951
    if (acquired) {
75✔
952
      this.leaseLeader = true;
55✔
953
      this.clearLeaseAcquireTimer();
55✔
954
      this.log.info(`Acquired queue lease (holder=${this.leaseHolderId}).`);
55✔
955
      this.scheduleLeaseHeartbeat();
55✔
956
      // Fire callback last so any exception from it doesn't leave timers stopped.
957
      try {
55✔
958
        this.onBecameLeader?.();
55✔
959
      } catch (err) {
NEW
960
        this.log.error(`onBecameLeader callback threw: ${normalizeErrorString(err)}`);
×
961
      }
962
      return;
55✔
963
    }
964

965
    // Someone else holds the lease — try again in a bit.
966
    this.scheduleLeaseAcquireRetry();
20✔
967
  }
968

969
  private scheduleLeaseAcquireRetry(): void {
970
    if (!this.leaseLoopActive || this.leaseAcquireTimer) {
25!
NEW
971
      return;
×
972
    }
973
    this.leaseAcquireTimer = setTimeout(() => {
25✔
974
      this.leaseAcquireTimer = undefined;
22✔
975
      this.tryAcquireDispatchLease();
22✔
976
    }, this.leaseAcquireRetryMs);
977
    if (typeof this.leaseAcquireTimer.unref === 'function') {
25!
978
      this.leaseAcquireTimer.unref();
25✔
979
    }
980
  }
981

982
  private scheduleLeaseHeartbeat(): void {
983
    if (!this.leaseLoopActive || this.leaseHeartbeatTimer) {
55!
NEW
984
      return;
×
985
    }
986
    this.leaseHeartbeatTimer = setInterval(() => this.leaseHeartbeat(), this.leaseHeartbeatMs);
55✔
987
    if (typeof this.leaseHeartbeatTimer.unref === 'function') {
55!
988
      this.leaseHeartbeatTimer.unref();
55✔
989
    }
990
  }
991

992
  private leaseHeartbeat(): void {
993
    if (!this.leaseLoopActive || !this.leaseLeader || !this.leaseHolderId) {
13!
NEW
994
      return;
×
995
    }
996
    let extended = false;
13✔
997
    try {
13✔
998
      extended = this.heartbeatLease(this.leaseHolderId, this.leaseTtlMs);
13✔
999
    } catch (err) {
NEW
1000
      this.log.warn(`Queue lease heartbeat threw: ${normalizeErrorString(err)}`);
×
NEW
1001
      return;
×
1002
    }
1003
    if (!extended) {
13✔
1004
      // We lost the lease — a peer took over after our TTL elapsed. Drop back to
1005
      // follower mode and start trying to reclaim it. We do NOT push a drain
1006
      // event: the dispatch ops are bound to our holder and now throw
1007
      // QueueLeaseError (a peer owns the lease), so the worker self-detects and
1008
      // drains on its next claim/in-flight check — typically well before this
1009
      // heartbeat even runs. (A small overlap window is inherent to TTL leases:
1010
      // the peer could only acquire after our TTL expired.)
1011
      this.log.error(`Lost queue lease (holder=${this.leaseHolderId}); peer took over.`);
5✔
1012
      this.leaseLeader = false;
5✔
1013
      this.clearLeaseHeartbeatTimer();
5✔
1014
      this.scheduleLeaseAcquireRetry();
5✔
1015
    }
1016
  }
1017

1018
  private clearLeaseAcquireTimer(): void {
1019
    if (this.leaseAcquireTimer) {
211✔
1020
      clearTimeout(this.leaseAcquireTimer);
3✔
1021
      this.leaseAcquireTimer = undefined;
3✔
1022
    }
1023
  }
1024

1025
  private clearLeaseHeartbeatTimer(): void {
1026
    if (this.leaseHeartbeatTimer) {
161✔
1027
      clearInterval(this.leaseHeartbeatTimer);
55✔
1028
      this.leaseHeartbeatTimer = undefined;
55✔
1029
    }
1030
  }
1031

1032
  /**
1033
   * @returns True when a lease row exists and a DIFFERENT holder owns it — i.e. a
1034
   * peer has taken over and this process has been demoted. False when no lease
1035
   * exists (no coordination in play) or the lease is ours. Note this intentionally
1036
   * does NOT consider expiry: a merely-expired-but-still-ours lease will be
1037
   * re-extended by our next heartbeat, so only a foreign holder means "demoted",
1038
   * which keeps this exactly aligned with {@link DurableQueue.isLeader}.
1039
   */
1040
  isLeaseHeldByPeer(): boolean {
1041
    const lease = this.getCurrentLease();
235✔
1042
    return lease !== null && lease.holder !== this.leaseHolderId;
235✔
1043
  }
1044

1045
  /**
1046
   * Throws {@link QueueLeaseError} when a peer holds the lease. Called at the top
1047
   * of every dispatch-class mutation so a demoted process can't claim, dispatch,
1048
   * or settle rows the new leader now owns — the authoritative, data-layer half of
1049
   * leadership enforcement (the {@link DurableQueue.isLeader} flag is the
1050
   * cheap optimistic half). Intake, maintenance, diagnostics, and the physical
1051
   * `markSent` marker are deliberately NOT gated.
1052
   */
1053
  private assertNotDemoted(): void {
1054
    if (this.isLeaseHeldByPeer()) {
233✔
1055
      throw new QueueLeaseError(this.leaseHolderId, this.getCurrentLease()?.holder);
2✔
1056
    }
1057
  }
1058

1059
  /**
1060
   * @returns Size of the underlying database file in bytes, computed from
1061
   * `page_count * page_size`. Used by the retention sweeper.
1062
   */
1063
  getDbSizeBytes(): number {
1064
    const row = this.dbSizeBytesStmt.get() as { bytes: number } | undefined;
138✔
1065
    return row?.bytes ?? 0;
138!
1066
  }
1067

1068
  /**
1069
   * @param channelName - The channel to search.
1070
   * @param msgControlId - MSH.10 to look up.
1071
   * @returns The most recent non-`nacked` row owning this `(channel, control_id)`, or null.
1072
   */
1073
  findSeenByControlId(channelName: string, msgControlId: string): InboundRow | null {
1074
    const raw = this.findSeenByControlIdStmt.get(channelName, msgControlId) as
1,322✔
1075
      | Record<string, SQLInputValue>
1076
      | undefined;
1077
    return raw ? rowFromSql(raw) : null;
1,322✔
1078
  }
1079

1080
  /**
1081
   * Returns the sequence number that {@link commitSeqNo} would next persist for
1082
   * a channel, WITHOUT advancing the counter. The first value for a channel is
1083
   * 0; thereafter it is the last committed value + 1. Read-only: callers stamp
1084
   * this candidate into MSH.13 and only {@link commitSeqNo} it once the row is
1085
   * durably enqueued, so a failed insert never burns a sequence number.
1086
   * @param channelName - The channel whose counter to peek.
1087
   * @returns The next sequence number to assign.
1088
   */
1089
  peekNextSeqNo(channelName: string): number {
1090
    const row = this.peekLastSeqNoStmt.get(channelName) as { last_seq_no: number } | undefined;
24✔
1091
    return row === undefined ? 0 : row.last_seq_no + 1;
24✔
1092
  }
1093

1094
  /**
1095
   * Persists `seqNo` as the channel's last assigned sequence number. Production
1096
   * intake commits this inside the insert transaction via
1097
   * {@link enqueue}'s `commitSeqNo` option, so the row and the counter advance
1098
   * atomically; call it directly only when not pairing it with an insert. The
1099
   * counter survives restarts, keeping MSH.13 sequence numbers monotonic.
1100
   * @param channelName - The channel whose counter to advance.
1101
   * @param seqNo - The sequence number that was assigned and successfully enqueued.
1102
   */
1103
  commitSeqNo(channelName: string, seqNo: number): void {
1104
    this.commitSeqNoStmt.run(channelName, seqNo);
13✔
1105
    this.walDirty = true;
13✔
1106
  }
1107
}
1108

1109
/**
1110
 * Decodes a raw SQL row into an {@link InboundRow}. Centralized so every read
1111
 * path produces the same shape — adding a column means touching one place.
1112
 * @param raw - The raw row object returned by `node:sqlite`.
1113
 * @returns The decoded row.
1114
 */
1115
function rowFromSql(raw: Record<string, SQLInputValue>): InboundRow {
1116
  return {
1,548✔
1117
    id: raw.id as number,
1118
    channelName: raw.channel_name as string,
1119
    remote: raw.remote as string,
1120
    msgControlId: (raw.msg_control_id as string | null) ?? null,
1,548!
1121
    msgType: (raw.msg_type as string | null) ?? null,
1,548!
1122
    originalMessage: toBuffer(raw.original_message),
1123
    finalizedMessage: toBuffer(raw.finalized_message),
1124
    encoding: (raw.encoding as string | null) ?? null,
1,548!
1125
    enhancedMode: (raw.enhanced_mode as 'standard' | 'aaMode' | null) ?? null,
1,548!
1126
    state: raw.state as MessageState,
1127
    attemptCount: raw.attempt_count as number,
1128
    callbackId: raw.callback_id as string,
1129
    serverResponseBody:
1130
      raw.server_response_body === null || raw.server_response_body === undefined
3,164✔
1131
        ? null
1132
        : toBuffer(raw.server_response_body),
1133
    serverStatusCode: (raw.server_status_code as number | null) ?? null,
3,028✔
1134
    ackOutcome: (raw.ack_outcome as AckOutcome | null) ?? AckOutcomeValues.PENDING,
1,548!
1135
    lastError: (raw.last_error as string | null) ?? null,
3,038✔
1136
    errorCode: (raw.error_code as QueueErrorCode | null) ?? null,
3,038✔
1137
    seqNo: (raw.seq_no as number | null) ?? null,
3,076✔
1138
    receivedAt: raw.received_at as number,
1139
    processingStartedAt: (raw.processing_started_at as number | null) ?? null,
2,861✔
1140
    sentAt: (raw.sent_at as number | null) ?? null,
3,053✔
1141
    processedAt: (raw.processed_at as number | null) ?? null,
3,030✔
1142
    erroredAt: (raw.errored_at as number | null) ?? null,
3,049✔
1143
  };
1144
}
1145

1146
function toBlob(value: Buffer | string): Buffer {
1147
  return typeof value === 'string' ? Buffer.from(value) : value;
1,566✔
1148
}
1149

1150
function toBuffer(value: SQLInputValue): Buffer {
1151
  if (Buffer.isBuffer(value)) {
3,164!
NEW
1152
    return value;
×
1153
  }
1154
  if (value instanceof Uint8Array) {
3,164!
1155
    return Buffer.from(value);
3,164✔
1156
  }
NEW
1157
  if (typeof value === 'string') {
×
NEW
1158
    return Buffer.from(value);
×
1159
  }
1160
  // SQLite NULL or unexpected — return empty to keep the shape stable; callers
1161
  // that needed a value should have checked for null upstream.
NEW
1162
  return Buffer.alloc(0);
×
1163
}
1164

1165
/**
1166
 * `node:sqlite` raises constraint failures as an Error whose message starts
1167
 * with "UNIQUE constraint failed". On recent Node versions a `code` property
1168
 * carries `ERR_SQLITE_CONSTRAINT_UNIQUE` (or `errcode = 2067`). We check all
1169
 * three so we keep working when the property shape evolves and across test
1170
 * runtimes (babel-jest can defeat `instanceof Error` checks).
1171
 * @param err - Value caught from a `node:sqlite` call.
1172
 * @returns True when the error represents a UNIQUE constraint failure.
1173
 */
1174
export function isUniqueConstraintError(err: unknown): boolean {
1175
  if (err === null || typeof err !== 'object') {
5✔
1176
    return false;
1✔
1177
  }
1178
  const asAny = err as { code?: string; errcode?: number; message?: string };
4✔
1179
  if (typeof asAny.code === 'string' && asAny.code.includes('CONSTRAINT_UNIQUE')) {
4✔
1180
    return true;
1✔
1181
  }
1182
  // SQLITE_CONSTRAINT_UNIQUE extended result code.
1183
  if (asAny.errcode === 2067) {
3!
NEW
1184
    return true;
×
1185
  }
1186
  if (typeof asAny.message === 'string' && /UNIQUE constraint failed/i.test(asAny.message)) {
3✔
1187
    return true;
1✔
1188
  }
1189
  return false;
2✔
1190
}
1191

1192
// Re-export so callers don't need to import from two modules.
1193
export { MessageStateValues as MessageState };
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc