15466403204

Committed 05 Jun 2025 11:57AM UTC coverage: 77.816% (+0.01%) from 77.803%

Build # 15466403204

Build Type

push

github

Committed by

web-flow

Commit Message

fix: sequentially register tasks (#703)

Run Details

1544 of 2146 branches covered (71.95%)

Branch coverage included in aggregate %.

1 of 5 new or added lines in 1 file covered. (20.0%)

1 existing line in 1 file now uncovered.

17394 of 22191 relevant lines covered (78.38%)

110.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

10.56

/src/internal/queue/queue.ts

import PgBoss, { Job, JobWithMetadata } from 'pg-boss'
import { ERRORS } from '@internal/errors'
import { QueueDB } from '@internal/queue/database'
import { getConfig } from '../../config'
import { logger, logSchema } from '../monitoring'
import { QueueJobRetryFailed, QueueJobCompleted, QueueJobError } from '../monitoring/metrics'
import { Event } from './event'
import { Semaphore } from '@shopify/semaphore'

//eslint-disable-next-line @typescript-eslint/no-explicit-any
type SubclassOfBaseClass = (new (payload: any) => Event<any>) & {
  [K in keyof typeof Event]: (typeof Event)[K]
}

export abstract class Queue {
  protected static events: SubclassOfBaseClass[] = []
  private static pgBoss?: PgBoss

  static async start(opts: {
    signal?: AbortSignal
    onMessage?: (job: Job) => void
    registerWorkers?: () => void
  }) {
    if (Queue.pgBoss) {
      return Queue.pgBoss
    }

    if (opts.signal?.aborted) {
      throw ERRORS.Aborted('Cannot start queue with aborted signal')
    }

    const {
      isMultitenant,
      databaseURL,
      multitenantDatabaseUrl,
      pgQueueConnectionURL,
      pgQueueArchiveCompletedAfterSeconds,
      pgQueueDeleteAfterDays,
      pgQueueDeleteAfterHours,
      pgQueueRetentionDays,
      pgQueueEnableWorkers,
      pgQueueReadWriteTimeout,
      pgQueueConcurrentTasksPerQueue,
      pgQueueMaxConnections,
    } = getConfig()

    let url = pgQueueConnectionURL ?? databaseURL

    if (isMultitenant && !pgQueueConnectionURL) {
      if (!multitenantDatabaseUrl) {
        throw new Error(
          'running storage in multi-tenant but DB_MULTITENANT_DATABASE_URL is not set'
        )
      }
      url = multitenantDatabaseUrl
    }

    Queue.pgBoss = new PgBoss({
      connectionString: url,

      db: new QueueDB({
        min: 0,
        max: pgQueueMaxConnections,
        connectionString: url,
        statement_timeout: pgQueueReadWriteTimeout > 0 ? pgQueueReadWriteTimeout : undefined,
      }),
      schema: 'pgboss_v10',
      application_name: 'storage-pgboss',
      ...(pgQueueDeleteAfterHours
        ? { deleteAfterHours: pgQueueDeleteAfterHours }
        : { deleteAfterDays: pgQueueDeleteAfterDays }),
      archiveCompletedAfterSeconds: pgQueueArchiveCompletedAfterSeconds,
      retentionDays: pgQueueRetentionDays,
      retryBackoff: true,
      retryLimit: 20,
      expireInHours: 23,
      maintenanceIntervalSeconds: 60 * 5,
      schedule: pgQueueEnableWorkers !== false,
      supervise: pgQueueEnableWorkers !== false,
    })

    Queue.pgBoss.on('error', (error) => {
      logSchema.error(logger, '[Queue] error', {
        type: 'queue',
        error,
      })
    })

    await Queue.pgBoss.start()

    if (opts.registerWorkers && pgQueueEnableWorkers) {
      opts.registerWorkers()
    }

    await Queue.callStart()
    await Queue.startWorkers({
      maxConcurrentTasks: pgQueueConcurrentTasksPerQueue,
      onMessage: opts.onMessage,
      signal: opts.signal,
    })

    if (opts.signal) {
      opts.signal.addEventListener(
        'abort',
        async () => {
          logSchema.info(logger, '[Queue] Stopping', {
            type: 'queue',
          })
          return Queue.stop()
            .then(async () => {
              logSchema.info(logger, '[Queue] Exited', {
                type: 'queue',
              })
            })
            .catch((e) => {
              logSchema.error(logger, '[Queue] Error while stopping queue', {
                error: e,
                type: 'queue',
              })
            })
            .finally(async () => {
              await Queue.callClose().catch(() => {
                // no-op
              })
            })
        },
        { once: true }
      )
    }

    return Queue.pgBoss
  }

  static getInstance() {
    if (!this.pgBoss) {
      throw new Error('pg boss not initialised')
    }

    return this.pgBoss
  }

  static register<T extends SubclassOfBaseClass>(event: T) {
    Queue.events.push(event)
  }

  static async stop() {
    if (!this.pgBoss) {
      return
    }

    const boss = this.pgBoss
    const { isProduction } = getConfig()

    await boss.stop({
      timeout: 20 * 1000,
      graceful: isProduction,
      wait: true,
    })

    await new Promise((resolve) => {
      boss.once('stopped', async () => {
        await this.callClose()
        resolve(null)
      })
    })

    Queue.pgBoss = undefined
  }

  protected static async startWorkers(opts: {
    maxConcurrentTasks: number
    signal?: AbortSignal
    onMessage?: (job: Job) => void
  }) {
    for (const event of Queue.events) {
      await Queue.registerTask(event, opts.maxConcurrentTasks, opts.onMessage, opts.signal)
    }
  }

  protected static callStart() {
    const events = Queue.events.map((event) => {
      return event.onStart()
    })

    return Promise.all(events)
  }

  protected static callClose() {
    const events = Queue.events.map((event) => {
      return event.onClose()
    })

    return Promise.all(events)
  }

  protected static async registerTask(
    event: SubclassOfBaseClass,
    maxConcurrentTasks: number,
    onMessage?: (job: Job) => void,
    signal?: AbortSignal
  ) {
    const queueName = event.getQueueName()
    const deadLetterName = event.deadLetterQueueName()

    const concurrentTaskCount = event.getWorkerOptions().concurrentTaskCount || maxConcurrentTasks
    try {
      // Create dead-letter queue and the normal queue
      const queueOptions = {
        policy: 'standard',
        ...event.getQueueOptions(),
      } as const

      // dead-letter
      await this.pgBoss?.createQueue(deadLetterName, {
        ...queueOptions,
        name: deadLetterName,
        retentionDays: 30,
        retryBackoff: true,
      })

      // // normal queue
      await this.pgBoss?.createQueue(queueName, {
        name: queueName,
        ...queueOptions,
        deadLetter: deadLetterName,
      })
    } catch {
      // no-op
    }

    return this.pollQueue(event, {
      concurrentTaskCount,
      onMessage,
      signal,
    })
  }

  protected static pollQueue(
    event: SubclassOfBaseClass,
    queueOpts: {
      concurrentTaskCount: number
      onMessage?: (job: Job) => void
      signal?: AbortSignal
    }
  ) {
    const semaphore = new Semaphore(queueOpts.concurrentTaskCount)
    const pollingInterval = (event.getWorkerOptions().pollingIntervalSeconds || 5) * 1000
    const batchSize =
      event.getWorkerOptions().batchSize ||
      queueOpts.concurrentTaskCount + Math.max(1, Math.floor(queueOpts.concurrentTaskCount * 1.2))

    let started = false
    const interval = setInterval(async () => {
      if (started) {
        return
      }

      try {
        started = true
        const defaultFetch = {
          includeMetadata: true,
          batchSize,
        }
        const jobs = await this.pgBoss?.fetch(event.getQueueName(), {
          ...event.getWorkerOptions(),
          ...defaultFetch,
        })

        if (queueOpts.signal?.aborted) {
          started = false
          return
        }

        if (!jobs || (jobs && jobs.length === 0)) {
          started = false
          return
        }

        await Promise.allSettled(
          jobs.map(async (job) => {
            const lock = await semaphore.acquire()
            try {
              queueOpts.onMessage?.(job as Job)

              await event.handle(job, { signal: queueOpts.signal })

              await this.pgBoss?.complete(event.getQueueName(), job.id)
              QueueJobCompleted.inc({
                name: event.getQueueName(),
              })
            } catch (e) {
              await this.pgBoss?.fail(event.getQueueName(), job.id)

              QueueJobRetryFailed.inc({
                name: event.getQueueName(),
              })

              try {
                const dbJob: JobWithMetadata | null =
                  (job as JobWithMetadata).priority !== undefined
                    ? (job as JobWithMetadata)
                    : await Queue.getInstance().getJobById(event.getQueueName(), job.id)

                if (!dbJob) {
                  return
                }
                if (dbJob.retryCount >= dbJob.retryLimit) {
                  QueueJobError.inc({
                    name: event.getQueueName(),
                  })
                }
              } catch (e) {
                logSchema.error(logger, `[Queue Handler] fetching job ${event.name}`, {
                  type: 'queue-task',
                  error: e,
                  metadata: JSON.stringify(job),
                })
              }

              logSchema.error(logger, `[Queue Handler] Error while processing job ${event.name}`, {
                type: 'queue-task',
                error: e,
                metadata: JSON.stringify(job),
              })

              throw e
            } finally {
              await lock.release()
            }
          })
        )
      } finally {
        started = false
      }
    }, pollingInterval)

    queueOpts.signal?.addEventListener('abort', () => {
      clearInterval(interval)
    })
  }
}

1	import PgBoss, { Job, JobWithMetadata } from 'pg-boss'	1✔
2	import { ERRORS } from '@internal/errors'	1✔
3	import { QueueDB } from '@internal/queue/database'	1✔
4	import { getConfig } from '../../config'	1✔
5	import { logger, logSchema } from '../monitoring'	1✔
6	import { QueueJobRetryFailed, QueueJobCompleted, QueueJobError } from '../monitoring/metrics'	1✔
7	import { Event } from './event'	1✔
8	import { Semaphore } from '@shopify/semaphore'	1✔
9		1✔
10	//eslint-disable-next-line @typescript-eslint/no-explicit-any	1✔
11	type SubclassOfBaseClass = (new (payload: any) => Event<any>) & {	1✔
12	[K in keyof typeof Event]: (typeof Event)[K]	1✔
13	}	1✔
14		1✔
15	export abstract class Queue {	1✔
16	protected static events: SubclassOfBaseClass[] = []	1✔
17	private static pgBoss?: PgBoss	1✔
18		1✔
19	static async start(opts: {	1✔
20	signal?: AbortSignal	×
21	onMessage?: (job: Job) => void	×
22	registerWorkers?: () => void	×
23	}) {	×
24	if (Queue.pgBoss) {	×
25	return Queue.pgBoss	×
26	}	×
27		×
28	if (opts.signal?.aborted) {	×
29	throw ERRORS.Aborted('Cannot start queue with aborted signal')	×
30	}	×
31		×
32	const {	×
33	isMultitenant,	×
34	databaseURL,	×
35	multitenantDatabaseUrl,	×
36	pgQueueConnectionURL,	×
37	pgQueueArchiveCompletedAfterSeconds,	×
38	pgQueueDeleteAfterDays,	×
39	pgQueueDeleteAfterHours,	×
40	pgQueueRetentionDays,	×
41	pgQueueEnableWorkers,	×
42	pgQueueReadWriteTimeout,	×
43	pgQueueConcurrentTasksPerQueue,	×
44	pgQueueMaxConnections,	×
45	} = getConfig()	×
46		×
47	let url = pgQueueConnectionURL ?? databaseURL	×
48		×
49	if (isMultitenant && !pgQueueConnectionURL) {	×
50	if (!multitenantDatabaseUrl) {	×
51	throw new Error(	×
52	'running storage in multi-tenant but DB_MULTITENANT_DATABASE_URL is not set'	×
53	)	×
54	}	×
55	url = multitenantDatabaseUrl	×
56	}	×
57		×
58	Queue.pgBoss = new PgBoss({	×
59	connectionString: url,	×
60		×
61	db: new QueueDB({	×
62	min: 0,	×
63	max: pgQueueMaxConnections,	×
64	connectionString: url,	×
65	statement_timeout: pgQueueReadWriteTimeout > 0 ? pgQueueReadWriteTimeout : undefined,	×
66	}),	×
67	schema: 'pgboss_v10',	×
68	application_name: 'storage-pgboss',	×
69	...(pgQueueDeleteAfterHours	×
70	? { deleteAfterHours: pgQueueDeleteAfterHours }	×
71	: { deleteAfterDays: pgQueueDeleteAfterDays }),	×
72	archiveCompletedAfterSeconds: pgQueueArchiveCompletedAfterSeconds,	×
73	retentionDays: pgQueueRetentionDays,	×
74	retryBackoff: true,	×
75	retryLimit: 20,	×
76	expireInHours: 23,	×
77	maintenanceIntervalSeconds: 60 * 5,	×
78	schedule: pgQueueEnableWorkers !== false,	×
79	supervise: pgQueueEnableWorkers !== false,	×
80	})	×
81		×
82	Queue.pgBoss.on('error', (error) => {	×
83	logSchema.error(logger, '[Queue] error', {	×
84	type: 'queue',	×
85	error,	×
86	})	×
87	})	×
88		×
89	await Queue.pgBoss.start()	×
90		×
91	if (opts.registerWorkers && pgQueueEnableWorkers) {	×
92	opts.registerWorkers()	×
93	}	×
94		×
95	await Queue.callStart()	×
96	await Queue.startWorkers({	×
97	maxConcurrentTasks: pgQueueConcurrentTasksPerQueue,	×
98	onMessage: opts.onMessage,	×
99	signal: opts.signal,	×
100	})	×
101		×
102	if (opts.signal) {	×
103	opts.signal.addEventListener(	×
104	'abort',	×
105	async () => {	×
106	logSchema.info(logger, '[Queue] Stopping', {	×
107	type: 'queue',	×
108	})	×
109	return Queue.stop()	×
110	.then(async () => {	×
111	logSchema.info(logger, '[Queue] Exited', {	×
112	type: 'queue',	×
113	})	×
114	})	×
115	.catch((e) => {	×
116	logSchema.error(logger, '[Queue] Error while stopping queue', {	×
117	error: e,	×
118	type: 'queue',	×
119	})	×
120	})	×
121	.finally(async () => {	×
122	await Queue.callClose().catch(() => {	×
123	// no-op	×
124	})	×
125	})	×
126	},	×
127	{ once: true }	×
128	)	×
129	}	×
130		×
131	return Queue.pgBoss	×
132	}	×
133		1✔
134	static getInstance() {	1✔
135	if (!this.pgBoss) {	×
136	throw new Error('pg boss not initialised')	×
137	}	×
138		×
139	return this.pgBoss	×
140	}	×
141		1✔
142	static register<T extends SubclassOfBaseClass>(event: T) {	1✔
143	Queue.events.push(event)	×
144	}	×
145		1✔
146	static async stop() {	1✔
147	if (!this.pgBoss) {	×
148	return	×
149	}	×
150		×
151	const boss = this.pgBoss	×
152	const { isProduction } = getConfig()	×
153		×
154	await boss.stop({	×
155	timeout: 20 * 1000,	×
156	graceful: isProduction,	×
157	wait: true,	×
158	})	×
159		×
160	await new Promise((resolve) => {	×
161	boss.once('stopped', async () => {	×
162	await this.callClose()	×
163	resolve(null)	×
164	})	×
165	})	×
166		×
167	Queue.pgBoss = undefined	×
168	}	×
169		1✔
170	protected static async startWorkers(opts: {	1✔
171	maxConcurrentTasks: number	×
172	signal?: AbortSignal	×
173	onMessage?: (job: Job) => void	×
174	}) {	×
NEW 175	for (const event of Queue.events) {	×
NEW 176	await Queue.registerTask(event, opts.maxConcurrentTasks, opts.onMessage, opts.signal)	×
NEW 177	}	×
UNCOV 178	}	×
179		1✔
180	protected static callStart() {	1✔
181	const events = Queue.events.map((event) => {	×
182	return event.onStart()	×
183	})	×
184		×
185	return Promise.all(events)	×
186	}	×
187		1✔
188	protected static callClose() {	1✔
189	const events = Queue.events.map((event) => {	×
190	return event.onClose()	×
191	})	×
192		×
193	return Promise.all(events)	×
194	}	×
195		1✔
196	protected static async registerTask(	1✔
197	event: SubclassOfBaseClass,	×
198	maxConcurrentTasks: number,	×
199	onMessage?: (job: Job) => void,	×
200	signal?: AbortSignal	×
201	) {	×
202	const queueName = event.getQueueName()	×
203	const deadLetterName = event.deadLetterQueueName()	×
204		×
205	const concurrentTaskCount = event.getWorkerOptions().concurrentTaskCount \|\| maxConcurrentTasks	×
206	try {	×
207	// Create dead-letter queue and the normal queue	×
208	const queueOptions = {	×
209	policy: 'standard',	×
210	...event.getQueueOptions(),	×
211	} as const	×
212		×
213	// dead-letter	×
214	await this.pgBoss?.createQueue(deadLetterName, {	×
215	...queueOptions,	×
216	name: deadLetterName,	×
217	retentionDays: 30,	×
218	retryBackoff: true,	×
219	})	×
220		×
NEW 221	// // normal queue	×
222	await this.pgBoss?.createQueue(queueName, {	×
223	name: queueName,	×
224	...queueOptions,	×
225	deadLetter: deadLetterName,	×
226	})	×
227	} catch {	×
228	// no-op	×
229	}	×
230		×
231	return this.pollQueue(event, {	×
232	concurrentTaskCount,	×
233	onMessage,	×
234	signal,	×
235	})	×
236	}	×
237		1✔
238	protected static pollQueue(	1✔
239	event: SubclassOfBaseClass,	×
240	queueOpts: {	×
241	concurrentTaskCount: number	×
242	onMessage?: (job: Job) => void	×
243	signal?: AbortSignal	×
244	}	×
245	) {	×
246	const semaphore = new Semaphore(queueOpts.concurrentTaskCount)	×
247	const pollingInterval = (event.getWorkerOptions().pollingIntervalSeconds \|\| 5) * 1000	×
248	const batchSize =	×
249	event.getWorkerOptions().batchSize \|\|	×
250	queueOpts.concurrentTaskCount + Math.max(1, Math.floor(queueOpts.concurrentTaskCount * 1.2))	×
251		×
252	let started = false	×
253	const interval = setInterval(async () => {	×
254	if (started) {	×
255	return	×
256	}	×
257		×
258	try {	×
259	started = true	×
260	const defaultFetch = {	×
261	includeMetadata: true,	×
262	batchSize,	×
263	}	×
264	const jobs = await this.pgBoss?.fetch(event.getQueueName(), {	×
265	...event.getWorkerOptions(),	×
266	...defaultFetch,	×
267	})	×
268		×
269	if (queueOpts.signal?.aborted) {	×
270	started = false	×
271	return	×
272	}	×
273		×
274	if (!jobs \|\| (jobs && jobs.length === 0)) {	×
275	started = false	×
276	return	×
277	}	×
278		×
279	await Promise.allSettled(	×
280	jobs.map(async (job) => {	×
281	const lock = await semaphore.acquire()	×
282	try {	×
283	queueOpts.onMessage?.(job as Job)	×
284		×
285	await event.handle(job, { signal: queueOpts.signal })	×
286		×
287	await this.pgBoss?.complete(event.getQueueName(), job.id)	×
288	QueueJobCompleted.inc({	×
289	name: event.getQueueName(),	×
290	})	×
291	} catch (e) {	×
292	await this.pgBoss?.fail(event.getQueueName(), job.id)	×
293		×
294	QueueJobRetryFailed.inc({	×
295	name: event.getQueueName(),	×
296	})	×
297		×
298	try {	×
299	const dbJob: JobWithMetadata \| null =	×
300	(job as JobWithMetadata).priority !== undefined	×
301	? (job as JobWithMetadata)	×
302	: await Queue.getInstance().getJobById(event.getQueueName(), job.id)	×
303		×
304	if (!dbJob) {	×
305	return	×
306	}	×
307	if (dbJob.retryCount >= dbJob.retryLimit) {	×
308	QueueJobError.inc({	×
309	name: event.getQueueName(),	×
310	})	×
311	}	×
312	} catch (e) {	×
313	logSchema.error(logger, `[Queue Handler] fetching job ${event.name}`, {	×
314	type: 'queue-task',	×
315	error: e,	×
316	metadata: JSON.stringify(job),	×
317	})	×
318	}	×
319		×
320	logSchema.error(logger, `[Queue Handler] Error while processing job ${event.name}`, {	×
321	type: 'queue-task',	×
322	error: e,	×
323	metadata: JSON.stringify(job),	×
324	})	×
325		×
326	throw e	×
327	} finally {	×
328	await lock.release()	×
329	}	×
330	})	×
331	)	×
332	} finally {	×
333	started = false	×
334	}	×
335	}, pollingInterval)	×
336		×
337	queueOpts.signal?.addEventListener('abort', () => {	×
338	clearInterval(interval)	×
339	})	×
340	}	×
341	}	1✔

supabase / storage / 15466403204

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous