11872108728

Committed 16 Nov 2024 05:56PM UTC coverage: 88.201% (-0.3%) from 88.479%

Build # 11872108728

Build Type

Pull #2769

github

Committed by

web-flow

Commit Message

Merge e2e3c9ded into 8204ea363

Pull Request Pull Request #2769: perf: keep jobs in waiting list when queue is paused

Run Details

1406 of 1707 branches covered (82.37%)

Branch coverage included in aggregate %.

57 of 75 new or added lines in 8 files covered. (76.0%)

1 existing line in 1 file now uncovered.

2623 of 2861 relevant lines covered (91.68%)

4121.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.74

/src/classes/redis-connection.ts

import { EventEmitter } from 'events';
import { default as IORedis } from 'ioredis';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { CONNECTION_CLOSED_ERROR_MSG } from 'ioredis/built/utils';
import { ConnectionOptions, RedisOptions, RedisClient } from '../interfaces';
import {
  decreaseMaxListeners,
  increaseMaxListeners,
  isNotConnectionError,
  isRedisCluster,
  isRedisInstance,
  isVersionLowerThan,
} from '../utils';
import { version as packageVersion } from '../version';
import * as scripts from '../scripts';

const overrideMessage = [
  'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
  'and will be overridden by BullMQ.',
].join(' ');

const deprecationMessage =
  'BullMQ: Your redis options maxRetriesPerRequest must be null.';

interface RedisCapabilities {
  canDoubleTimeout: boolean;
  canBlockFor1Ms: boolean;
}

export interface RawCommand {
  content: string;
  name: string;
  keys: number;
}

export class RedisConnection extends EventEmitter {
  static minimumVersion = '5.0.0';
  static recommendedMinimumVersion = '6.2.0';

  closing: boolean;
  capabilities: RedisCapabilities = {
    canDoubleTimeout: false,
    canBlockFor1Ms: true,
  };

  status: 'initializing' | 'ready' | 'closing' | 'closed' = 'initializing';

  protected _client: RedisClient;

  private readonly opts: RedisOptions;
  private readonly initializing: Promise<RedisClient>;

  private version: string;
  protected packageVersion = packageVersion;
  private skipVersionCheck: boolean;
  private handleClientError: (e: Error) => void;
  private handleClientClose: () => void;
  private handleClientReady: () => void;

  constructor(
    opts?: ConnectionOptions,
    private readonly shared: boolean = false,
    private readonly blocking = true,
    skipVersionCheck = false,
  ) {
    super();
    if (!isRedisInstance(opts)) {
      this.checkBlockingOptions(overrideMessage, opts);

      this.opts = {
        port: 6379,
        host: '127.0.0.1',
        retryStrategy: function (times: number) {
          return Math.max(Math.min(Math.exp(times), 20000), 1000);
        },
        ...opts,
      };

      if (this.blocking) {
        this.opts.maxRetriesPerRequest = null;
      }
    } else {
      this._client = opts;

      // Test if the redis instance is using keyPrefix
      // and if so, throw an error.
      if (this._client.options.keyPrefix) {
        throw new Error(
          'BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.',
        );
      }

      if (isRedisCluster(this._client)) {
        this.opts = this._client.options.redisOptions;
      } else {
        this.opts = this._client.options;
      }

      this.checkBlockingOptions(deprecationMessage, this.opts, true);
    }

    this.skipVersionCheck =
      skipVersionCheck || !!(this.opts && this.opts.skipVersionCheck);
    this.handleClientError = (err: Error): void => {
      this.emit('error', err);
    };

    this.handleClientClose = (): void => {
      this.emit('close');
    };

    this.handleClientReady = (): void => {
      this.emit('ready');
    };

    this.initializing = this.init();
    this.initializing.catch(err => this.emit('error', err));
  }

  private checkBlockingOptions(
    msg: string,
    options?: RedisOptions,
    throwError = false,
  ) {
    if (this.blocking && options && options.maxRetriesPerRequest) {
      if (throwError) {
        throw new Error(msg);
      } else {
        console.error(msg);
      }
    }
  }

  /**
   * Waits for a redis client to be ready.
   * @param redis - client
   */
  static async waitUntilReady(client: RedisClient): Promise<void> {
    if (client.status === 'ready') {
      return;
    }

    if (client.status === 'wait') {
      return client.connect();
    }

    if (client.status === 'end') {
      throw new Error(CONNECTION_CLOSED_ERROR_MSG);
    }

    let handleReady: () => void;
    let handleEnd: () => void;
    let handleError: (e: Error) => void;
    try {
      await new Promise<void>((resolve, reject) => {
        let lastError: Error;

        handleError = (err: Error) => {
          lastError = err;
        };

        handleReady = () => {
          resolve();
        };

        handleEnd = () => {
          if (client.status !== 'end') {
            reject(lastError || new Error(CONNECTION_CLOSED_ERROR_MSG));
          } else {
            if (lastError) {
              reject(lastError);
            } else {
              // when custon 'end' status is set we already closed
              resolve();
            }
          }
        };

        increaseMaxListeners(client, 3);

        client.once('ready', handleReady);
        client.on('end', handleEnd);
        client.once('error', handleError);
      });
    } finally {
      client.removeListener('end', handleEnd);
      client.removeListener('error', handleError);
      client.removeListener('ready', handleReady);

      decreaseMaxListeners(client, 3);
    }
  }

  get client(): Promise<RedisClient> {
    return this.initializing;
  }

  protected loadCommands(
    packageVersion: string,
    providedScripts?: Record<string, RawCommand>,
  ): void {
    const finalScripts =
      providedScripts || (scripts as Record<string, RawCommand>);
    for (const property in finalScripts as Record<string, RawCommand>) {
      // Only define the command if not already defined
      const commandName = `${finalScripts[property].name}:${packageVersion}`;
      if (!(<any>this._client)[commandName]) {
        (<any>this._client).defineCommand(commandName, {
          numberOfKeys: finalScripts[property].keys,
          lua: finalScripts[property].content,
        });
      }
    }
  }

  private async init() {
    if (!this._client) {
      const { url, ...rest } = this.opts;
      this._client = url ? new IORedis(url, rest) : new IORedis(rest);
    }

    increaseMaxListeners(this._client, 3);

    this._client.on('error', this.handleClientError);
    // ioredis treats connection errors as a different event ('close')
    this._client.on('close', this.handleClientClose);

    this._client.on('ready', this.handleClientReady);

    await RedisConnection.waitUntilReady(this._client);

    this.loadCommands(this.packageVersion);

    if (this._client['status'] !== 'end') {
      this.version = await this.getRedisVersion();
      if (this.skipVersionCheck !== true && !this.closing) {
        if (isVersionLowerThan(this.version, RedisConnection.minimumVersion)) {
          throw new Error(
            `Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} ` +
              `Current: ${this.version}`,
          );
        }

        if (
          isVersionLowerThan(
            this.version,
            RedisConnection.recommendedMinimumVersion,
          )
        ) {
          console.warn(
            `It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
             Current: ${this.version}`,
          );
        }
      }

      this.capabilities = {
        canDoubleTimeout: !isVersionLowerThan(this.version, '6.0.0'),
        canBlockFor1Ms: !isVersionLowerThan(this.version, '7.0.8'),
      };

      this.status = 'ready';
    }

    return this._client;
  }

  async disconnect(wait = true): Promise<void> {
    const client = await this.client;
    if (client.status !== 'end') {
      let _resolve, _reject;

      if (!wait) {
        return client.disconnect();
      }

      const disconnecting = new Promise<void>((resolve, reject) => {
        increaseMaxListeners(client, 2);

        client.once('end', resolve);
        client.once('error', reject);
        _resolve = resolve;
        _reject = reject;
      });

      client.disconnect();

      try {
        await disconnecting;
      } finally {
        decreaseMaxListeners(client, 2);

        client.removeListener('end', _resolve);
        client.removeListener('error', _reject);
      }
    }
  }

  async reconnect(): Promise<void> {
    const client = await this.client;
    return client.connect();
  }

  async close(force = false): Promise<void> {
    if (!this.closing) {
      const status = this.status;
      this.status = 'closing';
      this.closing = true;

      try {
        if (status === 'ready') {
          // Not sure if we need to wait for this
          await this.initializing;
        }
        if (!this.shared) {
          if (status == 'initializing' || force) {
            // If we have not still connected to Redis, we need to disconnect.
            this._client.disconnect();
          } else {
            await this._client.quit();
          }
          // As IORedis does not update this status properly, we do it ourselves.
          this._client['status'] = 'end';
        }
      } catch (error) {
        if (isNotConnectionError(error as Error)) {
          throw error;
        }
      } finally {
        this._client.off('error', this.handleClientError);
        this._client.off('close', this.handleClientClose);
        this._client.off('ready', this.handleClientReady);

        decreaseMaxListeners(this._client, 3);

        this.removeAllListeners();
        this.status = 'closed';
      }
    }
  }

  private async getRedisVersion() {
    const doc = await this._client.info();
    const redisPrefix = 'redis_version:';
    const maxMemoryPolicyPrefix = 'maxmemory_policy:';
    const lines = doc.split('\r\n');
    let redisVersion;

    for (let i = 0; i < lines.length; i++) {
      if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {
        const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);
        if (maxMemoryPolicy !== 'noeviction') {
          console.warn(
            `IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`,
          );
        }
      }

      if (lines[i].indexOf(redisPrefix) === 0) {
        redisVersion = lines[i].substr(redisPrefix.length);
      }
    }

    return redisVersion;
  }

  get redisVersion(): string {
    return this.version;
  }
}

1	import { EventEmitter } from 'events';	1✔
2	import { default as IORedis } from 'ioredis';	1✔
3	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
4	// @ts-ignore
5	import { CONNECTION_CLOSED_ERROR_MSG } from 'ioredis/built/utils';	1✔
6	import { ConnectionOptions, RedisOptions, RedisClient } from '../interfaces';
7	import {	1✔
8	decreaseMaxListeners,
9	increaseMaxListeners,
10	isNotConnectionError,
11	isRedisCluster,
12	isRedisInstance,
13	isVersionLowerThan,
14	} from '../utils';
15	import { version as packageVersion } from '../version';	1✔
16	import * as scripts from '../scripts';	1✔
17
18	const overrideMessage = [	1✔
19	'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
20	'and will be overridden by BullMQ.',
21	].join(' ');
22
23	const deprecationMessage =
24	'BullMQ: Your redis options maxRetriesPerRequest must be null.';	1✔
25
26	interface RedisCapabilities {
27	canDoubleTimeout: boolean;
28	canBlockFor1Ms: boolean;
29	}
30
31	export interface RawCommand {
32	content: string;
33	name: string;
34	keys: number;
35	}
36
37	export class RedisConnection extends EventEmitter {	1✔
38	static minimumVersion = '5.0.0';	1✔
39	static recommendedMinimumVersion = '6.2.0';	1✔
40
41	closing: boolean;
42	capabilities: RedisCapabilities = {	2,310✔
43	canDoubleTimeout: false,
44	canBlockFor1Ms: true,
45	};
46
47	status: 'initializing' \| 'ready' \| 'closing' \| 'closed' = 'initializing';	2,310✔
48
49	protected _client: RedisClient;
50
51	private readonly opts: RedisOptions;
52	private readonly initializing: Promise<RedisClient>;
53
54	private version: string;
55	protected packageVersion = packageVersion;	2,310✔
56	private skipVersionCheck: boolean;
57	private handleClientError: (e: Error) => void;
58	private handleClientClose: () => void;
59	private handleClientReady: () => void;
60
61	constructor(
62	opts?: ConnectionOptions,
63	private readonly shared: boolean = false,	2,310✔
64	private readonly blocking = true,	2,310✔
65	skipVersionCheck = false,	2,310✔
66	) {
67	super();	2,310✔
68	if (!isRedisInstance(opts)) {	2,310✔
69	this.checkBlockingOptions(overrideMessage, opts);	12✔
70
71	this.opts = {	12✔
72	port: 6379,
73	host: '127.0.0.1',
74	retryStrategy: function (times: number) {
75	return Math.max(Math.min(Math.exp(times), 20000), 1000);	×
76	},
77	...opts,
78	};
79
80	if (this.blocking) {	12✔
81	this.opts.maxRetriesPerRequest = null;	2✔
82	}
83	} else {
84	this._client = opts;	2,298✔
85
86	// Test if the redis instance is using keyPrefix
87	// and if so, throw an error.
88	if (this._client.options.keyPrefix) {	2,298✔
89	throw new Error(	2✔
90	'BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.',
91	);
92	}
93
94	if (isRedisCluster(this._client)) {	2,296✔
95	this.opts = this._client.options.redisOptions;	1✔
96	} else {
97	this.opts = this._client.options;	2,295✔
98	}
99
100	this.checkBlockingOptions(deprecationMessage, this.opts, true);	2,296✔
101	}
102
103	this.skipVersionCheck =	2,308✔
104	skipVersionCheck \|\| !!(this.opts && this.opts.skipVersionCheck);	9,231✔
105	this.handleClientError = (err: Error): void => {	2,308✔
106	this.emit('error', err);	14✔
107	};
108
109	this.handleClientClose = (): void => {	2,308✔
110	this.emit('close');	639✔
111	};
112
113	this.handleClientReady = (): void => {	2,308✔
114	this.emit('ready');	891✔
115	};
116
117	this.initializing = this.init();	2,308✔
118	this.initializing.catch(err => this.emit('error', err));	2,308✔
119	}
120
121	private checkBlockingOptions(
122	msg: string,
123	options?: RedisOptions,
124	throwError = false,	12✔
125	) {
126	if (this.blocking && options && options.maxRetriesPerRequest) {	2,308!
127	if (throwError) {	×
128	throw new Error(msg);	×
129	} else {
130	console.error(msg);	×
131	}
132	}
133	}
134
135	/**
136	* Waits for a redis client to be ready.
137	* @param redis - client
138	*/
139	static async waitUntilReady(client: RedisClient): Promise<void> {
140	if (client.status === 'ready') {	2,310✔
141	return;	1,410✔
142	}
143
144	if (client.status === 'wait') {	900!
145	return client.connect();	×
146	}
147
148	if (client.status === 'end') {	900!
149	throw new Error(CONNECTION_CLOSED_ERROR_MSG);	×
150	}
151
152	let handleReady: () => void;
153	let handleEnd: () => void;
154	let handleError: (e: Error) => void;
155	try {	900✔
156	await new Promise<void>((resolve, reject) => {	900✔
157	let lastError: Error;
158
159	handleError = (err: Error) => {	900✔
160	lastError = err;	7✔
161	};
162
163	handleReady = () => {	900✔
164	resolve();	885✔
165	};
166
167	handleEnd = () => {	900✔
168	if (client.status !== 'end') {	13!
169	reject(lastError \|\| new Error(CONNECTION_CLOSED_ERROR_MSG));	×
170	} else {
171	if (lastError) {	13✔
172	reject(lastError);	6✔
173	} else {
174	// when custon 'end' status is set we already closed
175	resolve();	7✔
176	}
177	}
178	};
179
180	increaseMaxListeners(client, 3);	900✔
181
182	client.once('ready', handleReady);	900✔
183	client.on('end', handleEnd);	900✔
184	client.once('error', handleError);	900✔
185	});
186	} finally {
187	client.removeListener('end', handleEnd);	898✔
188	client.removeListener('error', handleError);	898✔
189	client.removeListener('ready', handleReady);	898✔
190
191	decreaseMaxListeners(client, 3);	898✔
192	}
193	}
194
195	get client(): Promise<RedisClient> {
196	return this.initializing;	35,832✔
197	}
198
199	protected loadCommands(
200	packageVersion: string,
201	providedScripts?: Record<string, RawCommand>,
202	): void {
203	const finalScripts =
204	providedScripts \|\| (scripts as Record<string, RawCommand>);	2,300✔
205	for (const property in finalScripts as Record<string, RawCommand>) {	2,300✔
206	// Only define the command if not already defined
207	const commandName = `${finalScripts[property].name}:${packageVersion}`;	108,100✔
208	if (!(<any>this._client)[commandName]) {	108,100✔
209	(<any>this._client).defineCommand(commandName, {	41,689✔
210	numberOfKeys: finalScripts[property].keys,
211	lua: finalScripts[property].content,
212	});
213	}
214	}
215	}
216
217	private async init() {
218	if (!this._client) {	2,308✔
219	const { url, ...rest } = this.opts;	12✔
220	this._client = url ? new IORedis(url, rest) : new IORedis(rest);	12✔
221	}
222
223	increaseMaxListeners(this._client, 3);	2,308✔
224
225	this._client.on('error', this.handleClientError);	2,308✔
226	// ioredis treats connection errors as a different event ('close')
227	this._client.on('close', this.handleClientClose);	2,308✔
228
229	this._client.on('ready', this.handleClientReady);	2,308✔
230
231	await RedisConnection.waitUntilReady(this._client);	2,308✔
232
233	this.loadCommands(this.packageVersion);	2,300✔
234
235	if (this._client['status'] !== 'end') {	2,300✔
236	this.version = await this.getRedisVersion();	2,293✔
237	if (this.skipVersionCheck !== true && !this.closing) {	2,293✔
238	if (isVersionLowerThan(this.version, RedisConnection.minimumVersion)) {	2,287!
UNCOV 239	throw new Error(	×
240	`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} ` +
241	`Current: ${this.version}`,
242	);
243	}
244
245	if (	2,287!
246	isVersionLowerThan(
247	this.version,
248	RedisConnection.recommendedMinimumVersion,
249	)
250	) {
251	console.warn(	×
252	`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
253	Current: ${this.version}`,
254	);
255	}
256	}
257
258	this.capabilities = {	2,293✔
259	canDoubleTimeout: !isVersionLowerThan(this.version, '6.0.0'),
260	canBlockFor1Ms: !isVersionLowerThan(this.version, '7.0.8'),
261	};
262
263	this.status = 'ready';	2,293✔
264	}
265
266	return this._client;	2,300✔
267	}
268
269	async disconnect(wait = true): Promise<void> {	381✔
270	const client = await this.client;	515✔
271	if (client.status !== 'end') {	515!
272	let _resolve, _reject;
273
274	if (!wait) {	515✔
275	return client.disconnect();	132✔
276	}
277
278	const disconnecting = new Promise<void>((resolve, reject) => {	383✔
279	increaseMaxListeners(client, 2);	383✔
280
281	client.once('end', resolve);	383✔
282	client.once('error', reject);	383✔
283	_resolve = resolve;	383✔
284	_reject = reject;	383✔
285	});
286
287	client.disconnect();	383✔
288
289	try {	383✔
290	await disconnecting;	383✔
291	} finally {
292	decreaseMaxListeners(client, 2);	383✔
293
294	client.removeListener('end', _resolve);	383✔
295	client.removeListener('error', _reject);	383✔
296	}
297	}
298	}
299
300	async reconnect(): Promise<void> {
301	const client = await this.client;	2✔
302	return client.connect();	2✔
303	}
304
305	async close(force = false): Promise<void> {	867✔
306	if (!this.closing) {	1,799!
307	const status = this.status;	1,799✔
308	this.status = 'closing';	1,799✔
309	this.closing = true;	1,799✔
310
311	try {	1,799✔
312	if (status === 'ready') {	1,799✔
313	// Not sure if we need to wait for this
314	await this.initializing;	1,783✔
315	}
316	if (!this.shared) {	1,799✔
317	if (status == 'initializing' \|\| force) {	472✔
318	// If we have not still connected to Redis, we need to disconnect.
319	this._client.disconnect();	26✔
320	} else {
321	await this._client.quit();	446✔
322	}
323	// As IORedis does not update this status properly, we do it ourselves.
324	this._client['status'] = 'end';	341✔
325	}
326	} catch (error) {
327	if (isNotConnectionError(error as Error)) {	131!
328	throw error;	×
329	}
330	} finally {
331	this._client.off('error', this.handleClientError);	1,799✔
332	this._client.off('close', this.handleClientClose);	1,799✔
333	this._client.off('ready', this.handleClientReady);	1,799✔
334
335	decreaseMaxListeners(this._client, 3);	1,799✔
336
337	this.removeAllListeners();	1,799✔
338	this.status = 'closed';	1,799✔
339	}
340	}
341	}
342
343	private async getRedisVersion() {
344	const doc = await this._client.info();	2,293✔
345	const redisPrefix = 'redis_version:';	2,293✔
346	const maxMemoryPolicyPrefix = 'maxmemory_policy:';	2,293✔
347	const lines = doc.split('\r\n');	2,293✔
348	let redisVersion;
349
350	for (let i = 0; i < lines.length; i++) {	2,293✔
351	if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {	515,857✔
352	const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);	2,293✔
353	if (maxMemoryPolicy !== 'noeviction') {	2,293!
354	console.warn(	×
355	`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`,
356	);
357	}
358	}
359
360	if (lines[i].indexOf(redisPrefix) === 0) {	515,857✔
361	redisVersion = lines[i].substr(redisPrefix.length);	2,293✔
362	}
363	}
364
365	return redisVersion;	2,293✔
366	}
367
368	get redisVersion(): string {
369	return this.version;	409✔
370	}
371	}

taskforcesh / bullmq / 11872108728

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous