7101328086

Committed 05 Dec 2023 01:19PM UTC coverage: 89.572% (+0.01%) from 89.561%

Build # 7101328086

Build Type

Pull #2309

github

Committed by

web-flow

Commit Message

Merge a16b8fdfe into 76f9e8ffa

Pull Request Pull Request #2309: Feat/better handling of queue markers

Run Details

1018 of 1221 branches covered (0.0%)

Branch coverage included in aggregate %.

19 of 20 new or added lines in 3 files covered. (95.0%)

55 existing lines in 7 files now uncovered.

2040 of 2193 relevant lines covered (93.02%)

3037.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.22

/src/classes/redis-connection.ts

import { EventEmitter } from 'events';
import { default as IORedis } from 'ioredis';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { CONNECTION_CLOSED_ERROR_MSG } from 'ioredis/built/utils';
import { ConnectionOptions, RedisOptions, RedisClient } from '../interfaces';
import {
  decreaseMaxListeners,
  increaseMaxListeners,
  isNotConnectionError,
  isRedisCluster,
  isRedisInstance,
  isRedisVersionLowerThan,
} from '../utils';
import * as scripts from '../scripts';

const overrideMessage = [
  'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
  'and will be overridden by BullMQ.',
].join(' ');

const deprecationMessage = [
  'BullMQ: DEPRECATION WARNING! Your redis options maxRetriesPerRequest must be null.',
  'On the next versions having this settings will throw an exception',
].join(' ');

interface RedisCapabilities {
  canDoubleTimeout: boolean;
}

export interface RawCommand {
  content: string;
  name: string;
  keys: number;
}

export class RedisConnection extends EventEmitter {
  static minimumVersion = '5.0.0';
  static recommendedMinimumVersion = '6.2.0';

  closing: boolean;
  capabilities: RedisCapabilities = {
    canDoubleTimeout: false,
  };

  protected _client: RedisClient;

  private readonly opts: RedisOptions;
  private readonly initializing: Promise<RedisClient>;

  private version: string;
  private skipVersionCheck: boolean;
  private handleClientError: (e: Error) => void;
  private handleClientClose: () => void;
  private handleClientReady: () => void;

  constructor(
    opts?: ConnectionOptions,
    private readonly shared: boolean = false,
    private readonly blocking = true,
    skipVersionCheck = false,
  ) {
    super();

    if (!isRedisInstance(opts)) {
      this.checkBlockingOptions(overrideMessage, opts);

      this.opts = {
        port: 6379,
        host: '127.0.0.1',
        retryStrategy: function (times: number) {
          return Math.max(Math.min(Math.exp(times), 20000), 1000);
        },
        ...opts,
      };

      if (this.blocking) {
        this.opts.maxRetriesPerRequest = null;
      }
    } else {
      this._client = opts;

      // Test if the redis instance is using keyPrefix
      // and if so, throw an error.
      if (this._client.options.keyPrefix) {
        throw new Error(
          'BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.',
        );
      }

      if (isRedisCluster(this._client)) {
        this.opts = this._client.options.redisOptions;
      } else {
        this.opts = this._client.options;
      }

      this.checkBlockingOptions(deprecationMessage, this.opts);
    }

    this.skipVersionCheck =
      skipVersionCheck || !!(this.opts && this.opts.skipVersionCheck);
    this.handleClientError = (err: Error): void => {
      this.emit('error', err);
    };

    this.handleClientClose = (): void => {
      this.emit('close');
    };

    this.handleClientReady = (): void => {
      this.emit('ready');
    };

    this.initializing = this.init();
    this.initializing.catch(err => this.emit('error', err));
  }

  private checkBlockingOptions(msg: string, options?: RedisOptions) {
    if (this.blocking && options && options.maxRetriesPerRequest) {
      console.error(msg);
    }
  }

  /**
   * Waits for a redis client to be ready.
   * @param redis - client
   */
  static async waitUntilReady(client: RedisClient): Promise<void> {
    if (client.status === 'ready') {
      return;
    }

    if (client.status === 'wait') {
      return client.connect();
    }

    if (client.status === 'end') {
      throw new Error(CONNECTION_CLOSED_ERROR_MSG);
    }

    let handleReady: () => void;
    let handleEnd: () => void;
    let handleError: (e: Error) => void;
    try {
      await new Promise<void>((resolve, reject) => {
        let lastError: Error;

        handleError = (err: Error) => {
          lastError = err;
        };

        handleReady = () => {
          resolve();
        };

        handleEnd = () => {
          reject(lastError || new Error(CONNECTION_CLOSED_ERROR_MSG));
        };

        increaseMaxListeners(client, 3);

        client.once('ready', handleReady);
        client.on('end', handleEnd);
        client.once('error', handleError);
      });
    } finally {
      client.removeListener('end', handleEnd);
      client.removeListener('error', handleError);
      client.removeListener('ready', handleReady);

      decreaseMaxListeners(client, 3);
    }
  }

  get client(): Promise<RedisClient> {
    return this.initializing;
  }

  protected loadCommands(providedScripts?: Record<string, RawCommand>): void {
    const finalScripts =
      providedScripts || (scripts as Record<string, RawCommand>);
    for (const property in finalScripts as Record<string, RawCommand>) {
      // Only define the command if not already defined
      if (!(<any>this._client)[finalScripts[property].name]) {
        (<any>this._client).defineCommand(finalScripts[property].name, {
          numberOfKeys: finalScripts[property].keys,
          lua: finalScripts[property].content,
        });
      }
    }
  }

  private async init() {
    if (!this._client) {
      this._client = new IORedis(this.opts);
    }

    increaseMaxListeners(this._client, 3);

    this._client.on('error', this.handleClientError);
    // ioredis treats connection errors as a different event ('close')
    this._client.on('close', this.handleClientClose);

    this._client.on('ready', this.handleClientReady);

    await RedisConnection.waitUntilReady(this._client);
    this.loadCommands();

    this.version = await this.getRedisVersion();
    if (this.skipVersionCheck !== true && !this.closing) {
      if (
        isRedisVersionLowerThan(this.version, RedisConnection.minimumVersion)
      ) {
        throw new Error(
          `Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} Current: ${this.version}`,
        );
      }

      if (
        isRedisVersionLowerThan(
          this.version,
          RedisConnection.recommendedMinimumVersion,
        )
      ) {
        console.warn(
          `It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
           Current: ${this.version}`,
        );
      }
    }

    this.capabilities = {
      canDoubleTimeout: !isRedisVersionLowerThan(this.version, '6.0.0'),
    };

    return this._client;
  }

  async disconnect(wait = true): Promise<void> {
    const client = await this.client;
    if (client.status !== 'end') {
      let _resolve, _reject;

      if (!wait) {
        return client.disconnect();
      }

      const disconnecting = new Promise<void>((resolve, reject) => {
        increaseMaxListeners(client, 2);

        client.once('end', resolve);
        client.once('error', reject);
        _resolve = resolve;
        _reject = reject;
      });

      client.disconnect();

      try {
        await disconnecting;
      } finally {
        decreaseMaxListeners(client, 2);

        client.removeListener('end', _resolve);
        client.removeListener('error', _reject);
      }
    }
  }

  async reconnect(): Promise<void> {
    const client = await this.client;
    return client.connect();
  }

  async close(): Promise<void> {
    if (!this.closing) {
      this.closing = true;
      try {
        await this.initializing;
        if (!this.shared) {
          await this._client.quit();
        }
      } catch (error) {
        if (isNotConnectionError(error as Error)) {
          throw error;
        }
      } finally {
        this._client.off('error', this.handleClientError);
        this._client.off('close', this.handleClientClose);
        this._client.off('ready', this.handleClientReady);

        decreaseMaxListeners(this._client, 3);

        this.removeAllListeners();
      }
    }
  }

  private async getRedisVersion() {
    const doc = await this._client.info();
    const redisPrefix = 'redis_version:';
    const maxMemoryPolicyPrefix = 'maxmemory_policy:';
    const lines = doc.split('\r\n');
    let redisVersion;

    for (let i = 0; i < lines.length; i++) {
      if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {
        const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);
        if (maxMemoryPolicy !== 'noeviction') {
          console.warn(
            `IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`,
          );
        }
      }

      if (lines[i].indexOf(redisPrefix) === 0) {
        redisVersion = lines[i].substr(redisPrefix.length);
      }
    }

    return redisVersion;
  }

  get redisVersion(): string {
    return this.version;
  }
}

1	import { EventEmitter } from 'events';	1✔
2	import { default as IORedis } from 'ioredis';	1✔
3	// eslint-disable-next-line @typescript-eslint/ban-ts-comment
4	// @ts-ignore
5	import { CONNECTION_CLOSED_ERROR_MSG } from 'ioredis/built/utils';	1✔
6	import { ConnectionOptions, RedisOptions, RedisClient } from '../interfaces';
7	import {	1✔
8	decreaseMaxListeners,
9	increaseMaxListeners,
10	isNotConnectionError,
11	isRedisCluster,
12	isRedisInstance,
13	isRedisVersionLowerThan,
14	} from '../utils';
15	import * as scripts from '../scripts';	1✔
16
17	const overrideMessage = [	1✔
18	'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
19	'and will be overridden by BullMQ.',
20	].join(' ');
21
22	const deprecationMessage = [	1✔
23	'BullMQ: DEPRECATION WARNING! Your redis options maxRetriesPerRequest must be null.',
24	'On the next versions having this settings will throw an exception',
25	].join(' ');
26
27	interface RedisCapabilities {
28	canDoubleTimeout: boolean;
29	}
30
31	export interface RawCommand {
32	content: string;
33	name: string;
34	keys: number;
35	}
36
37	export class RedisConnection extends EventEmitter {	1✔
38	static minimumVersion = '5.0.0';	1✔
39	static recommendedMinimumVersion = '6.2.0';	1✔
40
41	closing: boolean;
42	capabilities: RedisCapabilities = {	1,723✔
43	canDoubleTimeout: false,
44	};
45
46	protected _client: RedisClient;
47
48	private readonly opts: RedisOptions;
49	private readonly initializing: Promise<RedisClient>;
50
51	private version: string;
52	private skipVersionCheck: boolean;
53	private handleClientError: (e: Error) => void;
54	private handleClientClose: () => void;
55	private handleClientReady: () => void;
56
57	constructor(
58	opts?: ConnectionOptions,
59	private readonly shared: boolean = false,	1,723✔
60	private readonly blocking = true,	1,723✔
61	skipVersionCheck = false,	1,723✔
62	) {
63	super();	1,723✔
64
65	if (!isRedisInstance(opts)) {	1,723✔
66	this.checkBlockingOptions(overrideMessage, opts);	7✔
67
68	this.opts = {	7✔
69	port: 6379,
70	host: '127.0.0.1',
71	retryStrategy: function (times: number) {
UNCOV 72	return Math.max(Math.min(Math.exp(times), 20000), 1000);	×
73	},
74	...opts,
75	};
76
77	if (this.blocking) {	7✔
78	this.opts.maxRetriesPerRequest = null;	3✔
79	}
80	} else {
81	this._client = opts;	1,716✔
82
83	// Test if the redis instance is using keyPrefix
84	// and if so, throw an error.
85	if (this._client.options.keyPrefix) {	1,716✔
86	throw new Error(	2✔
87	'BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.',
88	);
89	}
90
91	if (isRedisCluster(this._client)) {	1,714✔
92	this.opts = this._client.options.redisOptions;	1✔
93	} else {
94	this.opts = this._client.options;	1,713✔
95	}
96
97	this.checkBlockingOptions(deprecationMessage, this.opts);	1,714✔
98	}
99
100	this.skipVersionCheck =	1,721✔
101	skipVersionCheck \|\| !!(this.opts && this.opts.skipVersionCheck);	6,883✔
102	this.handleClientError = (err: Error): void => {	1,721✔
103	this.emit('error', err);	12✔
104	};
105
106	this.handleClientClose = (): void => {	1,721✔
107	this.emit('close');	751✔
108	};
109
110	this.handleClientReady = (): void => {	1,721✔
111	this.emit('ready');	690✔
112	};
113
114	this.initializing = this.init();	1,721✔
115	this.initializing.catch(err => this.emit('error', err));	1,721✔
116	}
117
118	private checkBlockingOptions(msg: string, options?: RedisOptions) {
119	if (this.blocking && options && options.maxRetriesPerRequest) {	1,721!
120	console.error(msg);	×
121	}
122	}
123
124	/**
125	* Waits for a redis client to be ready.
126	* @param redis - client
127	*/
128	static async waitUntilReady(client: RedisClient): Promise<void> {
129	if (client.status === 'ready') {	1,723✔
130	return;	1,035✔
131	}
132
133	if (client.status === 'wait') {	688!
UNCOV 134	return client.connect();	×
135	}
136
137	if (client.status === 'end') {	688!
UNCOV 138	throw new Error(CONNECTION_CLOSED_ERROR_MSG);	×
139	}
140
141	let handleReady: () => void;
142	let handleEnd: () => void;
143	let handleError: (e: Error) => void;
144	try {	688✔
145	await new Promise<void>((resolve, reject) => {	688✔
146	let lastError: Error;
147
148	handleError = (err: Error) => {	688✔
149	lastError = err;	4✔
150	};
151
152	handleReady = () => {	688✔
153	resolve();	683✔
154	};
155
156	handleEnd = () => {	688✔
157	reject(lastError \|\| new Error(CONNECTION_CLOSED_ERROR_MSG));	4!
158	};
159
160	increaseMaxListeners(client, 3);	688✔
161
162	client.once('ready', handleReady);	688✔
163	client.on('end', handleEnd);	688✔
164	client.once('error', handleError);	688✔
165	});
166	} finally {
167	client.removeListener('end', handleEnd);	687✔
168	client.removeListener('error', handleError);	687✔
169	client.removeListener('ready', handleReady);	687✔
170
171	decreaseMaxListeners(client, 3);	687✔
172	}
173	}
174
175	get client(): Promise<RedisClient> {
176	return this.initializing;	33,293✔
177	}
178
179	protected loadCommands(providedScripts?: Record<string, RawCommand>): void {
180	const finalScripts =
181	providedScripts \|\| (scripts as Record<string, RawCommand>);	1,716✔
182	for (const property in finalScripts as Record<string, RawCommand>) {	1,716✔
183	// Only define the command if not already defined
184	if (!(<any>this._client)[finalScripts[property].name]) {	56,628✔
185	(<any>this._client).defineCommand(finalScripts[property].name, {	22,341✔
186	numberOfKeys: finalScripts[property].keys,
187	lua: finalScripts[property].content,
188	});
189	}
190	}
191	}
192
193	private async init() {
194	if (!this._client) {	1,721✔
195	this._client = new IORedis(this.opts);	7✔
196	}
197
198	increaseMaxListeners(this._client, 3);	1,721✔
199
200	this._client.on('error', this.handleClientError);	1,721✔
201	// ioredis treats connection errors as a different event ('close')
202	this._client.on('close', this.handleClientClose);	1,721✔
203
204	this._client.on('ready', this.handleClientReady);	1,721✔
205
206	await RedisConnection.waitUntilReady(this._client);	1,721✔
207	this.loadCommands();	1,716✔
208
209	this.version = await this.getRedisVersion();	1,716✔
210	if (this.skipVersionCheck !== true && !this.closing) {	1,716✔
211	if (	1,714!
212	isRedisVersionLowerThan(this.version, RedisConnection.minimumVersion)
213	) {
UNCOV 214	throw new Error(	×
215	`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} Current: ${this.version}`,
216	);
217	}
218
219	if (	1,714!
220	isRedisVersionLowerThan(
221	this.version,
222	RedisConnection.recommendedMinimumVersion,
223	)
224	) {
UNCOV 225	console.warn(	×
226	`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
227	Current: ${this.version}`,
228	);
229	}
230	}
231
232	this.capabilities = {	1,716✔
233	canDoubleTimeout: !isRedisVersionLowerThan(this.version, '6.0.0'),
234	};
235
236	return this._client;	1,716✔
237	}
238
239	async disconnect(wait = true): Promise<void> {	284✔
240	const client = await this.client;	397✔
241	if (client.status !== 'end') {	397!
242	let _resolve, _reject;
243
244	if (!wait) {	397✔
245	return client.disconnect();	111✔
246	}
247
248	const disconnecting = new Promise<void>((resolve, reject) => {	286✔
249	increaseMaxListeners(client, 2);	286✔
250
251	client.once('end', resolve);	286✔
252	client.once('error', reject);	286✔
253	_resolve = resolve;	286✔
254	_reject = reject;	286✔
255	});
256
257	client.disconnect();	286✔
258
259	try {	286✔
260	await disconnecting;	286✔
261	} finally {
262	decreaseMaxListeners(client, 2);	286✔
263
264	client.removeListener('end', _resolve);	286✔
265	client.removeListener('error', _reject);	286✔
266	}
267	}
268	}
269
270	async reconnect(): Promise<void> {
271	const client = await this.client;	2✔
272	return client.connect();	2✔
273	}
274
275	async close(): Promise<void> {
276	if (!this.closing) {	975!
277	this.closing = true;	975✔
278	try {	975✔
279	await this.initializing;	975✔
280	if (!this.shared) {	973✔
281	await this._client.quit();	1✔
282	}
283	} catch (error) {
284	if (isNotConnectionError(error as Error)) {	2!
UNCOV 285	throw error;	×
286	}
287	} finally {
288	this._client.off('error', this.handleClientError);	975✔
289	this._client.off('close', this.handleClientClose);	975✔
290	this._client.off('ready', this.handleClientReady);	975✔
291
292	decreaseMaxListeners(this._client, 3);	975✔
293
294	this.removeAllListeners();	975✔
295	}
296	}
297	}
298
299	private async getRedisVersion() {
300	const doc = await this._client.info();	1,716✔
301	const redisPrefix = 'redis_version:';	1,716✔
302	const maxMemoryPolicyPrefix = 'maxmemory_policy:';	1,716✔
303	const lines = doc.split('\r\n');	1,716✔
304	let redisVersion;
305
306	for (let i = 0; i < lines.length; i++) {	1,716✔
307	if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {	370,628✔
308	const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);	1,716✔
309	if (maxMemoryPolicy !== 'noeviction') {	1,716!
UNCOV 310	console.warn(	×
311	`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`,
312	);
313	}
314	}
315
316	if (lines[i].indexOf(redisPrefix) === 0) {	370,628✔
317	redisVersion = lines[i].substr(redisPrefix.length);	1,716✔
318	}
319	}
320
321	return redisVersion;	1,716✔
322	}
323
324	get redisVersion(): string {
325	return this.version;	332✔
326	}
327	}

taskforcesh / bullmq / 7101328086

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous