15441519073

Committed 04 Jun 2025 11:50AM UTC coverage: 90.161% (-0.03%) from 90.195%

Build # 15441519073

Build Type

Pull #2608

github

Committed by

PooyaRaki

Commit Message

feat: enforce maximum TTL in RedisCacheService to prevent overflow errors

Pull Request Pull Request #2608: feat: deviate Redis expiration time

Run Details

3358 of 4066 branches covered (82.59%)

Branch coverage included in aggregate %.

17 of 17 new or added lines in 2 files covered. (100.0%)

71 existing lines in 11 files now uncovered.

11578 of 12500 relevant lines covered (92.62%)

554.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.38

/src/datasources/cache/cache.first.data.source.ts

import { Inject, Injectable } from '@nestjs/common';
import {
  CacheService,
  ICacheService,
} from '@/datasources/cache/cache.service.interface';
import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';
import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
import {
  INetworkService,
  NetworkService,
} from '@/datasources/network/network.service.interface';
import { ILoggingService, LoggingService } from '@/logging/logging.interface';
import { Page } from '@/domain/entities/page.entity';
import {
  isMultisigTransaction,
  isEthereumTransaction,
  isModuleTransaction,
  isCreationTransaction,
  Transaction,
} from '@/domain/safe/entities/transaction.entity';
import { IConfigurationService } from '@/config/configuration.service.interface';
import { Safe } from '@/domain/safe/entities/safe.entity';
import { Raw } from '@/validation/entities/raw.entity';
import { LogType } from '@/domain/common/entities/log-type.entity';

/**
 * A data source which tries to retrieve values from cache using
 * {@link CacheService} and fallbacks to {@link NetworkService}
 * if the cache entry expired or is not present.
 *
 * This is the recommended data source that should be used when
 * a feature requires both networking and caching the respective
 * responses.
 */
@Injectable()
export class CacheFirstDataSource {
  private readonly areDebugLogsEnabled: boolean;
  private readonly areConfigHooksDebugLogsEnabled: boolean;

  constructor(
    @Inject(CacheService) private readonly cacheService: ICacheService,
    @Inject(NetworkService) private readonly networkService: INetworkService,
    @Inject(LoggingService) private readonly loggingService: ILoggingService,
    @Inject(IConfigurationService)
    private readonly configurationService: IConfigurationService,
  ) {
    this.areDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>('features.debugLogs');
    this.areConfigHooksDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>(
        'features.configHooksDebugLogs',
      );
  }

  /**
   * Gets the cached value behind {@link CacheDir}.
   * If the value is not present, it tries to get the respective JSON
   * payload from {@link url}.
   * 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
   *
   * @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
   * @param args.url - the HTTP endpoint to retrieve the JSON payload
   * @param args.networkRequest - the HTTP request to be used if there is a cache miss
   * @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
   * @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
   */
  async get<T>(args: {
    cacheDir: CacheDir;
    url: string;
    notFoundExpireTimeSeconds: number;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<Raw<T>> {
    const cached = await this.cacheService.hGet(args.cacheDir);
    if (cached != null) return this._getFromCachedData(args.cacheDir, cached);

    try {
      return await this._getFromNetworkAndWriteCache(args);
    } catch (error) {
      if (
        error instanceof NetworkResponseError &&
        error.response.status === 404
      ) {
        await this.cacheNotFoundError(
          args.cacheDir,
          error,
          args.notFoundExpireTimeSeconds,
        );
      }
      throw error;
    }
  }

  /**
   * Gets the data from the contents stored in the cache.
   */
  private _getFromCachedData<T>(
    { key, field }: CacheDir,
    cached: string,
  ): Promise<T> {
    this.loggingService.debug({ type: LogType.CacheHit, key, field });
    const cachedData = JSON.parse(cached);
    if (cachedData?.response?.status === 404) {
      // TODO: create a CachedData type with guard to avoid these type assertions.
      const url: URL = cachedData.url;
      const response: Response = cachedData.response;
      throw new NetworkResponseError(url, response, cachedData?.data);
    }
    return cachedData;
  }

  /**
   * Gets the data from the network and caches the result.
   */
  private async _getFromNetworkAndWriteCache<T>(args: {
    cacheDir: CacheDir;
    url: string;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<Raw<T>> {
    const { key, field } = args.cacheDir;
    this.loggingService.debug({ type: LogType.CacheMiss, key, field });
    const startTimeMs = Date.now();
    const { data } = await this.networkService.get<T>({
      url: args.url,
      networkRequest: args.networkRequest,
    });

    const shouldBeCached = await this._shouldBeCached(key, startTimeMs);
    if (shouldBeCached) {
      await this.cacheService.hSet(
        args.cacheDir,
        JSON.stringify(data),
        args.expireTimeSeconds,
      );

      // TODO: transient logging for debugging
      if (
        this.areDebugLogsEnabled &&
        (args.url.includes('all-transactions') ||
          args.url.includes('multisig-transactions'))
      ) {
        this.logTransactionsCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as unknown as Page<Transaction>,
        );
      }

      if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {
        this.logSafeMetadataCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as unknown as Safe,
        );
      }

      if (
        this.areConfigHooksDebugLogsEnabled &&
        args.cacheDir.key.includes('chain')
      ) {
        this.logChainUpdateCacheWrite(startTimeMs, args.cacheDir, data);
      }
    }
    return data;
  }

  /**
   * Validates that the request is more recent than the last invalidation recorded for the item,
   * preventing a race condition where outdated data is stored due to the request being initiated
   * before the source communicated a change (via webhook or by other means).
   *
   * Returns true if (any of the following):
   * 1. An invalidationTimeMs entry for the key received as param is *not* found in the cache.
   * 2. An entry *is* found and contains an integer that is less than the received startTimeMs param.
   *
   * @param key key part of the {@link CacheDir} holding the requested item
   * @param startTimeMs Unix epoch timestamp in ms when the request was initiated
   * @returns true if any of the above conditions is met
   */
  private async _shouldBeCached(
    key: string,
    startTimeMs: number,
  ): Promise<boolean> {
    const invalidationTimeMsStr = await this.cacheService.hGet(
      new CacheDir(`invalidationTimeMs:${key}`, ''),
    );

    if (!invalidationTimeMsStr) return true;

    const invalidationTimeMs = Number(invalidationTimeMsStr);
    return (
      Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs
    );
  }

  /**
   * Caches a not found error.
   * @param cacheDir - {@link CacheDir} where the error should be placed
   */
  private async cacheNotFoundError(
    cacheDir: CacheDir,
    error: NetworkResponseError,
    notFoundExpireTimeSeconds?: number,
  ): Promise<void> {
    return this.cacheService.hSet(
      cacheDir,
      JSON.stringify({
        data: error.data,
        response: { status: error.response.status },
        url: error.url,
      }),
      notFoundExpireTimeSeconds,
    );
  }

  /**
   * Logs the type and the hash of the transactions present in the data parameter.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logTransactionsCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: Page<Transaction>,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      txHashes:
        Array.isArray(data?.results) && // no validation executed yet at this point
        data.results.map((transaction) => {
          if (isMultisigTransaction(transaction)) {
            return {
              txType: 'multisig',
              safeTxHash: transaction.safeTxHash,
              confirmations: transaction.confirmations,
              confirmationRequired: transaction.confirmationsRequired,
            };
          } else if (isEthereumTransaction(transaction)) {
            return {
              txType: 'ethereum',
              txHash: transaction.txHash,
            };
          } else if (isModuleTransaction(transaction)) {
            return {
              txType: 'module',
              transactionHash: transaction.transactionHash,
            };
          } else if (isCreationTransaction(transaction)) {
            return {
              txType: 'creation',
              transactionHash: transaction.transactionHash,
            };
          }
        }),
    });
  }

  /**
   * Logs the Safe metadata retrieved.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logSafeMetadataCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    safe: Safe,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      safe,
    });
  }

  /**
   * Logs the chain/chains retrieved.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logChainUpdateCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: unknown,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      data,
    });
  }
}

1	import { Inject, Injectable } from '@nestjs/common';	162✔
2	import {	162✔
3	CacheService,
4	ICacheService,
5	} from '@/datasources/cache/cache.service.interface';
6	import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';	162✔
7	import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';	162✔
8	import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
9	import {	162✔
10	INetworkService,
11	NetworkService,
12	} from '@/datasources/network/network.service.interface';
13	import { ILoggingService, LoggingService } from '@/logging/logging.interface';	162✔
14	import { Page } from '@/domain/entities/page.entity';
15	import {	162✔
16	isMultisigTransaction,
17	isEthereumTransaction,
18	isModuleTransaction,
19	isCreationTransaction,
20	Transaction,
21	} from '@/domain/safe/entities/transaction.entity';
22	import { IConfigurationService } from '@/config/configuration.service.interface';	162✔
23	import { Safe } from '@/domain/safe/entities/safe.entity';
24	import { Raw } from '@/validation/entities/raw.entity';
25	import { LogType } from '@/domain/common/entities/log-type.entity';	162✔
26
27	/**
28	* A data source which tries to retrieve values from cache using
29	* {@link CacheService} and fallbacks to {@link NetworkService}
30	* if the cache entry expired or is not present.
31	*
32	* This is the recommended data source that should be used when
33	* a feature requires both networking and caching the respective
34	* responses.
35	*/
36	@Injectable()
37	export class CacheFirstDataSource {	162✔
38	private readonly areDebugLogsEnabled: boolean;
39	private readonly areConfigHooksDebugLogsEnabled: boolean;
40
41	constructor(
42	@Inject(CacheService) private readonly cacheService: ICacheService,	1,634✔
43	@Inject(NetworkService) private readonly networkService: INetworkService,	1,634✔
44	@Inject(LoggingService) private readonly loggingService: ILoggingService,	1,634✔
45	@Inject(IConfigurationService)
46	private readonly configurationService: IConfigurationService,	1,634✔
47	) {
48	this.areDebugLogsEnabled =	1,634✔
49	this.configurationService.getOrThrow<boolean>('features.debugLogs');
50	this.areConfigHooksDebugLogsEnabled =	1,634✔
51	this.configurationService.getOrThrow<boolean>(
52	'features.configHooksDebugLogs',
53	);
54	}
55
56	/**
57	* Gets the cached value behind {@link CacheDir}.
58	* If the value is not present, it tries to get the respective JSON
59	* payload from {@link url}.
60	* 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
61	*
62	* @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
63	* @param args.url - the HTTP endpoint to retrieve the JSON payload
64	* @param args.networkRequest - the HTTP request to be used if there is a cache miss
65	* @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
66	* @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
67	*/
68	async get<T>(args: {
69	cacheDir: CacheDir;
70	url: string;
71	notFoundExpireTimeSeconds: number;
72	networkRequest?: NetworkRequest;
73	expireTimeSeconds?: number;
74	}): Promise<Raw<T>> {
75	const cached = await this.cacheService.hGet(args.cacheDir);	7,184✔
76	if (cached != null) return this._getFromCachedData(args.cacheDir, cached);	7,184✔
77
78	try {	5,722✔
79	return await this._getFromNetworkAndWriteCache(args);	5,722✔
80	} catch (error) {
81	if (	2,120✔
82	error instanceof NetworkResponseError &&	1,106✔
83	error.response.status === 404
84	) {
85	await this.cacheNotFoundError(	34✔
86	args.cacheDir,
87	error,
88	args.notFoundExpireTimeSeconds,
89	);
90	}
91	throw error;	2,120✔
92	}
93	}
94
95	/**
96	* Gets the data from the contents stored in the cache.
97	*/
98	private _getFromCachedData<T>(
99	{ key, field }: CacheDir,
100	cached: string,
101	): Promise<T> {
102	this.loggingService.debug({ type: LogType.CacheHit, key, field });	1,462✔
103	const cachedData = JSON.parse(cached);	1,462✔
104	if (cachedData?.response?.status === 404) {	1,462!
105	// TODO: create a CachedData type with guard to avoid these type assertions.
106	const url: URL = cachedData.url;	2✔
107	const response: Response = cachedData.response;	2✔
108	throw new NetworkResponseError(url, response, cachedData?.data);	2!
109	}
110	return cachedData;	1,460✔
111	}
112
113	/**
114	* Gets the data from the network and caches the result.
115	*/
116	private async _getFromNetworkAndWriteCache<T>(args: {
117	cacheDir: CacheDir;
118	url: string;
119	networkRequest?: NetworkRequest;
120	expireTimeSeconds?: number;
121	}): Promise<Raw<T>> {
122	const { key, field } = args.cacheDir;	5,722✔
123	this.loggingService.debug({ type: LogType.CacheMiss, key, field });	5,722✔
124	const startTimeMs = Date.now();	5,722✔
125	const { data } = await this.networkService.get<T>({	5,722✔
126	url: args.url,
127	networkRequest: args.networkRequest,
128	});
129
130	const shouldBeCached = await this._shouldBeCached(key, startTimeMs);	3,602✔
131	if (shouldBeCached) {	3,602✔
132	await this.cacheService.hSet(	3,588✔
133	args.cacheDir,
134	JSON.stringify(data),
135	args.expireTimeSeconds,
136	);
137
138	// TODO: transient logging for debugging
139	if (	3,588!
140	this.areDebugLogsEnabled &&	1,798✔
141	(args.url.includes('all-transactions') \|\|
142	args.url.includes('multisig-transactions'))
143	) {
UNCOV 144	this.logTransactionsCacheWrite(	×
145	startTimeMs,
146	args.cacheDir,
147	data as unknown as Page<Transaction>,
148	);
149	}
150
151	if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {	3,588!
UNCOV 152	this.logSafeMetadataCacheWrite(	×
153	startTimeMs,
154	args.cacheDir,
155	data as unknown as Safe,
156	);
157	}
158
159	if (	3,588!
160	this.areConfigHooksDebugLogsEnabled &&	1,794!
161	args.cacheDir.key.includes('chain')
162	) {
UNCOV 163	this.logChainUpdateCacheWrite(startTimeMs, args.cacheDir, data);	×
164	}
165	}
166	return data;	3,602✔
167	}
168
169	/**
170	* Validates that the request is more recent than the last invalidation recorded for the item,
171	* preventing a race condition where outdated data is stored due to the request being initiated
172	* before the source communicated a change (via webhook or by other means).
173	*
174	* Returns true if (any of the following):
175	* 1. An invalidationTimeMs entry for the key received as param is not found in the cache.
176	* 2. An entry is found and contains an integer that is less than the received startTimeMs param.
177	*
178	* @param key key part of the {@link CacheDir} holding the requested item
179	* @param startTimeMs Unix epoch timestamp in ms when the request was initiated
180	* @returns true if any of the above conditions is met
181	*/
182	private async _shouldBeCached(
183	key: string,
184	startTimeMs: number,
185	): Promise<boolean> {
186	const invalidationTimeMsStr = await this.cacheService.hGet(	3,602✔
187	new CacheDir(`invalidationTimeMs:${key}`, ''),
188	);
189
190	if (!invalidationTimeMsStr) return true;	3,602✔
191
192	const invalidationTimeMs = Number(invalidationTimeMsStr);	82✔
193	return (	82✔
194	Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs	82✔
195	);
196	}
197
198	/**
199	* Caches a not found error.
200	* @param cacheDir - {@link CacheDir} where the error should be placed
201	*/
202	private async cacheNotFoundError(
203	cacheDir: CacheDir,
204	error: NetworkResponseError,
205	notFoundExpireTimeSeconds?: number,
206	): Promise<void> {
207	return this.cacheService.hSet(	34✔
208	cacheDir,
209	JSON.stringify({
210	data: error.data,
211	response: { status: error.response.status },
212	url: error.url,
213	}),
214	notFoundExpireTimeSeconds,
215	);
216	}
217
218	/**
219	* Logs the type and the hash of the transactions present in the data parameter.
220	* NOTE: this is a debugging-only function.
221	* TODO: remove this function after debugging.
222	*/
223	private logTransactionsCacheWrite(
224	requestStartTime: number,
225	cacheDir: CacheDir,
226	data: Page<Transaction>,
227	): void {
UNCOV 228	this.loggingService.info({	×
229	type: 'cache_write',
230	cacheKey: cacheDir.key,
231	cacheField: cacheDir.field,
232	cacheWriteTime: new Date(),
233	requestStartTime: new Date(requestStartTime),
234	txHashes:
235	Array.isArray(data?.results) && // no validation executed yet at this point	×
236	data.results.map((transaction) => {
UNCOV 237	if (isMultisigTransaction(transaction)) {	×
UNCOV 238	return {	×
239	txType: 'multisig',
240	safeTxHash: transaction.safeTxHash,
241	confirmations: transaction.confirmations,
242	confirmationRequired: transaction.confirmationsRequired,
243	};
UNCOV 244	} else if (isEthereumTransaction(transaction)) {	×
UNCOV 245	return {	×
246	txType: 'ethereum',
247	txHash: transaction.txHash,
248	};
UNCOV 249	} else if (isModuleTransaction(transaction)) {	×
UNCOV 250	return {	×
251	txType: 'module',
252	transactionHash: transaction.transactionHash,
253	};
UNCOV 254	} else if (isCreationTransaction(transaction)) {	×
UNCOV 255	return {	×
256	txType: 'creation',
257	transactionHash: transaction.transactionHash,
258	};
259	}
260	}),
261	});
262	}
263
264	/**
265	* Logs the Safe metadata retrieved.
266	* NOTE: this is a debugging-only function.
267	* TODO: remove this function after debugging.
268	*/
269	private logSafeMetadataCacheWrite(
270	requestStartTime: number,
271	cacheDir: CacheDir,
272	safe: Safe,
273	): void {
UNCOV 274	this.loggingService.info({	×
275	type: 'cache_write',
276	cacheKey: cacheDir.key,
277	cacheField: cacheDir.field,
278	cacheWriteTime: new Date(),
279	requestStartTime: new Date(requestStartTime),
280	safe,
281	});
282	}
283
284	/**
285	* Logs the chain/chains retrieved.
286	* NOTE: this is a debugging-only function.
287	* TODO: remove this function after debugging.
288	*/
289	private logChainUpdateCacheWrite(
290	requestStartTime: number,
291	cacheDir: CacheDir,
292	data: unknown,
293	): void {
UNCOV 294	this.loggingService.info({	×
295	type: 'cache_write',
296	cacheKey: cacheDir.key,
297	cacheField: cacheDir.field,
298	cacheWriteTime: new Date(),
299	requestStartTime: new Date(requestStartTime),
300	data,
301	});
302	}
303	}

safe-global / safe-client-gateway / 15441519073

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous