11233174198

Committed 08 Oct 2024 10:05AM UTC coverage: 46.725% (-44.0%) from 90.725%

Build # 11233174198

Build Type

push

github

Committed by

web-flow

Commit Message

Increase relay rate limit TTL (#1971)

Increases the relay rate limit TTL fallback value to 24 hours:

- Change `relay.ttlSeconds` value to `60 * 60 * 24`

Run Details

498 of 3105 branches covered (16.04%)

Branch coverage included in aggregate %.

5024 of 8713 relevant lines covered (57.66%)

12.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/src/datasources/cache/cache.first.data.source.ts

import { Inject, Injectable } from '@nestjs/common';
import {
  CacheService,
  ICacheService,
} from '@/datasources/cache/cache.service.interface';
import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';
import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
import {
  INetworkService,
  NetworkService,
} from '@/datasources/network/network.service.interface';
import { ILoggingService, LoggingService } from '@/logging/logging.interface';
import { Page } from '@/domain/entities/page.entity';
import {
  isMultisigTransaction,
  isEthereumTransaction,
  isModuleTransaction,
  isCreationTransaction,
  Transaction,
} from '@/domain/safe/entities/transaction.entity';
import { IConfigurationService } from '@/config/configuration.service.interface';
import { isArray } from 'lodash';
import { Safe } from '@/domain/safe/entities/safe.entity';

/**
 * A data source which tries to retrieve values from cache using
 * {@link CacheService} and fallbacks to {@link NetworkService}
 * if the cache entry expired or is not present.
 *
 * This is the recommended data source that should be used when
 * a feature requires both networking and caching the respective
 * responses.
 */
@Injectable()
export class CacheFirstDataSource {
  private readonly areDebugLogsEnabled: boolean;
  private readonly areConfigHooksDebugLogsEnabled: boolean;

  constructor(
    @Inject(CacheService) private readonly cacheService: ICacheService,
    @Inject(NetworkService) private readonly networkService: INetworkService,
    @Inject(LoggingService) private readonly loggingService: ILoggingService,
    @Inject(IConfigurationService)
    private readonly configurationService: IConfigurationService,
  ) {
    this.areDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>('features.debugLogs');
    this.areConfigHooksDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>(
        'features.configHooksDebugLogs',
      );
  }

  /**
   * Gets the cached value behind {@link CacheDir}.
   * If the value is not present, it tries to get the respective JSON
   * payload from {@link url}.
   * 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
   *
   * @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
   * @param args.url - the HTTP endpoint to retrieve the JSON payload
   * @param args.networkRequest - the HTTP request to be used if there is a cache miss
   * @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
   * @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
   */
  async get<T>(args: {
    cacheDir: CacheDir;
    url: string;
    notFoundExpireTimeSeconds: number;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const cached = await this.cacheService.hGet(args.cacheDir);
    if (cached != null) return this._getFromCachedData(args.cacheDir, cached);

    try {
      return await this._getFromNetworkAndWriteCache(args);
    } catch (error) {
      if (
        error instanceof NetworkResponseError &&
        error.response.status === 404
      ) {
        await this.cacheNotFoundError(
          args.cacheDir,
          error,
          args.notFoundExpireTimeSeconds,
        );
      }
      throw error;
    }
  }

  /**
   * Gets the data from the contents stored in the cache.
   */
  private _getFromCachedData<T>(
    { key, field }: CacheDir,
    cached: string,
  ): Promise<T> {
    this.loggingService.debug({ type: 'cache_hit', key, field });
    const cachedData = JSON.parse(cached);
    if (cachedData?.response?.status === 404) {
      // TODO: create a CachedData type with guard to avoid these type assertions.
      const url: URL = cachedData.url;
      const response: Response = cachedData.response;
      throw new NetworkResponseError(url, response, cachedData?.data);
    }
    return cachedData;
  }

  /**
   * Gets the data from the network and caches the result.
   */
  private async _getFromNetworkAndWriteCache<T>(args: {
    cacheDir: CacheDir;
    url: string;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const { key, field } = args.cacheDir;
    this.loggingService.debug({ type: 'cache_miss', key, field });
    const startTimeMs = Date.now();
    const { data } = await this.networkService.get<T>({
      url: args.url,
      networkRequest: args.networkRequest,
    });

    const shouldBeCached = await this._shouldBeCached(key, startTimeMs);
    if (shouldBeCached) {
      await this.cacheService.hSet(
        args.cacheDir,
        JSON.stringify(data),
        args.expireTimeSeconds,
      );

      // TODO: transient logging for debugging
      if (
        this.areDebugLogsEnabled &&
        (args.url.includes('all-transactions') ||
          args.url.includes('multisig-transactions'))
      ) {
        this.logTransactionsCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as Page<Transaction>,
        );
      }

      if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {
        this.logSafeMetadataCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as Safe,
        );
      }

      if (
        this.areConfigHooksDebugLogsEnabled &&
        args.cacheDir.key.includes('chain')
      ) {
        this.logChainUpdateCacheWrite(startTimeMs, args.cacheDir, data);
      }
    }
    return data;
  }

  /**
   * Validates that the request is more recent than the last invalidation recorded for the item,
   * preventing a race condition where outdated data is stored due to the request being initiated
   * before the source communicated a change (via webhook or by other means).
   *
   * Returns true if (any of the following):
   * 1. An invalidationTimeMs entry for the key received as param is *not* found in the cache.
   * 2. An entry *is* found and contains an integer that is less than the received startTimeMs param.
   *
   * @param key key part of the {@link CacheDir} holding the requested item
   * @param startTimeMs Unix epoch timestamp in ms when the request was initiated
   * @returns true if any of the above conditions is met
   */
  private async _shouldBeCached(
    key: string,
    startTimeMs: number,
  ): Promise<boolean> {
    const invalidationTimeMsStr = await this.cacheService.hGet(
      new CacheDir(`invalidationTimeMs:${key}`, ''),
    );

    if (!invalidationTimeMsStr) return true;

    const invalidationTimeMs = Number(invalidationTimeMsStr);
    return (
      Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs
    );
  }

  /**
   * Caches a not found error.
   * @param cacheDir - {@link CacheDir} where the error should be placed
   */
  private async cacheNotFoundError(
    cacheDir: CacheDir,
    error: NetworkResponseError,
    notFoundExpireTimeSeconds?: number,
  ): Promise<void> {
    return this.cacheService.hSet(
      cacheDir,
      JSON.stringify({
        data: error.data,
        response: { status: error.response.status },
        url: error.url,
      }),
      notFoundExpireTimeSeconds,
    );
  }

  /**
   * Logs the type and the hash of the transactions present in the data parameter.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logTransactionsCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: Page<Transaction>,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      txHashes:
        isArray(data?.results) && // no validation executed yet at this point
        data.results.map((transaction) => {
          if (isMultisigTransaction(transaction)) {
            return {
              txType: 'multisig',
              safeTxHash: transaction.safeTxHash,
              confirmations: transaction.confirmations,
              confirmationRequired: transaction.confirmationsRequired,
            };
          } else if (isEthereumTransaction(transaction)) {
            return {
              txType: 'ethereum',
              txHash: transaction.txHash,
            };
          } else if (isModuleTransaction(transaction)) {
            return {
              txType: 'module',
              transactionHash: transaction.transactionHash,
            };
          } else if (isCreationTransaction(transaction)) {
            return {
              txType: 'creation',
              transactionHash: transaction.transactionHash,
            };
          }
        }),
    });
  }

  /**
   * Logs the Safe metadata retrieved.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logSafeMetadataCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    safe: Safe,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      safe,
    });
  }

  /**
   * Logs the chain/chains retrieved.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logChainUpdateCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: unknown,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      data,
    });
  }
}

1	import { Inject, Injectable } from '@nestjs/common';	16✔
2	import {	16✔
3	CacheService,
4	ICacheService,
5	} from '@/datasources/cache/cache.service.interface';
6	import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';	16✔
7	import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';	16✔
8	import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
9	import {	16✔
10	INetworkService,
11	NetworkService,
12	} from '@/datasources/network/network.service.interface';
13	import { ILoggingService, LoggingService } from '@/logging/logging.interface';	16✔
14	import { Page } from '@/domain/entities/page.entity';
15	import {	16✔
16	isMultisigTransaction,
17	isEthereumTransaction,
18	isModuleTransaction,
19	isCreationTransaction,
20	Transaction,
21	} from '@/domain/safe/entities/transaction.entity';
22	import { IConfigurationService } from '@/config/configuration.service.interface';	16✔
23	import { isArray } from 'lodash';	16✔
24	import { Safe } from '@/domain/safe/entities/safe.entity';
25
26	/**
27	* A data source which tries to retrieve values from cache using
28	* {@link CacheService} and fallbacks to {@link NetworkService}
29	* if the cache entry expired or is not present.
30	*
31	* This is the recommended data source that should be used when
32	* a feature requires both networking and caching the respective
33	* responses.
34	*/
35	@Injectable()
36	export class CacheFirstDataSource {	16✔
37	private readonly areDebugLogsEnabled: boolean;
38	private readonly areConfigHooksDebugLogsEnabled: boolean;
39
40	constructor(
41	@Inject(CacheService) private readonly cacheService: ICacheService,	16✔
42	@Inject(NetworkService) private readonly networkService: INetworkService,	16✔
43	@Inject(LoggingService) private readonly loggingService: ILoggingService,	16✔
44	@Inject(IConfigurationService)
45	private readonly configurationService: IConfigurationService,	16✔
46	) {
47	this.areDebugLogsEnabled =	16✔
48	this.configurationService.getOrThrow<boolean>('features.debugLogs');
49	this.areConfigHooksDebugLogsEnabled =	16✔
50	this.configurationService.getOrThrow<boolean>(
51	'features.configHooksDebugLogs',
52	);
53	}
54
55	/**
56	* Gets the cached value behind {@link CacheDir}.
57	* If the value is not present, it tries to get the respective JSON
58	* payload from {@link url}.
59	* 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
60	*
61	* @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
62	* @param args.url - the HTTP endpoint to retrieve the JSON payload
63	* @param args.networkRequest - the HTTP request to be used if there is a cache miss
64	* @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
65	* @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
66	*/
67	async get<T>(args: {
68	cacheDir: CacheDir;
69	url: string;
70	notFoundExpireTimeSeconds: number;
71	networkRequest?: NetworkRequest;
72	expireTimeSeconds?: number;
73	}): Promise<T> {
74	const cached = await this.cacheService.hGet(args.cacheDir);	52✔
75	if (cached != null) return this._getFromCachedData(args.cacheDir, cached);	52✔
76
77	try {	18✔
78	return await this._getFromNetworkAndWriteCache(args);	18✔
79	} catch (error) {
80	if (	×
81	error instanceof NetworkResponseError &&	×
82	error.response.status === 404
83	) {
84	await this.cacheNotFoundError(	×
85	args.cacheDir,
86	error,
87	args.notFoundExpireTimeSeconds,
88	);
89	}
90	throw error;	×
91	}
92	}
93
94	/**
95	* Gets the data from the contents stored in the cache.
96	*/
97	private _getFromCachedData<T>(
98	{ key, field }: CacheDir,
99	cached: string,
100	): Promise<T> {
101	this.loggingService.debug({ type: 'cache_hit', key, field });	34✔
102	const cachedData = JSON.parse(cached);	34✔
103	if (cachedData?.response?.status === 404) {	34!
104	// TODO: create a CachedData type with guard to avoid these type assertions.
105	const url: URL = cachedData.url;	×
106	const response: Response = cachedData.response;	×
107	throw new NetworkResponseError(url, response, cachedData?.data);	×
108	}
109	return cachedData;	34✔
110	}
111
112	/**
113	* Gets the data from the network and caches the result.
114	*/
115	private async _getFromNetworkAndWriteCache<T>(args: {
116	cacheDir: CacheDir;
117	url: string;
118	networkRequest?: NetworkRequest;
119	expireTimeSeconds?: number;
120	}): Promise<T> {
121	const { key, field } = args.cacheDir;	18✔
122	this.loggingService.debug({ type: 'cache_miss', key, field });	18✔
123	const startTimeMs = Date.now();	18✔
124	const { data } = await this.networkService.get<T>({	18✔
125	url: args.url,
126	networkRequest: args.networkRequest,
127	});
128
129	const shouldBeCached = await this._shouldBeCached(key, startTimeMs);	18✔
130	if (shouldBeCached) {	18✔
131	await this.cacheService.hSet(	18✔
132	args.cacheDir,
133	JSON.stringify(data),
134	args.expireTimeSeconds,
135	);
136
137	// TODO: transient logging for debugging
138	if (	18!
139	this.areDebugLogsEnabled &&	9!
140	(args.url.includes('all-transactions') \|\|
141	args.url.includes('multisig-transactions'))
142	) {
143	this.logTransactionsCacheWrite(	×
144	startTimeMs,
145	args.cacheDir,
146	data as Page<Transaction>,
147	);
148	}
149
150	if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {	18!
151	this.logSafeMetadataCacheWrite(	×
152	startTimeMs,
153	args.cacheDir,
154	data as Safe,
155	);
156	}
157
158	if (	18!
159	this.areConfigHooksDebugLogsEnabled &&	9!
160	args.cacheDir.key.includes('chain')
161	) {
162	this.logChainUpdateCacheWrite(startTimeMs, args.cacheDir, data);	×
163	}
164	}
165	return data;	18✔
166	}
167
168	/**
169	* Validates that the request is more recent than the last invalidation recorded for the item,
170	* preventing a race condition where outdated data is stored due to the request being initiated
171	* before the source communicated a change (via webhook or by other means).
172	*
173	* Returns true if (any of the following):
174	* 1. An invalidationTimeMs entry for the key received as param is not found in the cache.
175	* 2. An entry is found and contains an integer that is less than the received startTimeMs param.
176	*
177	* @param key key part of the {@link CacheDir} holding the requested item
178	* @param startTimeMs Unix epoch timestamp in ms when the request was initiated
179	* @returns true if any of the above conditions is met
180	*/
181	private async _shouldBeCached(
182	key: string,
183	startTimeMs: number,
184	): Promise<boolean> {
185	const invalidationTimeMsStr = await this.cacheService.hGet(	18✔
186	new CacheDir(`invalidationTimeMs:${key}`, ''),
187	);
188
189	if (!invalidationTimeMsStr) return true;	18✔
190
191	const invalidationTimeMs = Number(invalidationTimeMsStr);	2✔
192	return (	2✔
193	Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs	2✔
194	);
195	}
196
197	/**
198	* Caches a not found error.
199	* @param cacheDir - {@link CacheDir} where the error should be placed
200	*/
201	private async cacheNotFoundError(
202	cacheDir: CacheDir,
203	error: NetworkResponseError,
204	notFoundExpireTimeSeconds?: number,
205	): Promise<void> {
206	return this.cacheService.hSet(	×
207	cacheDir,
208	JSON.stringify({
209	data: error.data,
210	response: { status: error.response.status },
211	url: error.url,
212	}),
213	notFoundExpireTimeSeconds,
214	);
215	}
216
217	/**
218	* Logs the type and the hash of the transactions present in the data parameter.
219	* NOTE: this is a debugging-only function.
220	* TODO: remove this function after debugging.
221	*/
222	private logTransactionsCacheWrite(
223	requestStartTime: number,
224	cacheDir: CacheDir,
225	data: Page<Transaction>,
226	): void {
227	this.loggingService.info({	×
228	type: 'cache_write',
229	cacheKey: cacheDir.key,
230	cacheField: cacheDir.field,
231	cacheWriteTime: new Date(),
232	requestStartTime: new Date(requestStartTime),
233	txHashes:
234	isArray(data?.results) && // no validation executed yet at this point	×
235	data.results.map((transaction) => {
236	if (isMultisigTransaction(transaction)) {	×
237	return {	×
238	txType: 'multisig',
239	safeTxHash: transaction.safeTxHash,
240	confirmations: transaction.confirmations,
241	confirmationRequired: transaction.confirmationsRequired,
242	};
243	} else if (isEthereumTransaction(transaction)) {	×
244	return {	×
245	txType: 'ethereum',
246	txHash: transaction.txHash,
247	};
248	} else if (isModuleTransaction(transaction)) {	×
249	return {	×
250	txType: 'module',
251	transactionHash: transaction.transactionHash,
252	};
253	} else if (isCreationTransaction(transaction)) {	×
254	return {	×
255	txType: 'creation',
256	transactionHash: transaction.transactionHash,
257	};
258	}
259	}),
260	});
261	}
262
263	/**
264	* Logs the Safe metadata retrieved.
265	* NOTE: this is a debugging-only function.
266	* TODO: remove this function after debugging.
267	*/
268	private logSafeMetadataCacheWrite(
269	requestStartTime: number,
270	cacheDir: CacheDir,
271	safe: Safe,
272	): void {
273	this.loggingService.info({	×
274	type: 'cache_write',
275	cacheKey: cacheDir.key,
276	cacheField: cacheDir.field,
277	cacheWriteTime: new Date(),
278	requestStartTime: new Date(requestStartTime),
279	safe,
280	});
281	}
282
283	/**
284	* Logs the chain/chains retrieved.
285	* NOTE: this is a debugging-only function.
286	* TODO: remove this function after debugging.
287	*/
288	private logChainUpdateCacheWrite(
289	requestStartTime: number,
290	cacheDir: CacheDir,
291	data: unknown,
292	): void {
293	this.loggingService.info({	×
294	type: 'cache_write',
295	cacheKey: cacheDir.key,
296	cacheField: cacheDir.field,
297	cacheWriteTime: new Date(),
298	requestStartTime: new Date(requestStartTime),
299	data,
300	});
301	}
302	}

safe-global / safe-client-gateway / 11233174198

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous