10114076917

Committed 26 Jul 2024 03:40PM UTC coverage: 48.454% (-0.003%) from 48.457%

Build # 10114076917

Build Type

Pull #1788

github

Committed by

hectorgomezv

Commit Message

Add Postgres docs reference

Pull Request Pull Request #1788: Add DB_MAX_SAFE_INTEGER constant

Run Details

437 of 2633 branches covered (16.6%)

Branch coverage included in aggregate %.

4326 of 7197 relevant lines covered (60.11%)

13.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.22

/src/datasources/cache/cache.first.data.source.ts

import { Inject, Injectable } from '@nestjs/common';
import {
  CacheService,
  ICacheService,
} from '@/datasources/cache/cache.service.interface';
import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';
import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
import {
  INetworkService,
  NetworkService,
} from '@/datasources/network/network.service.interface';
import { ILoggingService, LoggingService } from '@/logging/logging.interface';
import { Page } from '@/domain/entities/page.entity';
import {
  isMultisigTransaction,
  isEthereumTransaction,
  isModuleTransaction,
  isCreationTransaction,
  Transaction,
} from '@/domain/safe/entities/transaction.entity';
import { IConfigurationService } from '@/config/configuration.service.interface';
import { isArray } from 'lodash';
import { Safe } from '@/domain/safe/entities/safe.entity';

/**
 * A data source which tries to retrieve values from cache using
 * {@link CacheService} and fallbacks to {@link NetworkService}
 * if the cache entry expired or is not present.
 *
 * This is the recommended data source that should be used when
 * a feature requires both networking and caching the respective
 * responses.
 */
@Injectable()
export class CacheFirstDataSource {
  private readonly areDebugLogsEnabled: boolean;

  constructor(
    @Inject(CacheService) private readonly cacheService: ICacheService,
    @Inject(NetworkService) private readonly networkService: INetworkService,
    @Inject(LoggingService) private readonly loggingService: ILoggingService,
    @Inject(IConfigurationService)
    private readonly configurationService: IConfigurationService,
  ) {
    this.areDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>('features.debugLogs');
  }

  /**
   * Gets the cached value behind {@link CacheDir}.
   * If the value is not present, it tries to get the respective JSON
   * payload from {@link url}.
   * 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
   *
   * @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
   * @param args.url - the HTTP endpoint to retrieve the JSON payload
   * @param args.networkRequest - the HTTP request to be used if there is a cache miss
   * @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
   * @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
   */
  async get<T>(args: {
    cacheDir: CacheDir;
    url: string;
    notFoundExpireTimeSeconds: number;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const cached = await this.cacheService.get(args.cacheDir);
    if (cached != null) return this._getFromCachedData(args.cacheDir, cached);

    try {
      return await this._getFromNetworkAndWriteCache(args);
    } catch (error) {
      if (
        error instanceof NetworkResponseError &&
        error.response.status === 404
      ) {
        await this.cacheNotFoundError(
          args.cacheDir,
          error,
          args.notFoundExpireTimeSeconds,
        );
      }
      throw error;
    }
  }

  /**
   * Gets the data from the contents stored in the cache.
   */
  private _getFromCachedData<T>(
    { key, field }: CacheDir,
    cached: string,
  ): Promise<T> {
    this.loggingService.debug({ type: 'cache_hit', key, field });
    const cachedData = JSON.parse(cached);
    if (cachedData?.response?.status === 404) {
      // TODO: create a CachedData type with guard to avoid these type assertions.
      const url: URL = cachedData.url;
      const response: Response = cachedData.response;
      throw new NetworkResponseError(url, response, cachedData?.data);
    }
    return cachedData;
  }

  /**
   * Gets the data from the network and caches the result.
   */
  private async _getFromNetworkAndWriteCache<T>(args: {
    cacheDir: CacheDir;
    url: string;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const { key, field } = args.cacheDir;
    this.loggingService.debug({ type: 'cache_miss', key, field });
    const startTimeMs = Date.now();
    const { data } = await this.networkService.get<T>({
      url: args.url,
      networkRequest: args.networkRequest,
    });

    const shouldBeCached = await this._shouldBeCached(key, startTimeMs);
    if (shouldBeCached) {
      await this.cacheService.set(
        args.cacheDir,
        JSON.stringify(data),
        args.expireTimeSeconds,
      );

      // TODO: transient logging for debugging
      if (
        this.areDebugLogsEnabled &&
        (args.url.includes('all-transactions') ||
          args.url.includes('multisig-transactions'))
      ) {
        this.logTransactionsCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as Page<Transaction>,
        );
      }

      if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {
        this.logSafeMetadataCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as Safe,
        );
      }
    }
    return data;
  }

  /**
   * Validates that the request is more recent than the last invalidation recorded for the item,
   * preventing a race condition where outdated data is stored due to the request being initiated
   * before the source communicated a change (via webhook or by other means).
   *
   * Returns true if (any of the following):
   * 1. An invalidationTimeMs entry for the key received as param is *not* found in the cache.
   * 2. An entry *is* found and contains an integer that is less than the received startTimeMs param.
   *
   * @param key key part of the {@link CacheDir} holding the requested item
   * @param startTimeMs Unix epoch timestamp in ms when the request was initiated
   * @returns true if any of the above conditions is met
   */
  private async _shouldBeCached(
    key: string,
    startTimeMs: number,
  ): Promise<boolean> {
    const invalidationTimeMsStr = await this.cacheService.get(
      new CacheDir(`invalidationTimeMs:${key}`, ''),
    );

    if (!invalidationTimeMsStr) return true;

    const invalidationTimeMs = Number(invalidationTimeMsStr);
    return (
      Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs
    );
  }

  /**
   * Caches a not found error.
   * @param cacheDir - {@link CacheDir} where the error should be placed
   */
  private async cacheNotFoundError(
    cacheDir: CacheDir,
    error: NetworkResponseError,
    notFoundExpireTimeSeconds?: number,
  ): Promise<void> {
    return this.cacheService.set(
      cacheDir,
      JSON.stringify({
        data: error.data,
        response: { status: error.response.status },
        url: error.url,
      }),
      notFoundExpireTimeSeconds,
    );
  }

  /**
   * Logs the type and the hash of the transactions present in the data parameter.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logTransactionsCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: Page<Transaction>,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      txHashes:
        isArray(data?.results) && // no validation executed yet at this point
        data.results.map((transaction) => {
          if (isMultisigTransaction(transaction)) {
            return {
              txType: 'multisig',
              safeTxHash: transaction.safeTxHash,
              confirmations: transaction.confirmations,
              confirmationRequired: transaction.confirmationsRequired,
            };
          } else if (isEthereumTransaction(transaction)) {
            return {
              txType: 'ethereum',
              txHash: transaction.txHash,
            };
          } else if (isModuleTransaction(transaction)) {
            return {
              txType: 'module',
              transactionHash: transaction.transactionHash,
            };
          } else if (isCreationTransaction(transaction)) {
            return {
              txType: 'creation',
              transactionHash: transaction.transactionHash,
            };
          }
        }),
    });
  }

  /**
   * Logs the Safe metadata retrieved.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logSafeMetadataCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    safe: Safe,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      safe,
    });
  }
}

1	import { Inject, Injectable } from '@nestjs/common';	16✔
2	import {	16✔
3	CacheService,
4	ICacheService,
5	} from '@/datasources/cache/cache.service.interface';
6	import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';	16✔
7	import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';	16✔
8	import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
9	import {	16✔
10	INetworkService,
11	NetworkService,
12	} from '@/datasources/network/network.service.interface';
13	import { ILoggingService, LoggingService } from '@/logging/logging.interface';	16✔
14	import { Page } from '@/domain/entities/page.entity';
15	import {	16✔
16	isMultisigTransaction,
17	isEthereumTransaction,
18	isModuleTransaction,
19	isCreationTransaction,
20	Transaction,
21	} from '@/domain/safe/entities/transaction.entity';
22	import { IConfigurationService } from '@/config/configuration.service.interface';	16✔
23	import { isArray } from 'lodash';	16✔
24	import { Safe } from '@/domain/safe/entities/safe.entity';
25
26	/**
27	* A data source which tries to retrieve values from cache using
28	* {@link CacheService} and fallbacks to {@link NetworkService}
29	* if the cache entry expired or is not present.
30	*
31	* This is the recommended data source that should be used when
32	* a feature requires both networking and caching the respective
33	* responses.
34	*/
35	@Injectable()
36	export class CacheFirstDataSource {	16✔
37	private readonly areDebugLogsEnabled: boolean;
38
39	constructor(
40	@Inject(CacheService) private readonly cacheService: ICacheService,	16✔
41	@Inject(NetworkService) private readonly networkService: INetworkService,	16✔
42	@Inject(LoggingService) private readonly loggingService: ILoggingService,	16✔
43	@Inject(IConfigurationService)
44	private readonly configurationService: IConfigurationService,	16✔
45	) {
46	this.areDebugLogsEnabled =	16✔
47	this.configurationService.getOrThrow<boolean>('features.debugLogs');
48	}
49
50	/**
51	* Gets the cached value behind {@link CacheDir}.
52	* If the value is not present, it tries to get the respective JSON
53	* payload from {@link url}.
54	* 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
55	*
56	* @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
57	* @param args.url - the HTTP endpoint to retrieve the JSON payload
58	* @param args.networkRequest - the HTTP request to be used if there is a cache miss
59	* @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
60	* @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
61	*/
62	async get<T>(args: {
63	cacheDir: CacheDir;
64	url: string;
65	notFoundExpireTimeSeconds: number;
66	networkRequest?: NetworkRequest;
67	expireTimeSeconds?: number;
68	}): Promise<T> {
69	const cached = await this.cacheService.get(args.cacheDir);	28✔
70	if (cached != null) return this._getFromCachedData(args.cacheDir, cached);	28✔
71
72	try {	26✔
73	return await this._getFromNetworkAndWriteCache(args);	26✔
74	} catch (error) {
75	if (	×
76	error instanceof NetworkResponseError &&	×
77	error.response.status === 404
78	) {
79	await this.cacheNotFoundError(	×
80	args.cacheDir,
81	error,
82	args.notFoundExpireTimeSeconds,
83	);
84	}
85	throw error;	×
86	}
87	}
88
89	/**
90	* Gets the data from the contents stored in the cache.
91	*/
92	private _getFromCachedData<T>(
93	{ key, field }: CacheDir,
94	cached: string,
95	): Promise<T> {
96	this.loggingService.debug({ type: 'cache_hit', key, field });	2✔
97	const cachedData = JSON.parse(cached);	2✔
98	if (cachedData?.response?.status === 404) {	2!
99	// TODO: create a CachedData type with guard to avoid these type assertions.
100	const url: URL = cachedData.url;	×
101	const response: Response = cachedData.response;	×
102	throw new NetworkResponseError(url, response, cachedData?.data);	×
103	}
104	return cachedData;	2✔
105	}
106
107	/**
108	* Gets the data from the network and caches the result.
109	*/
110	private async _getFromNetworkAndWriteCache<T>(args: {
111	cacheDir: CacheDir;
112	url: string;
113	networkRequest?: NetworkRequest;
114	expireTimeSeconds?: number;
115	}): Promise<T> {
116	const { key, field } = args.cacheDir;	26✔
117	this.loggingService.debug({ type: 'cache_miss', key, field });	26✔
118	const startTimeMs = Date.now();	26✔
119	const { data } = await this.networkService.get<T>({	26✔
120	url: args.url,
121	networkRequest: args.networkRequest,
122	});
123
124	const shouldBeCached = await this._shouldBeCached(key, startTimeMs);	26✔
125	if (shouldBeCached) {	26✔
126	await this.cacheService.set(	26✔
127	args.cacheDir,
128	JSON.stringify(data),
129	args.expireTimeSeconds,
130	);
131
132	// TODO: transient logging for debugging
133	if (	26!
134	this.areDebugLogsEnabled &&	13!
135	(args.url.includes('all-transactions') \|\|
136	args.url.includes('multisig-transactions'))
137	) {
138	this.logTransactionsCacheWrite(	×
139	startTimeMs,
140	args.cacheDir,
141	data as Page<Transaction>,
142	);
143	}
144
145	if (this.areDebugLogsEnabled && args.cacheDir.key.includes('_safe_')) {	26!
146	this.logSafeMetadataCacheWrite(	×
147	startTimeMs,
148	args.cacheDir,
149	data as Safe,
150	);
151	}
152	}
153	return data;	26✔
154	}
155
156	/**
157	* Validates that the request is more recent than the last invalidation recorded for the item,
158	* preventing a race condition where outdated data is stored due to the request being initiated
159	* before the source communicated a change (via webhook or by other means).
160	*
161	* Returns true if (any of the following):
162	* 1. An invalidationTimeMs entry for the key received as param is not found in the cache.
163	* 2. An entry is found and contains an integer that is less than the received startTimeMs param.
164	*
165	* @param key key part of the {@link CacheDir} holding the requested item
166	* @param startTimeMs Unix epoch timestamp in ms when the request was initiated
167	* @returns true if any of the above conditions is met
168	*/
169	private async _shouldBeCached(
170	key: string,
171	startTimeMs: number,
172	): Promise<boolean> {
173	const invalidationTimeMsStr = await this.cacheService.get(	26✔
174	new CacheDir(`invalidationTimeMs:${key}`, ''),
175	);
176
177	if (!invalidationTimeMsStr) return true;	26✔
178
179	const invalidationTimeMs = Number(invalidationTimeMsStr);	×
180	return (	×
181	Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs	×
182	);
183	}
184
185	/**
186	* Caches a not found error.
187	* @param cacheDir - {@link CacheDir} where the error should be placed
188	*/
189	private async cacheNotFoundError(
190	cacheDir: CacheDir,
191	error: NetworkResponseError,
192	notFoundExpireTimeSeconds?: number,
193	): Promise<void> {
194	return this.cacheService.set(	×
195	cacheDir,
196	JSON.stringify({
197	data: error.data,
198	response: { status: error.response.status },
199	url: error.url,
200	}),
201	notFoundExpireTimeSeconds,
202	);
203	}
204
205	/**
206	* Logs the type and the hash of the transactions present in the data parameter.
207	* NOTE: this is a debugging-only function.
208	* TODO: remove this function after debugging.
209	*/
210	private logTransactionsCacheWrite(
211	requestStartTime: number,
212	cacheDir: CacheDir,
213	data: Page<Transaction>,
214	): void {
215	this.loggingService.info({	×
216	type: 'cache_write',
217	cacheKey: cacheDir.key,
218	cacheField: cacheDir.field,
219	cacheWriteTime: new Date(),
220	requestStartTime: new Date(requestStartTime),
221	txHashes:
222	isArray(data?.results) && // no validation executed yet at this point	×
223	data.results.map((transaction) => {
224	if (isMultisigTransaction(transaction)) {	×
225	return {	×
226	txType: 'multisig',
227	safeTxHash: transaction.safeTxHash,
228	confirmations: transaction.confirmations,
229	confirmationRequired: transaction.confirmationsRequired,
230	};
231	} else if (isEthereumTransaction(transaction)) {	×
232	return {	×
233	txType: 'ethereum',
234	txHash: transaction.txHash,
235	};
236	} else if (isModuleTransaction(transaction)) {	×
237	return {	×
238	txType: 'module',
239	transactionHash: transaction.transactionHash,
240	};
241	} else if (isCreationTransaction(transaction)) {	×
242	return {	×
243	txType: 'creation',
244	transactionHash: transaction.transactionHash,
245	};
246	}
247	}),
248	});
249	}
250
251	/**
252	* Logs the Safe metadata retrieved.
253	* NOTE: this is a debugging-only function.
254	* TODO: remove this function after debugging.
255	*/
256	private logSafeMetadataCacheWrite(
257	requestStartTime: number,
258	cacheDir: CacheDir,
259	safe: Safe,
260	): void {
261	this.loggingService.info({	×
262	type: 'cache_write',
263	cacheKey: cacheDir.key,
264	cacheField: cacheDir.field,
265	cacheWriteTime: new Date(),
266	requestStartTime: new Date(requestStartTime),
267	safe,
268	});
269	}
270	}

safe-global / safe-client-gateway / 10114076917

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous