8347022186

Committed 19 Mar 2024 04:53PM UTC coverage: 93.282% (-0.4%) from 93.679%

Build # 8347022186

Build Type

push

github

Committed by

web-flow

Commit Message

Add optional debugging logs when retrieving transactions history (#1330)

- Adds `FF_HISTORY_DEBUG_LOGS` to (de)activate the logging.
- Implements a private function (`logTransactionsCacheWrite`) to log transactions following the above format.

Run Details

1827 of 2206 branches covered (82.82%)

Branch coverage included in aggregate %.

6 of 16 new or added lines in 1 file covered. (37.5%)

4 existing lines in 2 files now uncovered.

6476 of 6695 relevant lines covered (96.73%)

360.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.04

/src/datasources/cache/cache.first.data.source.ts

import { Inject, Injectable } from '@nestjs/common';
import {
  CacheService,
  ICacheService,
} from '@/datasources/cache/cache.service.interface';
import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';
import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
import {
  INetworkService,
  NetworkService,
} from '@/datasources/network/network.service.interface';
import { ILoggingService, LoggingService } from '@/logging/logging.interface';
import { Page } from '@/domain/entities/page.entity';
import {
  isMultisigTransaction,
  isEthereumTransaction,
  isModuleTransaction,
  isCreationTransaction,
  Transaction,
} from '@/domain/safe/entities/transaction.entity';
import { IConfigurationService } from '@/config/configuration.service.interface';
import { isArray } from 'lodash';

/**
 * A data source which tries to retrieve values from cache using
 * {@link CacheService} and fallbacks to {@link NetworkService}
 * if the cache entry expired or is not present.
 *
 * This is the recommended data source that should be used when
 * a feature requires both networking and caching the respective
 * responses.
 */
@Injectable()
export class CacheFirstDataSource {
  private readonly isHistoryDebugLogsEnabled: boolean;

  constructor(
    @Inject(CacheService) private readonly cacheService: ICacheService,
    @Inject(NetworkService) private readonly networkService: INetworkService,
    @Inject(LoggingService) private readonly loggingService: ILoggingService,
    @Inject(IConfigurationService)
    private readonly configurationService: IConfigurationService,
  ) {
    this.isHistoryDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>(
        'features.historyDebugLogs',
      );
  }

  /**
   * Gets the cached value behind {@link CacheDir}.
   * If the value is not present, it tries to get the respective JSON
   * payload from {@link url}.
   * 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
   *
   * @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
   * @param args.url - the HTTP endpoint to retrieve the JSON payload
   * @param args.networkRequest - the HTTP request to be used if there is a cache miss
   * @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
   * @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
   */
  async get<T>(args: {
    cacheDir: CacheDir;
    url: string;
    notFoundExpireTimeSeconds: number;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const cached = await this.cacheService.get(args.cacheDir);
    if (cached != null) return this._getFromCachedData(args.cacheDir, cached);

    try {
      return await this._getFromNetworkAndWriteCache(args);
    } catch (error) {
      if (
        error instanceof NetworkResponseError &&
        error.response.status === 404
      ) {
        await this.cacheNotFoundError(
          args.cacheDir,
          error,
          args.notFoundExpireTimeSeconds,
        );
      }
      throw error;
    }
  }

  /**
   * Gets the data from the contents stored in the cache.
   */
  private _getFromCachedData<T>(
    { key, field }: CacheDir,
    cached: string,
  ): Promise<T> {
    this.loggingService.debug({ type: 'cache_hit', key, field });
    const cachedData = JSON.parse(cached);
    if (cachedData?.response?.status === 404) {
      throw new NetworkResponseError(
        cachedData.url,
        cachedData.response,
        cachedData?.data,
      );
    }
    return cachedData;
  }

  /**
   * Gets the data from the network and caches the result.
   */
  private async _getFromNetworkAndWriteCache<T>(args: {
    cacheDir: CacheDir;
    url: string;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const { key, field } = args.cacheDir;
    this.loggingService.debug({ type: 'cache_miss', key, field });
    const startTimeMs = Date.now();
    const { data } = await this.networkService.get<T>({
      url: args.url,
      networkRequest: args.networkRequest,
    });

    const shouldBeCached = await this._shouldBeCached(key, startTimeMs);
    if (shouldBeCached) {
      await this.cacheService.set(
        args.cacheDir,
        JSON.stringify(data),
        args.expireTimeSeconds,
      );

      // TODO: transient logging for debugging
      if (
        this.isHistoryDebugLogsEnabled &&
        args.url.includes('all-transactions')
      ) {
        this.logTransactionsCacheWrite(
          args.cacheDir,
          data as Page<Transaction>,
        );
      }
    }
    return data;
  }

  /**
   * Validates that the request is more recent than the last invalidation recorded for the item,
   * preventing a race condition where outdated data is stored due to the request being initiated
   * before the source communicated a change (via webhook or by other means).
   *
   * Returns true if (any of the following):
   * 1. An invalidationTimeMs entry for the key received as param is *not* found in the cache.
   * 2. An entry *is* found and contains an integer that is less than the received startTimeMs param.
   *
   * @param key key part of the {@link CacheDir} holding the requested item
   * @param startTimeMs Unix epoch timestamp in ms when the request was initiated
   * @returns true if any of the above conditions is met
   */
  private async _shouldBeCached(
    key: string,
    startTimeMs: number,
  ): Promise<boolean> {
    const invalidationTimeMsStr = await this.cacheService.get(
      new CacheDir(`invalidationTimeMs:${key}`, ''),
    );

    if (!invalidationTimeMsStr) return true;

    const invalidationTimeMs = Number(invalidationTimeMsStr);
    return (
      Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs
    );
  }

  /**
   * Caches a not found error.
   * @param cacheDir - {@link CacheDir} where the error should be placed
   */
  private async cacheNotFoundError(
    cacheDir: CacheDir,
    error: NetworkResponseError,
    notFoundExpireTimeSeconds?: number,
  ): Promise<void> {
    return this.cacheService.set(
      cacheDir,
      JSON.stringify({
        data: error.data,
        response: { status: error.response.status },
        url: error.url,
      }),
      notFoundExpireTimeSeconds,
    );
  }

  /**
   * Logs the type and the hash of the transactions present in the data parameter.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logTransactionsCacheWrite(
    cacheDir: CacheDir,
    data: Page<Transaction>,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      timestamp: Date.now(),
      txHashes:
        isArray(data?.results) && // no validation executed yet at this point
        data.results.map((transaction) => {
          if (isMultisigTransaction(transaction)) {
            return {
              txType: 'multisig',
              safeTxHash: transaction.safeTxHash,
            };
          } else if (isEthereumTransaction(transaction)) {
            return {
              txType: 'ethereum',
              txHash: transaction.txHash,
            };
          } else if (isModuleTransaction(transaction)) {
            return {
              txType: 'module',
              transactionHash: transaction.transactionHash,
            };
          } else if (isCreationTransaction(transaction)) {
            return {
              txType: 'creation',
              transactionHash: transaction.transactionHash,
            };
          }
        }),
    });
  }
}

1	import { Inject, Injectable } from '@nestjs/common';	102✔
2	import {	102✔
3	CacheService,
4	ICacheService,
5	} from '@/datasources/cache/cache.service.interface';
6	import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';	102✔
7	import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';	102✔
8	import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
9	import {	102✔
10	INetworkService,
11	NetworkService,
12	} from '@/datasources/network/network.service.interface';
13	import { ILoggingService, LoggingService } from '@/logging/logging.interface';	102✔
14	import { Page } from '@/domain/entities/page.entity';
15	import {	102✔
16	isMultisigTransaction,
17	isEthereumTransaction,
18	isModuleTransaction,
19	isCreationTransaction,
20	Transaction,
21	} from '@/domain/safe/entities/transaction.entity';
22	import { IConfigurationService } from '@/config/configuration.service.interface';	102✔
23	import { isArray } from 'lodash';	102✔
24
25	/**
26	* A data source which tries to retrieve values from cache using
27	* {@link CacheService} and fallbacks to {@link NetworkService}
28	* if the cache entry expired or is not present.
29	*
30	* This is the recommended data source that should be used when
31	* a feature requires both networking and caching the respective
32	* responses.
33	*/
34	@Injectable()
35	export class CacheFirstDataSource {	102✔
36	private readonly isHistoryDebugLogsEnabled: boolean;
37
38	constructor(
39	@Inject(CacheService) private readonly cacheService: ICacheService,	1,208✔
40	@Inject(NetworkService) private readonly networkService: INetworkService,	1,208✔
41	@Inject(LoggingService) private readonly loggingService: ILoggingService,	1,208✔
42	@Inject(IConfigurationService)
43	private readonly configurationService: IConfigurationService,	1,208✔
44	) {
45	this.isHistoryDebugLogsEnabled =	1,208✔
46	this.configurationService.getOrThrow<boolean>(
47	'features.historyDebugLogs',
48	);
49	}
50
51	/**
52	* Gets the cached value behind {@link CacheDir}.
53	* If the value is not present, it tries to get the respective JSON
54	* payload from {@link url}.
55	* 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
56	*
57	* @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
58	* @param args.url - the HTTP endpoint to retrieve the JSON payload
59	* @param args.networkRequest - the HTTP request to be used if there is a cache miss
60	* @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
61	* @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
62	*/
63	async get<T>(args: {
64	cacheDir: CacheDir;
65	url: string;
66	notFoundExpireTimeSeconds: number;
67	networkRequest?: NetworkRequest;
68	expireTimeSeconds?: number;
69	}): Promise<T> {
70	const cached = await this.cacheService.get(args.cacheDir);	3,396✔
71	if (cached != null) return this._getFromCachedData(args.cacheDir, cached);	3,396✔
72
73	try {	3,286✔
74	return await this._getFromNetworkAndWriteCache(args);	3,286✔
75	} catch (error) {
76	if (	700✔
77	error instanceof NetworkResponseError &&	390✔
78	error.response.status === 404
79	) {
80	await this.cacheNotFoundError(	30✔
81	args.cacheDir,
82	error,
83	args.notFoundExpireTimeSeconds,
84	);
85	}
86	throw error;	700✔
87	}
88	}
89
90	/**
91	* Gets the data from the contents stored in the cache.
92	*/
93	private _getFromCachedData<T>(
94	{ key, field }: CacheDir,
95	cached: string,
96	): Promise<T> {
97	this.loggingService.debug({ type: 'cache_hit', key, field });	110✔
98	const cachedData = JSON.parse(cached);	110✔
99	if (cachedData?.response?.status === 404) {	110!
100	throw new NetworkResponseError(	2✔
101	cachedData.url,
102	cachedData.response,
103	cachedData?.data,	3!
104	);
105	}
106	return cachedData;	108✔
107	}
108
109	/**
110	* Gets the data from the network and caches the result.
111	*/
112	private async _getFromNetworkAndWriteCache<T>(args: {
113	cacheDir: CacheDir;
114	url: string;
115	networkRequest?: NetworkRequest;
116	expireTimeSeconds?: number;
117	}): Promise<T> {
118	const { key, field } = args.cacheDir;	3,286✔
119	this.loggingService.debug({ type: 'cache_miss', key, field });	3,286✔
120	const startTimeMs = Date.now();	3,286✔
121	const { data } = await this.networkService.get<T>({	3,286✔
122	url: args.url,
123	networkRequest: args.networkRequest,
124	});
125
126	const shouldBeCached = await this._shouldBeCached(key, startTimeMs);	2,586✔
127	if (shouldBeCached) {	2,586✔
128	await this.cacheService.set(	2,584✔
129	args.cacheDir,
130	JSON.stringify(data),
131	args.expireTimeSeconds,
132	);
133
134	// TODO: transient logging for debugging
135	if (	2,584!
136	this.isHistoryDebugLogsEnabled &&	1,294✔
137	args.url.includes('all-transactions')
138	) {
NEW 139	this.logTransactionsCacheWrite(	×
140	args.cacheDir,
141	data as Page<Transaction>,
142	);
143	}
144	}
145	return data;	2,586✔
146	}
147
148	/**
149	* Validates that the request is more recent than the last invalidation recorded for the item,
150	* preventing a race condition where outdated data is stored due to the request being initiated
151	* before the source communicated a change (via webhook or by other means).
152	*
153	* Returns true if (any of the following):
154	* 1. An invalidationTimeMs entry for the key received as param is not found in the cache.
155	* 2. An entry is found and contains an integer that is less than the received startTimeMs param.
156	*
157	* @param key key part of the {@link CacheDir} holding the requested item
158	* @param startTimeMs Unix epoch timestamp in ms when the request was initiated
159	* @returns true if any of the above conditions is met
160	*/
161	private async _shouldBeCached(
162	key: string,
163	startTimeMs: number,
164	): Promise<boolean> {
165	const invalidationTimeMsStr = await this.cacheService.get(	2,586✔
166	new CacheDir(`invalidationTimeMs:${key}`, ''),
167	);
168
169	if (!invalidationTimeMsStr) return true;	2,586✔
170
171	const invalidationTimeMs = Number(invalidationTimeMsStr);	4✔
172	return (	4✔
173	Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs	4✔
174	);
175	}
176
177	/**
178	* Caches a not found error.
179	* @param cacheDir - {@link CacheDir} where the error should be placed
180	*/
181	private async cacheNotFoundError(
182	cacheDir: CacheDir,
183	error: NetworkResponseError,
184	notFoundExpireTimeSeconds?: number,
185	): Promise<void> {
186	return this.cacheService.set(	30✔
187	cacheDir,
188	JSON.stringify({
189	data: error.data,
190	response: { status: error.response.status },
191	url: error.url,
192	}),
193	notFoundExpireTimeSeconds,
194	);
195	}
196
197	/**
198	* Logs the type and the hash of the transactions present in the data parameter.
199	* NOTE: this is a debugging-only function.
200	* TODO: remove this function after debugging.
201	*/
202	private logTransactionsCacheWrite(
203	cacheDir: CacheDir,
204	data: Page<Transaction>,
205	): void {
NEW 206	this.loggingService.info({	×
207	type: 'cache_write',
208	cacheKey: cacheDir.key,
209	cacheField: cacheDir.field,
210	timestamp: Date.now(),
211	txHashes:
212	isArray(data?.results) && // no validation executed yet at this point	×
213	data.results.map((transaction) => {
NEW 214	if (isMultisigTransaction(transaction)) {	×
NEW 215	return {	×
216	txType: 'multisig',
217	safeTxHash: transaction.safeTxHash,
218	};
NEW 219	} else if (isEthereumTransaction(transaction)) {	×
NEW 220	return {	×
221	txType: 'ethereum',
222	txHash: transaction.txHash,
223	};
NEW 224	} else if (isModuleTransaction(transaction)) {	×
NEW 225	return {	×
226	txType: 'module',
227	transactionHash: transaction.transactionHash,
228	};
NEW 229	} else if (isCreationTransaction(transaction)) {	×
NEW 230	return {	×
231	txType: 'creation',
232	transactionHash: transaction.transactionHash,
233	};
234	}
235	}),
236	});
237	}
238	}

safe-global / safe-client-gateway / 8347022186

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous