9546514333

Committed 17 Jun 2024 11:01AM UTC coverage: 49.473% (-42.8%) from 92.296%

Build # 9546514333

Build Type

push

github

Committed by

web-flow

Commit Message

Add `PostgresDatabaseMigrator` for (testing) migration (#1655)

Adds a new `PostgresDatabaseMigrator` that has the core logic of `postgres-shift`, as well as a testing method. The `migrate` method mirrors `postgres-shift` and the `test` method reuses part of it, executing each migration in a separate transaction. It allows us to interact with the database _before_ and _after_ a migration has executed, stopping after the desired migration:

- Create `PostgresDatabaseMigrator` and inject it
- Remove `postgres-shift` and associated definition/patch, replacing usage with the above
- Add appropriate test coverage

Run Details

394 of 2367 branches covered (16.65%)

Branch coverage included in aggregate %.

10 of 62 new or added lines in 3 files covered. (16.13%)

2191 existing lines in 221 files now uncovered.

3969 of 6452 relevant lines covered (61.52%)

12.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.0

/src/datasources/cache/cache.first.data.source.ts

import { Inject, Injectable } from '@nestjs/common';
import {
  CacheService,
  ICacheService,
} from '@/datasources/cache/cache.service.interface';
import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';
import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';
import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
import {
  INetworkService,
  NetworkService,
} from '@/datasources/network/network.service.interface';
import { ILoggingService, LoggingService } from '@/logging/logging.interface';
import { Page } from '@/domain/entities/page.entity';
import {
  isMultisigTransaction,
  isEthereumTransaction,
  isModuleTransaction,
  isCreationTransaction,
  Transaction,
} from '@/domain/safe/entities/transaction.entity';
import { IConfigurationService } from '@/config/configuration.service.interface';
import { isArray } from 'lodash';

/**
 * A data source which tries to retrieve values from cache using
 * {@link CacheService} and fallbacks to {@link NetworkService}
 * if the cache entry expired or is not present.
 *
 * This is the recommended data source that should be used when
 * a feature requires both networking and caching the respective
 * responses.
 */
@Injectable()
export class CacheFirstDataSource {
  private readonly isHistoryDebugLogsEnabled: boolean;

  constructor(
    @Inject(CacheService) private readonly cacheService: ICacheService,
    @Inject(NetworkService) private readonly networkService: INetworkService,
    @Inject(LoggingService) private readonly loggingService: ILoggingService,
    @Inject(IConfigurationService)
    private readonly configurationService: IConfigurationService,
  ) {
    this.isHistoryDebugLogsEnabled =
      this.configurationService.getOrThrow<boolean>(
        'features.historyDebugLogs',
      );
  }

  /**
   * Gets the cached value behind {@link CacheDir}.
   * If the value is not present, it tries to get the respective JSON
   * payload from {@link url}.
   * 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
   *
   * @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
   * @param args.url - the HTTP endpoint to retrieve the JSON payload
   * @param args.networkRequest - the HTTP request to be used if there is a cache miss
   * @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
   * @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
   */
  async get<T>(args: {
    cacheDir: CacheDir;
    url: string;
    notFoundExpireTimeSeconds: number;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const cached = await this.cacheService.get(args.cacheDir);
    if (cached != null) return this._getFromCachedData(args.cacheDir, cached);

    try {
      return await this._getFromNetworkAndWriteCache(args);
    } catch (error) {
      if (
        error instanceof NetworkResponseError &&
        error.response.status === 404
      ) {
        await this.cacheNotFoundError(
          args.cacheDir,
          error,
          args.notFoundExpireTimeSeconds,
        );
      }
      throw error;
    }
  }

  /**
   * Gets the data from the contents stored in the cache.
   */
  private _getFromCachedData<T>(
    { key, field }: CacheDir,
    cached: string,
  ): Promise<T> {
    this.loggingService.debug({ type: 'cache_hit', key, field });
    const cachedData = JSON.parse(cached);
    if (cachedData?.response?.status === 404) {
      // TODO: create a CachedData type with guard to avoid these type assertions.
      const url: URL = cachedData.url;
      const response: Response = cachedData.response;
      throw new NetworkResponseError(url, response, cachedData?.data);
    }
    return cachedData;
  }

  /**
   * Gets the data from the network and caches the result.
   */
  private async _getFromNetworkAndWriteCache<T>(args: {
    cacheDir: CacheDir;
    url: string;
    networkRequest?: NetworkRequest;
    expireTimeSeconds?: number;
  }): Promise<T> {
    const { key, field } = args.cacheDir;
    this.loggingService.debug({ type: 'cache_miss', key, field });
    const startTimeMs = Date.now();
    const { data } = await this.networkService.get<T>({
      url: args.url,
      networkRequest: args.networkRequest,
    });

    const shouldBeCached = await this._shouldBeCached(key, startTimeMs);
    if (shouldBeCached) {
      await this.cacheService.set(
        args.cacheDir,
        JSON.stringify(data),
        args.expireTimeSeconds,
      );

      // TODO: transient logging for debugging
      if (
        this.isHistoryDebugLogsEnabled &&
        args.url.includes('all-transactions')
      ) {
        this.logTransactionsCacheWrite(
          startTimeMs,
          args.cacheDir,
          data as Page<Transaction>,
        );
      }
    }
    return data;
  }

  /**
   * Validates that the request is more recent than the last invalidation recorded for the item,
   * preventing a race condition where outdated data is stored due to the request being initiated
   * before the source communicated a change (via webhook or by other means).
   *
   * Returns true if (any of the following):
   * 1. An invalidationTimeMs entry for the key received as param is *not* found in the cache.
   * 2. An entry *is* found and contains an integer that is less than the received startTimeMs param.
   *
   * @param key key part of the {@link CacheDir} holding the requested item
   * @param startTimeMs Unix epoch timestamp in ms when the request was initiated
   * @returns true if any of the above conditions is met
   */
  private async _shouldBeCached(
    key: string,
    startTimeMs: number,
  ): Promise<boolean> {
    const invalidationTimeMsStr = await this.cacheService.get(
      new CacheDir(`invalidationTimeMs:${key}`, ''),
    );

    if (!invalidationTimeMsStr) return true;

    const invalidationTimeMs = Number(invalidationTimeMsStr);
    return (
      Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs
    );
  }

  /**
   * Caches a not found error.
   * @param cacheDir - {@link CacheDir} where the error should be placed
   */
  private async cacheNotFoundError(
    cacheDir: CacheDir,
    error: NetworkResponseError,
    notFoundExpireTimeSeconds?: number,
  ): Promise<void> {
    return this.cacheService.set(
      cacheDir,
      JSON.stringify({
        data: error.data,
        response: { status: error.response.status },
        url: error.url,
      }),
      notFoundExpireTimeSeconds,
    );
  }

  /**
   * Logs the type and the hash of the transactions present in the data parameter.
   * NOTE: this is a debugging-only function.
   * TODO: remove this function after debugging.
   */
  private logTransactionsCacheWrite(
    requestStartTime: number,
    cacheDir: CacheDir,
    data: Page<Transaction>,
  ): void {
    this.loggingService.info({
      type: 'cache_write',
      cacheKey: cacheDir.key,
      cacheField: cacheDir.field,
      cacheWriteTime: new Date(),
      requestStartTime: new Date(requestStartTime),
      txHashes:
        isArray(data?.results) && // no validation executed yet at this point
        data.results.map((transaction) => {
          if (isMultisigTransaction(transaction)) {
            return {
              txType: 'multisig',
              safeTxHash: transaction.safeTxHash,
            };
          } else if (isEthereumTransaction(transaction)) {
            return {
              txType: 'ethereum',
              txHash: transaction.txHash,
            };
          } else if (isModuleTransaction(transaction)) {
            return {
              txType: 'module',
              transactionHash: transaction.transactionHash,
            };
          } else if (isCreationTransaction(transaction)) {
            return {
              txType: 'creation',
              transactionHash: transaction.transactionHash,
            };
          }
        }),
    });
  }
}

1	import { Inject, Injectable } from '@nestjs/common';	16✔
2	import {	16✔
3	CacheService,
4	ICacheService,
5	} from '@/datasources/cache/cache.service.interface';
6	import { CacheDir } from '@/datasources/cache/entities/cache-dir.entity';	16✔
7	import { NetworkResponseError } from '@/datasources/network/entities/network.error.entity';	16✔
8	import { NetworkRequest } from '@/datasources/network/entities/network.request.entity';
9	import {	16✔
10	INetworkService,
11	NetworkService,
12	} from '@/datasources/network/network.service.interface';
13	import { ILoggingService, LoggingService } from '@/logging/logging.interface';	16✔
14	import { Page } from '@/domain/entities/page.entity';
15	import {	16✔
16	isMultisigTransaction,
17	isEthereumTransaction,
18	isModuleTransaction,
19	isCreationTransaction,
20	Transaction,
21	} from '@/domain/safe/entities/transaction.entity';
22	import { IConfigurationService } from '@/config/configuration.service.interface';	16✔
23	import { isArray } from 'lodash';	16✔
24
25	/**
26	* A data source which tries to retrieve values from cache using
27	* {@link CacheService} and fallbacks to {@link NetworkService}
28	* if the cache entry expired or is not present.
29	*
30	* This is the recommended data source that should be used when
31	* a feature requires both networking and caching the respective
32	* responses.
33	*/
34	@Injectable()
35	export class CacheFirstDataSource {	16✔
36	private readonly isHistoryDebugLogsEnabled: boolean;
37
38	constructor(
39	@Inject(CacheService) private readonly cacheService: ICacheService,	16✔
40	@Inject(NetworkService) private readonly networkService: INetworkService,	16✔
41	@Inject(LoggingService) private readonly loggingService: ILoggingService,	16✔
42	@Inject(IConfigurationService)
43	private readonly configurationService: IConfigurationService,	16✔
44	) {
45	this.isHistoryDebugLogsEnabled =	16✔
46	this.configurationService.getOrThrow<boolean>(
47	'features.historyDebugLogs',
48	);
49	}
50
51	/**
52	* Gets the cached value behind {@link CacheDir}.
53	* If the value is not present, it tries to get the respective JSON
54	* payload from {@link url}.
55	* 404 errors are cached with {@link notFoundExpireTimeSeconds} seconds expiration time.
56	*
57	* @param args.cacheDir - {@link CacheDir} containing the key and field to be used to retrieve from cache
58	* @param args.url - the HTTP endpoint to retrieve the JSON payload
59	* @param args.networkRequest - the HTTP request to be used if there is a cache miss
60	* @param args.expireTimeSeconds - the time to live in seconds for the payload behind {@link CacheDir}
61	* @param args.notFoundExpireTimeSeconds - the time to live in seconds for the error when the item is not found
62	*/
63	async get<T>(args: {
64	cacheDir: CacheDir;
65	url: string;
66	notFoundExpireTimeSeconds: number;
67	networkRequest?: NetworkRequest;
68	expireTimeSeconds?: number;
69	}): Promise<T> {
70	const cached = await this.cacheService.get(args.cacheDir);	28✔
71	if (cached != null) return this._getFromCachedData(args.cacheDir, cached);	28✔
72
73	try {	26✔
74	return await this._getFromNetworkAndWriteCache(args);	26✔
75	} catch (error) {
UNCOV 76	if (	×
77	error instanceof NetworkResponseError &&	×
78	error.response.status === 404
79	) {
UNCOV 80	await this.cacheNotFoundError(	×
81	args.cacheDir,
82	error,
83	args.notFoundExpireTimeSeconds,
84	);
85	}
UNCOV 86	throw error;	×
87	}
88	}
89
90	/**
91	* Gets the data from the contents stored in the cache.
92	*/
93	private _getFromCachedData<T>(
94	{ key, field }: CacheDir,
95	cached: string,
96	): Promise<T> {
97	this.loggingService.debug({ type: 'cache_hit', key, field });	2✔
98	const cachedData = JSON.parse(cached);	2✔
99	if (cachedData?.response?.status === 404) {	2!
100	// TODO: create a CachedData type with guard to avoid these type assertions.
UNCOV 101	const url: URL = cachedData.url;	×
UNCOV 102	const response: Response = cachedData.response;	×
UNCOV 103	throw new NetworkResponseError(url, response, cachedData?.data);	×
104	}
105	return cachedData;	2✔
106	}
107
108	/**
109	* Gets the data from the network and caches the result.
110	*/
111	private async _getFromNetworkAndWriteCache<T>(args: {
112	cacheDir: CacheDir;
113	url: string;
114	networkRequest?: NetworkRequest;
115	expireTimeSeconds?: number;
116	}): Promise<T> {
117	const { key, field } = args.cacheDir;	26✔
118	this.loggingService.debug({ type: 'cache_miss', key, field });	26✔
119	const startTimeMs = Date.now();	26✔
120	const { data } = await this.networkService.get<T>({	26✔
121	url: args.url,
122	networkRequest: args.networkRequest,
123	});
124
125	const shouldBeCached = await this._shouldBeCached(key, startTimeMs);	26✔
126	if (shouldBeCached) {	26✔
127	await this.cacheService.set(	26✔
128	args.cacheDir,
129	JSON.stringify(data),
130	args.expireTimeSeconds,
131	);
132
133	// TODO: transient logging for debugging
134	if (	26!
135	this.isHistoryDebugLogsEnabled &&	13!
136	args.url.includes('all-transactions')
137	) {
138	this.logTransactionsCacheWrite(	×
139	startTimeMs,
140	args.cacheDir,
141	data as Page<Transaction>,
142	);
143	}
144	}
145	return data;	26✔
146	}
147
148	/**
149	* Validates that the request is more recent than the last invalidation recorded for the item,
150	* preventing a race condition where outdated data is stored due to the request being initiated
151	* before the source communicated a change (via webhook or by other means).
152	*
153	* Returns true if (any of the following):
154	* 1. An invalidationTimeMs entry for the key received as param is not found in the cache.
155	* 2. An entry is found and contains an integer that is less than the received startTimeMs param.
156	*
157	* @param key key part of the {@link CacheDir} holding the requested item
158	* @param startTimeMs Unix epoch timestamp in ms when the request was initiated
159	* @returns true if any of the above conditions is met
160	*/
161	private async _shouldBeCached(
162	key: string,
163	startTimeMs: number,
164	): Promise<boolean> {
165	const invalidationTimeMsStr = await this.cacheService.get(	26✔
166	new CacheDir(`invalidationTimeMs:${key}`, ''),
167	);
168
169	if (!invalidationTimeMsStr) return true;	26✔
170
UNCOV 171	const invalidationTimeMs = Number(invalidationTimeMsStr);	×
UNCOV 172	return (	×
173	Number.isInteger(invalidationTimeMs) && invalidationTimeMs < startTimeMs	×
174	);
175	}
176
177	/**
178	* Caches a not found error.
179	* @param cacheDir - {@link CacheDir} where the error should be placed
180	*/
181	private async cacheNotFoundError(
182	cacheDir: CacheDir,
183	error: NetworkResponseError,
184	notFoundExpireTimeSeconds?: number,
185	): Promise<void> {
UNCOV 186	return this.cacheService.set(	×
187	cacheDir,
188	JSON.stringify({
189	data: error.data,
190	response: { status: error.response.status },
191	url: error.url,
192	}),
193	notFoundExpireTimeSeconds,
194	);
195	}
196
197	/**
198	* Logs the type and the hash of the transactions present in the data parameter.
199	* NOTE: this is a debugging-only function.
200	* TODO: remove this function after debugging.
201	*/
202	private logTransactionsCacheWrite(
203	requestStartTime: number,
204	cacheDir: CacheDir,
205	data: Page<Transaction>,
206	): void {
207	this.loggingService.info({	×
208	type: 'cache_write',
209	cacheKey: cacheDir.key,
210	cacheField: cacheDir.field,
211	cacheWriteTime: new Date(),
212	requestStartTime: new Date(requestStartTime),
213	txHashes:
214	isArray(data?.results) && // no validation executed yet at this point	×
215	data.results.map((transaction) => {
216	if (isMultisigTransaction(transaction)) {	×
217	return {	×
218	txType: 'multisig',
219	safeTxHash: transaction.safeTxHash,
220	};
221	} else if (isEthereumTransaction(transaction)) {	×
222	return {	×
223	txType: 'ethereum',
224	txHash: transaction.txHash,
225	};
226	} else if (isModuleTransaction(transaction)) {	×
227	return {	×
228	txType: 'module',
229	transactionHash: transaction.transactionHash,
230	};
231	} else if (isCreationTransaction(transaction)) {	×
232	return {	×
233	txType: 'creation',
234	transactionHash: transaction.transactionHash,
235	};
236	}
237	}),
238	});
239	}
240	}

safe-global / safe-client-gateway / 9546514333

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous