10477225092

Committed 20 Aug 2024 06:33PM UTC coverage: 15.394% (+0.1%) from 15.248%

Build # 10477225092

Build Type

Pull #7499

github

Committed by

web-flow

Commit Message

Merge 988b70c61 into a7b7646a0

Pull Request Pull Request #7499: Discourse import policy

Run Details

3541 of 21612 branches covered (16.38%)

Branch coverage included in aggregate %.

42 of 385 new or added lines in 20 files covered. (10.91%)

169 existing lines in 11 files now uncovered.

4643 of 31553 relevant lines covered (14.71%)

5.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1.43

/libs/sitemaps/src/createDatabasePaginator.ts

import {
  config,
  models,
  ThreadInstance,
  UserAttributes,
} from '@hicommonwealth/model';
import { getThreadUrl } from '@hicommonwealth/shared';
import { Op } from 'sequelize';

export interface Link {
  readonly id: number;
  readonly url: string;
  readonly updated_at: string;
  readonly priority?: number;
}

/**
 * A single page which holds a batch of links that can be converted to a sitemap.
 */
export interface Page {
  readonly links: ReadonlyArray<Link>;
}

/**
 * Interface representing a Paginator.
 *
 * A Paginator is used to navigate through a collection of pages.  This way
 * we can avoid fetching all records into memory.
 */
export interface Paginator {
  readonly hasNext: () => Promise<boolean>;
  readonly next: () => Promise<Page>;
}

const NULL_PAGINATOR: Paginator = {
  hasNext: async () => Promise.resolve(false),
  next: async () => Promise.resolve(null!),
};

interface TableAdapter {
  /**
   * Covert the object to a record or undefined if we can't use it with the sitemaps.
   */
  toRecord: (obj: object) => Link | undefined;
  executeQuery: (ptr: number, limit: number) => Promise<ReadonlyArray<object>>;
}

export function createDatabasePaginatorDefault(limit: number = 50000) {
  const threads = createDatabasePaginatorWithAdapter(
    createThreadsTableAdapter(),
    limit,
  );

  const profiles =
    config.SITEMAP.PROFILE_PRIORITY !== -1
      ? createDatabasePaginatorWithAdapter(createProfilesTableAdapter(), limit)
      : NULL_PAGINATOR;

  return { threads, profiles };
}

function createThreadsTableAdapter(): TableAdapter {
  type ThreadInstancePartial = Pick<
    ThreadInstance,
    'id' | 'updated_at' | 'title' | 'community_id'
  >;

  function toRecord(obj: object): Link | undefined {
    const thread = obj as ThreadInstancePartial;

    if (!thread.updated_at || !thread.id) {
      return undefined;
    }

    const url = getThreadUrl({
      chain: thread.community_id,
      id: thread.id,
      title: thread.title,
    });
    return {
      id: thread.id,
      url,
      updated_at: thread.updated_at.toISOString(),
      priority: config.SITEMAP.THREAD_PRIORITY,
    };
  }

  async function executeQuery(
    ptr: number,
    limit: number,
  ): Promise<ReadonlyArray<object>> {
    return await models.Thread.findAll({
      attributes: ['id', 'updated_at', 'title', 'community_id'],
      where: {
        id: { [Op.gt]: ptr },
      },
      order: [['id', 'ASC']],
      limit,
    });
  }
  return { toRecord, executeQuery };
}

function createProfilesTableAdapter(): TableAdapter {
  type UserInstancePartial = Required<Pick<UserAttributes, 'id'>> &
    Pick<UserAttributes, 'updated_at'>;

  function toRecord(obj: object) {
    const user = obj as UserInstancePartial;

    if (!user.updated_at) {
      return undefined;
    }

    const url = `https://commonwealth.im/profile/id/${user.id}`;
    return {
      id: user.id,
      url,
      updated_at: user.updated_at.toISOString(),
      priority: config.SITEMAP.PROFILE_PRIORITY,
    };
  }

  async function executeQuery(
    ptr: number,
    limit: number,
  ): Promise<ReadonlyArray<object>> {
    return await models.User.findAll({
      attributes: ['id', 'updated_at'],
      where: {
        id: { [Op.gt]: ptr },
      },
      order: [['id', 'ASC']],
      limit,
    });
  }
  return { toRecord, executeQuery };
}

export function createDatabasePaginatorWithAdapter(
  adapter: TableAdapter,
  limit: number,
): Paginator {
  // the page we're on...
  let idx = 0;

  let ptr = -1;

  let records: ReadonlyArray<Link> = [];

  // eslint-disable-next-line @typescript-eslint/require-await
  async function hasNext() {
    return idx === 0 || records.length !== 0;
  }

  async function next(): Promise<Page> {
    ++idx;

    const raw = await adapter.executeQuery(ptr, limit);
    records = raw
      .map(adapter.toRecord)
      .filter((current) => !!current)
      .map((current) => current!);

    if (records.length > 0) {
      ptr = records[records.length - 1].id;
    }

    return {
      links: records,
    };
  }

  return { hasNext, next };
}

export function createDatabasePaginatorMock(
  nrRecords: number,
  pageSize: number,
) {
  const threads = createDatabasePaginatorMockForTable(
    'threads',
    nrRecords,
    pageSize,
  );
  const profiles = createDatabasePaginatorMockForTable(
    'profiles',
    nrRecords,
    pageSize,
  );

  return { threads, profiles };
}

export function createDatabasePaginatorMockForTable(
  table: string,
  nrRecords: number,
  pageSize: number,
): Paginator {
  let pageIdx = 0;
  const maxPages = Math.floor(nrRecords / pageSize);

  let idx = 0;

  // eslint-disable-next-line @typescript-eslint/require-await
  async function hasNext() {
    return pageIdx <= maxPages;
  }

  // eslint-disable-next-line @typescript-eslint/require-await
  async function next(): Promise<Page> {
    ++pageIdx;
    const links = [
      {
        id: 0,
        url: `http://www.example.com/${table}/` + idx++,
        updated_at: new Date().toISOString(),
      },
    ];

    return { links };
  }

  return { hasNext, next };
}

1	import {
2	config,
3	models,
4	ThreadInstance,
5	UserAttributes,
6	} from '@hicommonwealth/model';
7	import { getThreadUrl } from '@hicommonwealth/shared';
8	import { Op } from 'sequelize';
9
10	export interface Link {
11	readonly id: number;
12	readonly url: string;
13	readonly updated_at: string;
14	readonly priority?: number;
15	}
16
17	/**
18	* A single page which holds a batch of links that can be converted to a sitemap.
19	*/
20	export interface Page {
21	readonly links: ReadonlyArray<Link>;
22	}
23
24	/**
25	* Interface representing a Paginator.
26	*
27	* A Paginator is used to navigate through a collection of pages. This way
28	* we can avoid fetching all records into memory.
29	*/
30	export interface Paginator {
31	readonly hasNext: () => Promise<boolean>;
32	readonly next: () => Promise<Page>;
33	}
34
35	const NULL_PAGINATOR: Paginator = {	1✔
UNCOV 36	hasNext: async () => Promise.resolve(false),	×
UNCOV 37	next: async () => Promise.resolve(null!),	×
38	};
39
40	interface TableAdapter {
41	/**
42	* Covert the object to a record or undefined if we can't use it with the sitemaps.
43	*/
44	toRecord: (obj: object) => Link \| undefined;
45	executeQuery: (ptr: number, limit: number) => Promise<ReadonlyArray<object>>;
46	}
47
48	export function createDatabasePaginatorDefault(limit: number = 50000) {	×
UNCOV 49	const threads = createDatabasePaginatorWithAdapter(	×
50	createThreadsTableAdapter(),
51	limit,
52	);
53
54	const profiles =
UNCOV 55	config.SITEMAP.PROFILE_PRIORITY !== -1	×
56	? createDatabasePaginatorWithAdapter(createProfilesTableAdapter(), limit)
57	: NULL_PAGINATOR;
58
59	return { threads, profiles };	×
60	}
61
62	function createThreadsTableAdapter(): TableAdapter {
63	type ThreadInstancePartial = Pick<
64	ThreadInstance,
65	'id' \| 'updated_at' \| 'title' \| 'community_id'
66	>;
67
68	function toRecord(obj: object): Link \| undefined {
UNCOV 69	const thread = obj as ThreadInstancePartial;	×
70
UNCOV 71	if (!thread.updated_at \|\| !thread.id) {	×
UNCOV 72	return undefined;	×
73	}
74
UNCOV 75	const url = getThreadUrl({	×
76	chain: thread.community_id,
77	id: thread.id,
78	title: thread.title,
79	});
UNCOV 80	return {	×
81	id: thread.id,
82	url,
83	updated_at: thread.updated_at.toISOString(),
84	priority: config.SITEMAP.THREAD_PRIORITY,
85	};
86	}
87
88	async function executeQuery(
89	ptr: number,
90	limit: number,
91	): Promise<ReadonlyArray<object>> {
UNCOV 92	return await models.Thread.findAll({	×
93	attributes: ['id', 'updated_at', 'title', 'community_id'],
94	where: {
95	id: { [Op.gt]: ptr },
96	},
97	order: [['id', 'ASC']],
98	limit,
99	});
100	}
UNCOV 101	return { toRecord, executeQuery };	×
102	}
103
104	function createProfilesTableAdapter(): TableAdapter {
105	type UserInstancePartial = Required<Pick<UserAttributes, 'id'>> &
106	Pick<UserAttributes, 'updated_at'>;
107
108	function toRecord(obj: object) {
UNCOV 109	const user = obj as UserInstancePartial;	×
110
UNCOV 111	if (!user.updated_at) {	×
UNCOV 112	return undefined;	×
113	}
114
UNCOV 115	const url = `https://commonwealth.im/profile/id/${user.id}`;	×
UNCOV 116	return {	×
117	id: user.id,
118	url,
119	updated_at: user.updated_at.toISOString(),
120	priority: config.SITEMAP.PROFILE_PRIORITY,
121	};
122	}
123
124	async function executeQuery(
125	ptr: number,
126	limit: number,
127	): Promise<ReadonlyArray<object>> {
UNCOV 128	return await models.User.findAll({	×
129	attributes: ['id', 'updated_at'],
130	where: {
131	id: { [Op.gt]: ptr },
132	},
133	order: [['id', 'ASC']],
134	limit,
135	});
136	}
UNCOV 137	return { toRecord, executeQuery };	×
138	}
139
140	export function createDatabasePaginatorWithAdapter(
141	adapter: TableAdapter,
142	limit: number,
143	): Paginator {
144	// the page we're on...
145	let idx = 0;	×
146
UNCOV 147	let ptr = -1;	×
148
149	let records: ReadonlyArray<Link> = [];	×
150
151	// eslint-disable-next-line @typescript-eslint/require-await
152	async function hasNext() {
UNCOV 153	return idx === 0 \|\| records.length !== 0;	×
154	}
155
156	async function next(): Promise<Page> {
UNCOV 157	++idx;	×
158
UNCOV 159	const raw = await adapter.executeQuery(ptr, limit);	×
160	records = raw	×
161	.map(adapter.toRecord)
UNCOV 162	.filter((current) => !!current)	×
UNCOV 163	.map((current) => current!);	×
164
UNCOV 165	if (records.length > 0) {	×
UNCOV 166	ptr = records[records.length - 1].id;	×
167	}
168
UNCOV 169	return {	×
170	links: records,
171	};
172	}
173
UNCOV 174	return { hasNext, next };	×
175	}
176
177	export function createDatabasePaginatorMock(
178	nrRecords: number,
179	pageSize: number,
180	) {
UNCOV 181	const threads = createDatabasePaginatorMockForTable(	×
182	'threads',
183	nrRecords,
184	pageSize,
185	);
186	const profiles = createDatabasePaginatorMockForTable(	×
187	'profiles',
188	nrRecords,
189	pageSize,
190	);
191
UNCOV 192	return { threads, profiles };	×
193	}
194
195	export function createDatabasePaginatorMockForTable(
196	table: string,
197	nrRecords: number,
198	pageSize: number,
199	): Paginator {
UNCOV 200	let pageIdx = 0;	×
UNCOV 201	const maxPages = Math.floor(nrRecords / pageSize);	×
202
UNCOV 203	let idx = 0;	×
204
205	// eslint-disable-next-line @typescript-eslint/require-await
206	async function hasNext() {
207	return pageIdx <= maxPages;	×
208	}
209
210	// eslint-disable-next-line @typescript-eslint/require-await
211	async function next(): Promise<Page> {
UNCOV 212	++pageIdx;	×
UNCOV 213	const links = [	×
214	{
215	id: 0,
216	url: `http://www.example.com/${table}/` + idx++,
217	updated_at: new Date().toISOString(),
218	},
219	];
220
UNCOV 221	return { links };	×
222	}
223
UNCOV 224	return { hasNext, next };	×
225	}

hicommonwealth / commonwealth / 10477225092

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous