10669834679

Committed 02 Sep 2024 03:28PM UTC coverage: 31.812% (-0.05%) from 31.859%

Build # 10669834679

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9668 from ForumMagnum/proxy-repo-instantiation

Proxy the instantiation of repos

Run Details

5027 of 41223 branches covered (12.19%)

Branch coverage included in aggregate %.

6 of 7 new or added lines in 1 file covered. (85.71%)

34 existing lines in 32 files now uncovered.

29329 of 66774 relevant lines covered (43.92%)

176.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

23.81

/packages/lesswrong/server/repos/PostViewsRepo.ts

import { recordPerfMetrics } from "./perfMetricWrapper";
import PostViews from "../../lib/collections/postViews/collection";
import moment from "moment";
import { getAnalyticsConnectionOrThrow } from "../analytics/postgresConnection";
import { randomId } from "../../lib/random";
import chunk from "lodash/chunk";
import IncrementalViewRepo, { IncrementalViewDataType } from "./IncrementalViewRepo";
import { isE2E } from "../../lib/executionEnvironment";

class PostViewsRepo extends IncrementalViewRepo<"PostViews"> {
  constructor() {
    super(PostViews);
  }

  async calculateDataForDateRange({
    startDate,
    endDate,
  }: {
    startDate: Date;
    endDate: Date;
  }): Promise<
    IncrementalViewDataType<"PostViews">[]
  > {
    // Analytics DB is not available in e2e tests
    if (isE2E) {
      return [];
    }

    // Round startDate (endDate) down (up) to the nearest UTC date boundary
    const startDateFormatted = moment.utc(startDate).startOf("day").format("YYYY-MM-DD HH:mm:ss");
    const endDateFormatted = moment.utc(endDate).endOf("day").format("YYYY-MM-DD HH:mm:ss");

    const analyticsDb = getAnalyticsConnectionOrThrow();

    const results = await analyticsDb.any<
      Omit<DbPostViews, "_id" | "schemaVersion" | "createdAt" | "legacyData"> & {
        viewCount: string;
        uniqueViewCount: string;
      }
    >(
      `
      SELECT
        count(*) AS "viewCount",
        count(DISTINCT client_id) AS "uniqueViewCount",
        post_id as "postId",
        (date_trunc('day', timestamp)) AS "windowStart",
        (date_trunc('day', timestamp) + interval '1 day') AS "windowEnd"
      FROM
        page_view
      WHERE
        timestamp >= $1
        AND timestamp <= $2
        AND post_id IS NOT NULL
      GROUP BY
        post_id,
        date_trunc('day', timestamp)
    `,
      [startDateFormatted, endDateFormatted]
    );

    // Manually convert:
    // - The number fields from string to number (as this isn't handled automatically)
    // - The date fields to UTC strings (NOTE: this actually changes the time represented, because in the conversion
    //   from postgres to JS the timezone of the server is applied to the raw date, so 2023-04-01 00:00:00 becomes e.g.
    //   2023-04-01 00:00:00+0100 (BST), which is wrong)
    const typedResults = results.map((views) => ({
      ...views,
      windowStart: moment(views.windowStart)
        .add(moment(views.windowStart).utcOffset(), "minutes")
        .utc()
        .format("YYYY-MM-DD HH:mm:ss"),
      windowEnd: moment(views.windowEnd)
        .add(moment(views.windowEnd).utcOffset(), "minutes")
        .utc()
        .format("YYYY-MM-DD HH:mm:ss"),
      viewCount: parseInt(views.viewCount),
      uniqueViewCount: parseInt(views.uniqueViewCount),
    }));

    return typedResults;
  }

  /**
   * Delete rows where there is any overlap with the given (startDate, endDate) range
   */
  async deleteRange({
    startDate,
    endDate,
  }: {
    startDate: Date;
    endDate: Date;
  }): Promise<void> {
    // Round startDate (endDate) down (up) to the nearest UTC date boundary
    const startDateFormatted = moment.utc(startDate).startOf('day').format('YYYY-MM-DD HH:mm:ss');
    const endDateFormatted = moment.utc(endDate).endOf('day').format('YYYY-MM-DD HH:mm:ss');

    await this.getRawDb().none(
      `
      DELETE FROM "PostViews"
      WHERE
        ("windowEnd" > $1 AND "windowEnd" < $2)
        OR ("windowStart" > $1 AND "windowStart" < $2)
      `,
      [startDateFormatted, endDateFormatted]
    );
  }

  async upsertData({ data }: { data: IncrementalViewDataType<"PostViews">[] }) {
    const dataWithIds = data.map((item) => ({
      ...item,
      _id: randomId(),
    }));

    // Avoid doing too many at once, expect about 5000 total per day
    const chunks = chunk(dataWithIds, 1000);

    for (const chunk of chunks) {
      const values = chunk
        .map(
          (row) =>
            `('${row._id}', '${row.postId}', '${row.windowStart}', '${row.windowEnd}', ${
              row.viewCount
            }, ${row.uniqueViewCount}, NOW(), NOW())`
        )
        .join(",");

      const onConflictUpdate = `
        "postId" = EXCLUDED."postId",
        "windowStart" = EXCLUDED."windowStart",
        "windowEnd" = EXCLUDED."windowEnd",
        "viewCount" = EXCLUDED."viewCount",
        "uniqueViewCount" = EXCLUDED."uniqueViewCount",
        "updatedAt" = NOW()
      `;

      // If the row with this ("postId", "windowStart", "windowEnd") already exists, just update the numbers
      await this.none(`
        INSERT INTO "PostViews" (
          "_id",
          "postId",
          "windowStart",
          "windowEnd",
          "viewCount",
          "uniqueViewCount",
          "createdAt",
          "updatedAt"
        ) VALUES ${values}
        ON CONFLICT ("postId", "windowStart", "windowEnd")
        DO UPDATE SET ${onConflictUpdate}
      `);
    }
  }

  async getDateBounds(): Promise<{ earliestWindowStart: Date | null; latestWindowEnd: Date | null }> {
    return (
      (await this.getRawDb().oneOrNone<{ earliestWindowStart: Date | null; latestWindowEnd: Date | null }>(`
      SELECT
        min("windowStart") AS "earliestWindowStart",
        max("windowEnd") AS "latestWindowEnd"
      FROM
        "PostViews"
    `)) || {
        earliestWindowStart: null,
        latestWindowEnd: null,
      }
    );
  }

  async viewsByPost({
    postIds,
  }: {
    postIds: string[];
  }): Promise<{ postId: string; totalViews: number; totalUniqueViews: number }[]> {
    const results = await this.getRawDb().any<{ postId: string; totalViews: string; totalUniqueViews: string }>(`
      SELECT
        "postId",
        sum("viewCount") AS "totalViews",
        sum("uniqueViewCount") AS "totalUniqueViews"
      FROM
        "PostViews"
      WHERE
        "postId" IN ($1:csv)
      GROUP BY
        "postId"
      `,
      [postIds]
    )

    // Manually convert the number fields from string to number (as this isn't handled automatically)
    const typedResults = results.map((views) => ({
      ...views,
      totalViews: parseInt(views.totalViews),
      totalUniqueViews: parseInt(views.totalUniqueViews),
    }));

    return typedResults
  }

  async viewsByDate({
    postIds,
    startDate,
    endDate,
  }: {
    postIds: string[];
    startDate?: Date;
    endDate: Date;
  }): Promise<{ date: string; totalViews: number }[]> {
    const results = await this.getRawDb().any<{ date: string; viewCount: string }>(`
      SELECT
        to_char(date_trunc('day', "windowStart"), 'YYYY-MM-DD') AS "date",
        sum("viewCount") AS "viewCount"
      FROM
        "PostViews"
      WHERE
        "postId" IN ($1:csv)
        ${startDate ? `AND "windowStart" >= '${startDate.toISOString()}'` : ""}
        AND "windowEnd" <= '${endDate.toISOString()}'
      GROUP BY
        "date"
      ORDER BY
        "date"
      `,
      [postIds]
    );

    // Manually convert the number fields from string to number (as this isn't handled automatically)
    const typedResults = results.map((views) => ({
      date: views.date,
      totalViews: parseInt(views.viewCount),
    }));

    return typedResults;
  }
}

recordPerfMetrics(PostViewsRepo);

export default PostViewsRepo;

1	import { recordPerfMetrics } from "./perfMetricWrapper";	80✔
2	import PostViews from "../../lib/collections/postViews/collection";	20✔
3	import moment from "moment";	20✔
4	import { getAnalyticsConnectionOrThrow } from "../analytics/postgresConnection";	20✔
5	import { randomId } from "../../lib/random";	20✔
6	import chunk from "lodash/chunk";	20✔
7	import IncrementalViewRepo, { IncrementalViewDataType } from "./IncrementalViewRepo";	20✔
8	import { isE2E } from "../../lib/executionEnvironment";	20✔
9
10	class PostViewsRepo extends IncrementalViewRepo<"PostViews"> {
11	constructor() {
UNCOV 12	super(PostViews);	×
13	}
14
15	async calculateDataForDateRange({
16	startDate,
17	endDate,
18	}: {
19	startDate: Date;
20	endDate: Date;
21	}): Promise<
22	IncrementalViewDataType<"PostViews">[]
23	> {
24	// Analytics DB is not available in e2e tests
25	if (isE2E) {	×
26	return [];	×
27	}
28
29	// Round startDate (endDate) down (up) to the nearest UTC date boundary
30	const startDateFormatted = moment.utc(startDate).startOf("day").format("YYYY-MM-DD HH:mm:ss");	×
31	const endDateFormatted = moment.utc(endDate).endOf("day").format("YYYY-MM-DD HH:mm:ss");	×
32
33	const analyticsDb = getAnalyticsConnectionOrThrow();	×
34
35	const results = await analyticsDb.any<	×
36	Omit<DbPostViews, "_id" \| "schemaVersion" \| "createdAt" \| "legacyData"> & {
37	viewCount: string;
38	uniqueViewCount: string;
39	}
40	>(
41	`
42	SELECT
43	count(*) AS "viewCount",
44	count(DISTINCT client_id) AS "uniqueViewCount",
45	post_id as "postId",
46	(date_trunc('day', timestamp)) AS "windowStart",
47	(date_trunc('day', timestamp) + interval '1 day') AS "windowEnd"
48	FROM
49	page_view
50	WHERE
51	timestamp >= $1
52	AND timestamp <= $2
53	AND post_id IS NOT NULL
54	GROUP BY
55	post_id,
56	date_trunc('day', timestamp)
57	`,
58	[startDateFormatted, endDateFormatted]
59	);
60
61	// Manually convert:
62	// - The number fields from string to number (as this isn't handled automatically)
63	// - The date fields to UTC strings (NOTE: this actually changes the time represented, because in the conversion
64	// from postgres to JS the timezone of the server is applied to the raw date, so 2023-04-01 00:00:00 becomes e.g.
65	// 2023-04-01 00:00:00+0100 (BST), which is wrong)
66	const typedResults = results.map((views) => ({	×
67	...views,
68	windowStart: moment(views.windowStart)
69	.add(moment(views.windowStart).utcOffset(), "minutes")
70	.utc()
71	.format("YYYY-MM-DD HH:mm:ss"),
72	windowEnd: moment(views.windowEnd)
73	.add(moment(views.windowEnd).utcOffset(), "minutes")
74	.utc()
75	.format("YYYY-MM-DD HH:mm:ss"),
76	viewCount: parseInt(views.viewCount),
77	uniqueViewCount: parseInt(views.uniqueViewCount),
78	}));
79
80	return typedResults;	×
81	}
82
83	/**
84	* Delete rows where there is any overlap with the given (startDate, endDate) range
85	*/
86	async deleteRange({
87	startDate,
88	endDate,
89	}: {
90	startDate: Date;
91	endDate: Date;
92	}): Promise<void> {
93	// Round startDate (endDate) down (up) to the nearest UTC date boundary
94	const startDateFormatted = moment.utc(startDate).startOf('day').format('YYYY-MM-DD HH:mm:ss');	×
95	const endDateFormatted = moment.utc(endDate).endOf('day').format('YYYY-MM-DD HH:mm:ss');	×
96
97	await this.getRawDb().none(	×
98	`
99	DELETE FROM "PostViews"
100	WHERE
101	("windowEnd" > $1 AND "windowEnd" < $2)
102	OR ("windowStart" > $1 AND "windowStart" < $2)
103	`,
104	[startDateFormatted, endDateFormatted]
105	);
106	}
107
108	async upsertData({ data }: { data: IncrementalViewDataType<"PostViews">[] }) {
109	const dataWithIds = data.map((item) => ({	×
110	...item,
111	_id: randomId(),
112	}));
113
114	// Avoid doing too many at once, expect about 5000 total per day
115	const chunks = chunk(dataWithIds, 1000);	×
116
117	for (const chunk of chunks) {	×
118	const values = chunk	×
119	.map(
120	(row) =>
121	`('${row._id}', '${row.postId}', '${row.windowStart}', '${row.windowEnd}', ${	×
122	row.viewCount
123	}, ${row.uniqueViewCount}, NOW(), NOW())`
124	)
125	.join(",");
126
127	const onConflictUpdate = `	×
128	"postId" = EXCLUDED."postId",
129	"windowStart" = EXCLUDED."windowStart",
130	"windowEnd" = EXCLUDED."windowEnd",
131	"viewCount" = EXCLUDED."viewCount",
132	"uniqueViewCount" = EXCLUDED."uniqueViewCount",
133	"updatedAt" = NOW()
134	`;
135
136	// If the row with this ("postId", "windowStart", "windowEnd") already exists, just update the numbers
137	await this.none(`	×
138	INSERT INTO "PostViews" (
139	"_id",
140	"postId",
141	"windowStart",
142	"windowEnd",
143	"viewCount",
144	"uniqueViewCount",
145	"createdAt",
146	"updatedAt"
147	) VALUES ${values}
148	ON CONFLICT ("postId", "windowStart", "windowEnd")
149	DO UPDATE SET ${onConflictUpdate}
150	`);
151	}
152	}
153
154	async getDateBounds(): Promise<{ earliestWindowStart: Date \| null; latestWindowEnd: Date \| null }> {
155	return (	×
156	(await this.getRawDb().oneOrNone<{ earliestWindowStart: Date \| null; latestWindowEnd: Date \| null }>(`	×
157	SELECT
158	min("windowStart") AS "earliestWindowStart",
159	max("windowEnd") AS "latestWindowEnd"
160	FROM
161	"PostViews"
162	`)) \|\| {
163	earliestWindowStart: null,
164	latestWindowEnd: null,
165	}
166	);
167	}
168
169	async viewsByPost({
170	postIds,
171	}: {
172	postIds: string[];
173	}): Promise<{ postId: string; totalViews: number; totalUniqueViews: number }[]> {
174	const results = await this.getRawDb().any<{ postId: string; totalViews: string; totalUniqueViews: string }>(`	×
175	SELECT
176	"postId",
177	sum("viewCount") AS "totalViews",
178	sum("uniqueViewCount") AS "totalUniqueViews"
179	FROM
180	"PostViews"
181	WHERE
182	"postId" IN ($1:csv)
183	GROUP BY
184	"postId"
185	`,
186	[postIds]
187	)
188
189	// Manually convert the number fields from string to number (as this isn't handled automatically)
190	const typedResults = results.map((views) => ({	×
191	...views,
192	totalViews: parseInt(views.totalViews),
193	totalUniqueViews: parseInt(views.totalUniqueViews),
194	}));
195
196	return typedResults	×
197	}
198
199	async viewsByDate({
200	postIds,
201	startDate,
202	endDate,
203	}: {
204	postIds: string[];
205	startDate?: Date;
206	endDate: Date;
207	}): Promise<{ date: string; totalViews: number }[]> {
208	const results = await this.getRawDb().any<{ date: string; viewCount: string }>(`	×
209	SELECT
210	to_char(date_trunc('day', "windowStart"), 'YYYY-MM-DD') AS "date",
211	sum("viewCount") AS "viewCount"
212	FROM
213	"PostViews"
214	WHERE
215	"postId" IN ($1:csv)
216	${startDate ? `AND "windowStart" >= '${startDate.toISOString()}'` : ""}	×
217	AND "windowEnd" <= '${endDate.toISOString()}'
218	GROUP BY
219	"date"
220	ORDER BY
221	"date"
222	`,
223	[postIds]
224	);
225
226	// Manually convert the number fields from string to number (as this isn't handled automatically)
227	const typedResults = results.map((views) => ({	×
228	date: views.date,
229	totalViews: parseInt(views.viewCount),
230	}));
231
232	return typedResults;	×
233	}
234	}	20✔
235
236	recordPerfMetrics(PostViewsRepo);
237
238	export default PostViewsRepo;	20✔

ForumMagnum / ForumMagnum / 10669834679

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous