10584662094

Committed 27 Aug 2024 07:33PM UTC coverage: 32.011% (-0.1%) from 32.134%

Build # 10584662094

Build Type

Pull #9638

github

Committed by

web-flow

Commit Message

Merge d74c3ae3b into b0c53c180

Pull Request Pull Request #9638: Merge master to lw-deploy 2024 08 27

Run Details

5020 of 40800 branches covered (12.3%)

Branch coverage included in aggregate %.

202 of 690 new or added lines in 49 files covered. (29.28%)

5 existing lines in 4 files now uncovered.

29220 of 66164 relevant lines covered (44.16%)

189.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.13

/packages/lesswrong/server/repos/PostEmbeddingsRepo.ts

import AbstractRepo from "./AbstractRepo";
import PostEmbeddings from "../../lib/collections/postEmbeddings/collection";
import { randomId } from "../../lib/random";
import { recordPerfMetrics } from "./perfMetricWrapper";
import { getViewablePostsSelector } from "./helpers";

interface PostEmbeddingDistanceInfo {
  _id: string,
  title: string,
  raw_distance: unknown,
  quality_adjusted_score: unknown
}

class PostEmbeddingsRepo extends AbstractRepo<"PostEmbeddings"> {
  constructor() {
    super(PostEmbeddings);
  }

  setPostEmbeddings(
    postId: string,
    postHash: string,
    model: string,
    embeddings: number[],
  ): Promise<null> {
    if (!Array.isArray(embeddings) || embeddings.length < 1) {
      throw new Error("Cannot create post embeddings with empty array");
    }
    const now = new Date();
    return this.none(`
      -- PostEmbeddingsRepo.setPostEmbeddings
      INSERT INTO "PostEmbeddings" (
        "_id",
        "postId",
        "postHash",
        "lastGeneratedAt",
        "embeddings",
        "model",
        "createdAt"
      ) VALUES (
        $1, $2, $3, $4, $5, $6, $4
      ) ON CONFLICT ("postId", "model") DO UPDATE SET
        "postHash" = $3,
        "lastGeneratedAt" = $4,
        "embeddings" = $5
    `, [randomId(), postId, postHash, now, JSON.stringify(embeddings), model]);
  }

  private postIdsByEmbeddingDistanceSelector = `
    SELECT
      p._id,
      p.title,
      ed.distance AS raw_distance,
      (0.5 * (1 / (distance + 0.1)) + 0.5 * log(p."baseScore")) AS quality_adjusted_score
    FROM embedding_distances ed
    LEFT JOIN "Posts" p ON p._id = ed."postId"
    WHERE ${getViewablePostsSelector('p')}
    AND p."baseScore" > 0
    ORDER BY (0.5 * (1 / (distance + 0.1)) + 0.5 * log(p."baseScore")) DESC
    LIMIT $(limit)
  `;

  async getNearestPostIdsWeightedByQuality(
    inputEmbedding: number[],
    limit = 5,
  ): Promise<string[]> {
    const results = await this.getRawDb().any<PostEmbeddingDistanceInfo>(`
      -- PostEmbeddingsRepo.getNearestPostsWeightedByQuality
      WITH embedding_distances AS (
        SELECT
          pe."postId", 
          pe.embeddings <#> $(inputEmbedding)::VECTOR(1536) AS distance
        FROM public."PostEmbeddings" pe
        ORDER BY distance
        LIMIT 200 
      )
      ${this.postIdsByEmbeddingDistanceSelector}
    `, { inputEmbedding, limit });

    return results.map(({ _id }) => _id);
  }

  async getNearestPostIdsWeightedByQualityByPostId(
    postId: string,
    limit = 5
  ): Promise<string[]> {
    const results = await this.getRawDb().any<PostEmbeddingDistanceInfo>(`
      -- PostEmbeddingsRepo.getNearestPostsWeightedByQualityByPostId
      WITH source_embedding AS (
        SELECT embeddings
        FROM public."PostEmbeddings"
        WHERE "postId" = $(postId)
      ),
      embedding_distances AS (
        SELECT
          pe."postId", 
          pe.embeddings <#> (SELECT embeddings FROM source_embedding) AS distance
        FROM public."PostEmbeddings" pe
        WHERE pe."postId" != $(postId)
        ORDER BY distance
        LIMIT 200 
      )
      ${this.postIdsByEmbeddingDistanceSelector}
    `, { postId, limit });

    return results.map(({ _id }) => _id);
  }
}

recordPerfMetrics(PostEmbeddingsRepo);

export default PostEmbeddingsRepo;

1	import AbstractRepo from "./AbstractRepo";	1,917✔
2	import PostEmbeddings from "../../lib/collections/postEmbeddings/collection";	20✔
3	import { randomId } from "../../lib/random";	20✔
4	import { recordPerfMetrics } from "./perfMetricWrapper";	20✔
5	import { getViewablePostsSelector } from "./helpers";	20✔
6
7	interface PostEmbeddingDistanceInfo {
8	_id: string,
9	title: string,
10	raw_distance: unknown,
11	quality_adjusted_score: unknown
12	}
13
14	class PostEmbeddingsRepo extends AbstractRepo<"PostEmbeddings"> {
15	constructor() {
16	super(PostEmbeddings);	1,857✔
17	}
18
19	setPostEmbeddings(
20	postId: string,
21	postHash: string,
22	model: string,
23	embeddings: number[],
24	): Promise<null> {
25	if (!Array.isArray(embeddings) \|\| embeddings.length < 1) {	×
26	throw new Error("Cannot create post embeddings with empty array");	×
27	}
28	const now = new Date();	×
29	return this.none(`	×
30	-- PostEmbeddingsRepo.setPostEmbeddings
31	INSERT INTO "PostEmbeddings" (
32	"_id",
33	"postId",
34	"postHash",
35	"lastGeneratedAt",
36	"embeddings",
37	"model",
38	"createdAt"
39	) VALUES (
40	$1, $2, $3, $4, $5, $6, $4
41	) ON CONFLICT ("postId", "model") DO UPDATE SET
42	"postHash" = $3,
43	"lastGeneratedAt" = $4,
44	"embeddings" = $5
45	`, [randomId(), postId, postHash, now, JSON.stringify(embeddings), model]);
46	}
47
48	private postIdsByEmbeddingDistanceSelector = `	1,857✔
49	SELECT
50	p._id,
51	p.title,
52	ed.distance AS raw_distance,
53	(0.5 * (1 / (distance + 0.1)) + 0.5 * log(p."baseScore")) AS quality_adjusted_score
54	FROM embedding_distances ed
55	LEFT JOIN "Posts" p ON p._id = ed."postId"
56	WHERE ${getViewablePostsSelector('p')}
57	AND p."baseScore" > 0
58	ORDER BY (0.5 * (1 / (distance + 0.1)) + 0.5 * log(p."baseScore")) DESC
59	LIMIT $(limit)
60	`;
61
62	async getNearestPostIdsWeightedByQuality(
63	inputEmbedding: number[],
64	limit = 5,	×
65	): Promise<string[]> {
NEW 66	const results = await this.getRawDb().any<PostEmbeddingDistanceInfo>(`	×
67	-- PostEmbeddingsRepo.getNearestPostsWeightedByQuality
68	WITH embedding_distances AS (
69	SELECT
70	pe."postId",
71	pe.embeddings <#> $(inputEmbedding)::VECTOR(1536) AS distance
72	FROM public."PostEmbeddings" pe
73	ORDER BY distance
74	LIMIT 200
75	)
76	${this.postIdsByEmbeddingDistanceSelector}
77	`, { inputEmbedding, limit });
78
NEW 79	return results.map(({ _id }) => _id);	×
80	}
81
82	async getNearestPostIdsWeightedByQualityByPostId(
83	postId: string,
84	limit = 5	×
85	): Promise<string[]> {
NEW 86	const results = await this.getRawDb().any<PostEmbeddingDistanceInfo>(`	×
87	-- PostEmbeddingsRepo.getNearestPostsWeightedByQualityByPostId
88	WITH source_embedding AS (
89	SELECT embeddings
90	FROM public."PostEmbeddings"
91	WHERE "postId" = $(postId)
92	),
93	embedding_distances AS (
94	SELECT
95	pe."postId",
96	pe.embeddings <#> (SELECT embeddings FROM source_embedding) AS distance
97	FROM public."PostEmbeddings" pe
98	WHERE pe."postId" != $(postId)
99	ORDER BY distance
100	LIMIT 200
101	)
102	${this.postIdsByEmbeddingDistanceSelector}
103	`, { postId, limit });
104
NEW 105	return results.map(({ _id }) => _id);	×
106	}
107	}	20✔
108
109	recordPerfMetrics(PostEmbeddingsRepo);
110
111	export default PostEmbeddingsRepo;	20✔

ForumMagnum / ForumMagnum / 10584662094

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous