8895934749

Committed 30 Apr 2024 01:47PM UTC coverage: 31.95% (+0.01%) from 31.937%

Build # 8895934749

Build Type

Pull #9198

github

Committed by

oetherington

Commit Message

Tidy up elastic callbacks

Pull Request Pull Request #9198: Reindex content in elastic for deleted users

Run Details

4716 of 38553 branches covered (12.23%)

Branch coverage included in aggregate %.

13 of 40 new or added lines in 6 files covered. (32.5%)

445 existing lines in 10 files now uncovered.

27511 of 62315 relevant lines covered (44.15%)

179.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

21.1

/packages/lesswrong/server/google-vertex/client.ts

import { DocumentServiceClient, UserEventServiceClient, v1beta, protos } from "@google-cloud/discoveryengine";
import { onStartup } from "../../lib/executionEnvironment";
import { googleRecommendationsCredsPath } from "../../lib/instanceSettings";
import { filterNonnull } from "../../lib/utils/typeGuardUtils";
import { loadByIds } from "../../lib/loaders";
import { accessFilterMultiple } from "../../lib/utils/schemaUtils";
import { getInstanceSettingsFilePath } from "../commandLine";
import { vertexDocumentServiceParentPathSetting, vertexRecommendationServingConfigPathSetting, vertexUserEventServiceParentPathSetting } from "../databaseSettings";
import { postGetPageUrl } from "../../lib/collections/posts/helpers";
import path from "path";
import chunk from "lodash/chunk";
import type { ClientSettingDependencies, ClientSettings, CreateGoogleMediaDocumentMetadataArgs, GoogleMediaDocumentMetadata, InViewEvent, FrontpageViewEvent } from "./types";

const { RecommendationServiceClient } = v1beta;

onStartup(() => {
  // This is a weird hack which is necessary because this was the least crazy way to authenticate with Google Cloud from the outside.
  // The tl;dr is that Google's client libraries will read process.env.GOOGLE_APPLICATION_CREDENTIALS and expect to find a filepath to the credentials file.
  // The easiest way I thought of to solve the problem of differing relative paths between local env and production
  // was to put the path of the creds file _relative to the instance settings file_ into an instance setting, and then join it with the path to the instance settings file.
  // See https://cloud.google.com/docs/authentication/provide-credentials-adc#wlif-key for details
  const credsPath = googleRecommendationsCredsPath.get();
  const settingsFileName = getInstanceSettingsFilePath();
  if (credsPath && settingsFileName) {
    const settingsFileDirectory = path.dirname(settingsFileName);
    process.env.GOOGLE_APPLICATION_CREDENTIALS = path.join(settingsFileDirectory, credsPath);
  }
});

const getGoogleClientInstanceOrThrow = <T extends new () => InstanceType<T>, Deps extends ClientSettingDependencies>(
  constructor: T,
  /**
   * If the client depends on any settings at runtime, pass them in here to cause the getter to throw if any of the settings are missing.
   * The settings need to be provided via an object with the field names being formatted like `fooSetting` (ending in `Setting`), and returning a `T | null` value for arbitrary `T`.
   */
  runtimeSettingDependencies?: Deps
): () => { client: InstanceType<T>, settingValues: ClientSettings<Deps> } => ((ClientConstructor: T) => {
  let client: InstanceType<T>;
  return () => {
    let settingValues = {} as ClientSettings<Deps>;
    const settingDependencyTuples = Object.entries(runtimeSettingDependencies ?? {});
    if (settingDependencyTuples.length) {
      const settingValueTuples = settingDependencyTuples.map(([settingName, setting]) => [settingName.slice(0, -7), setting.get()]);
      const missingSettingTuples = settingValueTuples.filter(([_, settingValue]) => settingValue === null);
      if (missingSettingTuples.length) {
        const missingSettingNames = missingSettingTuples.map(([settingName]) => settingName).join(', ');
        throw new Error(`Missing setting(s) ${missingSettingNames} for ${constructor.name}!`);
      }

      settingValues = Object.fromEntries(settingValueTuples) as ClientSettings<Deps>;
    }

    if (!client) {
      client = new ClientConstructor();
    }

    return { client, settingValues };
  };
})(constructor);

const clients = {
  documents: getGoogleClientInstanceOrThrow(DocumentServiceClient, { vertexDocumentServiceParentPathSetting }),
  userEvents: getGoogleClientInstanceOrThrow(UserEventServiceClient, { vertexUserEventServiceParentPathSetting }),
  recommendations: getGoogleClientInstanceOrThrow(RecommendationServiceClient, { vertexRecommendationServingConfigPathSetting }),
};

const helpers = {
  createMediaDocument({ post, tags, authorIds }: CreateGoogleMediaDocumentMetadataArgs): protos.google.cloud.discoveryengine.v1.IDocument {
    const tagIds = tags?.map(tag => tag._id) ?? [];
    if (tagIds.length === 0) {
      tagIds.push('none')
    }
    const persons = authorIds?.map(authorId => ({ name: authorId, role: 'author' } as const));
  
    const metadata: GoogleMediaDocumentMetadata = {
      title: post.title,
      uri: `https://www.lesswrong.com${postGetPageUrl(post)}`,
      available_time: post.postedAt.toISOString(),
      categories: tagIds,
      filter_tags: tagIds,
      persons,
      media_type: 'articles',
    };
  
    return {
      id: post._id,
      schemaId: 'default_schema',
      jsonData: JSON.stringify(metadata),
    };
  },

  createViewItemEvent(eventType: 'view-item' | 'media-play', readStatus: DbReadStatus): protos.google.cloud.discoveryengine.v1.IUserEvent {
    const { userId, postId, lastUpdated: timestamp } = readStatus;
  
    return {
      eventType,
      // TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
      userPseudoId: userId, 
      eventTime: {
        seconds: timestamp.getTime() / 1000,
        nanos: 0
      },
      userInfo: { userId },
      documents: [{ id: postId }],
    };
  },

  createMediaCompleteEvent(inViewEvent: InViewEvent): protos.google.cloud.discoveryengine.v1.IUserEvent {
    const { userId, postId, timestamp } = inViewEvent;
  
    return {
      eventType: 'media-complete',
      // TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
      userPseudoId: userId, 
      eventTime: {
        seconds: timestamp.getTime() / 1000,
        nanos: 0
      },
      userInfo: { userId },
      documents: [{ id: postId }],
      mediaInfo: {
        mediaProgressPercentage: 1
      }
    };
  },

  createViewHomePageEvent(frontpageViewEvent: FrontpageViewEvent) {
    const { userId, timestamp } = frontpageViewEvent;

    return {
      eventType: 'view-home-page',
      // TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
      userPseudoId: userId,
      eventTime: {
        seconds: timestamp.getTime() / 1000,
        nanos: 0
      },
      userInfo: { userId }
    };
  }
};

const googleVertexApi = {
  async getRecommendations(limit: number, context: ResolverContext) {
    const { client, settingValues: { vertexRecommendationServingConfigPath } } = clients.recommendations();
    const [recommendationsResponse] = await client.recommend({
      validateOnly: false,
      servingConfig: vertexRecommendationServingConfigPath,
      pageSize: limit,
      userEvent: { eventType: 'view-home-page', userPseudoId: context.currentUser?._id },
    });
    if (!recommendationsResponse.results) {
      // TODO: error?
      // eslint-disable-next-line no-console
      console.log(`Didn't get back a response with any results from google!`);
      return [];
    }

    const postIds = filterNonnull(recommendationsResponse.results.map(result => result.document?.id ?? result.id));
    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context);

    const attributionId = recommendationsResponse.attributionToken;
    const postsWithAttribution = filteredPosts.map(post => ({ post, attributionId }));

    return postsWithAttribution;
  },

  async importPosts(postsWithMetadata: CreateGoogleMediaDocumentMetadataArgs[]) {
    const { client, settingValues: { vertexDocumentServiceParentPath } } = clients.documents();

    const googleMediaDocuments = postsWithMetadata.map(helpers.createMediaDocument);

    // Max batch size of 100 for media documents
    const chunkedDocuments = chunk(googleMediaDocuments, 100);
    for (const chunk of chunkedDocuments) {
      const [importDocumentsOperation] = await client.importDocuments({
        inlineSource: { documents: chunk },
        parent: vertexDocumentServiceParentPath
      });

      const [importDocumentsResponse] = await importDocumentsOperation.promise();
      if (importDocumentsResponse.errorSamples?.length) {
        // eslint-disable-next-line no-console
        console.log('Error importing documents', { error: importDocumentsResponse.errorSamples[0] });
      }
    }
  },

  async importUserEvents(userEvents: protos.google.cloud.discoveryengine.v1.IUserEvent[]) {
    const { client: userEventClient, settingValues: { vertexUserEventServiceParentPath } } = clients.userEvents();

    // Max batch size of 100k for user events
    const chunkedUserEvents = chunk(userEvents, 90000);
    for (const chunk of chunkedUserEvents) {
      const [importUserEventsOperation] = await userEventClient.importUserEvents({ inlineSource: { userEvents: chunk }, parent: vertexUserEventServiceParentPath });
      const [importUserEventsResponse] = await importUserEventsOperation.promise();
      if (importUserEventsResponse.errorSamples?.length) {
        // eslint-disable-next-line no-console
        console.log('Error importing user events', { error: importUserEventsResponse.errorSamples[0] });
      }
    }
  }
};

export { clients, helpers, googleVertexApi };

1	import { DocumentServiceClient, UserEventServiceClient, v1beta, protos } from "@google-cloud/discoveryengine";	54✔
2	import { onStartup } from "../../lib/executionEnvironment";	18✔
3	import { googleRecommendationsCredsPath } from "../../lib/instanceSettings";	18✔
4	import { filterNonnull } from "../../lib/utils/typeGuardUtils";	18✔
5	import { loadByIds } from "../../lib/loaders";	18✔
6	import { accessFilterMultiple } from "../../lib/utils/schemaUtils";	18✔
7	import { getInstanceSettingsFilePath } from "../commandLine";	18✔
8	import { vertexDocumentServiceParentPathSetting, vertexRecommendationServingConfigPathSetting, vertexUserEventServiceParentPathSetting } from "../databaseSettings";	18✔
9	import { postGetPageUrl } from "../../lib/collections/posts/helpers";	18✔
10	import path from "path";	18✔
11	import chunk from "lodash/chunk";	18✔
12	import type { ClientSettingDependencies, ClientSettings, CreateGoogleMediaDocumentMetadataArgs, GoogleMediaDocumentMetadata, InViewEvent, FrontpageViewEvent } from "./types";
13
14	const { RecommendationServiceClient } = v1beta;	18✔
15
16	onStartup(() => {
17	// This is a weird hack which is necessary because this was the least crazy way to authenticate with Google Cloud from the outside.
18	// The tl;dr is that Google's client libraries will read process.env.GOOGLE_APPLICATION_CREDENTIALS and expect to find a filepath to the credentials file.
19	// The easiest way I thought of to solve the problem of differing relative paths between local env and production
20	// was to put the path of the creds file _relative to the instance settings file_ into an instance setting, and then join it with the path to the instance settings file.
21	// See https://cloud.google.com/docs/authentication/provide-credentials-adc#wlif-key for details
22	const credsPath = googleRecommendationsCredsPath.get();	16✔
23	const settingsFileName = getInstanceSettingsFilePath();	16✔
24	if (credsPath && settingsFileName) {	16!
25	const settingsFileDirectory = path.dirname(settingsFileName);	×
26	process.env.GOOGLE_APPLICATION_CREDENTIALS = path.join(settingsFileDirectory, credsPath);	×
27	}
28	});
29
30	const getGoogleClientInstanceOrThrow = <T extends new () => InstanceType<T>, Deps extends ClientSettingDependencies>(	18✔
31	constructor: T,
32	/**
33	* If the client depends on any settings at runtime, pass them in here to cause the getter to throw if any of the settings are missing.
34	* The settings need to be provided via an object with the field names being formatted like `fooSetting` (ending in `Setting`), and returning a `T \| null` value for arbitrary `T`.
35	*/
36	runtimeSettingDependencies?: Deps	54✔
37	): () => { client: InstanceType<T>, settingValues: ClientSettings<Deps> } => ((ClientConstructor: T) => {
38	let client: InstanceType<T>;
39	return () => {	54✔
40	let settingValues = {} as ClientSettings<Deps>;	×
41	const settingDependencyTuples = Object.entries(runtimeSettingDependencies ?? {});	×
42	if (settingDependencyTuples.length) {	×
43	const settingValueTuples = settingDependencyTuples.map(([settingName, setting]) => [settingName.slice(0, -7), setting.get()]);	×
44	const missingSettingTuples = settingValueTuples.filter(([_, settingValue]) => settingValue === null);	×
45	if (missingSettingTuples.length) {	×
46	const missingSettingNames = missingSettingTuples.map(([settingName]) => settingName).join(', ');	×
47	throw new Error(`Missing setting(s) ${missingSettingNames} for ${constructor.name}!`);	×
48	}
49
50	settingValues = Object.fromEntries(settingValueTuples) as ClientSettings<Deps>;	×
51	}
52
53	if (!client) {	×
54	client = new ClientConstructor();	×
55	}
56
57	return { client, settingValues };	×
58	};
59	})(constructor);
60
61	const clients = {	18✔
62	documents: getGoogleClientInstanceOrThrow(DocumentServiceClient, { vertexDocumentServiceParentPathSetting }),
63	userEvents: getGoogleClientInstanceOrThrow(UserEventServiceClient, { vertexUserEventServiceParentPathSetting }),
64	recommendations: getGoogleClientInstanceOrThrow(RecommendationServiceClient, { vertexRecommendationServingConfigPathSetting }),
65	};
66
67	const helpers = {	18✔
68	createMediaDocument({ post, tags, authorIds }: CreateGoogleMediaDocumentMetadataArgs): protos.google.cloud.discoveryengine.v1.IDocument {
UNCOV 69	const tagIds = tags?.map(tag => tag._id) ?? [];	×
UNCOV 70	if (tagIds.length === 0) {	×
UNCOV 71	tagIds.push('none')	×
72	}
UNCOV 73	const persons = authorIds?.map(authorId => ({ name: authorId, role: 'author' } as const));	×
74
UNCOV 75	const metadata: GoogleMediaDocumentMetadata = {	×
76	title: post.title,
77	uri: `https://www.lesswrong.com${postGetPageUrl(post)}`,
78	available_time: post.postedAt.toISOString(),
79	categories: tagIds,
80	filter_tags: tagIds,
81	persons,
82	media_type: 'articles',
83	};
84
UNCOV 85	return {	×
86	id: post._id,
87	schemaId: 'default_schema',
88	jsonData: JSON.stringify(metadata),
89	};
90	},
91
92	createViewItemEvent(eventType: 'view-item' \| 'media-play', readStatus: DbReadStatus): protos.google.cloud.discoveryengine.v1.IUserEvent {
93	const { userId, postId, lastUpdated: timestamp } = readStatus;	×
94
95	return {	×
96	eventType,
97	// TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
98	userPseudoId: userId,
99	eventTime: {
100	seconds: timestamp.getTime() / 1000,
101	nanos: 0
102	},
103	userInfo: { userId },
104	documents: [{ id: postId }],
105	};
106	},
107
108	createMediaCompleteEvent(inViewEvent: InViewEvent): protos.google.cloud.discoveryengine.v1.IUserEvent {
UNCOV 109	const { userId, postId, timestamp } = inViewEvent;	×
110
UNCOV 111	return {	×
112	eventType: 'media-complete',
113	// TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
114	userPseudoId: userId,
115	eventTime: {
116	seconds: timestamp.getTime() / 1000,
117	nanos: 0
118	},
119	userInfo: { userId },
120	documents: [{ id: postId }],
121	mediaInfo: {
122	mediaProgressPercentage: 1
123	}
124	};
125	},
126
127	createViewHomePageEvent(frontpageViewEvent: FrontpageViewEvent) {
UNCOV 128	const { userId, timestamp } = frontpageViewEvent;	×
129
130	return {	×
131	eventType: 'view-home-page',
132	// TODO - this should maybe be clientId for logged out users if doing real stuff with it, but no logged out users for now
133	userPseudoId: userId,
134	eventTime: {
135	seconds: timestamp.getTime() / 1000,
136	nanos: 0
137	},
138	userInfo: { userId }
139	};
140	}
141	};
142
143	const googleVertexApi = {	18✔
144	async getRecommendations(limit: number, context: ResolverContext) {
UNCOV 145	const { client, settingValues: { vertexRecommendationServingConfigPath } } = clients.recommendations();	×
UNCOV 146	const [recommendationsResponse] = await client.recommend({	×
147	validateOnly: false,
148	servingConfig: vertexRecommendationServingConfigPath,
149	pageSize: limit,
150	userEvent: { eventType: 'view-home-page', userPseudoId: context.currentUser?._id },	×
151	});
UNCOV 152	if (!recommendationsResponse.results) {	×
153	// TODO: error?
154	// eslint-disable-next-line no-console
UNCOV 155	console.log(`Didn't get back a response with any results from google!`);	×
156	return [];	×
157	}
158
UNCOV 159	const postIds = filterNonnull(recommendationsResponse.results.map(result => result.document?.id ?? result.id));	×
UNCOV 160	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
UNCOV 161	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context);	×
162
163	const attributionId = recommendationsResponse.attributionToken;	×
UNCOV 164	const postsWithAttribution = filteredPosts.map(post => ({ post, attributionId }));	×
165
166	return postsWithAttribution;	×
167	},
168
169	async importPosts(postsWithMetadata: CreateGoogleMediaDocumentMetadataArgs[]) {
170	const { client, settingValues: { vertexDocumentServiceParentPath } } = clients.documents();	×
171
172	const googleMediaDocuments = postsWithMetadata.map(helpers.createMediaDocument);	×
173
174	// Max batch size of 100 for media documents
175	const chunkedDocuments = chunk(googleMediaDocuments, 100);	×
UNCOV 176	for (const chunk of chunkedDocuments) {	×
177	const [importDocumentsOperation] = await client.importDocuments({	×
178	inlineSource: { documents: chunk },
179	parent: vertexDocumentServiceParentPath
180	});
181
UNCOV 182	const [importDocumentsResponse] = await importDocumentsOperation.promise();	×
183	if (importDocumentsResponse.errorSamples?.length) {	×
184	// eslint-disable-next-line no-console
UNCOV 185	console.log('Error importing documents', { error: importDocumentsResponse.errorSamples[0] });	×
186	}
187	}
188	},
189
190	async importUserEvents(userEvents: protos.google.cloud.discoveryengine.v1.IUserEvent[]) {
UNCOV 191	const { client: userEventClient, settingValues: { vertexUserEventServiceParentPath } } = clients.userEvents();	×
192
193	// Max batch size of 100k for user events
194	const chunkedUserEvents = chunk(userEvents, 90000);	×
UNCOV 195	for (const chunk of chunkedUserEvents) {	×
196	const [importUserEventsOperation] = await userEventClient.importUserEvents({ inlineSource: { userEvents: chunk }, parent: vertexUserEventServiceParentPath });	×
UNCOV 197	const [importUserEventsResponse] = await importUserEventsOperation.promise();	×
UNCOV 198	if (importUserEventsResponse.errorSamples?.length) {	×
199	// eslint-disable-next-line no-console
UNCOV 200	console.log('Error importing user events', { error: importUserEventsResponse.errorSamples[0] });	×
201	}
202	}
203	}
204	};
205
206	export { clients, helpers, googleVertexApi };

ForumMagnum / ForumMagnum / 8895934749

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous