8804698607

Committed 23 Apr 2024 05:29PM UTC coverage: 32.044% (-0.02%) from 32.067%

Build # 8804698607

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9157 from ForumMagnum/master

[EA] Deploy LaTeX table of contents fix, hiding popular comments on drafts

Run Details

4689 of 38059 branches covered (12.32%)

Branch coverage included in aggregate %.

83 of 259 new or added lines in 21 files covered. (32.05%)

4 existing lines in 4 files now uncovered.

27195 of 61443 relevant lines covered (44.26%)

181.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.33

/packages/lesswrong/server/recombee/client.ts

import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';
import { HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
import { loadByIds } from '../../lib/loaders';
import { filterNonnull } from '../../lib/utils/typeGuardUtils';
import { htmlToTextDefault } from '../../lib/htmlToText';
import { truncate } from '../../lib/editor/ellipsize';
import findByIds from '../vulcan-lib/findbyids';
import ReadStatuses from '../../lib/collections/readStatus/collection';
import moment from 'moment';
import { accessFilterMultiple } from '../../lib/utils/schemaUtils';
import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';
import { viewTermsToQuery } from '../../lib/utils/viewUtils';
import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';
import groupBy from 'lodash/groupBy';

export const getRecombeeClientOrThrow = (() => {
  let client: ApiClient;

  return () => {
    if (!client) {
      const databaseId = recombeeDatabaseIdSetting.get();
      const apiToken = recombeePrivateApiTokenSetting.get();

      if (!databaseId || !apiToken) {
        throw new Error('Missing either databaseId or api token when initializing Recombee client!');
      }
      
      // TODO - pull out client options like region to db settings?
      client = new ApiClient(databaseId, apiToken, { region: 'us-west' });
    }

    return client;
  };
})();

const voteTypeRatingsMap: Partial<Record<string, number>> = {
  bigDownvote: -1,
  smallDownvote: -0.5,
  smallUpvote: 0.5,
  bigUpvote: 1,
};

const HYBRID_SCENARIO_MAP = {
  configurable: 'recombee-personal-latest',
  fixed: 'recombee-personal'
};

interface OnsitePostRecommendationsInfo {
  curatedPostIds: string[],
  stickiedPostIds: string[],
  excludedPostFilter?: string,
}

export interface RecombeeRecommendedPost {
  post: Partial<DbPost>,
  recommId: string,
  curated?: never,
  stickied?: never,
}

export type RecommendedPost = RecombeeRecommendedPost | {
  post: Partial<DbPost>,
  recommId?: never,
  curated: boolean,
  stickied: boolean,
};

const recombeeRequestHelpers = {
  createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
    const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;

    if (loadMore) {
      return new requests.RecommendNextItems(loadMore.prevRecommId, count);
    }

    const servedUserId = overrideUserId ?? userId;
    const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;

    // TODO: pass in scenario, exclude unread, etc, in options?
    const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';

    return new requests.RecommendItemsToUser(servedUserId, count, {
      ...settings,
      // Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
      booster: settings.booster || undefined,
      rotationTime: rotationTimeSeconds
    });
  },

  async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
    const { Tags } = context;

    const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)
    tags ??= filterNonnull(await findByIds(Tags, tagIds))
    const tagNames = tags.map(tag => tag.name)
    const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)

    const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))

    return new requests.SetItemValues(post._id, {
      title: post.title,
      author: post.author,
      authorId: post.userId,
      karma: post.baseScore,
      body: postText,
      postedAt: post.postedAt,
      tags: tagNames,
      coreTags: coreTagNames,
      curated: !!post.curatedDate,
      frontpage: !!post.frontpageDate,
      draft: !!post.draft,
      lastCommentedAt: post.lastCommentedAt,
    }, { cascadeCreate: true });
  },

  createReadStatusRequest(readStatus: DbReadStatus) {
    if (!readStatus.postId) {
      // eslint-disable-next-line no-console
      console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);
      return;
    }

    return new requests.AddDetailView(readStatus.userId, readStatus.postId, {
      timestamp: readStatus.lastUpdated.toISOString(),
      cascadeCreate: false
    });
  },

  createVoteRequest(vote: DbVote) {
    const rating = voteTypeRatingsMap[vote.voteType];
    if (typeof rating !== 'number') {
      // eslint-disable-next-line no-console
      console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);
      return;
    }

    return new requests.AddRating(vote.userId, vote.documentId, rating, {
      timestamp: vote.votedAt.toISOString(),
      cascadeCreate: false
    });
  },

  createUpsertUserDetailsRequest(user: DbUser) {
    const { displayName, karma, createdAt } = user;
    return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });
  },

  getBatchRequest(requestBatch: requests.Request[]) {
    return new requests.Batch(requestBatch);
  },

  async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration | RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
    if (lwAlgoSettings.loadMore) {
      return {
        curatedPostIds: [],
        stickiedPostIds: [],
        excludedPostFilter: undefined,
      };
    }

    const postPromises =  [curatedPostTerms, stickiedPostTerms]
      .map(terms => viewTermsToQuery("Posts", terms, undefined, context))
      .map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());

    const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);

    const curatedPostIds = curatedPosts.map(post => post._id);
    const stickiedPostIds = stickiedPosts.map(post => post._id);
    const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];
    const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;

    return {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter,
    };
  },

  convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof typeof HYBRID_SCENARIO_MAP, filter?: string) {
    const { loadMore, userId, ...rest } = hybridArgs;

    const scenario = HYBRID_SCENARIO_MAP[hybridArm];
    const isConfigurable = hybridArm === 'configurable';
    const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 144};
    const prevRecommIdIndex = isConfigurable ? 0 : 1;
    const loadMoreConfig = loadMore
      ? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }
      : {};

    return {
      userId,
      ...clientConfig,
      filter,
      scenario,
      ...loadMoreConfig
    };
  },

  interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
    const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);
    let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);

    const interleavedPosts: RecombeeRecommendedPost[] = [];

    while (interleavedPosts.length < recommendedPosts.length) {
      // Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
      const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);

      // Weighted random selection of next recommId to pick a post from
      let randomWeight = Math.random() * totalWeight;
      let selectedRecommId: string | undefined;
      for (const [recommId, recommIdWeight] of weights) {
        randomWeight -= recommIdWeight;
        if (randomWeight <= 0) {
          selectedRecommId = recommId;
          break;
        }
      }

      // Safety check, though selectedGroup should always be defined here
      if (!selectedRecommId) continue;

      // Add next post from the selected group to the interleaved array
      const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];
      const nextPost = postsWithRecommId.shift();
      if (!nextPost) {
        // We shouldn't ever get here, but just in case
        continue;
      } else {
        interleavedPosts.push(nextPost);

        // Update the group's state: remove it if it's out of posts
        if (postsWithRecommId.length === 0) {
          weights = weights.filter(([recommId]) => recommId !== selectedRecommId);
        }  
      }
    }

    return interleavedPosts;
  }
};

const curatedPostTerms: PostsViewTerms = {
  view: 'curated',
  limit: 3,
};

const recombeeApi = {
  async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });

    // We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
    const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;
    const { recomms, recommId } = recombeeResponse;

    const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);
    const recommendedPostIds = recomms.map(({ id }) => id);
    const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];

    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];
      if (recommId) {
        return { post, recommId };
      } else {
        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied: stickiedPostIds.includes(postId)  
        };
      }
    });

    return mappedPosts;
  },


  async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const split = 0.5;
    const firstCount = Math.floor(modifiedCount * split);
    const secondCount = modifiedCount - firstCount;

    const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);
    const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);

    const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);
    const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);
    const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);

    const batchResponse = await client.send(batchRequest);
    // We need the type cast here because recombee's type definitions don't provide response types for batch requests
    const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);

    // We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
    const recommendationIdPairs = recombeeResponses.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId] as const));
    const recommendedPostIds = recommendationIdPairs.map(([id]) => id);
    const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];
    
    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];
      if (recommId) {
        return { post, recommId };
      } else {
        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied: stickiedPostIds.includes(postId)  
        };
      }
    });

    const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);
    const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);

    const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);

    return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];
  },


  async upsertPost(post: DbPost, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();
    const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);

    await client.send(request);
  },

  async createReadStatus(readStatus: DbReadStatus) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createVote(vote: DbVote) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createVoteRequest(vote);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createUser(user: DbUser) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);
    await client.send(request);
  }
};

export { recombeeRequestHelpers, recombeeApi };

1	import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';	54✔
2	import { HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
3	import { loadByIds } from '../../lib/loaders';	18✔
4	import { filterNonnull } from '../../lib/utils/typeGuardUtils';	18✔
5	import { htmlToTextDefault } from '../../lib/htmlToText';	18✔
6	import { truncate } from '../../lib/editor/ellipsize';	18✔
7	import findByIds from '../vulcan-lib/findbyids';	18✔
8	import ReadStatuses from '../../lib/collections/readStatus/collection';
9	import moment from 'moment';
10	import { accessFilterMultiple } from '../../lib/utils/schemaUtils';	18✔
11	import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';	18✔
12	import { viewTermsToQuery } from '../../lib/utils/viewUtils';	18✔
13	import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';	18✔
14	import groupBy from 'lodash/groupBy';	18✔
15
16	export const getRecombeeClientOrThrow = (() => {	18✔
17	let client: ApiClient;
18
19	return () => {	18✔
20	if (!client) {	×
21	const databaseId = recombeeDatabaseIdSetting.get();	×
22	const apiToken = recombeePrivateApiTokenSetting.get();	×
23
24	if (!databaseId \|\| !apiToken) {	×
25	throw new Error('Missing either databaseId or api token when initializing Recombee client!');	×
26	}
27
28	// TODO - pull out client options like region to db settings?
29	client = new ApiClient(databaseId, apiToken, { region: 'us-west' });	×
30	}
31
32	return client;	×
33	};
34	})();
35
36	const voteTypeRatingsMap: Partial<Record<string, number>> = {	18✔
37	bigDownvote: -1,
38	smallDownvote: -0.5,
39	smallUpvote: 0.5,
40	bigUpvote: 1,
41	};
42
43	const HYBRID_SCENARIO_MAP = {	18✔
44	configurable: 'recombee-personal-latest',
45	fixed: 'recombee-personal'
46	};
47
48	interface OnsitePostRecommendationsInfo {
49	curatedPostIds: string[],
50	stickiedPostIds: string[],
51	excludedPostFilter?: string,
52	}
53
54	export interface RecombeeRecommendedPost {
55	post: Partial<DbPost>,
56	recommId: string,
57	curated?: never,
58	stickied?: never,
59	}
60
61	export type RecommendedPost = RecombeeRecommendedPost \| {
62	post: Partial<DbPost>,
63	recommId?: never,
64	curated: boolean,
65	stickied: boolean,
66	};
67
68	const recombeeRequestHelpers = {	18✔
69	createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
70	const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;	×
71
72	if (loadMore) {	×
73	return new requests.RecommendNextItems(loadMore.prevRecommId, count);	×
74	}
75
76	const servedUserId = overrideUserId ?? userId;	×
77	const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;	×
78
79	// TODO: pass in scenario, exclude unread, etc, in options?
80	const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';	×
81
82	return new requests.RecommendItemsToUser(servedUserId, count, {	×
83	...settings,
84	// Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
85	booster: settings.booster \|\| undefined,	×
86	rotationTime: rotationTimeSeconds
87	});
88	},
89
90	async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
91	const { Tags } = context;	×
92
93	const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)	×
94	tags ??= filterNonnull(await findByIds(Tags, tagIds))	×
95	const tagNames = tags.map(tag => tag.name)	×
96	const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)	×
97
98	const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))	×
99
100	return new requests.SetItemValues(post._id, {	×
101	title: post.title,
102	author: post.author,
103	authorId: post.userId,
104	karma: post.baseScore,
105	body: postText,
106	postedAt: post.postedAt,
107	tags: tagNames,
108	coreTags: coreTagNames,
109	curated: !!post.curatedDate,
110	frontpage: !!post.frontpageDate,
111	draft: !!post.draft,
112	lastCommentedAt: post.lastCommentedAt,
113	}, { cascadeCreate: true });
114	},
115
116	createReadStatusRequest(readStatus: DbReadStatus) {
117	if (!readStatus.postId) {	×
118	// eslint-disable-next-line no-console
119	console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);	×
120	return;	×
121	}
122
123	return new requests.AddDetailView(readStatus.userId, readStatus.postId, {	×
124	timestamp: readStatus.lastUpdated.toISOString(),
125	cascadeCreate: false
126	});
127	},
128
129	createVoteRequest(vote: DbVote) {
130	const rating = voteTypeRatingsMap[vote.voteType];	×
131	if (typeof rating !== 'number') {	×
132	// eslint-disable-next-line no-console
133	console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);	×
134	return;	×
135	}
136
137	return new requests.AddRating(vote.userId, vote.documentId, rating, {	×
138	timestamp: vote.votedAt.toISOString(),
139	cascadeCreate: false
140	});
141	},
142
143	createUpsertUserDetailsRequest(user: DbUser) {
144	const { displayName, karma, createdAt } = user;	×
145	return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });	×
146	},
147
148	getBatchRequest(requestBatch: requests.Request[]) {
149	return new requests.Batch(requestBatch);	×
150	},
151
152	async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration \| RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
153	if (lwAlgoSettings.loadMore) {	×
154	return {	×
155	curatedPostIds: [],
156	stickiedPostIds: [],
157	excludedPostFilter: undefined,
158	};
159	}
160
161	const postPromises = [curatedPostTerms, stickiedPostTerms]	×
162	.map(terms => viewTermsToQuery("Posts", terms, undefined, context))	×
163	.map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());	×
164
165	const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);	×
166
167	const curatedPostIds = curatedPosts.map(post => post._id);	×
168	const stickiedPostIds = stickiedPosts.map(post => post._id);	×
169	const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];	×
170	const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;	×
171
172	return {	×
173	curatedPostIds,
174	stickiedPostIds,
175	excludedPostFilter,
176	};
177	},
178
179	convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof typeof HYBRID_SCENARIO_MAP, filter?: string) {
180	const { loadMore, userId, ...rest } = hybridArgs;	×
181
182	const scenario = HYBRID_SCENARIO_MAP[hybridArm];	×
183	const isConfigurable = hybridArm === 'configurable';	×
184	const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"\|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 144};	×
185	const prevRecommIdIndex = isConfigurable ? 0 : 1;	×
186	const loadMoreConfig = loadMore	×
187	? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }	×
188	: {};
189
190	return {	×
191	userId,
192	...clientConfig,
193	filter,
194	scenario,
195	...loadMoreConfig
196	};
197	},
198
199	interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
200	const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);	×
201	let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);	×
202
203	const interleavedPosts: RecombeeRecommendedPost[] = [];	×
204
205	while (interleavedPosts.length < recommendedPosts.length) {	×
206	// Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
207	const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);	×
208
209	// Weighted random selection of next recommId to pick a post from
210	let randomWeight = Math.random() * totalWeight;	×
211	let selectedRecommId: string \| undefined;
212	for (const [recommId, recommIdWeight] of weights) {	×
213	randomWeight -= recommIdWeight;	×
214	if (randomWeight <= 0) {	×
215	selectedRecommId = recommId;	×
216	break;	×
217	}
218	}
219
220	// Safety check, though selectedGroup should always be defined here
221	if (!selectedRecommId) continue;	×
222
223	// Add next post from the selected group to the interleaved array
224	const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];	×
225	const nextPost = postsWithRecommId.shift();	×
226	if (!nextPost) {	×
227	// We shouldn't ever get here, but just in case
228	continue;	×
229	} else {
230	interleavedPosts.push(nextPost);	×
231
232	// Update the group's state: remove it if it's out of posts
233	if (postsWithRecommId.length === 0) {	×
234	weights = weights.filter(([recommId]) => recommId !== selectedRecommId);	×
235	}
236	}
237	}
238
239	return interleavedPosts;	×
240	}
241	};
242
243	const curatedPostTerms: PostsViewTerms = {	18✔
244	view: 'curated',
245	limit: 3,
246	};
247
248	const recombeeApi = {	18✔
249	async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
250	const client = getRecombeeClientOrThrow();	×
251
252	const {
253	curatedPostIds,
254	stickiedPostIds,
255	excludedPostFilter
256	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
257
NEW 258	const curatedPostReadStatuses = await (	×
259	lwAlgoSettings.loadMore
260	? Promise.resolve([])	×
261	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
262	);
263
NEW 264	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
NEW 265	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;	×
266
NEW 267	const modifiedCount = count - curatedAndStickiedPostCount;	×
NEW 268	const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });	×
269
270	// We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
NEW 271	const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;	×
UNCOV 272	const { recomms, recommId } = recombeeResponse;	×
273
274	const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);	×
275	const recommendedPostIds = recomms.map(({ id }) => id);	×
276	const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
277
278	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
279	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)	×
280
281	const mappedPosts = filteredPosts.map(post => {	×
282	// _id isn't going to be filtered out by `accessFilterMultiple`
283	const postId = post._id!;	×
284	const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];	×
285	if (recommId) {	×
286	return { post, recommId };	×
287	} else {
288	return {	×
289	post,
290	curated: curatedPostIds.includes(postId),
291	stickied: stickiedPostIds.includes(postId)
292	};
293	}
294	});
295
296	return mappedPosts;	×
297	},
298
299
300	async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
301	const client = getRecombeeClientOrThrow();	×
302
303	const {
304	curatedPostIds,
305	stickiedPostIds,
306	excludedPostFilter
307	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
308
NEW 309	const curatedPostReadStatuses = await (	×
310	lwAlgoSettings.loadMore
311	? Promise.resolve([])	×
312	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
313	);
314
NEW 315	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
NEW 316	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;	×
317
NEW 318	const modifiedCount = count - curatedAndStickiedPostCount;	×
319	const split = 0.5;	×
320	const firstCount = Math.floor(modifiedCount * split);	×
321	const secondCount = modifiedCount - firstCount;	×
322
323	const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);	×
324	const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);	×
325
326	const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);	×
327	const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);	×
328	const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);	×
329
NEW 330	const batchResponse = await client.send(batchRequest);	×
331	// We need the type cast here because recombee's type definitions don't provide response types for batch requests
332	const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);	×
333
334	// We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
335	const recommendationIdPairs = recombeeResponses.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId] as const));	×
336	const recommendedPostIds = recommendationIdPairs.map(([id]) => id);	×
337	const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
338
339	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
340	const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));	×
341	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);	×
342
343	const mappedPosts = filteredPosts.map(post => {	×
344	// _id isn't going to be filtered out by `accessFilterMultiple`
345	const postId = post._id!;	×
346	const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];	×
347	if (recommId) {	×
348	return { post, recommId };	×
349	} else {
350	return {	×
351	post,
352	curated: curatedPostIds.includes(postId),
353	stickied: stickiedPostIds.includes(postId)
354	};
355	}
356	});
357
358	const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);	×
359	const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);	×
360
361	const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);	×
362
363	return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];	×
364	},
365
366
367	async upsertPost(post: DbPost, context: ResolverContext) {
368	const client = getRecombeeClientOrThrow();	×
369	const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);	×
370
371	await client.send(request);	×
372	},
373
374	async createReadStatus(readStatus: DbReadStatus) {
375	const client = getRecombeeClientOrThrow();	×
376	const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);	×
377	if (!request) {	×
378	return;	×
379	}
380
381	await client.send(request);	×
382	},
383
384	async createVote(vote: DbVote) {
385	const client = getRecombeeClientOrThrow();	×
386	const request = recombeeRequestHelpers.createVoteRequest(vote);	×
387	if (!request) {	×
388	return;	×
389	}
390
391	await client.send(request);	×
392	},
393
394	async createUser(user: DbUser) {
395	const client = getRecombeeClientOrThrow();	×
396	const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);	×
397	await client.send(request);	×
398	}
399	};
400
401	export { recombeeRequestHelpers, recombeeApi };

ForumMagnum / ForumMagnum / 8804698607

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous