8838908930

Committed 25 Apr 2024 08:23PM UTC coverage: 32.032% (-0.01%) from 32.043%

Build # 8838908930

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9177 from ForumMagnum/master

LW Deploy, 2024-04-25

Run Details

4697 of 38110 branches covered (12.32%)

Branch coverage included in aggregate %.

6 of 35 new or added lines in 11 files covered. (17.14%)

5 existing lines in 5 files now uncovered.

27202 of 61476 relevant lines covered (44.25%)

181.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.23

/packages/lesswrong/server/recombee/client.ts

import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';
import { HybridArmsConfig, HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
import { loadByIds } from '../../lib/loaders';
import { filterNonnull } from '../../lib/utils/typeGuardUtils';
import { htmlToTextDefault } from '../../lib/htmlToText';
import { truncate } from '../../lib/editor/ellipsize';
import findByIds from '../vulcan-lib/findbyids';
import { accessFilterMultiple } from '../../lib/utils/schemaUtils';
import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';
import { viewTermsToQuery } from '../../lib/utils/viewUtils';
import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';
import groupBy from 'lodash/groupBy';
import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';

export const getRecombeeClientOrThrow = (() => {
  let client: ApiClient;

  return () => {
    if (!client) {
      const databaseId = recombeeDatabaseIdSetting.get();
      const apiToken = recombeePrivateApiTokenSetting.get();

      if (!databaseId || !apiToken) {
        throw new Error('Missing either databaseId or api token when initializing Recombee client!');
      }
      
      // TODO - pull out client options like region to db settings?
      client = new ApiClient(databaseId, apiToken, { region: 'us-west' });
    }

    return client;
  };
})();

const voteTypeRatingsMap: Partial<Record<string, number>> = {
  bigDownvote: -1,
  smallDownvote: -0.5,
  smallUpvote: 0.5,
  bigUpvote: 1,
};

const HYBRID_SCENARIO_MAP = {
  fixed: 'recombee-emulate-hacker-news',
  configurable: 'recombee-personal'
};

interface OnsitePostRecommendationsInfo {
  curatedPostIds: string[],
  stickiedPostIds: string[],
  excludedPostFilter?: string,
}

export interface RecombeeRecommendedPost {
  post: Partial<DbPost>,
  scenario: string,
  recommId: string,
  curated?: never,
  stickied?: never,
}

export type RecommendedPost = RecombeeRecommendedPost | {
  post: Partial<DbPost>,
  scenario?: never,
  recommId?: never,
  curated: boolean,
  stickied: boolean,
};

const recombeeRequestHelpers = {
  createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
    const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;

    if (loadMore) {
      return new requests.RecommendNextItems(loadMore.prevRecommId, count);
    }

    const servedUserId = overrideUserId ?? userId;
    const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;

    // TODO: pass in scenario, exclude unread, etc, in options?
    const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';

    return new requests.RecommendItemsToUser(servedUserId, count, {
      ...settings,
      // Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
      booster: settings.booster || undefined,
      rotationTime: rotationTimeSeconds
    });
  },

  async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
    const { Tags } = context;

    const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)
    tags ??= filterNonnull(await findByIds(Tags, tagIds))
    const tagNames = tags.map(tag => tag.name)
    const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)

    const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))

    return new requests.SetItemValues(post._id, {
      title: post.title,
      author: post.author,
      authorId: post.userId,
      karma: post.baseScore,
      body: postText,
      postedAt: post.postedAt,
      tags: tagNames,
      coreTags: coreTagNames,
      curated: !!post.curatedDate,
      frontpage: !!post.frontpageDate,
      draft: !!post.draft,
      lastCommentedAt: post.lastCommentedAt,
    }, { cascadeCreate: true });
  },

  createReadStatusRequest(readStatus: DbReadStatus) {
    if (!readStatus.postId) {
      // eslint-disable-next-line no-console
      console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);
      return;
    }

    return new requests.AddDetailView(readStatus.userId, readStatus.postId, {
      timestamp: readStatus.lastUpdated.toISOString(),
      cascadeCreate: false
    });
  },

  createVoteRequest(vote: DbVote) {
    const rating = voteTypeRatingsMap[vote.voteType];
    if (typeof rating !== 'number') {
      // eslint-disable-next-line no-console
      console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);
      return;
    }

    return new requests.AddRating(vote.userId, vote.documentId, rating, {
      timestamp: vote.votedAt.toISOString(),
      cascadeCreate: false
    });
  },

  createUpsertUserDetailsRequest(user: DbUser) {
    const { displayName, karma, createdAt } = user;
    return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });
  },

  getBatchRequest(requestBatch: requests.Request[]) {
    return new requests.Batch(requestBatch);
  },

  async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration | RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
    if (lwAlgoSettings.loadMore) {
      return {
        curatedPostIds: [],
        stickiedPostIds: [],
        excludedPostFilter: undefined,
      };
    }

    const postPromises =  [curatedPostTerms, stickiedPostTerms]
      .map(terms => viewTermsToQuery("Posts", terms, undefined, context))
      .map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());

    const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);

    const curatedPostIds = curatedPosts.map(post => post._id);
    const stickiedPostIds = stickiedPosts.map(post => post._id);
    const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];
    const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;

    return {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter,
    };
  },

  convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
    const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;

    const scenario = hybridScenarios[hybridArm];

    const isConfigurable = hybridArm === 'configurable';
    const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};
    const prevRecommIdIndex = isConfigurable ? 0 : 1;
    const loadMoreConfig = loadMore
      ? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }
      : {};

    return {
      userId,
      ...clientConfig,
      filter,
      scenario,
      ...loadMoreConfig
    };
  },

  interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
    const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);
    let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);

    const interleavedPosts: RecombeeRecommendedPost[] = [];

    while (interleavedPosts.length < recommendedPosts.length) {
      // Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
      const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);

      // Weighted random selection of next recommId to pick a post from
      let randomWeight = Math.random() * totalWeight;
      let selectedRecommId: string | undefined;
      for (const [recommId, recommIdWeight] of weights) {
        randomWeight -= recommIdWeight;
        if (randomWeight <= 0) {
          selectedRecommId = recommId;
          break;
        }
      }

      // Safety check, though selectedGroup should always be defined here
      if (!selectedRecommId) continue;

      // Add next post from the selected group to the interleaved array
      const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];
      const nextPost = postsWithRecommId.shift();
      if (!nextPost) {
        // We shouldn't ever get here, but just in case
        continue;
      } else {
        interleavedPosts.push(nextPost);

        // Update the group's state: remove it if it's out of posts
        if (postsWithRecommId.length === 0) {
          weights = weights.filter(([recommId]) => recommId !== selectedRecommId);
        }  
      }
    }

    return interleavedPosts;
  }
};

const curatedPostTerms: PostsViewTerms = {
  view: 'curated',
  limit: 3,
};

const recombeeApi = {
  async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });

    // We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
    const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;
    const { recomms, recommId } = recombeeResponse;

    const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);
    const recommendedPostIds = recomms.map(({ id }) => id);
    const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];

    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];
      if (recommId) {
        return { post, recommId, scenario: lwAlgoSettings.scenario };
      } else {
        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied: stickiedPostIds.includes(postId)  
        };
      }
    });

    return mappedPosts;
  },


  async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const split = 0.5;
    const firstCount = Math.floor(modifiedCount * split);
    const secondCount = modifiedCount - firstCount;

    const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);
    const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);

    const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);
    const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);
    const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);

    const batchResponse = await client.send(batchRequest);
    // We need the type cast here because recombee's type definitions don't provide response types for batch requests
    const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);
    const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));

    // We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
    const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));
    const recommendedPostIds = recommendationIdTuples.map(([id]) => id);
    const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];
    
    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];
      const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];

      if (recommId) {
        return { post, recommId, scenario };
      } else {
        const stickied = stickiedPostIds.includes(postId) || manuallyStickiedPostIds.includes(postId);
        if (stickied) {
          post.sticky = true;
        }

        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied  
        };
      }
    });

    const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);
    const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);

    const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);

    return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];
  },


  async upsertPost(post: DbPost, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();
    const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);

    await client.send(request);
  },

  async createReadStatus(readStatus: DbReadStatus) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createVote(vote: DbVote) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createVoteRequest(vote);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createUser(user: DbUser) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);
    await client.send(request);
  }
};

export { recombeeRequestHelpers, recombeeApi };

1	import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';	54✔
2	import { HybridArmsConfig, HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
3	import { loadByIds } from '../../lib/loaders';	18✔
4	import { filterNonnull } from '../../lib/utils/typeGuardUtils';	18✔
5	import { htmlToTextDefault } from '../../lib/htmlToText';	18✔
6	import { truncate } from '../../lib/editor/ellipsize';	18✔
7	import findByIds from '../vulcan-lib/findbyids';	18✔
8	import { accessFilterMultiple } from '../../lib/utils/schemaUtils';	18✔
9	import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';	18✔
10	import { viewTermsToQuery } from '../../lib/utils/viewUtils';	18✔
11	import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';	18✔
12	import groupBy from 'lodash/groupBy';	18✔
13	import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';	18✔
14
15	export const getRecombeeClientOrThrow = (() => {	18✔
16	let client: ApiClient;
17
18	return () => {	18✔
19	if (!client) {	×
20	const databaseId = recombeeDatabaseIdSetting.get();	×
21	const apiToken = recombeePrivateApiTokenSetting.get();	×
22
23	if (!databaseId \|\| !apiToken) {	×
24	throw new Error('Missing either databaseId or api token when initializing Recombee client!');	×
25	}
26
27	// TODO - pull out client options like region to db settings?
28	client = new ApiClient(databaseId, apiToken, { region: 'us-west' });	×
29	}
30
31	return client;	×
32	};
33	})();
34
35	const voteTypeRatingsMap: Partial<Record<string, number>> = {	18✔
36	bigDownvote: -1,
37	smallDownvote: -0.5,
38	smallUpvote: 0.5,
39	bigUpvote: 1,
40	};
41
42	const HYBRID_SCENARIO_MAP = {	18✔
43	fixed: 'recombee-emulate-hacker-news',
44	configurable: 'recombee-personal'
45	};
46
47	interface OnsitePostRecommendationsInfo {
48	curatedPostIds: string[],
49	stickiedPostIds: string[],
50	excludedPostFilter?: string,
51	}
52
53	export interface RecombeeRecommendedPost {
54	post: Partial<DbPost>,
55	scenario: string,
56	recommId: string,
57	curated?: never,
58	stickied?: never,
59	}
60
61	export type RecommendedPost = RecombeeRecommendedPost \| {
62	post: Partial<DbPost>,
63	scenario?: never,
64	recommId?: never,
65	curated: boolean,
66	stickied: boolean,
67	};
68
69	const recombeeRequestHelpers = {	18✔
70	createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
71	const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;	×
72
73	if (loadMore) {	×
74	return new requests.RecommendNextItems(loadMore.prevRecommId, count);	×
75	}
76
77	const servedUserId = overrideUserId ?? userId;	×
78	const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;	×
79
80	// TODO: pass in scenario, exclude unread, etc, in options?
81	const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';	×
82
83	return new requests.RecommendItemsToUser(servedUserId, count, {	×
84	...settings,
85	// Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
86	booster: settings.booster \|\| undefined,	×
87	rotationTime: rotationTimeSeconds
88	});
89	},
90
91	async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
92	const { Tags } = context;	×
93
94	const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)	×
95	tags ??= filterNonnull(await findByIds(Tags, tagIds))	×
96	const tagNames = tags.map(tag => tag.name)	×
97	const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)	×
98
99	const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))	×
100
101	return new requests.SetItemValues(post._id, {	×
102	title: post.title,
103	author: post.author,
104	authorId: post.userId,
105	karma: post.baseScore,
106	body: postText,
107	postedAt: post.postedAt,
108	tags: tagNames,
109	coreTags: coreTagNames,
110	curated: !!post.curatedDate,
111	frontpage: !!post.frontpageDate,
112	draft: !!post.draft,
113	lastCommentedAt: post.lastCommentedAt,
114	}, { cascadeCreate: true });
115	},
116
117	createReadStatusRequest(readStatus: DbReadStatus) {
118	if (!readStatus.postId) {	×
119	// eslint-disable-next-line no-console
120	console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);	×
121	return;	×
122	}
123
124	return new requests.AddDetailView(readStatus.userId, readStatus.postId, {	×
125	timestamp: readStatus.lastUpdated.toISOString(),
126	cascadeCreate: false
127	});
128	},
129
130	createVoteRequest(vote: DbVote) {
131	const rating = voteTypeRatingsMap[vote.voteType];	×
132	if (typeof rating !== 'number') {	×
133	// eslint-disable-next-line no-console
134	console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);	×
135	return;	×
136	}
137
138	return new requests.AddRating(vote.userId, vote.documentId, rating, {	×
139	timestamp: vote.votedAt.toISOString(),
140	cascadeCreate: false
141	});
142	},
143
144	createUpsertUserDetailsRequest(user: DbUser) {
145	const { displayName, karma, createdAt } = user;	×
146	return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });	×
147	},
148
149	getBatchRequest(requestBatch: requests.Request[]) {
150	return new requests.Batch(requestBatch);	×
151	},
152
153	async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration \| RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
154	if (lwAlgoSettings.loadMore) {	×
155	return {	×
156	curatedPostIds: [],
157	stickiedPostIds: [],
158	excludedPostFilter: undefined,
159	};
160	}
161
162	const postPromises = [curatedPostTerms, stickiedPostTerms]	×
163	.map(terms => viewTermsToQuery("Posts", terms, undefined, context))	×
164	.map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());	×
165
166	const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);	×
167
168	const curatedPostIds = curatedPosts.map(post => post._id);	×
169	const stickiedPostIds = stickiedPosts.map(post => post._id);	×
170	const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];	×
171	const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;	×
172
173	return {	×
174	curatedPostIds,
175	stickiedPostIds,
176	excludedPostFilter,
177	};
178	},
179
180	convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
NEW 181	const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;	×
182
NEW 183	const scenario = hybridScenarios[hybridArm];	×
184
185	const isConfigurable = hybridArm === 'configurable';	×
NEW 186	const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"\|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};	×
187	const prevRecommIdIndex = isConfigurable ? 0 : 1;	×
188	const loadMoreConfig = loadMore	×
189	? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }	×
190	: {};
191
192	return {	×
193	userId,
194	...clientConfig,
195	filter,
196	scenario,
197	...loadMoreConfig
198	};
199	},
200
201	interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
202	const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);	×
203	let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);	×
204
205	const interleavedPosts: RecombeeRecommendedPost[] = [];	×
206
207	while (interleavedPosts.length < recommendedPosts.length) {	×
208	// Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
209	const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);	×
210
211	// Weighted random selection of next recommId to pick a post from
212	let randomWeight = Math.random() * totalWeight;	×
213	let selectedRecommId: string \| undefined;
214	for (const [recommId, recommIdWeight] of weights) {	×
215	randomWeight -= recommIdWeight;	×
216	if (randomWeight <= 0) {	×
217	selectedRecommId = recommId;	×
218	break;	×
219	}
220	}
221
222	// Safety check, though selectedGroup should always be defined here
223	if (!selectedRecommId) continue;	×
224
225	// Add next post from the selected group to the interleaved array
226	const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];	×
227	const nextPost = postsWithRecommId.shift();	×
228	if (!nextPost) {	×
229	// We shouldn't ever get here, but just in case
230	continue;	×
231	} else {
232	interleavedPosts.push(nextPost);	×
233
234	// Update the group's state: remove it if it's out of posts
235	if (postsWithRecommId.length === 0) {	×
236	weights = weights.filter(([recommId]) => recommId !== selectedRecommId);	×
237	}
238	}
239	}
240
241	return interleavedPosts;	×
242	}
243	};
244
245	const curatedPostTerms: PostsViewTerms = {	18✔
246	view: 'curated',
247	limit: 3,
248	};
249
250	const recombeeApi = {	18✔
251	async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
252	const client = getRecombeeClientOrThrow();	×
253
254	const {
255	curatedPostIds,
256	stickiedPostIds,
257	excludedPostFilter
258	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
259
260	const curatedPostReadStatuses = await (	×
261	lwAlgoSettings.loadMore
262	? Promise.resolve([])	×
263	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
264	);
265
266	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
267	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;	×
268
269	const modifiedCount = count - curatedAndStickiedPostCount;	×
270	const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });	×
271
272	// We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
273	const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;	×
274	const { recomms, recommId } = recombeeResponse;	×
275
276	const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);	×
277	const recommendedPostIds = recomms.map(({ id }) => id);	×
278	const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
279
280	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
281	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)	×
282
283	const mappedPosts = filteredPosts.map(post => {	×
284	// _id isn't going to be filtered out by `accessFilterMultiple`
285	const postId = post._id!;	×
286	const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];	×
287	if (recommId) {	×
NEW 288	return { post, recommId, scenario: lwAlgoSettings.scenario };	×
289	} else {
290	return {	×
291	post,
292	curated: curatedPostIds.includes(postId),
293	stickied: stickiedPostIds.includes(postId)
294	};
295	}
296	});
297
298	return mappedPosts;	×
299	},
300
301
302	async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
303	const client = getRecombeeClientOrThrow();	×
304
305	const {
306	curatedPostIds,
307	stickiedPostIds,
308	excludedPostFilter
309	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
310
311	const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()	×
312
313	const curatedPostReadStatuses = await (	×
314	lwAlgoSettings.loadMore
315	? Promise.resolve([])	×
316	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
317	);
318
319	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
320	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;	×
321
322	const modifiedCount = count - curatedAndStickiedPostCount;	×
323	const split = 0.5;	×
324	const firstCount = Math.floor(modifiedCount * split);	×
325	const secondCount = modifiedCount - firstCount;	×
326
327	const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);	×
328	const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);	×
329
330	const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);	×
331	const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);	×
332	const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);	×
333
334	const batchResponse = await client.send(batchRequest);	×
335	// We need the type cast here because recombee's type definitions don't provide response types for batch requests
336	const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);	×
NEW 337	const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));	×
338
339	// We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
NEW 340	const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));	×
NEW 341	const recommendedPostIds = recommendationIdTuples.map(([id]) => id);	×
UNCOV 342	const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
343
344	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
345	const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));	×
346	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);	×
347
348	const mappedPosts = filteredPosts.map(post => {	×
349	// _id isn't going to be filtered out by `accessFilterMultiple`
350	const postId = post._id!;	×
NEW 351	const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];	×
NEW 352	const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];	×
353
354	if (recommId) {	×
NEW 355	return { post, recommId, scenario };	×
356	} else {
357	const stickied = stickiedPostIds.includes(postId) \|\| manuallyStickiedPostIds.includes(postId);	×
358	if (stickied) {	×
359	post.sticky = true;	×
360	}
361
362	return {	×
363	post,
364	curated: curatedPostIds.includes(postId),
365	stickied
366	};
367	}
368	});
369
370	const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);	×
371	const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);	×
372
373	const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);	×
374
375	return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];	×
376	},
377
378
379	async upsertPost(post: DbPost, context: ResolverContext) {
380	const client = getRecombeeClientOrThrow();	×
381	const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);	×
382
383	await client.send(request);	×
384	},
385
386	async createReadStatus(readStatus: DbReadStatus) {
387	const client = getRecombeeClientOrThrow();	×
388	const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);	×
389	if (!request) {	×
390	return;	×
391	}
392
393	await client.send(request);	×
394	},
395
396	async createVote(vote: DbVote) {
397	const client = getRecombeeClientOrThrow();	×
398	const request = recombeeRequestHelpers.createVoteRequest(vote);	×
399	if (!request) {	×
400	return;	×
401	}
402
403	await client.send(request);	×
404	},
405
406	async createUser(user: DbUser) {
407	const client = getRecombeeClientOrThrow();	×
408	const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);	×
409	await client.send(request);	×
410	}
411	};
412
413	export { recombeeRequestHelpers, recombeeApi };

ForumMagnum / ForumMagnum / 8838908930

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous