8838366332

Committed 25 Apr 2024 07:35PM UTC coverage: 32.042% (+0.02%) from 32.022%

Build # 8838366332

Build Type

Pull #9176

github

Committed by

darkruby501

Commit Message

revert loadmore index

Pull Request Pull Request #9176: [LW] Second Recombee Hybrid Tab for Admins

Run Details

4699 of 38106 branches covered (12.33%)

Branch coverage included in aggregate %.

1 of 5 new or added lines in 2 files covered. (20.0%)

47 existing lines in 5 files now uncovered.

27204 of 61460 relevant lines covered (44.26%)

181.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.23

/packages/lesswrong/server/recombee/client.ts

import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';
import { HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
import { loadByIds } from '../../lib/loaders';
import { filterNonnull } from '../../lib/utils/typeGuardUtils';
import { htmlToTextDefault } from '../../lib/htmlToText';
import { truncate } from '../../lib/editor/ellipsize';
import findByIds from '../vulcan-lib/findbyids';
import { accessFilterMultiple } from '../../lib/utils/schemaUtils';
import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';
import { viewTermsToQuery } from '../../lib/utils/viewUtils';
import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';
import groupBy from 'lodash/groupBy';
import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';
import { HybridArmsConfig } from '../../lib/collections/users/recommendationSettings';

export const getRecombeeClientOrThrow = (() => {
  let client: ApiClient;

  return () => {
    if (!client) {
      const databaseId = recombeeDatabaseIdSetting.get();
      const apiToken = recombeePrivateApiTokenSetting.get();

      if (!databaseId || !apiToken) {
        throw new Error('Missing either databaseId or api token when initializing Recombee client!');
      }
      
      // TODO - pull out client options like region to db settings?
      client = new ApiClient(databaseId, apiToken, { region: 'us-west' });
    }

    return client;
  };
})();

const voteTypeRatingsMap: Partial<Record<string, number>> = {
  bigDownvote: -1,
  smallDownvote: -0.5,
  smallUpvote: 0.5,
  bigUpvote: 1,
};

const HYBRID_SCENARIO_MAP = {
  fixed: 'recombee-emulate-hacker-news',
  configurable: 'recombee-personal'
};

interface OnsitePostRecommendationsInfo {
  curatedPostIds: string[],
  stickiedPostIds: string[],
  excludedPostFilter?: string,
}

export interface RecombeeRecommendedPost {
  post: Partial<DbPost>,
  scenario: string,
  recommId: string,
  curated?: never,
  stickied?: never,
}

export type RecommendedPost = RecombeeRecommendedPost | {
  post: Partial<DbPost>,
  scenario?: never,
  recommId?: never,
  curated: boolean,
  stickied: boolean,
};

const recombeeRequestHelpers = {
  createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
    const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;

    if (loadMore) {
      return new requests.RecommendNextItems(loadMore.prevRecommId, count);
    }

    const servedUserId = overrideUserId ?? userId;
    const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;

    // TODO: pass in scenario, exclude unread, etc, in options?
    const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';

    return new requests.RecommendItemsToUser(servedUserId, count, {
      ...settings,
      // Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
      booster: settings.booster || undefined,
      rotationTime: rotationTimeSeconds
    });
  },

  async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
    const { Tags } = context;

    const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)
    tags ??= filterNonnull(await findByIds(Tags, tagIds))
    const tagNames = tags.map(tag => tag.name)
    const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)

    const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))

    return new requests.SetItemValues(post._id, {
      title: post.title,
      author: post.author,
      authorId: post.userId,
      karma: post.baseScore,
      body: postText,
      postedAt: post.postedAt,
      tags: tagNames,
      coreTags: coreTagNames,
      curated: !!post.curatedDate,
      frontpage: !!post.frontpageDate,
      draft: !!post.draft,
      lastCommentedAt: post.lastCommentedAt,
    }, { cascadeCreate: true });
  },

  createReadStatusRequest(readStatus: DbReadStatus) {
    if (!readStatus.postId) {
      // eslint-disable-next-line no-console
      console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);
      return;
    }

    return new requests.AddDetailView(readStatus.userId, readStatus.postId, {
      timestamp: readStatus.lastUpdated.toISOString(),
      cascadeCreate: false
    });
  },

  createVoteRequest(vote: DbVote) {
    const rating = voteTypeRatingsMap[vote.voteType];
    if (typeof rating !== 'number') {
      // eslint-disable-next-line no-console
      console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);
      return;
    }

    return new requests.AddRating(vote.userId, vote.documentId, rating, {
      timestamp: vote.votedAt.toISOString(),
      cascadeCreate: false
    });
  },

  createUpsertUserDetailsRequest(user: DbUser) {
    const { displayName, karma, createdAt } = user;
    return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });
  },

  getBatchRequest(requestBatch: requests.Request[]) {
    return new requests.Batch(requestBatch);
  },

  async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration | RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
    if (lwAlgoSettings.loadMore) {
      return {
        curatedPostIds: [],
        stickiedPostIds: [],
        excludedPostFilter: undefined,
      };
    }

    const postPromises =  [curatedPostTerms, stickiedPostTerms]
      .map(terms => viewTermsToQuery("Posts", terms, undefined, context))
      .map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());

    const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);

    const curatedPostIds = curatedPosts.map(post => post._id);
    const stickiedPostIds = stickiedPosts.map(post => post._id);
    const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];
    const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;

    return {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter,
    };
  },

  convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
    const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;

    const scenario = hybridScenarios[hybridArm];

    const isConfigurable = hybridArm === 'configurable';
    const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};
    const prevRecommIdIndex = isConfigurable ? 0 : 1;
    const loadMoreConfig = loadMore
      ? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }
      : {};

    return {
      userId,
      ...clientConfig,
      filter,
      scenario,
      ...loadMoreConfig
    };
  },

  interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
    const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);
    let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);

    const interleavedPosts: RecombeeRecommendedPost[] = [];

    while (interleavedPosts.length < recommendedPosts.length) {
      // Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
      const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);

      // Weighted random selection of next recommId to pick a post from
      let randomWeight = Math.random() * totalWeight;
      let selectedRecommId: string | undefined;
      for (const [recommId, recommIdWeight] of weights) {
        randomWeight -= recommIdWeight;
        if (randomWeight <= 0) {
          selectedRecommId = recommId;
          break;
        }
      }

      // Safety check, though selectedGroup should always be defined here
      if (!selectedRecommId) continue;

      // Add next post from the selected group to the interleaved array
      const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];
      const nextPost = postsWithRecommId.shift();
      if (!nextPost) {
        // We shouldn't ever get here, but just in case
        continue;
      } else {
        interleavedPosts.push(nextPost);

        // Update the group's state: remove it if it's out of posts
        if (postsWithRecommId.length === 0) {
          weights = weights.filter(([recommId]) => recommId !== selectedRecommId);
        }  
      }
    }

    return interleavedPosts;
  }
};

const curatedPostTerms: PostsViewTerms = {
  view: 'curated',
  limit: 3,
};

const recombeeApi = {
  async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });

    // We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
    const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;
    const { recomms, recommId } = recombeeResponse;

    const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);
    const recommendedPostIds = recomms.map(({ id }) => id);
    const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];

    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];
      if (recommId) {
        return { post, recommId, scenario: lwAlgoSettings.scenario };
      } else {
        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied: stickiedPostIds.includes(postId)  
        };
      }
    });

    return mappedPosts;
  },


  async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const split = 0.5;
    const firstCount = Math.floor(modifiedCount * split);
    const secondCount = modifiedCount - firstCount;

    const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);
    const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);

    const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);
    const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);
    const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);

    const batchResponse = await client.send(batchRequest);
    // We need the type cast here because recombee's type definitions don't provide response types for batch requests
    const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);
    const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));

    // We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
    const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));
    const recommendedPostIds = recommendationIdTuples.map(([id]) => id);
    const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];
    
    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];
      const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];

      if (recommId) {
        return { post, recommId, scenario };
      } else {
        const stickied = stickiedPostIds.includes(postId) || manuallyStickiedPostIds.includes(postId);
        if (stickied) {
          post.sticky = true;
        }

        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied  
        };
      }
    });

    const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);
    const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);

    const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);

    return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];
  },


  async upsertPost(post: DbPost, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();
    const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);

    await client.send(request);
  },

  async createReadStatus(readStatus: DbReadStatus) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createVote(vote: DbVote) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createVoteRequest(vote);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createUser(user: DbUser) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);
    await client.send(request);
  }
};

export { recombeeRequestHelpers, recombeeApi };

1	import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';	54✔
2	import { HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
3	import { loadByIds } from '../../lib/loaders';	18✔
4	import { filterNonnull } from '../../lib/utils/typeGuardUtils';	18✔
5	import { htmlToTextDefault } from '../../lib/htmlToText';	18✔
6	import { truncate } from '../../lib/editor/ellipsize';	18✔
7	import findByIds from '../vulcan-lib/findbyids';	18✔
8	import { accessFilterMultiple } from '../../lib/utils/schemaUtils';	18✔
9	import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';	18✔
10	import { viewTermsToQuery } from '../../lib/utils/viewUtils';	18✔
11	import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';	18✔
12	import groupBy from 'lodash/groupBy';	18✔
13	import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';	18✔
14	import { HybridArmsConfig } from '../../lib/collections/users/recommendationSettings';
15
16	export const getRecombeeClientOrThrow = (() => {	18✔
17	let client: ApiClient;
18
19	return () => {	18✔
20	if (!client) {	×
21	const databaseId = recombeeDatabaseIdSetting.get();	×
22	const apiToken = recombeePrivateApiTokenSetting.get();	×
23
24	if (!databaseId \|\| !apiToken) {	×
25	throw new Error('Missing either databaseId or api token when initializing Recombee client!');	×
26	}
27
28	// TODO - pull out client options like region to db settings?
29	client = new ApiClient(databaseId, apiToken, { region: 'us-west' });	×
30	}
31
32	return client;	×
33	};
34	})();
35
36	const voteTypeRatingsMap: Partial<Record<string, number>> = {	18✔
37	bigDownvote: -1,
38	smallDownvote: -0.5,
39	smallUpvote: 0.5,
40	bigUpvote: 1,
41	};
42
43	const HYBRID_SCENARIO_MAP = {	18✔
44	fixed: 'recombee-emulate-hacker-news',
45	configurable: 'recombee-personal'
46	};
47
48	interface OnsitePostRecommendationsInfo {
49	curatedPostIds: string[],
50	stickiedPostIds: string[],
51	excludedPostFilter?: string,
52	}
53
54	export interface RecombeeRecommendedPost {
55	post: Partial<DbPost>,
56	scenario: string,
57	recommId: string,
58	curated?: never,
59	stickied?: never,
60	}
61
62	export type RecommendedPost = RecombeeRecommendedPost \| {
63	post: Partial<DbPost>,
64	scenario?: never,
65	recommId?: never,
66	curated: boolean,
67	stickied: boolean,
68	};
69
70	const recombeeRequestHelpers = {	18✔
71	createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
72	const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;	×
73
74	if (loadMore) {	×
75	return new requests.RecommendNextItems(loadMore.prevRecommId, count);	×
76	}
77
78	const servedUserId = overrideUserId ?? userId;	×
79	const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;	×
80
81	// TODO: pass in scenario, exclude unread, etc, in options?
82	const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';	×
83
84	return new requests.RecommendItemsToUser(servedUserId, count, {	×
85	...settings,
86	// Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
87	booster: settings.booster \|\| undefined,	×
88	rotationTime: rotationTimeSeconds
89	});
90	},
91
92	async createUpsertPostRequest(post: DbPost, context: ResolverContext, tags?: { name: string, core: boolean }[]) {
93	const { Tags } = context;	×
94
95	const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId)	×
96	tags ??= filterNonnull(await findByIds(Tags, tagIds))	×
97	const tagNames = tags.map(tag => tag.name)	×
98	const coreTagNames = tags.filter(tag => tag.core).map(tag => tag.name)	×
99
100	const postText = htmlToTextDefault(truncate(post.contents?.html, 2000, 'words'))	×
101
102	return new requests.SetItemValues(post._id, {	×
103	title: post.title,
104	author: post.author,
105	authorId: post.userId,
106	karma: post.baseScore,
107	body: postText,
108	postedAt: post.postedAt,
109	tags: tagNames,
110	coreTags: coreTagNames,
111	curated: !!post.curatedDate,
112	frontpage: !!post.frontpageDate,
113	draft: !!post.draft,
114	lastCommentedAt: post.lastCommentedAt,
115	}, { cascadeCreate: true });
116	},
117
118	createReadStatusRequest(readStatus: DbReadStatus) {
119	if (!readStatus.postId) {	×
120	// eslint-disable-next-line no-console
121	console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);	×
122	return;	×
123	}
124
125	return new requests.AddDetailView(readStatus.userId, readStatus.postId, {	×
126	timestamp: readStatus.lastUpdated.toISOString(),
127	cascadeCreate: false
128	});
129	},
130
131	createVoteRequest(vote: DbVote) {
132	const rating = voteTypeRatingsMap[vote.voteType];	×
133	if (typeof rating !== 'number') {	×
134	// eslint-disable-next-line no-console
135	console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);	×
136	return;	×
137	}
138
139	return new requests.AddRating(vote.userId, vote.documentId, rating, {	×
140	timestamp: vote.votedAt.toISOString(),
141	cascadeCreate: false
142	});
143	},
144
145	createUpsertUserDetailsRequest(user: DbUser) {
146	const { displayName, karma, createdAt } = user;	×
147	return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });	×
148	},
149
150	getBatchRequest(requestBatch: requests.Request[]) {
151	return new requests.Batch(requestBatch);	×
152	},
153
154	async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration \| RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
155	if (lwAlgoSettings.loadMore) {	×
156	return {	×
157	curatedPostIds: [],
158	stickiedPostIds: [],
159	excludedPostFilter: undefined,
160	};
161	}
162
163	const postPromises = [curatedPostTerms, stickiedPostTerms]	×
164	.map(terms => viewTermsToQuery("Posts", terms, undefined, context))	×
165	.map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());	×
166
167	const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);	×
168
169	const curatedPostIds = curatedPosts.map(post => post._id);	×
170	const stickiedPostIds = stickiedPosts.map(post => post._id);	×
171	const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];	×
172	const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;	×
173
174	return {	×
175	curatedPostIds,
176	stickiedPostIds,
177	excludedPostFilter,
178	};
179	},
180
181	convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
NEW 182	const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;	×
183
NEW 184	const scenario = hybridScenarios[hybridArm];	×
185
186	const isConfigurable = hybridArm === 'configurable';	×
NEW 187	const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"\|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};	×
188	const prevRecommIdIndex = isConfigurable ? 0 : 1;	×
189	const loadMoreConfig = loadMore	×
190	? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }	×
191	: {};
192
193	return {	×
194	userId,
195	...clientConfig,
196	filter,
197	scenario,
198	...loadMoreConfig
199	};
200	},
201
202	interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
203	const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);	×
204	let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);	×
205
206	const interleavedPosts: RecombeeRecommendedPost[] = [];	×
207
208	while (interleavedPosts.length < recommendedPosts.length) {	×
209	// Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
210	const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);	×
211
212	// Weighted random selection of next recommId to pick a post from
213	let randomWeight = Math.random() * totalWeight;	×
214	let selectedRecommId: string \| undefined;
215	for (const [recommId, recommIdWeight] of weights) {	×
216	randomWeight -= recommIdWeight;	×
217	if (randomWeight <= 0) {	×
218	selectedRecommId = recommId;	×
219	break;	×
220	}
221	}
222
223	// Safety check, though selectedGroup should always be defined here
224	if (!selectedRecommId) continue;	×
225
226	// Add next post from the selected group to the interleaved array
227	const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];	×
228	const nextPost = postsWithRecommId.shift();	×
229	if (!nextPost) {	×
230	// We shouldn't ever get here, but just in case
231	continue;	×
232	} else {
233	interleavedPosts.push(nextPost);	×
234
235	// Update the group's state: remove it if it's out of posts
236	if (postsWithRecommId.length === 0) {	×
237	weights = weights.filter(([recommId]) => recommId !== selectedRecommId);	×
238	}
239	}
240	}
241
242	return interleavedPosts;	×
243	}
244	};
245
246	const curatedPostTerms: PostsViewTerms = {	18✔
247	view: 'curated',
248	limit: 3,
249	};
250
251	const recombeeApi = {	18✔
252	async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
253	const client = getRecombeeClientOrThrow();	×
254
255	const {
256	curatedPostIds,
257	stickiedPostIds,
258	excludedPostFilter
259	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
260
261	const curatedPostReadStatuses = await (	×
262	lwAlgoSettings.loadMore
263	? Promise.resolve([])	×
264	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
265	);
266
267	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
268	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;	×
269
270	const modifiedCount = count - curatedAndStickiedPostCount;	×
271	const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });	×
272
273	// We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
274	const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;	×
275	const { recomms, recommId } = recombeeResponse;	×
276
277	const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);	×
278	const recommendedPostIds = recomms.map(({ id }) => id);	×
279	const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
280
281	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
282	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)	×
283
284	const mappedPosts = filteredPosts.map(post => {	×
285	// _id isn't going to be filtered out by `accessFilterMultiple`
286	const postId = post._id!;	×
287	const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];	×
288	if (recommId) {	×
289	return { post, recommId, scenario: lwAlgoSettings.scenario };	×
290	} else {
291	return {	×
292	post,
293	curated: curatedPostIds.includes(postId),
294	stickied: stickiedPostIds.includes(postId)
295	};
296	}
297	});
298
299	return mappedPosts;	×
300	},
301
302
303	async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
304	const client = getRecombeeClientOrThrow();	×
305
306	const {
307	curatedPostIds,
308	stickiedPostIds,
309	excludedPostFilter
310	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
311
312	const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()	×
313
314	const curatedPostReadStatuses = await (	×
315	lwAlgoSettings.loadMore
316	? Promise.resolve([])	×
317	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
318	);
319
320	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
321	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;	×
322
323	const modifiedCount = count - curatedAndStickiedPostCount;	×
324	const split = 0.5;	×
325	const firstCount = Math.floor(modifiedCount * split);	×
326	const secondCount = modifiedCount - firstCount;	×
327
328	const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);	×
329	const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);	×
330
331	const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);	×
332	const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);	×
333	const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);	×
334
335	const batchResponse = await client.send(batchRequest);	×
336	// We need the type cast here because recombee's type definitions don't provide response types for batch requests
337	const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);	×
338	const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));	×
339
340	// We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
341	const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));	×
342	const recommendedPostIds = recommendationIdTuples.map(([id]) => id);	×
343	const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
344
345	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
346	const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));	×
347	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);	×
348
349	const mappedPosts = filteredPosts.map(post => {	×
350	// _id isn't going to be filtered out by `accessFilterMultiple`
351	const postId = post._id!;	×
352	const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];	×
353	const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];	×
354
355	if (recommId) {	×
356	return { post, recommId, scenario };	×
357	} else {
358	const stickied = stickiedPostIds.includes(postId) \|\| manuallyStickiedPostIds.includes(postId);	×
359	if (stickied) {	×
360	post.sticky = true;	×
361	}
362
363	return {	×
364	post,
365	curated: curatedPostIds.includes(postId),
366	stickied
367	};
368	}
369	});
370
371	const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);	×
372	const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);	×
373
374	const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);	×
375
376	return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];	×
377	},
378
379
380	async upsertPost(post: DbPost, context: ResolverContext) {
381	const client = getRecombeeClientOrThrow();	×
382	const request = await recombeeRequestHelpers.createUpsertPostRequest(post, context);	×
383
384	await client.send(request);	×
385	},
386
387	async createReadStatus(readStatus: DbReadStatus) {
388	const client = getRecombeeClientOrThrow();	×
389	const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);	×
390	if (!request) {	×
391	return;	×
392	}
393
394	await client.send(request);	×
395	},
396
397	async createVote(vote: DbVote) {
398	const client = getRecombeeClientOrThrow();	×
399	const request = recombeeRequestHelpers.createVoteRequest(vote);	×
400	if (!request) {	×
401	return;	×
402	}
403
404	await client.send(request);	×
405	},
406
407	async createUser(user: DbUser) {
408	const client = getRecombeeClientOrThrow();	×
409	const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);	×
410	await client.send(request);	×
411	}
412	};
413
414	export { recombeeRequestHelpers, recombeeApi };

ForumMagnum / ForumMagnum / 8838366332

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous