8884450389

Committed 29 Apr 2024 07:41PM UTC coverage: 31.958% (-0.01%) from 31.971%

Build # 8884450389

Build Type

Pull #9194

github

Committed by

b0b3rt

Commit Message

rename view portion event

Pull Request Pull Request #9194: add google vertex real-time events and attribution id handling

Run Details

4720 of 38518 branches covered (12.25%)

Branch coverage included in aggregate %.

36 of 144 new or added lines in 16 files covered. (25.0%)

6 existing lines in 4 files now uncovered.

27517 of 62355 relevant lines covered (44.13%)

179.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.16

/packages/lesswrong/server/recombee/client.ts

import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';
import { HybridArmsConfig, HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
import { loadByIds } from '../../lib/loaders';
import { filterNonnull } from '../../lib/utils/typeGuardUtils';
import { htmlToTextDefault } from '../../lib/htmlToText';
import { truncate } from '../../lib/editor/ellipsize';
import { accessFilterMultiple } from '../../lib/utils/schemaUtils';
import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';
import { viewTermsToQuery } from '../../lib/utils/viewUtils';
import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';
import groupBy from 'lodash/groupBy';
import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';

export const getRecombeeClientOrThrow = (() => {
  let client: ApiClient;

  return () => {
    if (!client) {
      const databaseId = recombeeDatabaseIdSetting.get();
      const apiToken = recombeePrivateApiTokenSetting.get();

      if (!databaseId || !apiToken) {
        throw new Error('Missing either databaseId or api token when initializing Recombee client!');
      }
      
      // TODO - pull out client options like region to db settings?
      client = new ApiClient(databaseId, apiToken, { region: 'us-west' });
    }

    return client;
  };
})();

const voteTypeRatingsMap: Partial<Record<string, number>> = {
  bigDownvote: -1,
  smallDownvote: -0.5,
  smallUpvote: 0.5,
  bigUpvote: 1,
};

const HYBRID_SCENARIO_MAP = {
  fixed: 'recombee-emulate-hacker-news',
  configurable: 'recombee-personal'
};

interface OnsitePostRecommendationsInfo {
  curatedPostIds: string[],
  stickiedPostIds: string[],
  excludedPostFilter?: string,
}

export interface RecombeeRecommendedPost {
  post: Partial<DbPost>,
  scenario: string,
  recommId: string,
  curated?: never,
  stickied?: never,
}

export type RecommendedPost = RecombeeRecommendedPost | {
  post: Partial<DbPost>,
  scenario?: never,
  recommId?: never,
  curated: boolean,
  stickied: boolean,
};

interface PostFieldDependencies {
  title: 'title',
  author: 'author',
  authorId: 'userId',
  karma: 'baseScore',
  body: 'contents',
  postedAt: 'postedAt',
  curated: 'curatedDate',
  frontpage: 'frontpageDate',
  draft: 'draft',
  lastCommentedAt: 'lastCommentedAt',
  shortform: 'shortform',
}

interface UpsertPostData<FieldMask extends PostFieldDependencies[keyof PostFieldDependencies] = PostFieldDependencies[keyof PostFieldDependencies]> {
  post: Pick<DbPost, FieldMask | '_id'>,
  tags: Pick<DbTag, 'name' | 'core'>[],
}

const recombeePostFieldMappings = {
  title:            ({ post }: UpsertPostData) => post.title,
  author:           ({ post }: UpsertPostData) => post.author,
  authorId:         ({ post }: UpsertPostData) => post.userId,
  karma:            ({ post }: UpsertPostData) => post.baseScore,
  body:             ({ post }: UpsertPostData) => htmlToTextDefault(truncate(post.contents?.html, 2000, 'words')),
  postedAt:         ({ post }: UpsertPostData) => post.postedAt,
  tags:             ({ tags }: UpsertPostData) => tags.map(tag => tag.name),
  coreTags:         ({ tags }: UpsertPostData) => tags.filter(tag => tag.core).map(tag => tag.name),
  curated:          ({ post }: UpsertPostData) => !!post.curatedDate,
  frontpage:        ({ post }: UpsertPostData) => !!post.frontpageDate,
  draft:            ({ post }: UpsertPostData) => !!post.draft,
  lastCommentedAt:  ({ post }: UpsertPostData) => post.lastCommentedAt,
  shortform:        ({ post }: UpsertPostData) => post.shortform,
};

type RecombeePostFields = keyof typeof recombeePostFieldMappings;

const ALL_RECOMBEE_POST_FIELDS = [
  'title', 'author', 'authorId', 'karma', 'body', 'postedAt', 'tags', 'coreTags',
  'curated', 'frontpage', 'draft', 'lastCommentedAt', 'shortform'
] as const;

const recombeeRequestHelpers = {
  createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
    const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;

    if (loadMore) {
      return new requests.RecommendNextItems(loadMore.prevRecommId, count);
    }

    const servedUserId = overrideUserId ?? userId;
    const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;

    // TODO: pass in scenario, exclude unread, etc, in options?
    const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';

    return new requests.RecommendItemsToUser(servedUserId, count, {
      ...settings,
      // Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
      booster: settings.booster || undefined,
      rotationTime: rotationTimeSeconds
    });
  },

  createUpsertPostRequest<Fields extends ReadonlyArray<RecombeePostFields>>(
    postData: UpsertPostData,
    fieldMask?: Fields,
    overrideOptions?: requests.SetValuesOptions
  ) {
    const { post } = postData;
    
    const recombeePostInfo = Object.fromEntries(
      (fieldMask ?? ALL_RECOMBEE_POST_FIELDS).map(field => [
        field,
        recombeePostFieldMappings[field](postData)
      ] as const)
    );

    return new requests.SetItemValues(post._id, recombeePostInfo, { cascadeCreate: true, ...overrideOptions });
  },

  createReadStatusRequest(readStatus: DbReadStatus) {
    if (!readStatus.postId) {
      // eslint-disable-next-line no-console
      console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);
      return;
    }

    return new requests.AddDetailView(readStatus.userId, readStatus.postId, {
      timestamp: readStatus.lastUpdated.toISOString(),
      cascadeCreate: false
    });
  },

  createVoteRequest(vote: DbVote) {
    const rating = voteTypeRatingsMap[vote.voteType];
    if (typeof rating !== 'number') {
      // eslint-disable-next-line no-console
      console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);
      return;
    }

    return new requests.AddRating(vote.userId, vote.documentId, rating, {
      timestamp: vote.votedAt.toISOString(),
      cascadeCreate: false
    });
  },

  createUpsertUserDetailsRequest(user: DbUser) {
    const { displayName, karma, createdAt } = user;
    return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });
  },

  getBatchRequest(requestBatch: requests.Request[]) {
    return new requests.Batch(requestBatch);
  },

  async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration | RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
    if (lwAlgoSettings.loadMore) {
      return {
        curatedPostIds: [],
        stickiedPostIds: [],
        excludedPostFilter: undefined,
      };
    }

    const postPromises =  [curatedPostTerms, stickiedPostTerms]
      .map(terms => viewTermsToQuery("Posts", terms, undefined, context))
      .map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());

    const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);

    const curatedPostIds = curatedPosts.map(post => post._id);
    const stickiedPostIds = stickiedPosts.map(post => post._id);
    const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];
    const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;

    return {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter,
    };
  },

  convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
    const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;

    const scenario = hybridScenarios[hybridArm];

    const isConfigurable = hybridArm === 'configurable';
    const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};
    const prevRecommIdIndex = isConfigurable ? 0 : 1;
    const loadMoreConfig = loadMore
      ? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }
      : {};

    return {
      userId,
      ...clientConfig,
      filter,
      scenario,
      ...loadMoreConfig
    };
  },

  interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
    const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);
    let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);

    const interleavedPosts: RecombeeRecommendedPost[] = [];

    while (interleavedPosts.length < recommendedPosts.length) {
      // Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
      const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);

      // Weighted random selection of next recommId to pick a post from
      let randomWeight = Math.random() * totalWeight;
      let selectedRecommId: string | undefined;
      for (const [recommId, recommIdWeight] of weights) {
        randomWeight -= recommIdWeight;
        if (randomWeight <= 0) {
          selectedRecommId = recommId;
          break;
        }
      }

      // Safety check, though selectedGroup should always be defined here
      if (!selectedRecommId) continue;

      // Add next post from the selected group to the interleaved array
      const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];
      const nextPost = postsWithRecommId.shift();
      if (!nextPost) {
        // We shouldn't ever get here, but just in case
        continue;
      } else {
        interleavedPosts.push(nextPost);

        // Update the group's state: remove it if it's out of posts
        if (postsWithRecommId.length === 0) {
          weights = weights.filter(([recommId]) => recommId !== selectedRecommId);
        }  
      }
    }

    return interleavedPosts;
  }
};

const curatedPostTerms: PostsViewTerms = {
  view: 'curated',
  limit: 3,
};

const recombeeApi = {
  async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });

    // We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
    const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;
    const { recomms, recommId } = recombeeResponse;

    const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);
    const recommendedPostIds = recomms.map(({ id }) => id);
    const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];

    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];
      if (recommId) {
        return { post, recommId, scenario: lwAlgoSettings.scenario };
      } else {
        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied: stickiedPostIds.includes(postId)  
        };
      }
    });

    return mappedPosts;
  },


  async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();

    const {
      curatedPostIds,
      stickiedPostIds,
      excludedPostFilter
    } = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);

    const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()

    const curatedPostReadStatuses = await (
      lwAlgoSettings.loadMore
        ? Promise.resolve([])
        : context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
    );

    const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));
    const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;

    const modifiedCount = count - curatedAndStickiedPostCount;
    const split = 0.5;
    const firstCount = Math.floor(modifiedCount * split);
    const secondCount = modifiedCount - firstCount;

    const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);
    const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);

    const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);
    const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);
    const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);

    const batchResponse = await client.send(batchRequest);
    // We need the type cast here because recombee's type definitions don't provide response types for batch requests
    const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);
    const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));

    // We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
    const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));
    const recommendedPostIds = recommendationIdTuples.map(([id]) => id);
    const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];
    
    const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));
    const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));
    const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);

    const mappedPosts = filteredPosts.map(post => {
      // _id isn't going to be filtered out by `accessFilterMultiple`
      const postId = post._id!;
      const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];
      const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];

      if (recommId) {
        return { post, recommId, scenario };
      } else {
        const stickied = stickiedPostIds.includes(postId) || manuallyStickiedPostIds.includes(postId);
        if (stickied) {
          post.sticky = true;
        }

        return {
          post,
          curated: curatedPostIds.includes(postId),
          stickied  
        };
      }
    });

    const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);
    const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);

    const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);

    return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];
  },


  async upsertPost(post: DbPost, context: ResolverContext) {
    const client = getRecombeeClientOrThrow();
    const { Tags } = context;

    const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId);
    const tags = filterNonnull(await loadByIds(context, 'Tags', tagIds));

    const request = recombeeRequestHelpers.createUpsertPostRequest({ post, tags });

    await client.send(request);
  },

  async createReadStatus(readStatus: DbReadStatus) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createVote(vote: DbVote) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createVoteRequest(vote);
    if (!request) {
      return;
    }

    await client.send(request);
  },

  async createUser(user: DbUser) {
    const client = getRecombeeClientOrThrow();
    const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);
    await client.send(request);
  }
};

export { recombeeRequestHelpers, recombeeApi };

1	import { ApiClient, RecommendationResponse, requests } from 'recombee-api-client';	54✔
2	import { HybridArmsConfig, HybridRecombeeConfiguration, RecombeeConfiguration, RecombeeRecommendationArgs } from '../../lib/collections/users/recommendationSettings';
3	import { loadByIds } from '../../lib/loaders';	18✔
4	import { filterNonnull } from '../../lib/utils/typeGuardUtils';	18✔
5	import { htmlToTextDefault } from '../../lib/htmlToText';	18✔
6	import { truncate } from '../../lib/editor/ellipsize';	18✔
7	import { accessFilterMultiple } from '../../lib/utils/schemaUtils';	18✔
8	import { recombeeDatabaseIdSetting, recombeePrivateApiTokenSetting } from '../../lib/instanceSettings';	18✔
9	import { viewTermsToQuery } from '../../lib/utils/viewUtils';	18✔
10	import { stickiedPostTerms } from '../../components/posts/RecombeePostsList';	18✔
11	import groupBy from 'lodash/groupBy';	18✔
12	import { recommendationsTabManuallyStickiedPostIdsSetting } from '../../lib/publicSettings';	18✔
13
14	export const getRecombeeClientOrThrow = (() => {	18✔
15	let client: ApiClient;
16
17	return () => {	18✔
18	if (!client) {	×
19	const databaseId = recombeeDatabaseIdSetting.get();	×
20	const apiToken = recombeePrivateApiTokenSetting.get();	×
21
22	if (!databaseId \|\| !apiToken) {	×
23	throw new Error('Missing either databaseId or api token when initializing Recombee client!');	×
24	}
25
26	// TODO - pull out client options like region to db settings?
27	client = new ApiClient(databaseId, apiToken, { region: 'us-west' });	×
28	}
29
30	return client;	×
31	};
32	})();
33
34	const voteTypeRatingsMap: Partial<Record<string, number>> = {	18✔
35	bigDownvote: -1,
36	smallDownvote: -0.5,
37	smallUpvote: 0.5,
38	bigUpvote: 1,
39	};
40
41	const HYBRID_SCENARIO_MAP = {	18✔
42	fixed: 'recombee-emulate-hacker-news',
43	configurable: 'recombee-personal'
44	};
45
46	interface OnsitePostRecommendationsInfo {
47	curatedPostIds: string[],
48	stickiedPostIds: string[],
49	excludedPostFilter?: string,
50	}
51
52	export interface RecombeeRecommendedPost {
53	post: Partial<DbPost>,
54	scenario: string,
55	recommId: string,
56	curated?: never,
57	stickied?: never,
58	}
59
60	export type RecommendedPost = RecombeeRecommendedPost \| {
61	post: Partial<DbPost>,
62	scenario?: never,
63	recommId?: never,
64	curated: boolean,
65	stickied: boolean,
66	};
67
68	interface PostFieldDependencies {
69	title: 'title',
70	author: 'author',
71	authorId: 'userId',
72	karma: 'baseScore',
73	body: 'contents',
74	postedAt: 'postedAt',
75	curated: 'curatedDate',
76	frontpage: 'frontpageDate',
77	draft: 'draft',
78	lastCommentedAt: 'lastCommentedAt',
79	shortform: 'shortform',
80	}
81
82	interface UpsertPostData<FieldMask extends PostFieldDependencies[keyof PostFieldDependencies] = PostFieldDependencies[keyof PostFieldDependencies]> {
83	post: Pick<DbPost, FieldMask \| '_id'>,
84	tags: Pick<DbTag, 'name' \| 'core'>[],
85	}
86
87	const recombeePostFieldMappings = {	18✔
88	title: ({ post }: UpsertPostData) => post.title,	×
89	author: ({ post }: UpsertPostData) => post.author,	×
90	authorId: ({ post }: UpsertPostData) => post.userId,	×
91	karma: ({ post }: UpsertPostData) => post.baseScore,	×
92	body: ({ post }: UpsertPostData) => htmlToTextDefault(truncate(post.contents?.html, 2000, 'words')),	×
93	postedAt: ({ post }: UpsertPostData) => post.postedAt,	×
94	tags: ({ tags }: UpsertPostData) => tags.map(tag => tag.name),	×
95	coreTags: ({ tags }: UpsertPostData) => tags.filter(tag => tag.core).map(tag => tag.name),	×
96	curated: ({ post }: UpsertPostData) => !!post.curatedDate,	×
97	frontpage: ({ post }: UpsertPostData) => !!post.frontpageDate,	×
98	draft: ({ post }: UpsertPostData) => !!post.draft,	×
99	lastCommentedAt: ({ post }: UpsertPostData) => post.lastCommentedAt,	×
100	shortform: ({ post }: UpsertPostData) => post.shortform,	×
101	};
102
103	type RecombeePostFields = keyof typeof recombeePostFieldMappings;
104
105	const ALL_RECOMBEE_POST_FIELDS = [	18✔
106	'title', 'author', 'authorId', 'karma', 'body', 'postedAt', 'tags', 'coreTags',
107	'curated', 'frontpage', 'draft', 'lastCommentedAt', 'shortform'
108	] as const;
109
110	const recombeeRequestHelpers = {	18✔
111	createRecommendationsForUserRequest(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs) {
112	const { userId: overrideUserId, rotationTime, lwRationalityOnly, onlyUnread, loadMore, ...settings } = lwAlgoSettings;	×
113
114	if (loadMore) {	×
115	return new requests.RecommendNextItems(loadMore.prevRecommId, count);	×
116	}
117
118	const servedUserId = overrideUserId ?? userId;	×
119	const rotationTimeSeconds = typeof rotationTime === 'number' ? rotationTime * 3600 : undefined;	×
120
121	// TODO: pass in scenario, exclude unread, etc, in options?
122	const lwRationalityFilter = lwRationalityOnly ? ` and ("Rationality" in 'core_tags' or "World Modeling" in 'core_tags')` : '';	×
123
124	return new requests.RecommendItemsToUser(servedUserId, count, {	×
125	...settings,
126	// Explicitly coalesce empty strings to undefined, since empty strings aren't valid booster expressions
127	booster: settings.booster \|\| undefined,	×
128	rotationTime: rotationTimeSeconds
129	});
130	},
131
132	createUpsertPostRequest<Fields extends ReadonlyArray<RecombeePostFields>>(
133	postData: UpsertPostData,
134	fieldMask?: Fields,
135	overrideOptions?: requests.SetValuesOptions
136	) {
137	const { post } = postData;	×
138
139	const recombeePostInfo = Object.fromEntries(	×
140	(fieldMask ?? ALL_RECOMBEE_POST_FIELDS).map(field => [	×
141	field,
142	recombeePostFieldMappings[field](postData)
143	] as const)
144	);
145
146	return new requests.SetItemValues(post._id, recombeePostInfo, { cascadeCreate: true, ...overrideOptions });	×
147	},
148
149	createReadStatusRequest(readStatus: DbReadStatus) {
150	if (!readStatus.postId) {	×
151	// eslint-disable-next-line no-console
152	console.error(`Missing postId for read status ${readStatus._id} when trying to add detail view to recombee`);	×
153	return;	×
154	}
155
156	return new requests.AddDetailView(readStatus.userId, readStatus.postId, {	×
157	timestamp: readStatus.lastUpdated.toISOString(),
158	cascadeCreate: false
159	});
160	},
161
162	createVoteRequest(vote: DbVote) {
163	const rating = voteTypeRatingsMap[vote.voteType];	×
164	if (typeof rating !== 'number') {	×
165	// eslint-disable-next-line no-console
166	console.log(`Attempted to create a recombee rating request for a non-karma vote with id ${vote._id}`);	×
167	return;	×
168	}
169
170	return new requests.AddRating(vote.userId, vote.documentId, rating, {	×
171	timestamp: vote.votedAt.toISOString(),
172	cascadeCreate: false
173	});
174	},
175
176	createUpsertUserDetailsRequest(user: DbUser) {
177	const { displayName, karma, createdAt } = user;	×
178	return new requests.SetUserValues(user._id, { displayName, karma, createdAt }, { cascadeCreate: true });	×
179	},
180
181	getBatchRequest(requestBatch: requests.Request[]) {
182	return new requests.Batch(requestBatch);	×
183	},
184
185	async getOnsitePostInfo(lwAlgoSettings: HybridRecombeeConfiguration \| RecombeeConfiguration, context: ResolverContext): Promise<OnsitePostRecommendationsInfo> {
186	if (lwAlgoSettings.loadMore) {	×
187	return {	×
188	curatedPostIds: [],
189	stickiedPostIds: [],
190	excludedPostFilter: undefined,
191	};
192	}
193
194	const postPromises = [curatedPostTerms, stickiedPostTerms]	×
195	.map(terms => viewTermsToQuery("Posts", terms, undefined, context))	×
196	.map(postsQuery => context.Posts.find(postsQuery.selector, postsQuery.options, { _id: 1 }).fetch());	×
197
198	const [curatedPosts, stickiedPosts] = await Promise.all(postPromises);	×
199
200	const curatedPostIds = curatedPosts.map(post => post._id);	×
201	const stickiedPostIds = stickiedPosts.map(post => post._id);	×
202	const excludedPostIds = [...curatedPostIds, ...stickiedPostIds];	×
203	const excludedPostFilter = `'itemId' not in {${excludedPostIds.map(id => `"${id}"`).join(', ')}}`;	×
204
205	return {	×
206	curatedPostIds,
207	stickiedPostIds,
208	excludedPostFilter,
209	};
210	},
211
212	convertHybridToRecombeeArgs(hybridArgs: HybridRecombeeConfiguration, hybridArm: keyof HybridArmsConfig, filter?: string) {
213	const { loadMore, userId, hybridScenarios, ...rest } = hybridArgs;	×
214
215	const scenario = hybridScenarios[hybridArm];	×
216
217	const isConfigurable = hybridArm === 'configurable';	×
218	const clientConfig: Partial<Omit<HybridRecombeeConfiguration,"loadMore"\|"userId">> = isConfigurable ? rest : {rotationRate: 0.1, rotationTime: 12};	×
219	const prevRecommIdIndex = isConfigurable ? 0 : 1;	×
220	const loadMoreConfig = loadMore	×
221	? { loadMore: { prevRecommId: loadMore.prevRecommIds[prevRecommIdIndex] } }	×
222	: {};
223
224	return {	×
225	userId,
226	...clientConfig,
227	filter,
228	scenario,
229	...loadMoreConfig
230	};
231	},
232
233	interleaveHybridRecommendedPosts(recommendedPosts: RecombeeRecommendedPost[]) {
234	const recommendationGroupedPosts = groupBy(recommendedPosts, (result) => result.recommId);	×
235	let weights = Object.entries(recommendationGroupedPosts).map(([recommId, posts]) => [recommId, posts.length / recommendedPosts.length] as const);	×
236
237	const interleavedPosts: RecombeeRecommendedPost[] = [];	×
238
239	while (interleavedPosts.length < recommendedPosts.length) {	×
240	// Calculate current total weight (since it changes when we remove a recommId group for running out of posts)
241	const totalWeight = weights.reduce((sum, [_, weight]) => sum + weight, 0);	×
242
243	// Weighted random selection of next recommId to pick a post from
244	let randomWeight = Math.random() * totalWeight;	×
245	let selectedRecommId: string \| undefined;
246	for (const [recommId, recommIdWeight] of weights) {	×
247	randomWeight -= recommIdWeight;	×
248	if (randomWeight <= 0) {	×
249	selectedRecommId = recommId;	×
250	break;	×
251	}
252	}
253
254	// Safety check, though selectedGroup should always be defined here
255	if (!selectedRecommId) continue;	×
256
257	// Add next post from the selected group to the interleaved array
258	const postsWithRecommId = recommendationGroupedPosts[selectedRecommId];	×
259	const nextPost = postsWithRecommId.shift();	×
260	if (!nextPost) {	×
261	// We shouldn't ever get here, but just in case
262	continue;	×
263	} else {
264	interleavedPosts.push(nextPost);	×
265
266	// Update the group's state: remove it if it's out of posts
267	if (postsWithRecommId.length === 0) {	×
268	weights = weights.filter(([recommId]) => recommId !== selectedRecommId);	×
269	}
270	}
271	}
272
273	return interleavedPosts;	×
274	}
275	};
276
277	const curatedPostTerms: PostsViewTerms = {	18✔
278	view: 'curated',
279	limit: 3,
280	};
281
282	const recombeeApi = {	18✔
283	async getRecommendationsForUser(userId: string, count: number, lwAlgoSettings: RecombeeRecommendationArgs, context: ResolverContext) {
284	const client = getRecombeeClientOrThrow();	×
285
286	const {
287	curatedPostIds,
288	stickiedPostIds,
289	excludedPostFilter
290	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
291
292	const curatedPostReadStatuses = await (	×
293	lwAlgoSettings.loadMore
294	? Promise.resolve([])	×
295	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
296	);
297
298	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
299	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length;	×
300
301	const modifiedCount = count - curatedAndStickiedPostCount;	×
302	const recommendationsRequestBody = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, modifiedCount, { ...lwAlgoSettings, filter: excludedPostFilter });	×
303
304	// We need the type cast here because recombee's type definitions can't handle inferring response types for union request types, even if they have the same response type
305	const recombeeResponse = await client.send(recommendationsRequestBody) as RecommendationResponse;	×
306	const { recomms, recommId } = recombeeResponse;	×
307
308	const recommendationIdPairs = recomms.map(rec => [rec.id, recommId] as const);	×
309	const recommendedPostIds = recomms.map(({ id }) => id);	×
310	const postIds = [...includedCuratedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
311
312	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
313	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, posts, context)	×
314
315	const mappedPosts = filteredPosts.map(post => {	×
316	// _id isn't going to be filtered out by `accessFilterMultiple`
317	const postId = post._id!;	×
318	const recommId = recommendationIdPairs.find(([id]) => id === postId)?.[1];	×
319	if (recommId) {	×
320	return { post, recommId, scenario: lwAlgoSettings.scenario };	×
321	} else {
322	return {	×
323	post,
324	curated: curatedPostIds.includes(postId),
325	stickied: stickiedPostIds.includes(postId)
326	};
327	}
328	});
329
330	return mappedPosts;	×
331	},
332
333
334	async getHybridRecommendationsForUser(userId: string, count: number, lwAlgoSettings: HybridRecombeeConfiguration, context: ResolverContext) {
335	const client = getRecombeeClientOrThrow();	×
336
337	const {
338	curatedPostIds,
339	stickiedPostIds,
340	excludedPostFilter
341	} = await recombeeRequestHelpers.getOnsitePostInfo(lwAlgoSettings, context);	×
342
343	const manuallyStickiedPostIds = recommendationsTabManuallyStickiedPostIdsSetting.get()	×
344
345	const curatedPostReadStatuses = await (	×
346	lwAlgoSettings.loadMore
347	? Promise.resolve([])	×
348	: context.ReadStatuses.find({ postId: { $in: curatedPostIds.slice(1) }, userId, isRead: true }).fetch()
349	);
350
351	const includedCuratedPostIds = curatedPostIds.filter(id => !curatedPostReadStatuses.find(readStatus => readStatus.postId === id));	×
352	const curatedAndStickiedPostCount = includedCuratedPostIds.length + stickiedPostIds.length + manuallyStickiedPostIds.length;	×
353
354	const modifiedCount = count - curatedAndStickiedPostCount;	×
355	const split = 0.5;	×
356	const firstCount = Math.floor(modifiedCount * split);	×
357	const secondCount = modifiedCount - firstCount;	×
358
359	const firstRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'configurable', excludedPostFilter);	×
360	const secondRequestSettings = recombeeRequestHelpers.convertHybridToRecombeeArgs(lwAlgoSettings, 'fixed', excludedPostFilter);	×
361
362	const firstRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, firstCount, firstRequestSettings);	×
363	const secondRequest = recombeeRequestHelpers.createRecommendationsForUserRequest(userId, secondCount, secondRequestSettings);	×
364	const batchRequest = recombeeRequestHelpers.getBatchRequest([firstRequest, secondRequest]);	×
365
366	const batchResponse = await client.send(batchRequest);	×
367	// We need the type cast here because recombee's type definitions don't provide response types for batch requests
368	const recombeeResponses = batchResponse.map(({json}) => json as RecommendationResponse);	×
369	const recombeeResponsesWithScenario = recombeeResponses.map((response, index) => ({ ...response, scenario: index === 0 ? firstRequestSettings.scenario : secondRequestSettings.scenario }));	×
370
371	// We explicitly avoid deduplicating postIds because we want to see how often the same post is recommended by both arms of the hybrid recommender
372	const recommendationIdTuples = recombeeResponsesWithScenario.flatMap(response => response.recomms.map(rec => [rec.id, response.recommId, response.scenario] as const));	×
373	const recommendedPostIds = recommendationIdTuples.map(([id]) => id);	×
374	const postIds = [...includedCuratedPostIds, ...manuallyStickiedPostIds, ...stickiedPostIds, ...recommendedPostIds];	×
375
376	const posts = filterNonnull(await loadByIds(context, 'Posts', postIds));	×
377	const orderedPosts = filterNonnull(postIds.map(id => posts.find(post => post._id === id)));	×
378	const filteredPosts = await accessFilterMultiple(context.currentUser, context.Posts, orderedPosts, context);	×
379
380	const mappedPosts = filteredPosts.map(post => {	×
381	// _id isn't going to be filtered out by `accessFilterMultiple`
382	const postId = post._id!;	×
383	const recommId = recommendationIdTuples.find(([id]) => id === postId)?.[1];	×
384	const scenario = recommendationIdTuples.find(([id]) => id === postId)?.[2];	×
385
386	if (recommId) {	×
387	return { post, recommId, scenario };	×
388	} else {
389	const stickied = stickiedPostIds.includes(postId) \|\| manuallyStickiedPostIds.includes(postId);	×
390	if (stickied) {	×
391	post.sticky = true;	×
392	}
393
394	return {	×
395	post,
396	curated: curatedPostIds.includes(postId),
397	stickied
398	};
399	}
400	});
401
402	const curatedOrStickiedPosts = mappedPosts.filter((result): result is Exclude<RecommendedPost, RecombeeRecommendedPost> => !result.recommId);	×
403	const recommendedPosts = mappedPosts.filter((result): result is RecombeeRecommendedPost => !!result.recommId);	×
404
405	const interleavedRecommendedPosts = recombeeRequestHelpers.interleaveHybridRecommendedPosts(recommendedPosts);	×
406
407	return [...curatedOrStickiedPosts, ...interleavedRecommendedPosts];	×
408	},
409
410
411	async upsertPost(post: DbPost, context: ResolverContext) {
412	const client = getRecombeeClientOrThrow();	×
413	const { Tags } = context;	×
414
415	const tagIds = Object.entries(post.tagRelevance ?? {}).filter(([_, relevance]: [string, number]) => relevance > 0).map(([tagId]) => tagId);	×
NEW 416	const tags = filterNonnull(await loadByIds(context, 'Tags', tagIds));	×
417
418	const request = recombeeRequestHelpers.createUpsertPostRequest({ post, tags });	×
419
420	await client.send(request);	×
421	},
422
423	async createReadStatus(readStatus: DbReadStatus) {
424	const client = getRecombeeClientOrThrow();	×
425	const request = recombeeRequestHelpers.createReadStatusRequest(readStatus);	×
426	if (!request) {	×
427	return;	×
428	}
429
430	await client.send(request);	×
431	},
432
433	async createVote(vote: DbVote) {
434	const client = getRecombeeClientOrThrow();	×
435	const request = recombeeRequestHelpers.createVoteRequest(vote);	×
436	if (!request) {	×
437	return;	×
438	}
439
440	await client.send(request);	×
441	},
442
443	async createUser(user: DbUser) {
444	const client = getRecombeeClientOrThrow();	×
445	const request = recombeeRequestHelpers.createUpsertUserDetailsRequest(user);	×
446	await client.send(request);	×
447	}
448	};
449
450	export { recombeeRequestHelpers, recombeeApi };

ForumMagnum / ForumMagnum / 8884450389

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous