9406031023

Committed 06 Jun 2024 06:30PM UTC coverage: 31.94% (-0.003%) from 31.943%

Build # 9406031023

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9370 from ForumMagnum/fix-subscribed-feed-scroll-loading

fix loading more items when scrolling down subscribed feed

Run Details

4815 of 39261 branches covered (12.26%)

Branch coverage included in aggregate %.

2 of 12 new or added lines in 5 files covered. (16.67%)

1 existing line in 1 file now uncovered.

27975 of 63401 relevant lines covered (44.12%)

180.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.48

/packages/lesswrong/server/utils/feedUtil.ts

import _ from 'underscore';
import { addGraphQLResolvers, addGraphQLQuery, addGraphQLSchema } from '../../lib/vulcan-lib/graphql';
import { accessFilterMultiple } from '../../lib/utils/schemaUtils';
import { getDefaultViewSelector, mergeSelectors, replaceSpecialFieldSelectors } from '../../lib/utils/viewUtils';

type FeedSubquery<ResultType extends DbObject, SortKeyType> = {
  type: string,
  getSortKey: (item: ResultType) => SortKeyType,
  isNumericallyPositioned?: boolean,
  doQuery: (limit: number, cutoff?: SortKeyType) => Promise<Partial<ResultType>[]>,
}

export type SortDirection = "asc" | "desc";

export type SubquerySortField<ResultType extends DbObject, SortFieldName extends keyof ResultType> = {
  sortField: SortFieldName,
  sortDirection?: SortDirection,
}

type Sortable<SortKey extends number | Date> = { sortKey: SortKey };

type ViewBasedSubqueryProps<
  N extends CollectionNameString,
  SortFieldName extends keyof ObjectsByCollectionName[N]
> = {
  type: string,
  collection: CollectionBase<N>,
  context: ResolverContext,
  selector: MongoSelector<ObjectsByCollectionName[N]>,
  sticky?: boolean,
} & SubquerySortField<ObjectsByCollectionName[N], SortFieldName>;

export function viewBasedSubquery<
  N extends CollectionNameString,
  SortKeyType,
  SortFieldName extends keyof ObjectsByCollectionName[N]
>(props: ViewBasedSubqueryProps<N, SortFieldName>): FeedSubquery<ObjectsByCollectionName[N], SortKeyType> {
  props.sortDirection ??= "desc";
  const {type, collection, context, selector, sticky, sortField, sortDirection} = props;
  return {
    type,
    getSortKey: (item: ObjectsByCollectionName[N]) => item[props.sortField] as unknown as SortKeyType,
    isNumericallyPositioned: !!sticky,
    doQuery: async (limit: number, cutoff: SortKeyType): Promise<Partial<ObjectsByCollectionName[N]>[]> => {
      return queryWithCutoff({context, collection, selector, limit, cutoffField: sortField, cutoff, sortDirection});
    }
  };
}

export function fixedResultSubquery<ResultType extends DbObject, SortKeyType>({type, result, sortKey}: {
  type: string,
  result: Partial<ResultType>,
  sortKey: SortKeyType,
}): FeedSubquery<ResultType, SortKeyType> {
  return {
    type,
    getSortKey: (_item: ResultType): SortKeyType => sortKey,
    doQuery: async (_limit: number, _cutoff: SortKeyType): Promise<Partial<ResultType>[]> => {
      return [result];
    }
  };
}

export function fixedIndexSubquery<ResultType extends DbObject>({type, index, result}: {
  type: string,
  index: number,
  result: any,
}): FeedSubquery<ResultType, number> {
  return {
    type,
    getSortKey: () => index,
    isNumericallyPositioned: true,
    doQuery: async () => [result],
  };
}

export function defineFeedResolver<CutoffType>({name, resolver, args, cutoffTypeGraphQL, resultTypesGraphQL}: {
  name: string,
  resolver: ({limit, cutoff, args, context}: {
    limit?: number,
    cutoff?: CutoffType|null,
    offset?: number,
    args: any, context: ResolverContext
  }) => Promise<{
    cutoff: CutoffType|null,
    results: Array<any>
  }>,
  args: string,
  cutoffTypeGraphQL: string,
  resultTypesGraphQL: string,
}) {
  addGraphQLSchema(`
    type ${name}QueryResults {
      cutoff: ${cutoffTypeGraphQL}
      endOffset: Int!
      results: [${name}EntryType!]
    }
    type ${name}EntryType {
      type: String!
      ${resultTypesGraphQL}
    }
  `);
  addGraphQLQuery(`${name}(
    limit: Int,
    cutoff: ${cutoffTypeGraphQL},
    offset: Int
    ${(args && args.length>0) ? ", " : ""}${args}
  ): ${name}QueryResults!`);
  
  addGraphQLResolvers({
    Query: {
      [name]: async (_root: void, args: any, context: ResolverContext) => {
        const {limit, cutoff, offset, ...rest} = args;
        return {
          __typename: `${name}QueryResults`,
          ...await resolver({
            limit, cutoff, offset,
            args: rest,
            context
          })
        };
      }
    },
  });
}

const applyCutoff = <T extends Sortable<SortKeyType>, SortKeyType extends number | Date>(
  sortedResults: T[],
  cutoff: SortKeyType,
  sortDirection: SortDirection,
) => {
  const cutoffFilter = sortDirection === "asc"
    ? ({sortKey}: { sortKey: SortKeyType }) => sortKey > cutoff
    : ({sortKey}: { sortKey: SortKeyType }) => sortKey < cutoff;
  return _.filter<T>(sortedResults, cutoffFilter);
}

export async function mergeFeedQueries<SortKeyType extends number | Date>({limit, cutoff, offset, sortDirection, subqueries}: {
  limit: number
  cutoff?: SortKeyType,
  offset?: number,
  sortDirection?: SortDirection,
  subqueries: Array<FeedSubquery<DbObject, any>>,
}) {
  sortDirection ??= "desc";

  // Perform the subqueries
  const unsortedSubqueryResults = await Promise.all(
    subqueries.map(async (subquery) => {
      const subqueryResults = await subquery.doQuery(limit, cutoff)
      return subqueryResults.map((result: DbObject) => ({
        type: subquery.type,
        sortKey: subquery.getSortKey(result),
        isNumericallyPositioned: subquery.isNumericallyPositioned,
        [subquery.type]: result,
      }))
    })
  );
  
  // Merge the result lists
  const unsortedResults = unsortedSubqueryResults.flat();
  
  // Split into results with numeric indexes and results with sort-key indexes
  const [
    numericallyPositionedResults,
    orderedResults,
  ] = _.partition(unsortedResults, ({isNumericallyPositioned}) => !!isNumericallyPositioned);
  
  // Sort by shared sort key
  const sortedResults = _.sortBy(orderedResults, r=>r.sortKey);
  if (sortDirection === "desc") {
    sortedResults.reverse();
  }
  
  // Apply cutoff
  const withCutoffApplied = cutoff
    ? applyCutoff(sortedResults, cutoff, sortDirection)
    : sortedResults;
  
  // Merge in the numerically positioned results
  const bothResultKinds = mergeSortedAndNumericallyPositionedResults(withCutoffApplied, numericallyPositionedResults, offset||0);
  
  // Apply limit
  const withLimitApplied = _.first(bothResultKinds, limit);
  
  // Find the last result that wasn't numerically positioned (after the limit
  // is applied), and get its sortKey to use as the page cutoff
  const nonNumericallyPositionedResults = _.filter(withLimitApplied, r => !_.some(numericallyPositionedResults, r2=>r===r2)) as Sortable<SortKeyType>[];
  const nextCutoff = (nonNumericallyPositionedResults.length>0) ? nonNumericallyPositionedResults[nonNumericallyPositionedResults.length-1].sortKey : null;
  
  return {
    results: withLimitApplied,
    cutoff: nextCutoff,
    endOffset: (offset||0)+withLimitApplied.length
  };
}

// Take a list of results that are sorted by a sort key (ie a date), and a list
// of results that have numeric indexes instead, and merge them. Eg, Recent
// Discussion contains posts sorted by date, but with some things mixed in
// with their position defined as "index 5".
function mergeSortedAndNumericallyPositionedResults(sortedResults: Array<Sortable<Date>>, numericallyPositionedResults: Array<Sortable<number>>, offset: number) {
  // Take the numerically positioned results. Sort them by index, discard ones
  // from below the offset, and resolve collisions.
  const sortedNumericallyPositionedResults = _.sortBy(numericallyPositionedResults, r=>r.sortKey);
  
  let mergedResults: Sortable<number | Date>[] = [...sortedResults];
  for (let i=0; i<sortedNumericallyPositionedResults.length; i++) {
    const insertedResult = sortedNumericallyPositionedResults[i];
    const insertionPosition = insertedResult.sortKey-offset;
    
    if (insertionPosition >= 0) {
      if (insertionPosition < mergedResults.length) {
        mergedResults.splice(insertionPosition, 0, insertedResult);
      } else {
        mergedResults.push(insertedResult);
      }
    }
  }
  
  return mergedResults;
}

function isNonEmptyObject(obj: {}): boolean {
  return Object.keys(obj).length > 0;
}

async function queryWithCutoff<N extends CollectionNameString>({
  context, collection, selector, limit, cutoffField, cutoff, sortDirection,
}: {
  context: ResolverContext,
  collection: CollectionBase<N>,
  selector: MongoSelector<ObjectsByCollectionName[N]>,
  limit: number,
  cutoffField: keyof ObjectsByCollectionName[N],
  cutoff: any,
  sortDirection: SortDirection,
}) {
  const collectionName = collection.collectionName;
  const {currentUser} = context;

  const sort = {[cutoffField]: sortDirection === "asc" ? 1 : -1, _id: 1};
  const cutoffSelector = cutoff
    ? {[cutoffField]: {[sortDirection === "asc" ? "$gt" : "$lt"]: cutoff}}
    : {};
  const mergedSelector = mergeSelectors(
    getDefaultViewSelector(collectionName),
    selector,
    cutoffSelector
  )
  const finalizedSelector = replaceSpecialFieldSelectors(mergedSelector);
  const resultsRaw = await collection.find(finalizedSelector, {
    sort: sort as Partial<Record<keyof ObjectsByCollectionName[N], number>>,
    limit,
  }).fetch();

  return await accessFilterMultiple(currentUser, collection, resultsRaw, context);
}

1	import _ from 'underscore';	216✔
2	import { addGraphQLResolvers, addGraphQLQuery, addGraphQLSchema } from '../../lib/vulcan-lib/graphql';	18✔
3	import { accessFilterMultiple } from '../../lib/utils/schemaUtils';	18✔
4	import { getDefaultViewSelector, mergeSelectors, replaceSpecialFieldSelectors } from '../../lib/utils/viewUtils';	18✔
5
6	type FeedSubquery<ResultType extends DbObject, SortKeyType> = {
7	type: string,
8	getSortKey: (item: ResultType) => SortKeyType,
9	isNumericallyPositioned?: boolean,
10	doQuery: (limit: number, cutoff?: SortKeyType) => Promise<Partial<ResultType>[]>,
11	}
12
13	export type SortDirection = "asc" \| "desc";
14
15	export type SubquerySortField<ResultType extends DbObject, SortFieldName extends keyof ResultType> = {
16	sortField: SortFieldName,
17	sortDirection?: SortDirection,
18	}
19
20	type Sortable<SortKey extends number \| Date> = { sortKey: SortKey };
21
22	type ViewBasedSubqueryProps<
23	N extends CollectionNameString,
24	SortFieldName extends keyof ObjectsByCollectionName[N]
25	> = {
26	type: string,
27	collection: CollectionBase<N>,
28	context: ResolverContext,
29	selector: MongoSelector<ObjectsByCollectionName[N]>,
30	sticky?: boolean,
31	} & SubquerySortField<ObjectsByCollectionName[N], SortFieldName>;
32
33	export function viewBasedSubquery<
34	N extends CollectionNameString,
35	SortKeyType,
36	SortFieldName extends keyof ObjectsByCollectionName[N]
37	>(props: ViewBasedSubqueryProps<N, SortFieldName>): FeedSubquery<ObjectsByCollectionName[N], SortKeyType> {
38	props.sortDirection ??= "desc";	×
39	const {type, collection, context, selector, sticky, sortField, sortDirection} = props;	×
40	return {	×
41	type,
42	getSortKey: (item: ObjectsByCollectionName[N]) => item[props.sortField] as unknown as SortKeyType,	×
43	isNumericallyPositioned: !!sticky,
44	doQuery: async (limit: number, cutoff: SortKeyType): Promise<Partial<ObjectsByCollectionName[N]>[]> => {
45	return queryWithCutoff({context, collection, selector, limit, cutoffField: sortField, cutoff, sortDirection});	×
46	}
47	};
48	}
49
50	export function fixedResultSubquery<ResultType extends DbObject, SortKeyType>({type, result, sortKey}: {
51	type: string,
52	result: Partial<ResultType>,
53	sortKey: SortKeyType,
54	}): FeedSubquery<ResultType, SortKeyType> {
55	return {	×
56	type,
57	getSortKey: (_item: ResultType): SortKeyType => sortKey,	×
58	doQuery: async (_limit: number, _cutoff: SortKeyType): Promise<Partial<ResultType>[]> => {
59	return [result];	×
60	}
61	};
62	}
63
64	export function fixedIndexSubquery<ResultType extends DbObject>({type, index, result}: {
65	type: string,
66	index: number,
67	result: any,
68	}): FeedSubquery<ResultType, number> {
69	return {	×
70	type,
71	getSortKey: () => index,	×
72	isNumericallyPositioned: true,
73	doQuery: async () => [result],	×
74	};
75	}
76
77	export function defineFeedResolver<CutoffType>({name, resolver, args, cutoffTypeGraphQL, resultTypesGraphQL}: {
78	name: string,
79	resolver: ({limit, cutoff, args, context}: {
80	limit?: number,
81	cutoff?: CutoffType\|null,
82	offset?: number,
83	args: any, context: ResolverContext
84	}) => Promise<{
85	cutoff: CutoffType\|null,
86	results: Array<any>
87	}>,
88	args: string,
89	cutoffTypeGraphQL: string,
90	resultTypesGraphQL: string,
91	}) {
92	addGraphQLSchema(`	162✔
93	type ${name}QueryResults {
94	cutoff: ${cutoffTypeGraphQL}
95	endOffset: Int!
96	results: [${name}EntryType!]
97	}
98	type ${name}EntryType {
99	type: String!
100	${resultTypesGraphQL}
101	}
102	`);
103	addGraphQLQuery(`${name}(	162✔
104	limit: Int,
105	cutoff: ${cutoffTypeGraphQL},
106	offset: Int
107	${(args && args.length>0) ? ", " : ""}${args}	468✔
108	): ${name}QueryResults!`);
109
110	addGraphQLResolvers({	162✔
111	Query: {
112	[name]: async (_root: void, args: any, context: ResolverContext) => {
113	const {limit, cutoff, offset, ...rest} = args;	×
114	return {	×
115	__typename: `${name}QueryResults`,
116	...await resolver({
117	limit, cutoff, offset,
118	args: rest,
119	context
120	})
121	};
122	}
123	},
124	});
125	}
126
127	const applyCutoff = <T extends Sortable<SortKeyType>, SortKeyType extends number \| Date>(	18✔
128	sortedResults: T[],
129	cutoff: SortKeyType,
130	sortDirection: SortDirection,
131	) => {
132	const cutoffFilter = sortDirection === "asc"	×
133	? ({sortKey}: { sortKey: SortKeyType }) => sortKey > cutoff	×
134	: ({sortKey}: { sortKey: SortKeyType }) => sortKey < cutoff;	×
NEW 135	return _.filter<T>(sortedResults, cutoffFilter);	×
136	}
137
138	export async function mergeFeedQueries<SortKeyType extends number \| Date>({limit, cutoff, offset, sortDirection, subqueries}: {
139	limit: number
140	cutoff?: SortKeyType,
141	offset?: number,
142	sortDirection?: SortDirection,
143	subqueries: Array<FeedSubquery<DbObject, any>>,
144	}) {
145	sortDirection ??= "desc";	×
146
147	// Perform the subqueries
148	const unsortedSubqueryResults = await Promise.all(	×
149	subqueries.map(async (subquery) => {
150	const subqueryResults = await subquery.doQuery(limit, cutoff)	×
151	return subqueryResults.map((result: DbObject) => ({	×
152	type: subquery.type,
153	sortKey: subquery.getSortKey(result),
154	isNumericallyPositioned: subquery.isNumericallyPositioned,
155	[subquery.type]: result,
156	}))
157	})
158	);
159
160	// Merge the result lists
NEW 161	const unsortedResults = unsortedSubqueryResults.flat();	×
162
163	// Split into results with numeric indexes and results with sort-key indexes
164	const [
165	numericallyPositionedResults,
166	orderedResults,
NEW 167	] = _.partition(unsortedResults, ({isNumericallyPositioned}) => !!isNumericallyPositioned);	×
168
169	// Sort by shared sort key
170	const sortedResults = _.sortBy(orderedResults, r=>r.sortKey);	×
171	if (sortDirection === "desc") {	×
172	sortedResults.reverse();	×
173	}
174
175	// Apply cutoff
176	const withCutoffApplied = cutoff	×
177	? applyCutoff(sortedResults, cutoff, sortDirection)	×
178	: sortedResults;
179
180	// Merge in the numerically positioned results
181	const bothResultKinds = mergeSortedAndNumericallyPositionedResults(withCutoffApplied, numericallyPositionedResults, offset\|\|0);	×
182
183	// Apply limit
184	const withLimitApplied = _.first(bothResultKinds, limit);	×
185
186	// Find the last result that wasn't numerically positioned (after the limit
187	// is applied), and get its sortKey to use as the page cutoff
NEW 188	const nonNumericallyPositionedResults = _.filter(withLimitApplied, r => !_.some(numericallyPositionedResults, r2=>r===r2)) as Sortable<SortKeyType>[];	×
189	const nextCutoff = (nonNumericallyPositionedResults.length>0) ? nonNumericallyPositionedResults[nonNumericallyPositionedResults.length-1].sortKey : null;	×
190
191	return {	×
192	results: withLimitApplied,
193	cutoff: nextCutoff,
194	endOffset: (offset\|\|0)+withLimitApplied.length	×
195	};
196	}
197
198	// Take a list of results that are sorted by a sort key (ie a date), and a list
199	// of results that have numeric indexes instead, and merge them. Eg, Recent
200	// Discussion contains posts sorted by date, but with some things mixed in
201	// with their position defined as "index 5".
202	function mergeSortedAndNumericallyPositionedResults(sortedResults: Array<Sortable<Date>>, numericallyPositionedResults: Array<Sortable<number>>, offset: number) {
203	// Take the numerically positioned results. Sort them by index, discard ones
204	// from below the offset, and resolve collisions.
205	const sortedNumericallyPositionedResults = _.sortBy(numericallyPositionedResults, r=>r.sortKey);	×
206
NEW 207	let mergedResults: Sortable<number \| Date>[] = [...sortedResults];	×
208	for (let i=0; i<sortedNumericallyPositionedResults.length; i++) {	×
209	const insertedResult = sortedNumericallyPositionedResults[i];	×
210	const insertionPosition = insertedResult.sortKey-offset;	×
211
212	if (insertionPosition >= 0) {	×
213	if (insertionPosition < mergedResults.length) {	×
214	mergedResults.splice(insertionPosition, 0, insertedResult);	×
215	} else {
216	mergedResults.push(insertedResult);	×
217	}
218	}
219	}
220
221	return mergedResults;	×
222	}
223
224	function isNonEmptyObject(obj: {}): boolean {
225	return Object.keys(obj).length > 0;	×
226	}
227
228	async function queryWithCutoff<N extends CollectionNameString>({
229	context, collection, selector, limit, cutoffField, cutoff, sortDirection,
230	}: {
231	context: ResolverContext,
232	collection: CollectionBase<N>,
233	selector: MongoSelector<ObjectsByCollectionName[N]>,
234	limit: number,
235	cutoffField: keyof ObjectsByCollectionName[N],
236	cutoff: any,
237	sortDirection: SortDirection,
238	}) {
239	const collectionName = collection.collectionName;	×
240	const {currentUser} = context;	×
241
242	const sort = {[cutoffField]: sortDirection === "asc" ? 1 : -1, _id: 1};	×
243	const cutoffSelector = cutoff	×
244	? {[cutoffField]: {[sortDirection === "asc" ? "$gt" : "$lt"]: cutoff}}	×
245	: {};
246	const mergedSelector = mergeSelectors(	×
247	getDefaultViewSelector(collectionName),
248	selector,
249	cutoffSelector
250	)
251	const finalizedSelector = replaceSpecialFieldSelectors(mergedSelector);	×
252	const resultsRaw = await collection.find(finalizedSelector, {	×
253	sort: sort as Partial<Record<keyof ObjectsByCollectionName[N], number>>,
254	limit,
255	}).fetch();
256
257	return await accessFilterMultiple(currentUser, collection, resultsRaw, context);	×
258	}

ForumMagnum / ForumMagnum / 9406031023

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous