11525885691

Committed 25 Oct 2024 09:50PM UTC coverage: 31.469% (-0.2%) from 31.657%

Build # 11525885691

Build Type

Pull #9875

github

Committed by

web-flow

Commit Message

Merge 0c5fcb2a5 into 787ddb6be

Pull Request Pull Request #9875: Merge master into lw-deploy

Run Details

5018 of 41815 branches covered (12.0%)

Branch coverage included in aggregate %.

63 of 202 new or added lines in 35 files covered. (31.19%)

15 existing lines in 8 files now uncovered.

29439 of 67679 relevant lines covered (43.5%)

174.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.85

/packages/lesswrong/server/search/elastic/ElasticQuery.ts

import type {
  QueryDslQueryContainer,
  SearchRequest as SearchRequestInfo,
  Sort,
} from "@elastic/elasticsearch/lib/api/types";
import type {
  SearchRequest as SearchRequestBody,
} from "@elastic/elasticsearch/lib/api/typesWithBodyKey";
import {
  IndexConfig,
  Ranking,
  isFullTextField,
  indexToCollectionName,
  collectionNameToConfig,
} from "./ElasticConfig";
import { parseQuery, QueryToken } from "./parseQuery";
import { searchOriginDate } from "./elasticSettings";
import { SearchIndexCollectionName } from "../../../lib/search/searchUtil";

/**
 * There a couple of places where we need a rough origin date
 * for scaling/faceting/etc. which is defined here. This needn't
 * be exact - just a date a little older than the oldest searchable
 * records.
 */
export const SEARCH_ORIGIN_DATE = new Date(searchOriginDate.get());

export type QueryFilterOperator = "gt" | "gte" | "lt" | "lte" | "eq";

export type QueryFilter = {
  field: string,
} & ({
  type: "facet",
  value: boolean | string | null,
  negated: boolean,
} | {
  type: "numeric",
  value: number,
  op: QueryFilterOperator,
} | {
  type: "exists"
});

export type QueryData = {
  index: string,
  sorting?: string,
  search: string
  offset?: number,
  limit?: number,
  preTag?: string,
  postTag?: string,
  filters: QueryFilter[],
  // Providing coordinates will trigger a special case, which sorts results by distance and ignores relevance
  coordinates?: number[],
}

export type Fuzziness = "AUTO" | number;

type CompiledQuery = {
  tokens: QueryToken[],
  searchQuery: QueryDslQueryContainer,
  snippetName: string,
  snippetQuery?: QueryDslQueryContainer,
  highlightName?: string,
  highlightQuery?: QueryDslQueryContainer,
}

class ElasticQuery {
  private collectionName: SearchIndexCollectionName;
  private config: IndexConfig;

  constructor(
    private queryData: QueryData,
    private fuzziness: Fuzziness = 1,
  ) {
    this.collectionName = indexToCollectionName(queryData.index);
    this.config = collectionNameToConfig(this.collectionName);
  }

  compileRanking({field, order, weight, scoring}: Ranking): string {
    let expr: string;
    switch (scoring.type) {
    case "numeric":
      const min = scoring.min ?? 1;
      expr = `saturation(Math.max(${min}, doc['${field}'].value), ${scoring.pivot}L)`;
      break;
    case "date":
      const start = SEARCH_ORIGIN_DATE;
      const delta = Date.now() - start.getTime();
      const dayRange = Math.ceil(delta / (1000 * 60 * 60 * 24));
      expr = `1 - decayDateLinear('${start.toISOString()}', '${dayRange}d', '0', 0.5, doc['${field}'].value)`;
      break;
    case "bool":
      expr = `doc['${field}'].value == true ? 0.75 : 0.25`;
      break;
    }
    if (weight) {
      expr = `((${expr}) * ${weight})`;
    }
    expr = `(doc['${field}'].size() == 0 ? 0 : (${expr}))`;
    return order === "asc" ? `(1 - ${expr})` : expr;
  }

  private compileScoreExpression(): string {
    let expr = "_score";
    for (const ranking of this.config.ranking ?? []) {
      expr += " * " + this.compileRanking(ranking);
    }
    return expr;
  }

  private compileFacet(
    field: string,
    value: boolean | string | null,
    negated: boolean,
  ): QueryDslQueryContainer {
    if (value === null) {
      const term: QueryDslQueryContainer = {
        exists: {
          field,
        },
      };
      return negated
        ? term
        : {
          bool: {
            should: [],
            must_not: [term],
          },
        };
    }

    const term: QueryDslQueryContainer = isFullTextField(this.collectionName, field)
      ? {
        match: {
          [`${field}.exact`]: {
            query: value,
            analyzer: "fm_exact_analyzer",
            operator: "AND",
            fuzziness: 0,
          },
        },
      }
      : {
        term: {
          [field]: value,
        },
      };
    return negated
      ? {
        bool: {
          should: [],
          must_not: [term],
        },
      }
      : term;
  }

  private compileFilterTermForField(filter: QueryFilter): QueryDslQueryContainer {
    switch (filter.type) {
    case "facet":
      return this.compileFacet(filter.field, filter.value, filter.negated);

    case "numeric":
      return {
        range: {
          [filter.field]: {
            [filter.op]: filter.value,
          },
        },
      };

    case "exists":
      return {
        bool: {
          should: [],
          must: [{
            exists: {
              field: filter.field
            }
          }]
        }
      };

    default:
      return {};
    }
  }

  private compileFilters(tokens: QueryToken[]) {
    const filtersByField: Record<string, QueryFilter[]> = {};
    for (const filter of this.queryData.filters) {
      filtersByField[filter.field] ??= [];
      filtersByField[filter.field].push(filter);
    }

    const terms = [...(this.config.filters ?? [])];

    for (const filterField in filtersByField) {
      const filters = filtersByField[filterField];
      if (!filters.length) {
        continue;
      }
      const fieldTerms = filters.map(
        (filter) => this.compileFilterTermForField(filter),
      );
      if (fieldTerms.length > 1) {
        terms.push({
          bool: {
            should: fieldTerms,
          },
        });
      } else {
        terms.push(fieldTerms[0]);
      }
    }

    const userFilters: QueryDslQueryContainer[] = [];
    const tagFilters: QueryDslQueryContainer[] = [];
    for (const {type, token} of tokens) {
      if (type === "user") {
        userFilters.push(
          {term: {"authorSlug.sort": token}},
          {term: {"authorDisplayName.sort": token}},
          {term: {userId: token}},
        );
      } else if (type === "tag") {
        tagFilters.push(
          {term: {"tags._id": token}},
          {term: {"tags.slug": {value: token, case_insensitive: true}}},
          {term: {"tags.name": {value: token, case_insensitive: true}}},
        );
      }
    }
    if (userFilters.length) {
      terms.push({bool: {should: userFilters}});
    }
    if (tagFilters.length) {
      terms.push({bool: {should: tagFilters}});
    }

    return terms.length ? terms : undefined;
  }

  private getDefaultQuery(
    search: string,
    fields: string[],
  ): QueryDslQueryContainer {
    return {
      multi_match: {
        query: search,
        fields,
        fuzziness: this.fuzziness,
        max_expansions: 10,
        prefix_length: 3,
        minimum_should_match: "75%",
        operator: "or",
      },
    };
  }

  private compileSimpleQuery(tokens: QueryToken[]): CompiledQuery {
    const {fields, snippet, highlight} = this.config;
    const {search} = this.queryData;
    const mainField = this.textFieldToExactField(fields[0], false);
    return {
      tokens,
      searchQuery: {
        bool: {
          should: [
            {
              term: {
                objectID: {
                  value: search,
                },
              },
            },
            this.getDefaultQuery(search, fields),
            {
              multi_match: {
                query: search,
                fields,
                type: "phrase",
                slop: 2,
                boost: 100,
              },
            },
            {
              match_phrase_prefix: {
                [mainField]: {
                  query: search,
                  boost: 1000,
                },
              },
            },
          ],
        },
      },
      snippetName: snippet,
      highlightName: highlight,
    };
  }

  private textFieldToExactField(
    textField: string,
    keepRelevance = true,
  ): string {
    const [fieldName, relevance] = textField.split("^");
    const exactField = `${fieldName}.exact`;
    return relevance && keepRelevance
      ? `${exactField}^${relevance}`
      : exactField;
  }

  private getAdvancedHighlightQuery(
    mustToken: string,
  ): Omit<CompiledQuery, "searchQuery"> {
    const {snippet, highlight} = this.config;
    const snippetName = `${snippet}.exact`;
    const highlightName = `${highlight}.exact`;
    const buildQuery = (fieldName: string) => ({
      match_phrase: {
        [fieldName]: {
          query: mustToken,
          analyzer: "simple",
        },
      },
    });
    return {
      tokens: [{ type: "must", token: mustToken }],
      snippetName,
      snippetQuery: buildQuery(snippetName),
      ...(highlight && {
        highlightName,
        highlightQuery: buildQuery(highlightName),
      }),
    };
  }

  private compileAdvancedQuery(tokens: QueryToken[]): CompiledQuery {
    const {fields, snippet, highlight} = this.config;

    const must: QueryDslQueryContainer[] = [];
    const must_not: QueryDslQueryContainer[] = [];
    const should: QueryDslQueryContainer[] = [];

    for (const {type, token} of tokens) {
      switch (type) {
      case "must":
        must.push({
          multi_match: {
            query: token,
            fields: fields.map((field) => this.textFieldToExactField(field)),
            type: "phrase",
          },
        });
        break;
      case "not":
        must_not.push({
          multi_match: {
            query: token,
            fields,
          },
        });
        break;
      case "should":
        should.push(this.getDefaultQuery(token, fields));
        break;
      case "user":
      case "tag":
        // Do nothing - this is handled by `this.compileFilters`
        break;
      }
    }

    const searchQuery: QueryDslQueryContainer = {
      bool: {
        must,
        must_not,
        should,
      },
    };

    if (must.length) {
      return {
        searchQuery,
        ...this.getAdvancedHighlightQuery(must[0].multi_match!.query),
      };
    }

    const highlightQueryString = tokens.filter(
      ({type}) => type !== "user" && type !== "tag",
    ).map(({token}) => token).join(" ");
    const highlightQuery = this.getDefaultQuery(
      highlightQueryString,
      this.config.fields,
    );

    return {
      tokens,
      searchQuery,
      snippetName: snippet,
      snippetQuery: highlightQuery,
      highlightName: highlight,
      highlightQuery,
    };
  }

  private compileEmptyQuery(): CompiledQuery {
    return {
      tokens: [],
      searchQuery: {
        match_all: {},
      },
      snippetName: "",
    };
  }

  private compileQuery(): CompiledQuery {
    const {search} = this.queryData;
    if (!search) {
      return this.compileEmptyQuery();
    }
    const {tokens, isAdvanced} = parseQuery(search);
    return isAdvanced
      ? this.compileAdvancedQuery(tokens)
      : this.compileSimpleQuery(tokens);
  }

  private compileSort(sorting?: string, coordinates?: number[]): Sort {
    // Special case:
    // When providing coordinates in the format [lng, lat], we sort by distance
    // and ignore the relevance score. See also parseLatLng()
    if (coordinates) {
      if (!this.config.locationField) {
        throw new Error("Index cannot be sorted by location");
      }
      return [
        {
          _geo_distance : {
            [this.config.locationField]: coordinates,
            order : "asc",
          },
        },
        {[this.config.tiebreaker]: {order: "desc"}},
      ];
    }

    const sort: Sort = [
      {[this.config.tiebreaker]: {order: "desc"}},
    ];

    // If we specify a custom sort (such as from the people directory) then
    // we should ignore the search _score completely when sorting
    if (sorting && sorting.indexOf(":") > 0) {
      const [field, order] = sorting.split(":");
      if (order !== "asc" && order !== "desc") {
        throw new Error("Invalid sorting order");
      }
      sort.unshift({[field]: {order}});
      return sort;
    } else {
      sort.unshift({_score: {order: "desc"}});
    }

    // There are several special sortings builtin to the main search page.
    // Note that these are used in conjunction with the search _score
    switch (sorting) {
    case "karma":
      const field = this.config.karmaField ?? "baseScore";
      sort.unshift({[field]: {order: "desc"}});
      break;
    case "newest_first":
      sort.unshift({publicDateMs: {order: "desc"}});
      break;
    case "oldest_first":
      sort.unshift({publicDateMs: {order: "asc"}});
      break;
    case "relevance":
      break;
    default:
      if (sorting) {
        throw new Error("Invalid sorting: " + sorting);
      }
    }

    return sort;
  }

  compile(): SearchRequestInfo | SearchRequestBody {
    const {
      preTag,
      postTag,
      index,
      sorting,
      offset = 0,
      limit = 10,
      coordinates,
    } = this.queryData;
    const {privateFields} = this.config;

    // When sorting by nearest-geographically we disable custom highlighting as
    // this isn't supported by elastic and causes an exception
    const hasCustomHighlight = !coordinates;

    const {
      tokens,
      searchQuery,
      snippetName,
      snippetQuery,
      highlightName,
      highlightQuery,
    } = this.compileQuery();
    const highlightConfig =  {
      type: "plain",
      pre_tags: [preTag ?? "<em>"],
      post_tags: [postTag ?? "</em>"],

      // This is the default value for index.highlight.max_analyzed_offset
      // which we haven't customized. If this wasn't set here or was set
      // larger than the corresponding setting on the index, then search
      // would fail entirely when results contain a poss where the
      // plain-text version of the body is larger than this.
      max_analyzed_offset: 1000000,
    };
    return {
      index,
      from: offset,
      size: limit,
      body: {
        track_scores: true,
        track_total_hits: true,
        ...(hasCustomHighlight && {
          highlight: {
            fields: {
              [snippetName]: {
                ...highlightConfig,
                highlight_query: snippetQuery,
              },
              ...(highlightName && {
                [highlightName]: {
                  ...highlightConfig,
                  highlight_query: highlightQuery,
                },
              }),
            },
            number_of_fragments: 1,
            fragment_size: 140,
            no_match_size: 140,
          },
        }),
        query: {
          script_score: {
            query: {
              bool: {
                must: searchQuery,
                should: [],
                filter: this.compileFilters(tokens),
              },
            },
            script: {
              source: this.compileScoreExpression(),
            },
          },
        },
        sort: this.compileSort(sorting, coordinates),
        _source: {
          excludes: ["exportedAt", ...privateFields],
        },
      },
    };
  }
}

export default ElasticQuery;

1	import type {	72✔
2	QueryDslQueryContainer,
3	SearchRequest as SearchRequestInfo,
4	Sort,
5	} from "@elastic/elasticsearch/lib/api/types";
6	import type {
7	SearchRequest as SearchRequestBody,
8	} from "@elastic/elasticsearch/lib/api/typesWithBodyKey";
9	import {
10	IndexConfig,
11	Ranking,
12	isFullTextField,
13	indexToCollectionName,
14	collectionNameToConfig,
15	} from "./ElasticConfig";	21✔
16	import { parseQuery, QueryToken } from "./parseQuery";	21✔
17	import { searchOriginDate } from "./elasticSettings";	21✔
18	import { SearchIndexCollectionName } from "../../../lib/search/searchUtil";
19
20	/**
21	* There a couple of places where we need a rough origin date
22	* for scaling/faceting/etc. which is defined here. This needn't
23	* be exact - just a date a little older than the oldest searchable
24	* records.
25	*/
26	export const SEARCH_ORIGIN_DATE = new Date(searchOriginDate.get());	21✔
27
28	export type QueryFilterOperator = "gt" \| "gte" \| "lt" \| "lte" \| "eq";
29
30	export type QueryFilter = {
31	field: string,
32	} & ({
33	type: "facet",
34	value: boolean \| string \| null,
35	negated: boolean,
36	} \| {
37	type: "numeric",
38	value: number,
39	op: QueryFilterOperator,
40	} \| {
41	type: "exists"
42	});
43
44	export type QueryData = {
45	index: string,
46	sorting?: string,
47	search: string
48	offset?: number,
49	limit?: number,
50	preTag?: string,
51	postTag?: string,
52	filters: QueryFilter[],
53	// Providing coordinates will trigger a special case, which sorts results by distance and ignores relevance
54	coordinates?: number[],
55	}
56
57	export type Fuzziness = "AUTO" \| number;
58
59	type CompiledQuery = {
60	tokens: QueryToken[],
61	searchQuery: QueryDslQueryContainer,
62	snippetName: string,
63	snippetQuery?: QueryDslQueryContainer,
64	highlightName?: string,
65	highlightQuery?: QueryDslQueryContainer,
66	}
67
68	class ElasticQuery {
69	private collectionName: SearchIndexCollectionName;
70	private config: IndexConfig;
71
72	constructor(
73	private queryData: QueryData,	7✔
74	private fuzziness: Fuzziness = 1,	7✔
75	) {
76	this.collectionName = indexToCollectionName(queryData.index);	7✔
77	this.config = collectionNameToConfig(this.collectionName);	7✔
78	}
79
80	compileRanking({field, order, weight, scoring}: Ranking): string {
81	let expr: string;
82	switch (scoring.type) {	7✔
83	case "numeric":
84	const min = scoring.min ?? 1;	3!
85	expr = `saturation(Math.max(${min}, doc['${field}'].value), ${scoring.pivot}L)`;	3✔
86	break;	3✔
87	case "date":
88	const start = SEARCH_ORIGIN_DATE;	2✔
89	const delta = Date.now() - start.getTime();	2✔
90	const dayRange = Math.ceil(delta / (1000 * 60 * 60 * 24));	2✔
91	expr = `1 - decayDateLinear('${start.toISOString()}', '${dayRange}d', '0', 0.5, doc['${field}'].value)`;	2✔
92	break;	2✔
93	case "bool":
94	expr = `doc['${field}'].value == true ? 0.75 : 0.25`;	2✔
95	break;	2✔
96	}
97	if (weight) {	7✔
98	expr = `((${expr}) * ${weight})`;	1✔
99	}
100	expr = `(doc['${field}'].size() == 0 ? 0 : (${expr}))`;	7✔
101	return order === "asc" ? `(1 - ${expr})` : expr;	7✔
102	}
103
104	private compileScoreExpression(): string {
105	let expr = "_score";	×
106	for (const ranking of this.config.ranking ?? []) {	×
107	expr += " * " + this.compileRanking(ranking);	×
108	}
109	return expr;	×
110	}
111
112	private compileFacet(
113	field: string,
114	value: boolean \| string \| null,
115	negated: boolean,
116	): QueryDslQueryContainer {
117	if (value === null) {	×
118	const term: QueryDslQueryContainer = {	×
119	exists: {
120	field,
121	},
122	};
123	return negated	×
124	? term	×
125	: {
126	bool: {
127	should: [],
128	must_not: [term],
129	},
130	};
131	}
132
133	const term: QueryDslQueryContainer = isFullTextField(this.collectionName, field)	×
134	? {	×
135	match: {
136	[`${field}.exact`]: {
137	query: value,
138	analyzer: "fm_exact_analyzer",
139	operator: "AND",
140	fuzziness: 0,
141	},
142	},
143	}
144	: {
145	term: {
146	[field]: value,
147	},
148	};
149	return negated	×
150	? {	×
151	bool: {
152	should: [],
153	must_not: [term],
154	},
155	}
156	: term;
157	}
158
159	private compileFilterTermForField(filter: QueryFilter): QueryDslQueryContainer {
160	switch (filter.type) {	×
161	case "facet":
162	return this.compileFacet(filter.field, filter.value, filter.negated);	×
163
164	case "numeric":
165	return {	×
166	range: {
167	[filter.field]: {
168	[filter.op]: filter.value,
169	},
170	},
171	};
172
173	case "exists":
174	return {	×
175	bool: {
176	should: [],
177	must: [{
178	exists: {
179	field: filter.field
180	}
181	}]
182	}
183	};
184
185	default:
186	return {};	×
187	}
188	}
189
190	private compileFilters(tokens: QueryToken[]) {
191	const filtersByField: Record<string, QueryFilter[]> = {};	×
192	for (const filter of this.queryData.filters) {	×
193	filtersByField[filter.field] ??= [];	×
194	filtersByField[filter.field].push(filter);	×
195	}
196
197	const terms = [...(this.config.filters ?? [])];	×
198
199	for (const filterField in filtersByField) {	×
200	const filters = filtersByField[filterField];	×
201	if (!filters.length) {	×
202	continue;	×
203	}
204	const fieldTerms = filters.map(	×
205	(filter) => this.compileFilterTermForField(filter),	×
206	);
207	if (fieldTerms.length > 1) {	×
208	terms.push({	×
209	bool: {
210	should: fieldTerms,
211	},
212	});
213	} else {
214	terms.push(fieldTerms[0]);	×
215	}
216	}
217
NEW 218	const userFilters: QueryDslQueryContainer[] = [];	×
NEW 219	const tagFilters: QueryDslQueryContainer[] = [];	×
NEW 220	for (const {type, token} of tokens) {	×
NEW 221	if (type === "user") {	×
NEW 222	userFilters.push(	×
223	{term: {"authorSlug.sort": token}},
224	{term: {"authorDisplayName.sort": token}},
225	{term: {userId: token}},
226	);
NEW 227	} else if (type === "tag") {	×
NEW 228	tagFilters.push(	×
229	{term: {"tags._id": token}},
230	{term: {"tags.slug": {value: token, case_insensitive: true}}},
231	{term: {"tags.name": {value: token, case_insensitive: true}}},
232	);
233	}
234	}
NEW 235	if (userFilters.length) {	×
NEW 236	terms.push({bool: {should: userFilters}});	×
237	}
NEW 238	if (tagFilters.length) {	×
NEW 239	terms.push({bool: {should: tagFilters}});	×
240	}
241
UNCOV 242	return terms.length ? terms : undefined;	×
243	}
244
245	private getDefaultQuery(
246	search: string,
247	fields: string[],
248	): QueryDslQueryContainer {
249	return {	×
250	multi_match: {
251	query: search,
252	fields,
253	fuzziness: this.fuzziness,
254	max_expansions: 10,
255	prefix_length: 3,
256	minimum_should_match: "75%",
257	operator: "or",
258	},
259	};
260	}
261
262	private compileSimpleQuery(tokens: QueryToken[]): CompiledQuery {
263	const {fields, snippet, highlight} = this.config;	×
264	const {search} = this.queryData;	×
265	const mainField = this.textFieldToExactField(fields[0], false);	×
266	return {	×
267	tokens,
268	searchQuery: {
269	bool: {
270	should: [
271	{
272	term: {
273	objectID: {
274	value: search,
275	},
276	},
277	},
278	this.getDefaultQuery(search, fields),
279	{
280	multi_match: {
281	query: search,
282	fields,
283	type: "phrase",
284	slop: 2,
285	boost: 100,
286	},
287	},
288	{
289	match_phrase_prefix: {
290	[mainField]: {
291	query: search,
292	boost: 1000,
293	},
294	},
295	},
296	],
297	},
298	},
299	snippetName: snippet,
300	highlightName: highlight,
301	};
302	}
303
304	private textFieldToExactField(
305	textField: string,
306	keepRelevance = true,	×
307	): string {
308	const [fieldName, relevance] = textField.split("^");	×
309	const exactField = `${fieldName}.exact`;	×
310	return relevance && keepRelevance	×
311	? `${exactField}^${relevance}`	×
312	: exactField;
313	}
314
315	private getAdvancedHighlightQuery(
316	mustToken: string,
317	): Omit<CompiledQuery, "searchQuery"> {
318	const {snippet, highlight} = this.config;	×
319	const snippetName = `${snippet}.exact`;	×
320	const highlightName = `${highlight}.exact`;	×
321	const buildQuery = (fieldName: string) => ({	×
322	match_phrase: {
323	[fieldName]: {
324	query: mustToken,
325	analyzer: "simple",
326	},
327	},
328	});
329	return {	×
330	tokens: [{ type: "must", token: mustToken }],
331	snippetName,
332	snippetQuery: buildQuery(snippetName),
333	...(highlight && {	×
334	highlightName,
335	highlightQuery: buildQuery(highlightName),
336	}),
337	};
338	}
339
340	private compileAdvancedQuery(tokens: QueryToken[]): CompiledQuery {
341	const {fields, snippet, highlight} = this.config;	×
342
343	const must: QueryDslQueryContainer[] = [];	×
344	const must_not: QueryDslQueryContainer[] = [];	×
345	const should: QueryDslQueryContainer[] = [];	×
346
347	for (const {type, token} of tokens) {	×
348	switch (type) {	×
349	case "must":
350	must.push({	×
351	multi_match: {
352	query: token,
353	fields: fields.map((field) => this.textFieldToExactField(field)),	×
354	type: "phrase",
355	},
356	});
357	break;	×
358	case "not":
359	must_not.push({	×
360	multi_match: {
361	query: token,
362	fields,
363	},
364	});
365	break;	×
366	case "should":
367	should.push(this.getDefaultQuery(token, fields));	×
368	break;	×
369	case "user":
370	case "tag":
371	// Do nothing - this is handled by `this.compileFilters`
NEW 372	break;	×
373	}
374	}
375
376	const searchQuery: QueryDslQueryContainer = {	×
377	bool: {
378	must,
379	must_not,
380	should,
381	},
382	};
383
384	if (must.length) {	×
385	return {	×
386	searchQuery,
387	...this.getAdvancedHighlightQuery(must[0].multi_match!.query),
388	};
389	}
390
NEW 391	const highlightQueryString = tokens.filter(	×
NEW 392	({type}) => type !== "user" && type !== "tag",	×
NEW 393	).map(({token}) => token).join(" ");	×
NEW 394	const highlightQuery = this.getDefaultQuery(	×
395	highlightQueryString,
396	this.config.fields,
397	);
398
UNCOV 399	return {	×
400	tokens,
401	searchQuery,
402	snippetName: snippet,
403	snippetQuery: highlightQuery,
404	highlightName: highlight,
405	highlightQuery,
406	};
407	}
408
409	private compileEmptyQuery(): CompiledQuery {
410	return {	×
411	tokens: [],
412	searchQuery: {
413	match_all: {},
414	},
415	snippetName: "",
416	};
417	}
418
419	private compileQuery(): CompiledQuery {
420	const {search} = this.queryData;	×
421	if (!search) {	×
422	return this.compileEmptyQuery();	×
423	}
424	const {tokens, isAdvanced} = parseQuery(search);	×
425	return isAdvanced	×
426	? this.compileAdvancedQuery(tokens)	×
427	: this.compileSimpleQuery(tokens);
428	}
429
430	private compileSort(sorting?: string, coordinates?: number[]): Sort {
431	// Special case:
432	// When providing coordinates in the format [lng, lat], we sort by distance
433	// and ignore the relevance score. See also parseLatLng()
434	if (coordinates) {	×
435	if (!this.config.locationField) {	×
436	throw new Error("Index cannot be sorted by location");	×
437	}
438	return [	×
439	{
440	_geo_distance : {
441	[this.config.locationField]: coordinates,
442	order : "asc",
443	},
444	},
445	{[this.config.tiebreaker]: {order: "desc"}},
446	];
447	}
448
449	const sort: Sort = [	×
450	{[this.config.tiebreaker]: {order: "desc"}},
451	];
452
453	// If we specify a custom sort (such as from the people directory) then
454	// we should ignore the search _score completely when sorting
455	if (sorting && sorting.indexOf(":") > 0) {	×
456	const [field, order] = sorting.split(":");	×
457	if (order !== "asc" && order !== "desc") {	×
458	throw new Error("Invalid sorting order");	×
459	}
460	sort.unshift({[field]: {order}});	×
461	return sort;	×
462	} else {
463	sort.unshift({_score: {order: "desc"}});	×
464	}
465
466	// There are several special sortings builtin to the main search page.
467	// Note that these are used in conjunction with the search _score
468	switch (sorting) {	×
469	case "karma":
470	const field = this.config.karmaField ?? "baseScore";	×
471	sort.unshift({[field]: {order: "desc"}});	×
472	break;	×
473	case "newest_first":
474	sort.unshift({publicDateMs: {order: "desc"}});	×
475	break;	×
476	case "oldest_first":
477	sort.unshift({publicDateMs: {order: "asc"}});	×
478	break;	×
479	case "relevance":
480	break;	×
481	default:
482	if (sorting) {	×
483	throw new Error("Invalid sorting: " + sorting);	×
484	}
485	}
486
487	return sort;	×
488	}
489
490	compile(): SearchRequestInfo \| SearchRequestBody {
491	const {
492	preTag,
493	postTag,
494	index,
495	sorting,
496	offset = 0,	×
497	limit = 10,	×
498	coordinates,
499	} = this.queryData;	×
500	const {privateFields} = this.config;	×
501
502	// When sorting by nearest-geographically we disable custom highlighting as
503	// this isn't supported by elastic and causes an exception
504	const hasCustomHighlight = !coordinates;	×
505
506	const {
507	tokens,
508	searchQuery,
509	snippetName,
510	snippetQuery,
511	highlightName,
512	highlightQuery,
513	} = this.compileQuery();	×
514	const highlightConfig = {	×
515	type: "plain",
516	pre_tags: [preTag ?? "<em>"],	×
517	post_tags: [postTag ?? "</em>"],	×
518
519	// This is the default value for index.highlight.max_analyzed_offset
520	// which we haven't customized. If this wasn't set here or was set
521	// larger than the corresponding setting on the index, then search
522	// would fail entirely when results contain a poss where the
523	// plain-text version of the body is larger than this.
524	max_analyzed_offset: 1000000,
525	};
526	return {	×
527	index,
528	from: offset,
529	size: limit,
530	body: {
531	track_scores: true,
532	track_total_hits: true,
533	...(hasCustomHighlight && {	×
534	highlight: {
535	fields: {
536	[snippetName]: {
537	...highlightConfig,
538	highlight_query: snippetQuery,
539	},
540	...(highlightName && {	×
541	[highlightName]: {
542	...highlightConfig,
543	highlight_query: highlightQuery,
544	},
545	}),
546	},
547	number_of_fragments: 1,
548	fragment_size: 140,
549	no_match_size: 140,
550	},
551	}),
552	query: {
553	script_score: {
554	query: {
555	bool: {
556	must: searchQuery,
557	should: [],
558	filter: this.compileFilters(tokens),
559	},
560	},
561	script: {
562	source: this.compileScoreExpression(),
563	},
564	},
565	},
566	sort: this.compileSort(sorting, coordinates),
567	_source: {
568	excludes: ["exportedAt", ...privateFields],
569	},
570	},
571	};
572	}
573	}
574
575	export default ElasticQuery;	21✔

ForumMagnum / ForumMagnum / 11525885691

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous