9010223155

Committed 09 May 2024 12:12AM UTC coverage: 31.914% (+0.002%) from 31.912%

Build # 9010223155

Build Type

Pull #9252

github

Committed by

b0b3rt

Commit Message

remove update custom indexes call from migration

Pull Request Pull Request #9252: cache recombee recommendations

Run Details

4762 of 38927 branches covered (12.23%)

Branch coverage included in aggregate %.

31 of 63 new or added lines in 11 files covered. (49.21%)

2 existing lines in 1 file now uncovered.

27724 of 62866 relevant lines covered (44.1%)

178.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.37

/packages/lesswrong/server/scripts/recomputeDenormalized.ts

import { Vulcan, Collections, getCollection } from '../vulcan-lib';
import { getFieldsWithAttribute } from './utils';
import { migrateDocuments } from '../manualMigrations/migrationUtils'
import { createAdminContext } from '../vulcan-lib/query';
import { getSchema } from '../../lib/utils/getSchema';
import * as _ from 'underscore';
import { filterNonnull } from '../../lib/utils/typeGuardUtils';


export const recomputeAllDenormalizedValues = async () => {
  for(let collection of Collections) {
    await Vulcan.recomputeDenormalizedValues({
      collectionName: collection.options.collectionName
    })
  }
}
Vulcan.recomputeAllDenormalizedValues = recomputeAllDenormalizedValues;

export const validateAllDenormalizedValues = async () => {
  for(let collection of Collections) {
    await Vulcan.recomputeDenormalizedValues({
      collectionName: collection.options.collectionName,
      validateOnly: true
    })
  }
}
Vulcan.validateAllDenormalizedValues = validateAllDenormalizedValues;

// Recompute the value of denormalized fields (that are tagged with canAutoDenormalize).
// If validateOnly is true, compare them with the existing values in the database and
// report how many differ; otherwise update them to the correct values. If fieldName
// is given, recompute a single field; otherwise recompute all fields on the collection.
export const recomputeDenormalizedValues = async <N extends CollectionNameString>({collectionName, fieldName=null, validateOnly=false, projection, nullToZero=false}: {
  collectionName: N,
  fieldName?: (keyof ObjectsByCollectionName[N] & string)|null,
  validateOnly?: boolean,
  projection?: MongoProjection<ObjectsByCollectionName[N]>,
  nullToZero?: boolean
}) => {
  // eslint-disable-next-line no-console
  console.log(`Recomputing denormalize values for ${collectionName} ${fieldName ? `and ${fieldName}` : ""}`)

  const collection = getCollection(collectionName)
  const schema: any = getSchema(collection);
  if (!schema) {
    // eslint-disable-next-line no-console
    console.log(`${collectionName} does not have a schema defined, not computing denormalized values`)
    return
  }

  if (fieldName) {
    if (!schema[fieldName]) {
      // eslint-disable-next-line no-console
      throw new Error(`${collectionName} does not have field ${fieldName}, not computing denormalized values`)
    }
    if (!schema[fieldName].denormalized) {
      throw new Error(`${collectionName}.${fieldName} is not marked as a denormalized field`)
    }
    if (!schema[fieldName].canAutoDenormalize) {
      throw new Error(`${collectionName}.${fieldName} is not marked as canAutoDenormalize`)
    }
    const getValue = schema[fieldName].getValue
    if (!getValue) {
      throw new Error(`${collectionName}.${fieldName} is missing its getValue function`)
    }

    await runDenormalizedFieldMigration({ collection, fieldName, getValue, projection, validateOnly, nullToZero })
  } else {
    const denormalizedFields = getFieldsWithAttribute(schema, 'canAutoDenormalize')
    if (denormalizedFields.length === 0) {
      // eslint-disable-next-line no-console
      console.log(`${collectionName} does not have any fields with "canAutoDenormalize", not computing denormalized values`)
      return;
    }

    // eslint-disable-next-line no-console
    console.log(`Recomputing denormalized values for ${collection.collectionName} in fields: ${denormalizedFields}`);

    for (let j=0; j<denormalizedFields.length; j++) {
      const fieldName = denormalizedFields[j] as keyof ObjectsByCollectionName[N] & string;
      const getValue = schema[fieldName].getValue
      await runDenormalizedFieldMigration({ collection, fieldName, getValue, projection, validateOnly, nullToZero })
    }
  }

  // eslint-disable-next-line no-console
  console.log(`Finished recomputing denormalized values for ${collectionName} ${fieldName ? `and ${fieldName}` : ""}`)
}
Vulcan.recomputeDenormalizedValues = recomputeDenormalizedValues;

async function runDenormalizedFieldMigration<N extends CollectionNameString>({
  collection,
  fieldName,
  getValue,
  unmigratedDocumentQuery,
  projection,
  validateOnly,
  nullToZero,
}: {
  collection: CollectionBase<N>,
  fieldName: keyof ObjectsByCollectionName[N],
  getValue: AnyBecauseTodo,
  unmigratedDocumentQuery?: MongoSelector<ObjectsByCollectionName[N]>,
  projection?: MongoProjection<ObjectsByCollectionName[N]>,
  validateOnly: boolean,
  nullToZero: boolean
}) {
  let numDifferent = 0;

  await migrateDocuments({
    description: `Recomputing denormalized values for ${collection.collectionName} field ${String(fieldName)}`,
    collection,
    unmigratedDocumentQuery,
    projection,
    batchSize: 100,
    migrate: async (documents) => {
      const context = createAdminContext();
      
      // eslint-disable-next-line no-console
      const updates = await Promise.all(documents.map(async doc => {
        const newValue = await getValue(doc, context)
        // If the correct value is already present, don't make a database update
        if ((isNullOrDefined(newValue) && isNullOrDefined(doc[fieldName])) || doc[fieldName] === newValue) return null
        // If we don't want to update null or missing values to 0, don't make a database update
        if (!nullToZero && isNullOrDefined(doc[fieldName]) && newValue === 0) return null
        return {
          updateOne: {
            filter: {_id: doc._id},
            update: {
              $set: {
                [fieldName]: newValue
              }
            },
          }
        }
      }))

      const nonEmptyUpdates = filterNonnull(updates)
      numDifferent += nonEmptyUpdates.length;

      // eslint-disable-next-line no-console
      console.log(`${nonEmptyUpdates.length} documents in batch with changing denormalized value`)
      if (!validateOnly) {
        if (nonEmptyUpdates.length > 0)  {
          await collection.rawCollection().bulkWrite(
            nonEmptyUpdates,
            { ordered: false }
          );
        }
      } else {
        // TODO: This is a hack, but better than leaving it. We're basically
        // breaking the expected API from migrateDocuments by supporting a
        // validateOnly option, so it does not offer us good hooks to do this.
        throw new Error([
          'Abort! validateOnly means the document will not change and the migration will never',
          'complete. This error is expected behavior to cause the migration to end.'
        ].join(' '))
      }
    },
  });

  // eslint-disable-next-line no-console
  console.log(`${numDifferent} total documents had wrong denormalized value`)
}

function isNullOrDefined(value: AnyBecauseTodo) {
  return value === null || value === undefined
}

1	import { Vulcan, Collections, getCollection } from '../vulcan-lib';	54✔
2	import { getFieldsWithAttribute } from './utils';	18✔
3	import { migrateDocuments } from '../manualMigrations/migrationUtils'	18✔
4	import { createAdminContext } from '../vulcan-lib/query';	18✔
5	import { getSchema } from '../../lib/utils/getSchema';	18✔
6	import * as _ from 'underscore';
7	import { filterNonnull } from '../../lib/utils/typeGuardUtils';	18✔
8
9
10	export const recomputeAllDenormalizedValues = async () => {	18✔
11	for(let collection of Collections) {	×
12	await Vulcan.recomputeDenormalizedValues({	×
13	collectionName: collection.options.collectionName
14	})
15	}
16	}
17	Vulcan.recomputeAllDenormalizedValues = recomputeAllDenormalizedValues;	18✔
18
19	export const validateAllDenormalizedValues = async () => {	18✔
20	for(let collection of Collections) {	×
21	await Vulcan.recomputeDenormalizedValues({	×
22	collectionName: collection.options.collectionName,
23	validateOnly: true
24	})
25	}
26	}
27	Vulcan.validateAllDenormalizedValues = validateAllDenormalizedValues;	18✔
28
29	// Recompute the value of denormalized fields (that are tagged with canAutoDenormalize).
30	// If validateOnly is true, compare them with the existing values in the database and
31	// report how many differ; otherwise update them to the correct values. If fieldName
32	// is given, recompute a single field; otherwise recompute all fields on the collection.
33	export const recomputeDenormalizedValues = async <N extends CollectionNameString>({collectionName, fieldName=null, validateOnly=false, projection, nullToZero=false}: {	18!
34	collectionName: N,
35	fieldName?: (keyof ObjectsByCollectionName[N] & string)\|null,
36	validateOnly?: boolean,
37	projection?: MongoProjection<ObjectsByCollectionName[N]>,
38	nullToZero?: boolean
39	}) => {
40	// eslint-disable-next-line no-console
41	console.log(`Recomputing denormalize values for ${collectionName} ${fieldName ? `and ${fieldName}` : ""}`)	×
42
43	const collection = getCollection(collectionName)	×
44	const schema: any = getSchema(collection);	×
45	if (!schema) {	×
46	// eslint-disable-next-line no-console
47	console.log(`${collectionName} does not have a schema defined, not computing denormalized values`)	×
48	return	×
49	}
50
51	if (fieldName) {	×
52	if (!schema[fieldName]) {	×
53	// eslint-disable-next-line no-console
54	throw new Error(`${collectionName} does not have field ${fieldName}, not computing denormalized values`)	×
55	}
56	if (!schema[fieldName].denormalized) {	×
57	throw new Error(`${collectionName}.${fieldName} is not marked as a denormalized field`)	×
58	}
59	if (!schema[fieldName].canAutoDenormalize) {	×
60	throw new Error(`${collectionName}.${fieldName} is not marked as canAutoDenormalize`)	×
61	}
62	const getValue = schema[fieldName].getValue	×
63	if (!getValue) {	×
64	throw new Error(`${collectionName}.${fieldName} is missing its getValue function`)	×
65	}
66
67	await runDenormalizedFieldMigration({ collection, fieldName, getValue, projection, validateOnly, nullToZero })	×
68	} else {
69	const denormalizedFields = getFieldsWithAttribute(schema, 'canAutoDenormalize')	×
70	if (denormalizedFields.length === 0) {	×
71	// eslint-disable-next-line no-console
72	console.log(`${collectionName} does not have any fields with "canAutoDenormalize", not computing denormalized values`)	×
73	return;	×
74	}
75
76	// eslint-disable-next-line no-console
77	console.log(`Recomputing denormalized values for ${collection.collectionName} in fields: ${denormalizedFields}`);	×
78
79	for (let j=0; j<denormalizedFields.length; j++) {	×
80	const fieldName = denormalizedFields[j] as keyof ObjectsByCollectionName[N] & string;	×
81	const getValue = schema[fieldName].getValue	×
82	await runDenormalizedFieldMigration({ collection, fieldName, getValue, projection, validateOnly, nullToZero })	×
83	}
84	}
85
86	// eslint-disable-next-line no-console
87	console.log(`Finished recomputing denormalized values for ${collectionName} ${fieldName ? `and ${fieldName}` : ""}`)	×
88	}
89	Vulcan.recomputeDenormalizedValues = recomputeDenormalizedValues;	18✔
90
91	async function runDenormalizedFieldMigration<N extends CollectionNameString>({
92	collection,
93	fieldName,
94	getValue,
95	unmigratedDocumentQuery,
96	projection,
97	validateOnly,
98	nullToZero,
99	}: {
100	collection: CollectionBase<N>,
101	fieldName: keyof ObjectsByCollectionName[N],
102	getValue: AnyBecauseTodo,
103	unmigratedDocumentQuery?: MongoSelector<ObjectsByCollectionName[N]>,
104	projection?: MongoProjection<ObjectsByCollectionName[N]>,
105	validateOnly: boolean,
106	nullToZero: boolean
107	}) {
108	let numDifferent = 0;	×
109
110	await migrateDocuments({	×
111	description: `Recomputing denormalized values for ${collection.collectionName} field ${String(fieldName)}`,
112	collection,
113	unmigratedDocumentQuery,
114	projection,
115	batchSize: 100,
116	migrate: async (documents) => {
117	const context = createAdminContext();	×
118
119	// eslint-disable-next-line no-console
120	const updates = await Promise.all(documents.map(async doc => {	×
121	const newValue = await getValue(doc, context)	×
122	// If the correct value is already present, don't make a database update
123	if ((isNullOrDefined(newValue) && isNullOrDefined(doc[fieldName])) \|\| doc[fieldName] === newValue) return null	×
124	// If we don't want to update null or missing values to 0, don't make a database update
125	if (!nullToZero && isNullOrDefined(doc[fieldName]) && newValue === 0) return null	×
126	return {	×
127	updateOne: {
128	filter: {_id: doc._id},
129	update: {
130	$set: {
131	[fieldName]: newValue
132	}
133	},
134	}
135	}
136	}))
137
NEW 138	const nonEmptyUpdates = filterNonnull(updates)	×
139	numDifferent += nonEmptyUpdates.length;	×
140
141	// eslint-disable-next-line no-console
142	console.log(`${nonEmptyUpdates.length} documents in batch with changing denormalized value`)	×
143	if (!validateOnly) {	×
144	if (nonEmptyUpdates.length > 0) {	×
145	await collection.rawCollection().bulkWrite(	×
146	nonEmptyUpdates,
147	{ ordered: false }
148	);
149	}
150	} else {
151	// TODO: This is a hack, but better than leaving it. We're basically
152	// breaking the expected API from migrateDocuments by supporting a
153	// validateOnly option, so it does not offer us good hooks to do this.
154	throw new Error([	×
155	'Abort! validateOnly means the document will not change and the migration will never',
156	'complete. This error is expected behavior to cause the migration to end.'
157	].join(' '))
158	}
159	},
160	});
161
162	// eslint-disable-next-line no-console
163	console.log(`${numDifferent} total documents had wrong denormalized value`)	×
164	}
165
166	function isNullOrDefined(value: AnyBecauseTodo) {
167	return value === null \|\| value === undefined	×
168	}

ForumMagnum / ForumMagnum / 9010223155

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous