10149877155

Committed 29 Jul 2024 06:34PM UTC coverage: 32.021% (+0.2%) from 31.857%

Build # 10149877155

Build Type

Pull #9301

github

Committed by

jpaddison3

Commit Message

Merge branch 'master' into normalized-posts-contents

Pull Request Pull Request #9301: Normalized posts contents

Run Details

5018 of 40681 branches covered (12.33%)

Branch coverage included in aggregate %.

142 of 246 new or added lines in 44 files covered. (57.72%)

11 existing lines in 10 files now uncovered.

29032 of 65657 relevant lines covered (44.22%)

191.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.95

/packages/lesswrong/server/scripts/validateMakeEditableDenormalization.ts

import { editableCollections, editableCollectionsFields } from '../../lib/editor/make_editable'
import { Vulcan, getCollection } from '../vulcan-lib';
import { Revisions } from '../../lib/collections/revisions/collection';
import { forEachDocumentBatchInCollection } from '../manualMigrations/migrationUtils';
import { editableFieldIsNormalized } from '@/lib/editor/makeEditableOptions';
import * as _ from 'underscore';

// Check that the denormalized contents field of objects with make_editable match
// the newest revision in the revisions table. This is important because we're
// going to be dropping those denormalized fields, but there's a risk that for
// some subset of content (eg particular types of important posts), they may
// contain real unique content rather than just being denormalized copies.
Vulcan.validateMakeEditableDenormalization = async () => {
  function recordError(err: string) {
    // eslint-disable-next-line no-console
    console.error("    "+err);
  }
  
  for (let collectionName of editableCollections) {
    for (let editableField of editableCollectionsFields[collectionName]!) {
      if (editableFieldIsNormalized(collectionName, editableField)) {
        continue;
      }

      // eslint-disable-next-line no-console
      console.log(`Checking ${collectionName}.${editableField}...`);
      
      const collection = getCollection(collectionName);
      await forEachDocumentBatchInCollection({
        collection: collection, batchSize: 100,
        callback: async (documents: Array<any>) => {
          const documentIds = _.map(documents, d=>d._id);
          const revs = await Revisions.find({
            documentId: {$in: documentIds},
            fieldName: editableField,
          }).fetch();
          const revsByDocument = _.groupBy(revs, rev=>rev.documentId);
          
          for (let doc of documents) {
            if (!doc[editableField]) {
              // Editable field is optional, and not present in the denormalized version
              continue;
            }
            if (!(doc._id in revsByDocument)) {
              recordError(`Document ${doc._id} has no revisions`);
              continue;
            }
            const latestRev: DbRevision|undefined = _.last(_.sortBy(revsByDocument[doc._id], r=>r.version));
            
            const denormalizedContents = doc[editableField].originalContents;
            const revContents = latestRev?.originalContents;
            if (JSON.stringify(denormalizedContents) !== JSON.stringify(revContents)) {
              recordError(`Document ${doc._id} denormalized contents don't match latest rev contents`);
            }
          }
        }
      });
    }
  }
}

1	import { editableCollections, editableCollectionsFields } from '../../lib/editor/make_editable'	18✔
2	import { Vulcan, getCollection } from '../vulcan-lib';	18✔
3	import { Revisions } from '../../lib/collections/revisions/collection';	18✔
4	import { forEachDocumentBatchInCollection } from '../manualMigrations/migrationUtils';	18✔
5	import { editableFieldIsNormalized } from '@/lib/editor/makeEditableOptions';	18✔
6	import * as _ from 'underscore';	18✔
7
8	// Check that the denormalized contents field of objects with make_editable match
9	// the newest revision in the revisions table. This is important because we're
10	// going to be dropping those denormalized fields, but there's a risk that for
11	// some subset of content (eg particular types of important posts), they may
12	// contain real unique content rather than just being denormalized copies.
13	Vulcan.validateMakeEditableDenormalization = async () => {	18✔
14	function recordError(err: string) {
15	// eslint-disable-next-line no-console
16	console.error(" "+err);	×
17	}
18
19	for (let collectionName of editableCollections) {	×
20	for (let editableField of editableCollectionsFields[collectionName]!) {	×
NEW 21	if (editableFieldIsNormalized(collectionName, editableField)) {	×
NEW 22	continue;	×
23	}
24
25	// eslint-disable-next-line no-console
26	console.log(`Checking ${collectionName}.${editableField}...`);	×
27
28	const collection = getCollection(collectionName);	×
29	await forEachDocumentBatchInCollection({	×
30	collection: collection, batchSize: 100,
31	callback: async (documents: Array<any>) => {
32	const documentIds = _.map(documents, d=>d._id);	×
33	const revs = await Revisions.find({	×
34	documentId: {$in: documentIds},
35	fieldName: editableField,
36	}).fetch();
37	const revsByDocument = _.groupBy(revs, rev=>rev.documentId);	×
38
39	for (let doc of documents) {	×
40	if (!doc[editableField]) {	×
41	// Editable field is optional, and not present in the denormalized version
42	continue;	×
43	}
44	if (!(doc._id in revsByDocument)) {	×
45	recordError(`Document ${doc._id} has no revisions`);	×
46	continue;	×
47	}
48	const latestRev: DbRevision\|undefined = _.last(_.sortBy(revsByDocument[doc._id], r=>r.version));	×
49
50	const denormalizedContents = doc[editableField].originalContents;	×
51	const revContents = latestRev?.originalContents;	×
52	if (JSON.stringify(denormalizedContents) !== JSON.stringify(revContents)) {	×
53	recordError(`Document ${doc._id} denormalized contents don't match latest rev contents`);	×
54	}
55	}
56	}
57	});
58	}
59	}
60	}

ForumMagnum / ForumMagnum / 10149877155

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous