9022893161

Committed 09 May 2024 08:06PM UTC coverage: 31.946% (+0.03%) from 31.914%

Build # 9022893161

Build Type

Pull #9256

github

Committed by

oetherington

Commit Message

Detect dependency cycles

Pull Request Pull Request #9256: Better schema dependencies

Run Details

4781 of 38947 branches covered (12.28%)

Branch coverage included in aggregate %.

106 of 113 new or added lines in 5 files covered. (93.81%)

3 existing lines in 3 files now uncovered.

27753 of 62893 relevant lines covered (44.13%)

181.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.67

/packages/lesswrong/server/scripts/makeMigrations.ts

/* eslint-disable no-console */
import { format as sqlFormatter } from 'sql-formatter';
import { Vulcan } from "../vulcan-lib";
import { getAllCollections, isValidCollectionName } from "../../lib/vulcan-lib/getCollection";
import Table from "../../lib/sql/Table";
import CreateTableQuery from "../../lib/sql/CreateTableQuery";
import md5 from 'md5';
import { unlink, writeFile } from 'node:fs/promises'
import path from 'path';
import { exec } from 'child_process';
import { acceptMigrations, migrationsPath } from './acceptMigrations';
import { existsSync } from 'node:fs';
import { ForumTypeString } from '../../lib/instanceSettings';
import { PostgresFunction, postgresFunctions } from '../postgresFunctions';
import { PostgresExtension, postgresExtensions } from '../postgresExtensions';
import CreateExtensionQuery from '../../lib/sql/CreateExtensionQuery';
import CreateIndexQuery from '../../lib/sql/CreateIndexQuery';
import { sqlInterpolateArgs } from '../../lib/sql/Type';
import { CustomPgIndex, expectedCustomPgIndexes } from '../../lib/collectionIndexUtils';
import { PostgresView, getAllPostgresViews } from '../postgresView';
import TableIndex from '../../lib/sql/TableIndex';

const ROOT_PATH = path.join(__dirname, "../../../");
const acceptedSchemePath = (rootPath: string) => path.join(rootPath, "schema/accepted_schema.sql");
const schemaToAcceptPath = (rootPath: string) => path.join(rootPath, "schema/schema_to_accept.sql");

const migrationTemplateHeader = `/**
 * Generated on %TIMESTAMP% by \`yarn makemigrations\`
 * The following schema changes were detected:
 * -------------------------------------------
`

const migrationTemplateFooter = `
 * -------------------------------------------
 * (run \`git diff --no-index schema/accepted_schema.sql schema/schema_to_accept.sql\` to see this more clearly)
 *
 * - [ ] Write a migration to represent these changes
 * - [ ] Rename this file to something more readable
 * - [ ] Uncomment \`acceptsSchemaHash\` below
 * - [ ] Run \`yarn acceptmigrations\` to update the accepted schema hash (running makemigrations again will also do this)
 */
// export const acceptsSchemaHash = "%HASH%";

export const up = async ({db}: MigrationContext) => {
  // TODO
}

export const down = async ({db}: MigrationContext) => {
  // TODO, not required
}
`

const schemaFileHeaderTemplate = `-- GENERATED FILE
-- Do not edit this file directly. Instead, start a server and run "yarn makemigrations"
-- as described in the README. This file should nevertheless be checked in to version control.
--
`

declare global {
  type SchemaDependency =
    {type: "extension", name: PostgresExtension} |
    {type: "collection", name: CollectionNameString} |
    {type: "function", name: string} |
    {type: "view", name: string};
}

abstract class Node {
  public dependencies: SchemaDependency[] = [];

  addDependency(dependency: SchemaDependency) {
    this.dependencies.push(dependency);
  }

  abstract getName(): string;
  abstract getQuery(): {compile(): {sql: string, args: any[]}};

  getHeader(): string {
    const type = this.constructor.name.replace(/(_|Node)/g, "");
    return `-- ${type} "${this.getName()}", hash ${this.getHash()}`;
  }

  getSource(): string {
    const {sql, args} = this.getQuery().compile();
    return sqlInterpolateArgs(sql, args).trim();
  }

  getAnnotatedSource(): string {
    const source = this.getSource();
    const hasSemi = source[source.length - 1] === ";";
    return `${this.getHeader()}\n${source}${hasSemi ? "" : ";"}\n`;
  }

  getHash(): string {
    return md5(this.getSource());
  }
}

class ExtensionNode extends Node {
  constructor(private extension: PostgresExtension) {
    super();
  }

  getName() {
    return this.extension;
  }

  getQuery() {
    return new CreateExtensionQuery(this.extension);
  }
}

class TableNode extends Node {
  constructor(private table: Table<DbObject>) {
    super();
  }

  getName() {
    return this.table.getName();
  }

  getQuery() {
    return new CreateTableQuery(this.table);
  }
}

class IndexNode extends Node {
  constructor(
    private table: Table<DbObject>,
    private index: TableIndex<DbObject>,
  ) {
    super();
    this.addDependency({
      type: "collection",
      name: table.getName() as CollectionNameString,
    });
  }

  getName() {
    return this.index.getName();
  }

  getQuery() {
    return new CreateIndexQuery(this.table, this.index);
  }
}

class CustomIndexNode extends Node {
  private static nameRegex = /^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(CONCURRENTLY\s+)?(IF\s+NOT\s+EXISTS\s+)?"?([a-zA-Z0-9_]+)/i;
  private static targetRegex = /.*ON\s+(public\.)?"([A-Za-z0-9_]+)"/i;
  private static concurrentRegex = /\s+CONCURRENTLY\s+/gi;

  private name: string;

  constructor(private index: CustomPgIndex) {
    super();
    const {source, options} = index;
    const name = source.match(CustomIndexNode.nameRegex)?.[4];
    if (!name) {
      throw new Error(`Can't parse name for custom index: ${source}`);
    }
    this.name = name;

    const target = source.match(CustomIndexNode.targetRegex)?.[2];
    if (!target) {
      throw new Error(`Can't parse target for custom index "${name}"`);
    }
    const dependency: SchemaDependency = isValidCollectionName(target)
      ? {type: "collection", name: target}
      : {type: "view", name: target};
    this.addDependency(dependency);

    this.dependencies = this.dependencies.concat(options?.dependencies ?? []);
  }

  getName() {
    return this.name;
  }

  getQuery() {
    return {
      compile: () => ({
        sql: this.index.source.trim().replace(CustomIndexNode.concurrentRegex, " "),
        args: [],
      }),
    };
  }
}

class FunctionNode extends Node {
  private static nameRegex = /^\s*CREATE\s+(OR\s+REPLACE\s+)?FUNCTION\s+([a-zA-Z0-9_]+)/i;

  private name: string;

  constructor(private func: PostgresFunction) {
    super();
    const name = func.source.match(FunctionNode.nameRegex)?.[2];
    if (!name) {
      throw new Error(`Can't find name for function: ${func.source}`);
    }
    const overload = func.overload ? `_${func.overload}` : "";
    this.name = name + overload;
    this.dependencies = this.dependencies.concat(func.dependencies ?? []);
  }

  getName() {
    return this.name;
  }

  getQuery() {
    return {
      compile: () => ({
        sql: this.func.source.trim(),
        args: [],
      }),
    };
  }
}

class ViewNode extends Node {
  constructor(private view: PostgresView) {
    super();
  }

  getName() {
    return this.view.getName();
  }

  getQuery() {
    return {
      compile: () => ({
        sql: this.view.getCreateViewQuery().trim(),
        args: [],
      }),
    };
  }
}

class Graph {
  private nodes: Record<string, Node> = {};

  addNode(node: Node) {
    const name = node.getName();
    if (this.nodes[name]) {
      throw new Error(`Duplicate node names: "${name}"`);
    }
    this.nodes[name] = node;
  }

  addNodes(nodes: Node[]) {
    for (const node of nodes) {
      this.addNode(node);
    }
  }

  getOverallHash() {
    return md5(Object.values(this.nodes).map((n) => n.getHash()).sort().join());
  }

  linearize(): Node[] {
    const stack: Node[] = [];
    const unvisited = new Set<string>(Object.keys(this.nodes));

    const depthFirstSearch = (nodeName: string, parents: string[]) => {
      if (parents.includes(nodeName)) {
        throw new Error(`Dependency cycle detected for ${nodeName} via ${parents}`);
      }

      const node = this.nodes[nodeName];
      if (!node) {
        throw new Error(`Invalid node: "${nodeName}"`);
      }

      if (!unvisited.has(nodeName)) {
        return;
      }
      unvisited.delete(nodeName);

      for (const dependency of node.dependencies) {
        depthFirstSearch(dependency.name, [...parents, nodeName]);
      }

      stack.push(node);
    }

    unvisited.forEach((node) => depthFirstSearch(node, []));

    return stack;
  }
}

const generateMigration = async ({
  acceptedSchemaFile, toAcceptSchemaFile, toAcceptHash, rootPath,
}: {
  acceptedSchemaFile: string,
  toAcceptSchemaFile: string,
  toAcceptHash: string,
  rootPath: string,
}) => {
  const execRun = (cmd: string) => {
    return new Promise((resolve) => {
      // git diff exits with an error code if there are differences, ignore that and just always return stdout
      exec(cmd, (_error, stdout, _stderr) => resolve(stdout))
    })
  }

  // bit of a hack but using `git diff` for everything makes the changes easy to read
  const diff: string = await execRun(`git diff --no-index ${acceptedSchemaFile} ${toAcceptSchemaFile} --unified=1`) as string;
  const paddedDiff = diff.replace(/^/gm, ' * ');

  let contents = "";
  contents += migrationTemplateHeader.replace("%TIMESTAMP%", new Date().toISOString());
  contents += paddedDiff.length < 30000 ? paddedDiff : ` * ***Diff too large to display***`;
  contents += migrationTemplateFooter.replace("%HASH%", toAcceptHash);

  const fileTimestamp = new Date().toISOString().replace(/[-:]/g, "").split(".")[0];
  const fileName = `${fileTimestamp}.auto.ts`;

  await writeFile(path.join(migrationsPath(rootPath), fileName), contents);
}

/**
 * Update the `./schema/` files to match the current database schema, and generate a migration if there are changes which need to be accepted.
 *
 * Implementation details which may be useful to know:
 * This function (and `acceptMigrations`) generates a hash of the current schema (as defined in code) and uses it to maintain three files
 * in the `./schema` directory, `schema_changelog.json`, `accepted_schema.sql`, `schema_to_accept.sql`:
 * - `schema_changelog.json`: This is the file that actually determines whether the current schema is "accepted" or not.
 *   It contains a list of hashes of schema files that have been accepted. If the current schema hash is the most recent entry in this file, then the schema is accepted.
 * - `accepted_schema.sql`: This is a SQL view of the schema that has been accepted.
 * - `schema_to_accept.sql`: If the current schema is not accepted, this file will be generated to contain a SQL view of the "unaccepted" schema.
 *   This is useful for comparing against the accepted schema to see what changes need to be made in the migration that is generated. It is automatically deleted when the schema is accepted.
 */
export const makeMigrations = async ({
  writeSchemaChangelog=true,
  writeAcceptedSchema=true,
  generateMigrations=true,
  rootPath=ROOT_PATH,
  forumType,
  silent=false,
}: {
  /** If true, update the schema_changelog.json file before checking for changes */
  writeSchemaChangelog: boolean,
  /** If true, update the `accepted_schema.sql` and `schema_to_accept.sql` */
  writeAcceptedSchema: boolean,
  /** If true, generate a template migration file if the schema has changed */
  generateMigrations: boolean,
  /** The root path of the project, this is annoying but required because this script is sometimes run from the server bundle, and sometimes from a test. */
  rootPath: string,
  /** The optional forumType to switch to */
  forumType?: ForumTypeString,
  silent?: boolean,
}) => {
  const log = silent ? (..._args: any[]) => {} : console.log;
  log(`=== Checking for schema changes ===`);
  // Get the most recent accepted schema hash from `schema_changelog.json`
  const {acceptsSchemaHash: acceptedHash, acceptedByMigration, timestamp} = await acceptMigrations({write: writeSchemaChangelog, rootPath});
  log(`-- Using accepted hash ${acceptedHash}`);

  const graph = new Graph();
  graph.addNodes(postgresExtensions.map((e) => new ExtensionNode(e)));
  graph.addNodes(getAllCollections().flatMap((collection) => {
    const table = Table.fromCollection(collection, forumType);
    const indexes: Node[] = table.getIndexes().map((i) => new IndexNode(table, i));
    return indexes.concat(new TableNode(table));
  }));
  graph.addNodes(expectedCustomPgIndexes.map((i) => new CustomIndexNode(i)));
  graph.addNodes(postgresFunctions.map((f) => new FunctionNode(f)));
  graph.addNodes(getAllPostgresViews().flatMap((view) => {
    const indexQueries = view.getCreateIndexQueries();
    const indexes: Node[] = indexQueries.map((source) => new CustomIndexNode({source}));
    return indexes.concat(new ViewNode(view));
  }));

  const nodes = graph.linearize();
  const rawSchema = nodes.map((n) => n.getAnnotatedSource()).join("\n");
  const schemaFileContents = sqlFormatter(rawSchema, {
    language: "postgresql",
    linesBetweenQueries: 1,
    tabWidth: 2,
    useTabs: false,
    keywordCase: "upper",
    dataTypeCase: "upper",
    functionCase: "upper",
    identifierCase: "preserve",
    logicalOperatorNewline: "after",
    paramTypes: {
      positional: false,
      numbered: [],
      named: [],
      quoted: [],
      custom: [],
    },
  }) + "\n";

  const overallHash = graph.getOverallHash();

  let schemaFileHeader = schemaFileHeaderTemplate + `-- Overall schema hash: ${overallHash}\n\n`;

  const toAcceptSchemaFile = schemaToAcceptPath(rootPath);
  const acceptedSchemaFile = acceptedSchemePath(rootPath);

  if (overallHash !== acceptedHash) {
    if (writeAcceptedSchema) {
      await writeFile(toAcceptSchemaFile, schemaFileHeader + schemaFileContents);
    }
    if (generateMigrations) {
      await generateMigration({acceptedSchemaFile, toAcceptSchemaFile, toAcceptHash: overallHash, rootPath});
    }
    throw new Error(`Schema has changed, write a migration to accept the new hash: ${overallHash}`);
  }

  if (writeAcceptedSchema) {
    schemaFileHeader += `-- Accepted on ${timestamp}${acceptedByMigration ? " by " + acceptedByMigration : ''}\n\n`;
    await writeFile(acceptedSchemaFile, schemaFileHeader + schemaFileContents);
    if (existsSync(toAcceptSchemaFile)) {
      await unlink(toAcceptSchemaFile);
    }
  }

  log("=== Done ===");
}

Vulcan.makeMigrations = makeMigrations;

1	/* eslint-disable no-console */	55✔
2	import { format as sqlFormatter } from 'sql-formatter';	18✔
3	import { Vulcan } from "../vulcan-lib";	18✔
4	import { getAllCollections, isValidCollectionName } from "../../lib/vulcan-lib/getCollection";	18✔
5	import Table from "../../lib/sql/Table";	18✔
6	import CreateTableQuery from "../../lib/sql/CreateTableQuery";	18✔
7	import md5 from 'md5';	18✔
8	import { unlink, writeFile } from 'node:fs/promises'	18✔
9	import path from 'path';	18✔
10	import { exec } from 'child_process';	18✔
11	import { acceptMigrations, migrationsPath } from './acceptMigrations';	18✔
12	import { existsSync } from 'node:fs';	18✔
13	import { ForumTypeString } from '../../lib/instanceSettings';
14	import { PostgresFunction, postgresFunctions } from '../postgresFunctions';	18✔
15	import { PostgresExtension, postgresExtensions } from '../postgresExtensions';	18✔
16	import CreateExtensionQuery from '../../lib/sql/CreateExtensionQuery';	18✔
17	import CreateIndexQuery from '../../lib/sql/CreateIndexQuery';	18✔
18	import { sqlInterpolateArgs } from '../../lib/sql/Type';	18✔
19	import { CustomPgIndex, expectedCustomPgIndexes } from '../../lib/collectionIndexUtils';	18✔
20	import { PostgresView, getAllPostgresViews } from '../postgresView';	18✔
21	import TableIndex from '../../lib/sql/TableIndex';
22
23	const ROOT_PATH = path.join(__dirname, "../../../");	18✔
24	const acceptedSchemePath = (rootPath: string) => path.join(rootPath, "schema/accepted_schema.sql");	18✔
25	const schemaToAcceptPath = (rootPath: string) => path.join(rootPath, "schema/schema_to_accept.sql");	18✔
26
27	const migrationTemplateHeader = `/**	18✔
28	* Generated on %TIMESTAMP% by \`yarn makemigrations\`
29	* The following schema changes were detected:
30	* -------------------------------------------
31	`
32
33	const migrationTemplateFooter = `	18✔
34	* -------------------------------------------
35	* (run \`git diff --no-index schema/accepted_schema.sql schema/schema_to_accept.sql\` to see this more clearly)
36	*
37	* - [ ] Write a migration to represent these changes
38	* - [ ] Rename this file to something more readable
39	* - [ ] Uncomment \`acceptsSchemaHash\` below
40	* - [ ] Run \`yarn acceptmigrations\` to update the accepted schema hash (running makemigrations again will also do this)
41	*/
42	// export const acceptsSchemaHash = "%HASH%";
43
44	export const up = async ({db}: MigrationContext) => {
45	// TODO
46	}
47
48	export const down = async ({db}: MigrationContext) => {
49	// TODO, not required
50	}
51	`
52
53	const schemaFileHeaderTemplate = `-- GENERATED FILE	18✔
54	-- Do not edit this file directly. Instead, start a server and run "yarn makemigrations"
55	-- as described in the README. This file should nevertheless be checked in to version control.
56	--
57	`
58
59	declare global {
60	type SchemaDependency =
61	{type: "extension", name: PostgresExtension} \|
62	{type: "collection", name: CollectionNameString} \|
63	{type: "function", name: string} \|
64	{type: "view", name: string};
65	}
66
67	abstract class Node {
68	public dependencies: SchemaDependency[] = [];	409✔
69
70	addDependency(dependency: SchemaDependency) {
71	this.dependencies.push(dependency);	315✔
72	}
73
74	abstract getName(): string;
75	abstract getQuery(): {compile(): {sql: string, args: any[]}};
76
77	getHeader(): string {
78	const type = this.constructor.name.replace(/(_\|Node)/g, "");	409✔
79	return `-- ${type} "${this.getName()}", hash ${this.getHash()}`;	409✔
80	}
81
82	getSource(): string {
83	const {sql, args} = this.getQuery().compile();	1,227✔
84	return sqlInterpolateArgs(sql, args).trim();	1,227✔
85	}
86
87	getAnnotatedSource(): string {
88	const source = this.getSource();	409✔
89	const hasSemi = source[source.length - 1] === ";";	409✔
90	return `${this.getHeader()}\n${source}${hasSemi ? "" : ";"}\n`;	409✔
91	}
92
93	getHash(): string {
94	return md5(this.getSource());	818✔
95	}
96	}
97
98	class ExtensionNode extends Node {
99	constructor(private extension: PostgresExtension) {	5✔
100	super();	5✔
101	}
102
103	getName() {
104	return this.extension;	10✔
105	}
106
107	getQuery() {
108	return new CreateExtensionQuery(this.extension);	15✔
109	}
110	}
111
112	class TableNode extends Node {
113	constructor(private table: Table<DbObject>) {	74✔
114	super();	74✔
115	}
116
117	getName() {
118	return this.table.getName();	148✔
119	}
120
121	getQuery() {
122	return new CreateTableQuery(this.table);	222✔
123	}
124	}
125
126	class IndexNode extends Node {
127	constructor(
128	private table: Table<DbObject>,	299✔
129	private index: TableIndex<DbObject>,	299✔
130	) {
131	super();	299✔
132	this.addDependency({	299✔
133	type: "collection",
134	name: table.getName() as CollectionNameString,
135	});
136	}
137
138	getName() {
139	return this.index.getName();	598✔
140	}
141
142	getQuery() {
143	return new CreateIndexQuery(this.table, this.index);	897✔
144	}
145	}
146
147	class CustomIndexNode extends Node {	18✔
148	private static nameRegex = /^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(CONCURRENTLY\s+)?(IF\s+NOT\s+EXISTS\s+)?"?([a-zA-Z0-9_]+)/i;
149	private static targetRegex = /.*ON\s+(public\.)?"([A-Za-z0-9_]+)"/i;
150	private static concurrentRegex = /\s+CONCURRENTLY\s+/gi;
151
152	private name: string;
153
154	constructor(private index: CustomPgIndex) {	16✔
155	super();	16✔
156	const {source, options} = index;	16✔
157	const name = source.match(CustomIndexNode.nameRegex)?.[4];	16!
158	if (!name) {	16!
NEW 159	throw new Error(`Can't parse name for custom index: ${source}`);	×
160	}
161	this.name = name;	16✔
162
163	const target = source.match(CustomIndexNode.targetRegex)?.[2];	16!
164	if (!target) {	16!
NEW 165	throw new Error(`Can't parse target for custom index "${name}"`);	×
166	}
167	const dependency: SchemaDependency = isValidCollectionName(target)	16✔
168	? {type: "collection", name: target}	16✔
169	: {type: "view", name: target};
170	this.addDependency(dependency);	16✔
171
172	this.dependencies = this.dependencies.concat(options?.dependencies ?? []);	16✔
173	}
174
175	getName() {
176	return this.name;	32✔
177	}
178
179	getQuery() {
180	return {	48✔
181	compile: () => ({	48✔
182	sql: this.index.source.trim().replace(CustomIndexNode.concurrentRegex, " "),
183	args: [],
184	}),
185	};
186	}
187	}
188
189	class FunctionNode extends Node {	18✔
190	private static nameRegex = /^\s*CREATE\s+(OR\s+REPLACE\s+)?FUNCTION\s+([a-zA-Z0-9_]+)/i;
191
192	private name: string;
193
194	constructor(private func: PostgresFunction) {	12✔
195	super();	12✔
196	const name = func.source.match(FunctionNode.nameRegex)?.[2];	12!
197	if (!name) {	12!
NEW 198	throw new Error(`Can't find name for function: ${func.source}`);	×
199	}
200	const overload = func.overload ? `_${func.overload}` : "";	12✔
201	this.name = name + overload;	12✔
202	this.dependencies = this.dependencies.concat(func.dependencies ?? []);	12✔
203	}
204
205	getName() {
206	return this.name;	24✔
207	}
208
209	getQuery() {
210	return {	36✔
211	compile: () => ({	36✔
212	sql: this.func.source.trim(),
213	args: [],
214	}),
215	};
216	}
217	}
218
219	class ViewNode extends Node {
220	constructor(private view: PostgresView) {	3✔
221	super();	3✔
222	}
223
224	getName() {
225	return this.view.getName();	6✔
226	}
227
228	getQuery() {
229	return {	9✔
230	compile: () => ({	9✔
231	sql: this.view.getCreateViewQuery().trim(),
232	args: [],
233	}),
234	};
235	}
236	}
237
238	class Graph {
239	private nodes: Record<string, Node> = {};	1✔
240
241	addNode(node: Node) {
242	const name = node.getName();	409✔
243	if (this.nodes[name]) {	409!
NEW 244	throw new Error(`Duplicate node names: "${name}"`);	×
245	}
246	this.nodes[name] = node;	409✔
247	}
248
249	addNodes(nodes: Node[]) {
250	for (const node of nodes) {	5✔
251	this.addNode(node);	409✔
252	}
253	}
254
255	getOverallHash() {
256	return md5(Object.values(this.nodes).map((n) => n.getHash()).sort().join());	409✔
257	}
258
259	linearize(): Node[] {
260	const stack: Node[] = [];	1✔
261	const unvisited = new Set<string>(Object.keys(this.nodes));	1✔
262
263	const depthFirstSearch = (nodeName: string, parents: string[]) => {	1✔
264	if (parents.includes(nodeName)) {	656!
NEW 265	throw new Error(`Dependency cycle detected for ${nodeName} via ${parents}`);	×
266	}
267
268	const node = this.nodes[nodeName];	656✔
269	if (!node) {	656!
NEW 270	throw new Error(`Invalid node: "${nodeName}"`);	×
271	}
272
273	if (!unvisited.has(nodeName)) {	656✔
274	return;	247✔
275	}
276	unvisited.delete(nodeName);	409✔
277
278	for (const dependency of node.dependencies) {	409✔
279	depthFirstSearch(dependency.name, [...parents, nodeName]);	322✔
280	}
281
282	stack.push(node);	409✔
283	}
284
285	unvisited.forEach((node) => depthFirstSearch(node, []));	334✔
286
287	return stack;	1✔
288	}
289	}
290
291	const generateMigration = async ({	18✔
292	acceptedSchemaFile, toAcceptSchemaFile, toAcceptHash, rootPath,
293	}: {
294	acceptedSchemaFile: string,
295	toAcceptSchemaFile: string,
296	toAcceptHash: string,
297	rootPath: string,
298	}) => {
299	const execRun = (cmd: string) => {	×
300	return new Promise((resolve) => {	×
301	// git diff exits with an error code if there are differences, ignore that and just always return stdout
302	exec(cmd, (_error, stdout, _stderr) => resolve(stdout))	×
303	})
304	}
305
306	// bit of a hack but using `git diff` for everything makes the changes easy to read
307	const diff: string = await execRun(`git diff --no-index ${acceptedSchemaFile} ${toAcceptSchemaFile} --unified=1`) as string;	×
308	const paddedDiff = diff.replace(/^/gm, ' * ');	×
309
310	let contents = "";	×
311	contents += migrationTemplateHeader.replace("%TIMESTAMP%", new Date().toISOString());	×
312	contents += paddedDiff.length < 30000 ? paddedDiff : ` * *Diff too large to display*`;	×
313	contents += migrationTemplateFooter.replace("%HASH%", toAcceptHash);	×
314
315	const fileTimestamp = new Date().toISOString().replace(/[-:]/g, "").split(".")[0];	×
316	const fileName = `${fileTimestamp}.auto.ts`;	×
317
318	await writeFile(path.join(migrationsPath(rootPath), fileName), contents);	×
319	}
320
321	/**
322	* Update the `./schema/` files to match the current database schema, and generate a migration if there are changes which need to be accepted.
323	*
324	* Implementation details which may be useful to know:
325	* This function (and `acceptMigrations`) generates a hash of the current schema (as defined in code) and uses it to maintain three files
326	* in the `./schema` directory, `schema_changelog.json`, `accepted_schema.sql`, `schema_to_accept.sql`:
327	* - `schema_changelog.json`: This is the file that actually determines whether the current schema is "accepted" or not.
328	* It contains a list of hashes of schema files that have been accepted. If the current schema hash is the most recent entry in this file, then the schema is accepted.
329	* - `accepted_schema.sql`: This is a SQL view of the schema that has been accepted.
330	* - `schema_to_accept.sql`: If the current schema is not accepted, this file will be generated to contain a SQL view of the "unaccepted" schema.
331	* This is useful for comparing against the accepted schema to see what changes need to be made in the migration that is generated. It is automatically deleted when the schema is accepted.
332	*/
333	export const makeMigrations = async ({	18✔
334	writeSchemaChangelog=true,	×
335	writeAcceptedSchema=true,	×
336	generateMigrations=true,	×
337	rootPath=ROOT_PATH,	×
338	forumType,
339	silent=false,	×
340	}: {
341	/** If true, update the schema_changelog.json file before checking for changes */
342	writeSchemaChangelog: boolean,
343	/** If true, update the `accepted_schema.sql` and `schema_to_accept.sql` */
344	writeAcceptedSchema: boolean,
345	/** If true, generate a template migration file if the schema has changed */
346	generateMigrations: boolean,
347	/** The root path of the project, this is annoying but required because this script is sometimes run from the server bundle, and sometimes from a test. */
348	rootPath: string,
349	/** The optional forumType to switch to */
350	forumType?: ForumTypeString,
351	silent?: boolean,
352	}) => {
353	const log = silent ? (..._args: any[]) => {} : console.log;	1!
354	log(`=== Checking for schema changes ===`);	1✔
355	// Get the most recent accepted schema hash from `schema_changelog.json`
356	const {acceptsSchemaHash: acceptedHash, acceptedByMigration, timestamp} = await acceptMigrations({write: writeSchemaChangelog, rootPath});	1✔
357	log(`-- Using accepted hash ${acceptedHash}`);	1✔
358
359	const graph = new Graph();	1✔
360	graph.addNodes(postgresExtensions.map((e) => new ExtensionNode(e)));	5✔
361	graph.addNodes(getAllCollections().flatMap((collection) => {	1✔
362	const table = Table.fromCollection(collection, forumType);	74✔
363	const indexes: Node[] = table.getIndexes().map((i) => new IndexNode(table, i));	299✔
364	return indexes.concat(new TableNode(table));	74✔
365	}));
366	graph.addNodes(expectedCustomPgIndexes.map((i) => new CustomIndexNode(i)));	14✔
367	graph.addNodes(postgresFunctions.map((f) => new FunctionNode(f)));	12✔
368	graph.addNodes(getAllPostgresViews().flatMap((view) => {	1✔
369	const indexQueries = view.getCreateIndexQueries();	3✔
370	const indexes: Node[] = indexQueries.map((source) => new CustomIndexNode({source}));	3✔
371	return indexes.concat(new ViewNode(view));	3✔
372	}));
373
374	const nodes = graph.linearize();	1✔
375	const rawSchema = nodes.map((n) => n.getAnnotatedSource()).join("\n");	409✔
376	const schemaFileContents = sqlFormatter(rawSchema, {	1✔
377	language: "postgresql",
378	linesBetweenQueries: 1,
379	tabWidth: 2,
380	useTabs: false,
381	keywordCase: "upper",
382	dataTypeCase: "upper",
383	functionCase: "upper",
384	identifierCase: "preserve",
385	logicalOperatorNewline: "after",
386	paramTypes: {
387	positional: false,
388	numbered: [],
389	named: [],
390	quoted: [],
391	custom: [],
392	},
393	}) + "\n";
394
395	const overallHash = graph.getOverallHash();	1✔
396
397	let schemaFileHeader = schemaFileHeaderTemplate + `-- Overall schema hash: ${overallHash}\n\n`;	1✔
398
399	const toAcceptSchemaFile = schemaToAcceptPath(rootPath);	1✔
400	const acceptedSchemaFile = acceptedSchemePath(rootPath);	1✔
401
402	if (overallHash !== acceptedHash) {	1!
403	if (writeAcceptedSchema) {	×
404	await writeFile(toAcceptSchemaFile, schemaFileHeader + schemaFileContents);	×
405	}
406	if (generateMigrations) {	×
407	await generateMigration({acceptedSchemaFile, toAcceptSchemaFile, toAcceptHash: overallHash, rootPath});	×
408	}
409	throw new Error(`Schema has changed, write a migration to accept the new hash: ${overallHash}`);	×
410	}
411
412	if (writeAcceptedSchema) {	1!
413	schemaFileHeader += `-- Accepted on ${timestamp}${acceptedByMigration ? " by " + acceptedByMigration : ''}\n\n`;	1!
414	await writeFile(acceptedSchemaFile, schemaFileHeader + schemaFileContents);	1✔
415	if (existsSync(toAcceptSchemaFile)) {	1!
416	await unlink(toAcceptSchemaFile);	×
417	}
418	}
419
420	log("=== Done ===");	1✔
421	}
422
423	Vulcan.makeMigrations = makeMigrations;	18✔

ForumMagnum / ForumMagnum / 9022893161

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous