8881983748

Committed 29 Apr 2024 04:04PM UTC coverage: 31.951% (-0.001%) from 31.952%

Build # 8881983748

Build Type

Pull #8980

github

Committed by

oetherington

Commit Message

Fix soft merge conflict for unread messages postgres view

Pull Request Pull Request #8980: Materialized view for BTREE login token lookup

Run Details

4714 of 38543 branches covered (12.23%)

Branch coverage included in aggregate %.

9 of 28 new or added lines in 6 files covered. (32.14%)

1 existing line in 1 file now uncovered.

27506 of 62299 relevant lines covered (44.15%)

179.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/packages/lesswrong/server/serverStartup.ts

/* eslint-disable no-console */
import './datadog/tracer';
import { createSqlConnection } from './sqlConnection';
import { getSqlClientOrThrow, replaceDbNameInPgConnectionString, setSqlClient } from '../lib/sql/sqlClient';
import PgCollection, { DbTarget } from '../lib/sql/PgCollection';
import { Collections } from '../lib/vulcan-lib/getCollection';
import { runStartupFunctions, isAnyTest, isMigrations, CommandLineArguments } from '../lib/executionEnvironment';
import { PublicInstanceSetting } from "../lib/instanceSettings";
import { refreshSettingsCaches } from './loadDatabaseSettings';
import { getCommandLineArguments } from './commandLine';
import { startWebserver } from './apolloServer';
import { initGraphQL } from './vulcan-lib/apollo-server/initGraphQL';
import { createVoteableUnionType } from './votingGraphQL';
import { Globals, Vulcan } from '../lib/vulcan-lib/config';
import { getBranchDbName } from "./branchDb";
import { dropAndCreatePg } from './testingSqlClient';
import process from 'process';
import chokidar from 'chokidar';
import fs from 'fs';
import { basename, join } from 'path';
import { filterConsoleLogSpam, wrapConsoleLogFunctions } from '../lib/consoleFilters';
import { ensurePostgresViewsExist } from './postgresView';
import cluster from 'node:cluster';
import { cpus } from 'node:os';
import { panic } from './utils/errorUtil';
import { addCronJob } from './cronUtil';

const numCPUs = cpus().length;

/**
 * Whether to run multiple node processes in a cluster.
 * The main reason this is a PublicInstanceSetting because it would be annoying and disruptive for other devs to change this while you're running the server.
 */
export const clusterSetting = new PublicInstanceSetting<boolean>('cluster.enabled', false, 'optional')
export const numWorkersSetting = new PublicInstanceSetting<number>('cluster.numWorkers', numCPUs, 'optional')

// Do this here to avoid a dependency cycle
Globals.dropAndCreatePg = dropAndCreatePg;

const initConsole = () => {
  const isTTY = process.stdout.isTTY;
  const CSI = "\x1b[";
  const blue = isTTY ? `${CSI}34m` : "";
  const endBlue = isTTY ? `${CSI}39m` : "";

  filterConsoleLogSpam();
  wrapConsoleLogFunctions((log, ...message) => {
    const pidString = clusterSetting.get() ? ` (pid: ${process.pid})` : "";
    process.stdout.write(`${blue}${new Date().toISOString()}${pidString}:${endBlue} `);
    log(...message);

    // Uncomment to add stacktraces to every console.log, for debugging where
    // mysterious output came from.
    //var stack = new Error().stack
    //log(stack)
  });
}

const connectToPostgres = async (connectionString: string, target: DbTarget = "write") => {
  try {
    if (connectionString) {
      const branchDb = await getBranchDbName();
      if (branchDb) {
        connectionString = replaceDbNameInPgConnectionString(connectionString, branchDb);
      }
      const dbName = /.*\/(.*)/.exec(connectionString)?.[1];
      // eslint-disable-next-line no-console
      console.log(`Connecting to postgres (${dbName})`);
      const sql = await createSqlConnection(connectionString, false);
      setSqlClient(sql, target);
    }
  } catch(err) {
    // eslint-disable-next-line no-console
    console.error("Failed to connect to postgres: ", err.message);
    process.exit(1);
  }
}

const initDatabases = ({postgresUrl, postgresReadUrl}: CommandLineArguments) =>
  Promise.all([
    connectToPostgres(postgresUrl),
    connectToPostgres(postgresReadUrl, "read"),
  ]);

const initSettings = () => {
  if (!isAnyTest) {
    setInterval(refreshSettingsCaches, 1000 * 60 * 5) // We refresh the cache every 5 minutes on all servers
  }
  return refreshSettingsCaches();
}

const initPostgres = async () => {
  if (Collections.some(collection => collection instanceof PgCollection)) {
    for (const collection of Collections) {
      if (collection instanceof PgCollection) {
        collection.buildPostgresTable();
      }
    }
  }

  // If we're migrating up, we might be migrating from the start on a fresh database, so skip the check
  // for whether postgres views exist
  const migrating = process.argv.some(arg => arg.indexOf('migrate') > -1)
  const migratingUp = process.argv.some(arg => arg === 'up')
  if (migrating && migratingUp) return;

  try {
    await ensurePostgresViewsExist(getSqlClientOrThrow(), addCronJob);
  } catch (e) {
    // eslint-disable-next-line no-console
    console.error("Failed to ensure Postgres views exist:", e);
  }
}

const executeServerWithArgs = async ({shellMode, command}: CommandLineArguments) => {
  await runStartupFunctions();

  // define executableSchema
  createVoteableUnionType();
  initGraphQL();

  if (shellMode) {
    initShell();
  } else if (command) {
    const func = compileWithGlobals(command);
    const result = await func();
    // eslint-disable-next-line no-console
    console.log("Finished. Result: ", result);
    process.kill(estrellaPid, 'SIGQUIT');
  } else if (!isAnyTest && !isMigrations) {
    watchForShellCommands();
    startWebserver();
  }
}

export const initServer = async (commandLineArguments?: CommandLineArguments) => {
  initConsole();
  const args = commandLineArguments ?? getCommandLineArguments();
  if (!args.postgresUrl) {
    panic("Missing postgresUrl");
  }
  await initDatabases(args);
  await initSettings();
  require('../server.ts');
  await initPostgres();
  return args;
}

export const serverStartup = async () => {
  // Run server directly if not in cluster mode
  if (!clusterSetting.get()) {
    await serverStartupWorker();
    return;
  }

  // Use OS load balancing (as opposed to round-robin)
  // In principle, this should give better performance because it is aware of resource (cpu) usage
  cluster.schedulingPolicy = cluster.SCHED_NONE
  if (cluster.isPrimary) {
    // Initialize db connection and a few other things such as settings, but don't start a webserver.
    console.log("Initializing primary process");
    await initServer();

    const numWorkers = numWorkersSetting.get();

    console.log(`Running in cluster mode with ${numWorkers} workers (vs ${numCPUs} cpus)`);
    console.log(`Primary ${process.pid} is running, about to fork workers`);

    // Fork workers.
    for (let i = 0; i < numWorkers; i++) {
      cluster.fork();
    }

    cluster.on('exit', (worker, code, signal) => {
      console.log(`Worker ${worker.process.pid} died`);
    });
  } else {
    console.log(`Starting worker ${process.pid}`);
    await serverStartupWorker();
    console.log(`Worker ${process.pid} started`);
  }
}

export const serverStartupWorker = async () => {
  const commandLineArguments = await initServer();
  await executeServerWithArgs(commandLineArguments);
}

function initShell() {
  const repl = require('repl');
  /*const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout,
    prompt: "> ",
  });
  rl.on('line', line => {
    console.log(`Got input: ${line}`);
    rl.prompt();
  });
  rl.prompt();*/

  const r = repl.start({
    prompt: "> ",
    terminal: true,
    preview: true,
    breakEvalOnSigint: true,
    useGlobal: true,
  });
  r.context.Globals = Globals;
  r.context.Vulcan = Globals;
}

const compileWithGlobals = (code: string) => {
  // This is basically just eval() but done in a way that:
  //   1) Allows us to define our own global scope
  //   2) Doesn't upset esbuild
  const callable = (async function () {}).constructor(`with(this) { await ${code} }`);
  const scope = {Globals, Vulcan};
  return () => {
    return callable.call(new Proxy({}, {
      has () { return true; },
      get (_target, key) {
        if (typeof key !== "symbol") {
          return global[key as keyof Global] ?? scope[key as "Globals"|"Vulcan"];
        }
      }
    }));
  }
}

// Monitor ./tmp/pendingShellCommands for shell commands. If a JS file is
// written there, run it then delete it. Security-wise this is okay because
// write-access inside the repo directory is already equivalent to script
// execution.
const watchForShellCommands = () => {
  const watcher = chokidar.watch('./tmp/pendingShellCommands');
  watcher.on('add', async (path) => {
    const fileContents = fs.readFileSync(path, 'utf8');
    // eslint-disable-next-line no-console
    console.log(`Running shell command: ${fileContents}`);
    const newPath = join("tmp/runningShellCommands", basename(path));
    fs.renameSync(path, newPath);
    try {
      const func = compileWithGlobals(fileContents);
      const result = await func();
      // eslint-disable-next-line no-console
      console.log("Finished. Result: ", result);
    } catch(e) {
      // eslint-disable-next-line no-console
      console.log("Failed.");
      // eslint-disable-next-line no-console
      console.log(e);
    } finally {
      fs.unlinkSync(newPath);
    }
  });
}

1	/* eslint-disable no-console */	×
2	import './datadog/tracer';	×
3	import { createSqlConnection } from './sqlConnection';	×
4	import { getSqlClientOrThrow, replaceDbNameInPgConnectionString, setSqlClient } from '../lib/sql/sqlClient';	×
5	import PgCollection, { DbTarget } from '../lib/sql/PgCollection';	×
6	import { Collections } from '../lib/vulcan-lib/getCollection';	×
7	import { runStartupFunctions, isAnyTest, isMigrations, CommandLineArguments } from '../lib/executionEnvironment';	×
8	import { PublicInstanceSetting } from "../lib/instanceSettings";	×
9	import { refreshSettingsCaches } from './loadDatabaseSettings';	×
10	import { getCommandLineArguments } from './commandLine';	×
11	import { startWebserver } from './apolloServer';	×
12	import { initGraphQL } from './vulcan-lib/apollo-server/initGraphQL';	×
13	import { createVoteableUnionType } from './votingGraphQL';	×
14	import { Globals, Vulcan } from '../lib/vulcan-lib/config';	×
15	import { getBranchDbName } from "./branchDb";	×
16	import { dropAndCreatePg } from './testingSqlClient';	×
17	import process from 'process';	×
18	import chokidar from 'chokidar';	×
19	import fs from 'fs';	×
20	import { basename, join } from 'path';	×
21	import { filterConsoleLogSpam, wrapConsoleLogFunctions } from '../lib/consoleFilters';	×
22	import { ensurePostgresViewsExist } from './postgresView';	×
23	import cluster from 'node:cluster';	×
24	import { cpus } from 'node:os';	×
25	import { panic } from './utils/errorUtil';	×
NEW 26	import { addCronJob } from './cronUtil';	×
27
28	const numCPUs = cpus().length;	×
29
30	/**
31	* Whether to run multiple node processes in a cluster.
32	* The main reason this is a PublicInstanceSetting because it would be annoying and disruptive for other devs to change this while you're running the server.
33	*/
34	export const clusterSetting = new PublicInstanceSetting<boolean>('cluster.enabled', false, 'optional')	×
35	export const numWorkersSetting = new PublicInstanceSetting<number>('cluster.numWorkers', numCPUs, 'optional')	×
36
37	// Do this here to avoid a dependency cycle
38	Globals.dropAndCreatePg = dropAndCreatePg;	×
39
40	const initConsole = () => {	×
41	const isTTY = process.stdout.isTTY;	×
42	const CSI = "\x1b[";	×
43	const blue = isTTY ? `${CSI}34m` : "";	×
44	const endBlue = isTTY ? `${CSI}39m` : "";	×
45
46	filterConsoleLogSpam();	×
47	wrapConsoleLogFunctions((log, ...message) => {	×
48	const pidString = clusterSetting.get() ? ` (pid: ${process.pid})` : "";	×
49	process.stdout.write(`${blue}${new Date().toISOString()}${pidString}:${endBlue} `);	×
50	log(...message);	×
51
52	// Uncomment to add stacktraces to every console.log, for debugging where
53	// mysterious output came from.
54	//var stack = new Error().stack
55	//log(stack)
56	});
57	}
58
59	const connectToPostgres = async (connectionString: string, target: DbTarget = "write") => {	×
60	try {	×
61	if (connectionString) {	×
62	const branchDb = await getBranchDbName();	×
63	if (branchDb) {	×
64	connectionString = replaceDbNameInPgConnectionString(connectionString, branchDb);	×
65	}
66	const dbName = /.\/(.)/.exec(connectionString)?.[1];	×
67	// eslint-disable-next-line no-console
68	console.log(`Connecting to postgres (${dbName})`);	×
69	const sql = await createSqlConnection(connectionString, false);	×
70	setSqlClient(sql, target);	×
71	}
72	} catch(err) {
73	// eslint-disable-next-line no-console
74	console.error("Failed to connect to postgres: ", err.message);	×
75	process.exit(1);	×
76	}
77	}
78
79	const initDatabases = ({postgresUrl, postgresReadUrl}: CommandLineArguments) =>	×
80	Promise.all([	×
81	connectToPostgres(postgresUrl),
82	connectToPostgres(postgresReadUrl, "read"),
83	]);
84
85	const initSettings = () => {	×
86	if (!isAnyTest) {	×
87	setInterval(refreshSettingsCaches, 1000 * 60 * 5) // We refresh the cache every 5 minutes on all servers	×
88	}
89	return refreshSettingsCaches();	×
90	}
91
92	const initPostgres = async () => {	×
93	if (Collections.some(collection => collection instanceof PgCollection)) {	×
94	for (const collection of Collections) {	×
95	if (collection instanceof PgCollection) {	×
96	collection.buildPostgresTable();	×
97	}
98	}
99	}
100
101	// If we're migrating up, we might be migrating from the start on a fresh database, so skip the check
102	// for whether postgres views exist
103	const migrating = process.argv.some(arg => arg.indexOf('migrate') > -1)	×
104	const migratingUp = process.argv.some(arg => arg === 'up')	×
105	if (migrating && migratingUp) return;	×
106
107	try {	×
NEW 108	await ensurePostgresViewsExist(getSqlClientOrThrow(), addCronJob);	×
109	} catch (e) {
110	// eslint-disable-next-line no-console
111	console.error("Failed to ensure Postgres views exist:", e);	×
112	}
113	}
114
115	const executeServerWithArgs = async ({shellMode, command}: CommandLineArguments) => {	×
116	await runStartupFunctions();	×
117
118	// define executableSchema
119	createVoteableUnionType();	×
120	initGraphQL();	×
121
122	if (shellMode) {	×
123	initShell();	×
124	} else if (command) {	×
125	const func = compileWithGlobals(command);	×
126	const result = await func();	×
127	// eslint-disable-next-line no-console
128	console.log("Finished. Result: ", result);	×
129	process.kill(estrellaPid, 'SIGQUIT');	×
130	} else if (!isAnyTest && !isMigrations) {	×
131	watchForShellCommands();	×
132	startWebserver();	×
133	}
134	}
135
136	export const initServer = async (commandLineArguments?: CommandLineArguments) => {	×
137	initConsole();	×
138	const args = commandLineArguments ?? getCommandLineArguments();	×
139	if (!args.postgresUrl) {	×
140	panic("Missing postgresUrl");	×
141	}
142	await initDatabases(args);	×
143	await initSettings();	×
144	require('../server.ts');	×
145	await initPostgres();	×
146	return args;	×
147	}
148
149	export const serverStartup = async () => {	×
150	// Run server directly if not in cluster mode
151	if (!clusterSetting.get()) {	×
152	await serverStartupWorker();	×
153	return;	×
154	}
155
156	// Use OS load balancing (as opposed to round-robin)
157	// In principle, this should give better performance because it is aware of resource (cpu) usage
158	cluster.schedulingPolicy = cluster.SCHED_NONE	×
159	if (cluster.isPrimary) {	×
160	// Initialize db connection and a few other things such as settings, but don't start a webserver.
161	console.log("Initializing primary process");	×
162	await initServer();	×
163
164	const numWorkers = numWorkersSetting.get();	×
165
166	console.log(`Running in cluster mode with ${numWorkers} workers (vs ${numCPUs} cpus)`);	×
167	console.log(`Primary ${process.pid} is running, about to fork workers`);	×
168
169	// Fork workers.
170	for (let i = 0; i < numWorkers; i++) {	×
171	cluster.fork();	×
172	}
173
174	cluster.on('exit', (worker, code, signal) => {	×
175	console.log(`Worker ${worker.process.pid} died`);	×
176	});
177	} else {
178	console.log(`Starting worker ${process.pid}`);	×
179	await serverStartupWorker();	×
180	console.log(`Worker ${process.pid} started`);	×
181	}
182	}
183
184	export const serverStartupWorker = async () => {	×
185	const commandLineArguments = await initServer();	×
186	await executeServerWithArgs(commandLineArguments);	×
187	}
188
189	function initShell() {
190	const repl = require('repl');	×
191	/*const rl = readline.createInterface({
192	input: process.stdin,
193	output: process.stdout,
194	prompt: "> ",
195	});
196	rl.on('line', line => {
197	console.log(`Got input: ${line}`);
198	rl.prompt();
199	});
200	rl.prompt();*/
201
202	const r = repl.start({	×
203	prompt: "> ",
204	terminal: true,
205	preview: true,
206	breakEvalOnSigint: true,
207	useGlobal: true,
208	});
209	r.context.Globals = Globals;	×
210	r.context.Vulcan = Globals;	×
211	}
212
213	const compileWithGlobals = (code: string) => {	×
214	// This is basically just eval() but done in a way that:
215	// 1) Allows us to define our own global scope
216	// 2) Doesn't upset esbuild
217	const callable = (async function () {}).constructor(`with(this) { await ${code} }`);	×
218	const scope = {Globals, Vulcan};	×
219	return () => {	×
220	return callable.call(new Proxy({}, {	×
221	has () { return true; },	×
222	get (_target, key) {
223	if (typeof key !== "symbol") {	×
224	return global[key as keyof Global] ?? scope[key as "Globals"\|"Vulcan"];	×
225	}
226	}
227	}));
228	}
229	}
230
231	// Monitor ./tmp/pendingShellCommands for shell commands. If a JS file is
232	// written there, run it then delete it. Security-wise this is okay because
233	// write-access inside the repo directory is already equivalent to script
234	// execution.
235	const watchForShellCommands = () => {	×
236	const watcher = chokidar.watch('./tmp/pendingShellCommands');	×
237	watcher.on('add', async (path) => {	×
238	const fileContents = fs.readFileSync(path, 'utf8');	×
239	// eslint-disable-next-line no-console
240	console.log(`Running shell command: ${fileContents}`);	×
241	const newPath = join("tmp/runningShellCommands", basename(path));	×
242	fs.renameSync(path, newPath);	×
243	try {	×
244	const func = compileWithGlobals(fileContents);	×
245	const result = await func();	×
246	// eslint-disable-next-line no-console
247	console.log("Finished. Result: ", result);	×
248	} catch(e) {
249	// eslint-disable-next-line no-console
250	console.log("Failed.");	×
251	// eslint-disable-next-line no-console
252	console.log(e);	×
253	} finally {
254	fs.unlinkSync(newPath);	×
255	}
256	});
257	}

ForumMagnum / ForumMagnum / 8881983748

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous