13048977500

Committed 30 Jan 2025 09:07AM UTC coverage: 98.336% (+0.04%) from 98.297%

Build # 13048977500

Build Type

Pull #277

github

Committed by

web-flow

Commit Message

Merge 4ccb3710c into 121036fcb

Pull Request Pull Request #277: perf(export): Speed up and reduce memory usage

Coverage Stats

804 of 840 branches covered (95.71%)

4 of 6 new or added lines in 1 file covered. (66.67%)

4432 of 4507 relevant lines covered (98.34%)

541.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.39

/src/database.ts

import { parse as createJsonParseStream } from './lib/jsonstream';
import BluebirdPromise from 'bluebird';
import { promises as fsPromises, createReadStream } from 'graceful-fs';
import { pipeline, Stream } from 'stream';
import Model from './model';
import Schema from './schema';
import SchemaType from './schematype';
import WarehouseError from './error';
import { logger } from 'hexo-log';
import type { AddSchemaTypeOptions, NodeJSLikeCallback } from './types';

const log = logger();
const pkg = require('../package.json');
const { open } = fsPromises;
const pipelineAsync = BluebirdPromise.promisify(pipeline) as unknown as (...args: Stream[]) => BluebirdPromise<unknown>;

async function exportAsync(database: Database, path: string): Promise<void> {
  const handle = await open(path, 'w');

  try {
    // Start body & Meta & Start models
    await handle.write(`{"meta":${JSON.stringify({
      version: database.options.version,
      warehouse: pkg.version
    })},"models":{`);

    const models = database._models;
    const keys = Object.keys(models);
    const { length } = keys;

    // models body
    for (let i = 0; i < length; i++) {
      const key = keys[i];

      if (!models[key]) continue;

      let prefix = '';
      if (i) {
        prefix = ',';
      }
      await handle.write(`${prefix}"${key}":${models[key]._export()}`);
    }
    // End models
    await handle.write('}}');
  } catch (e) {
    log.error(e);
    if (e instanceof RangeError && e.message.includes('Invalid string length')) {
      // NOTE:  Currently, we can't deal with anything about this issue.
      //        If do not `catch` the exception after the process will not work (e.g: `after_generate` filter.)
      //        A side-effect of this workaround is the `db.json` will not generate.
      log.warn('see: https://github.com/nodejs/node/issues/35973');
    } else {
      throw e;
    }
  } finally {
    await handle.close();
  }
}

interface DatabaseOptions {
  version: number,
  path: string,
  onUpgrade: (oldVersion: number, newVersion: number) => any,
  onDowngrade: (oldVersion: number, newVersion: number) => any
}

class Database {
  options: DatabaseOptions;
  _models: Record<string, Model<any>>;
  Model: typeof Model;

  /**
   * Database constructor.
   *
   * @param {object} [options]
   *   @param {number} [options.version=0] Database version
   *   @param {string} [options.path] Database path
   *   @param {function} [options.onUpgrade] Triggered when the database is upgraded
   *   @param {function} [options.onDowngrade] Triggered when the database is downgraded
   */
  constructor(options?: { path: string } & Partial<DatabaseOptions>) {
    this.options = {
      version: 0,
      // eslint-disable-next-line @typescript-eslint/no-empty-function
      onUpgrade() {},
      // eslint-disable-next-line @typescript-eslint/no-empty-function
      onDowngrade() {},
      ...options
    };

    this._models = {};

    class _Model extends Model<any> {}

    this.Model = _Model;

    _Model.prototype._database = this;
  }

  /**
   * Creates a new model.
   *
   * @param {string} name
   * @param {Schema|object} [schema]
   * @return {Model}
   */
  model<T = any>(name: string, schema?: Schema<T> | Record<string, AddSchemaTypeOptions>): Model<any> {
    if (this._models[name]) {
      return this._models[name];
    }

    this._models[name] = new this.Model(name, schema);
    const model = this._models[name];
    return model;
  }

  /**
   * Loads database.
   *
   * @param {function} [callback]
   * @return {BluebirdPromise}
   */
  load(callback?: NodeJSLikeCallback<any>): BluebirdPromise<any> {
    const { path, onUpgrade, onDowngrade, version: newVersion } = this.options;

    if (!path) throw new WarehouseError('options.path is required');

    let oldVersion = 0;

    const getMetaCallBack = data => {
      if (data.meta && data.meta.version) {
        oldVersion = data.meta.version;
      }
    };

    // data event arg0 wrap key/value pair.
    const parseStream = createJsonParseStream('models.$*');

    parseStream.once('header', getMetaCallBack);
    parseStream.once('footer', getMetaCallBack);

    parseStream.on('data', data => {
      this.model(data.key)._import(data.value);
    });

    const rs = createReadStream(path, 'utf8');

    return pipelineAsync(rs, parseStream).then(() => {
      if (newVersion > oldVersion) {
        return onUpgrade(oldVersion, newVersion);
      } else if (newVersion < oldVersion) {
        return onDowngrade(oldVersion, newVersion);
      }
    }).asCallback(callback);
  }

  /**
   * Saves database.
   *
   * @param {function} [callback]
   * @return {BluebirdPromise}
   */
  save(callback?: NodeJSLikeCallback<any>): BluebirdPromise<void> {
    const { path } = this.options;

    if (!path) throw new WarehouseError('options.path is required');
    return BluebirdPromise.resolve(exportAsync(this, path)).asCallback(callback);
  }

  toJSON(): { meta: { version: number, warehouse: string }, models: Record<string, Model<any>> } {
    const models = Object.keys(this._models)
      .reduce((obj, key) => {
        const value = this._models[key];
        if (value != null) obj[key] = value;
        return obj;
      }, {});

    return {
      meta: {
        version: this.options.version,
        warehouse: pkg.version
      }, models
    };
  }
  static Schema = Schema;
  Schema: typeof Schema;
  static SchemaType = SchemaType;
  SchemaType: typeof SchemaType;
  static version: number;
}

Database.prototype.Schema = Schema;
Database.prototype.SchemaType = SchemaType;
Database.version = pkg.version;

export default Database;

1	import { parse as createJsonParseStream } from './lib/jsonstream';	6✔
2	import BluebirdPromise from 'bluebird';	6✔
3	import { promises as fsPromises, createReadStream } from 'graceful-fs';	6✔
4	import { pipeline, Stream } from 'stream';	6✔
5	import Model from './model';	6✔
6	import Schema from './schema';	6✔
7	import SchemaType from './schematype';	6✔
8	import WarehouseError from './error';	6✔
9	import { logger } from 'hexo-log';	6✔
10	import type { AddSchemaTypeOptions, NodeJSLikeCallback } from './types';	6✔
11		6✔
12	const log = logger();	6✔
13	const pkg = require('../package.json');	6✔
14	const { open } = fsPromises;	6✔
15	const pipelineAsync = BluebirdPromise.promisify(pipeline) as unknown as (...args: Stream[]) => BluebirdPromise<unknown>;	6✔
16		6✔
17	async function exportAsync(database: Database, path: string): Promise<void> {	6✔
18	const handle = await open(path, 'w');	6✔
19		6✔
20	try {	6✔
21	// Start body & Meta & Start models	6✔
22	await handle.write(`{"meta":${JSON.stringify({	6✔
23	version: database.options.version,	6✔
24	warehouse: pkg.version	6✔
25	})},"models":{`);	6✔
26		6✔
27	const models = database._models;	6✔
28	const keys = Object.keys(models);	6✔
29	const { length } = keys;	6✔
30		6✔
31	// models body	6✔
32	for (let i = 0; i < length; i++) {	6✔
33	const key = keys[i];	12✔
34		12✔
35	if (!models[key]) continue;	12✔
36		6✔
37	let prefix = '';	6✔
38	if (i) {	12!
NEW 39	prefix = ',';	×
NEW 40	}	×
41	await handle.write(`${prefix}"${key}":${models[key]._export()}`);	6✔
42	}	6✔
43	// End models	6✔
44	await handle.write('}}');	6✔
45	} catch (e) {	6!
46	log.error(e);	×
47	if (e instanceof RangeError && e.message.includes('Invalid string length')) {	×
48	// NOTE: Currently, we can't deal with anything about this issue.	×
49	// If do not `catch` the exception after the process will not work (e.g: `after_generate` filter.)	×
50	// A side-effect of this workaround is the `db.json` will not generate.	×
51	log.warn('see: https://github.com/nodejs/node/issues/35973');	×
52	} else {	×
53	throw e;	×
54	}	×
55	} finally {	6✔
56	await handle.close();	6✔
57	}	6✔
58	}	6✔
59		6✔
60	interface DatabaseOptions {	6✔
61	version: number,	6✔
62	path: string,	6✔
63	onUpgrade: (oldVersion: number, newVersion: number) => any,	6✔
64	onDowngrade: (oldVersion: number, newVersion: number) => any	6✔
65	}	6✔
66		6✔
67	class Database {	6✔
68	options: DatabaseOptions;	6✔
69	_models: Record<string, Model<any>>;	6✔
70	Model: typeof Model;	6✔
71		6✔
72	/**	6✔
73	* Database constructor.	6✔
74	*	6✔
75	* @param {object} [options]	6✔
76	* @param {number} [options.version=0] Database version	6✔
77	* @param {string} [options.path] Database path	6✔
78	* @param {function} [options.onUpgrade] Triggered when the database is upgraded	6✔
79	* @param {function} [options.onDowngrade] Triggered when the database is downgraded	6✔
80	*/	6✔
81	constructor(options?: { path: string } & Partial<DatabaseOptions>) {	6✔
82	this.options = {	84✔
83	version: 0,	84✔
84	// eslint-disable-next-line @typescript-eslint/no-empty-function	84✔
85	onUpgrade() {},	84✔
86	// eslint-disable-next-line @typescript-eslint/no-empty-function	84✔
87	onDowngrade() {},	84✔
88	...options	84✔
89	};	84✔
90		84✔
91	this._models = {};	84✔
92		84✔
93	class _Model extends Model<any> {}	84✔
94		84✔
95	this.Model = _Model;	84✔
96		84✔
97	_Model.prototype._database = this;	84✔
98	}	84✔
99		6✔
100	/**	6✔
101	* Creates a new model.	6✔
102	*	6✔
103	* @param {string} name	6✔
104	* @param {Schema\|object} [schema]	6✔
105	* @return {Model}	6✔
106	*/	6✔
107	model<T = any>(name: string, schema?: Schema<T> \| Record<string, AddSchemaTypeOptions>): Model<any> {	6✔
108	if (this._models[name]) {	168✔
109	return this._models[name];	18✔
110	}	18✔
111		150✔
112	this._models[name] = new this.Model(name, schema);	150✔
113	const model = this._models[name];	150✔
114	return model;	150✔
115	}	150✔
116		6✔
117	/**	6✔
118	* Loads database.	6✔
119	*	6✔
120	* @param {function} [callback]	6✔
121	* @return {BluebirdPromise}	6✔
122	*/	6✔
123	load(callback?: NodeJSLikeCallback<any>): BluebirdPromise<any> {	6✔
124	const { path, onUpgrade, onDowngrade, version: newVersion } = this.options;	24✔
125		24✔
126	if (!path) throw new WarehouseError('options.path is required');	24!
127		24✔
128	let oldVersion = 0;	24✔
129		24✔
130	const getMetaCallBack = data => {	24✔
131	if (data.meta && data.meta.version) {	24✔
132	oldVersion = data.meta.version;	24✔
133	}	24✔
134	};	24✔
135		24✔
136	// data event arg0 wrap key/value pair.	24✔
137	const parseStream = createJsonParseStream('models.$*');	24✔
138		24✔
139	parseStream.once('header', getMetaCallBack);	24✔
140	parseStream.once('footer', getMetaCallBack);	24✔
141		24✔
142	parseStream.on('data', data => {	24✔
143	this.model(data.key)._import(data.value);	24✔
144	});	24✔
145		24✔
146	const rs = createReadStream(path, 'utf8');	24✔
147		24✔
148	return pipelineAsync(rs, parseStream).then(() => {	24✔
149	if (newVersion > oldVersion) {	24✔
150	return onUpgrade(oldVersion, newVersion);	6✔
151	} else if (newVersion < oldVersion) {	24✔
152	return onDowngrade(oldVersion, newVersion);	18✔
153	}	18✔
154	}).asCallback(callback);	24✔
155	}	24✔
156		6✔
157	/**	6✔
158	* Saves database.	6✔
159	*	6✔
160	* @param {function} [callback]	6✔
161	* @return {BluebirdPromise}	6✔
162	*/	6✔
163	save(callback?: NodeJSLikeCallback<any>): BluebirdPromise<void> {	6✔
164	const { path } = this.options;	6✔
165		6✔
166	if (!path) throw new WarehouseError('options.path is required');	6!
167	return BluebirdPromise.resolve(exportAsync(this, path)).asCallback(callback);	6✔
168	}	6✔
169		6✔
170	toJSON(): { meta: { version: number, warehouse: string }, models: Record<string, Model<any>> } {	6✔
171	const models = Object.keys(this._models)	12✔
172	.reduce((obj, key) => {	12✔
173	const value = this._models[key];	12✔
174	if (value != null) obj[key] = value;	12✔
175	return obj;	12✔
176	}, {});	12✔
177		12✔
178	return {	12✔
179	meta: {	12✔
180	version: this.options.version,	12✔
181	warehouse: pkg.version	12✔
182	}, models	12✔
183	};	12✔
184	}	12✔
185	static Schema = Schema;	6✔
186	Schema: typeof Schema;	6✔
187	static SchemaType = SchemaType;	6✔
188	SchemaType: typeof SchemaType;	6✔
189	static version: number;	6✔
190	}	6✔
191		6✔
192	Database.prototype.Schema = Schema;	6✔
193	Database.prototype.SchemaType = SchemaType;	6✔
194	Database.version = pkg.version;	6✔
195		6✔
196	export default Database;	6✔

hexojs / warehouse / 13048977500

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous