8800035659

Committed 23 Apr 2024 12:03PM UTC coverage: 98.978% (-0.05%) from 99.029%

Build # 8800035659

Build Type

push

github

Committed by

web-flow

Commit Message

perf(processor/post): improve processing speed when `config.post_asset_folder` is enabled (#5473)

Run Details

2292 of 2384 branches covered (96.14%)

21 of 24 new or added lines in 1 file covered. (87.5%)

2 existing lines in 1 file now uncovered.

9293 of 9389 relevant lines covered (98.98%)

52.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.73

/lib/plugins/processor/post.ts

import { toDate, timezone, isExcludedFile, isTmpFile, isHiddenFile, isMatch } from './common';
import Promise from 'bluebird';
import { parse as yfm } from 'hexo-front-matter';
import { extname, join, posix, sep } from 'path';
import { stat, listDir } from 'hexo-fs';
import { slugize, Pattern, Permalink } from 'hexo-util';
import { magenta } from 'picocolors';
import type { _File } from '../../box';
import type Hexo from '../../hexo';
import type { Stats } from 'fs';
import { PostSchema } from '../../types';

const postDir = '_posts/';
const draftDir = '_drafts/';
let permalink;

const preservedKeys = {
  title: true,
  year: true,
  month: true,
  day: true,
  i_month: true,
  i_day: true,
  hash: true
};

export = (ctx: Hexo) => {
  return {
    pattern: new Pattern(path => {
      if (isTmpFile(path)) return;

      let result;

      if (path.startsWith(postDir)) {
        result = {
          published: true,
          path: path.substring(postDir.length)
        };
      } else if (path.startsWith(draftDir)) {
        result = {
          published: false,
          path: path.substring(draftDir.length)
        };
      }

      if (!result || isHiddenFile(result.path)) return;

      // checks only if there is a renderer for the file type or if is included in skip_render
      result.renderable = ctx.render.isRenderable(path) && !isMatch(path, ctx.config.skip_render);

      // if post_asset_folder is set, restrict renderable files to default file extension
      if (result.renderable && ctx.config.post_asset_folder) {
        result.renderable = (extname(ctx.config.new_post_name) === extname(path));
      }

      return result;
    }),

    process: function postProcessor(file: _File) {
      if (file.params.renderable) {
        return processPost(ctx, file);
      } else if (ctx.config.post_asset_folder) {
        return processAsset(ctx, file);
      }
    }
  };
};

function processPost(ctx: Hexo, file: _File) {
  const Post = ctx.model('Post');
  const { path } = file.params;
  const doc = Post.findOne({source: file.path});
  const { config } = ctx;
  const { timezone: timezoneCfg } = config;
  const updated_option = config.updated_option;
  let categories, tags;

  if (file.type === 'skip' && doc) {
    return;
  }

  if (file.type === 'delete') {
    if (doc) {
      return doc.remove();
    }

    return;
  }

  return Promise.all([
    file.stat(),
    file.read()
  ]).spread((stats: Stats, content: string) => {
    const data = yfm(content);
    const info = parseFilename(config.new_post_name, path);
    const keys = Object.keys(info);

    data.source = file.path;
    data.raw = content;
    data.slug = info.title;

    if (file.params.published) {
      if (!Object.prototype.hasOwnProperty.call(data, 'published')) data.published = true;
    } else {
      data.published = false;
    }

    for (let i = 0, len = keys.length; i < len; i++) {
      const key = keys[i];
      if (!preservedKeys[key]) data[key] = info[key];
    }

    if (data.date) {
      data.date = toDate(data.date);
    } else if (info && info.year && (info.month || info.i_month) && (info.day || info.i_day)) {
      data.date = new Date(
        info.year,
        parseInt(info.month || info.i_month, 10) - 1,
        parseInt(info.day || info.i_day, 10)
      );
    }

    if (data.date) {
      if (timezoneCfg) data.date = timezone(data.date, timezoneCfg);
    } else {
      data.date = stats.birthtime;
    }

    data.updated = toDate(data.updated);

    if (data.updated) {
      if (timezoneCfg) data.updated = timezone(data.updated, timezoneCfg);
    } else if (updated_option === 'date') {
      data.updated = data.date;
    } else if (updated_option === 'empty') {
      data.updated = undefined;
    } else {
      data.updated = stats.mtime;
    }

    if (data.category && !data.categories) {
      data.categories = data.category;
      data.category = undefined;
    }

    if (data.tag && !data.tags) {
      data.tags = data.tag;
      data.tag = undefined;
    }

    categories = data.categories || [];
    tags = data.tags || [];

    if (!Array.isArray(categories)) categories = [categories];
    if (!Array.isArray(tags)) tags = [tags];

    if (data.photo && !data.photos) {
      data.photos = data.photo;
      data.photo = undefined;
    }

    if (data.photos && !Array.isArray(data.photos)) {
      data.photos = [data.photos];
    }

    if (data.permalink) {
      data.__permalink = data.permalink;
      data.permalink = undefined;
    }

    if (doc) {
      if (file.type !== 'update') {
        ctx.log.warn(`Trying to "create" ${magenta(file.path)}, but the file already exists!`);
      }
      return doc.replace(data);
    }

    return Post.insert(data);
  }).then(doc => Promise.all([
    doc.setCategories(categories),
    doc.setTags(tags),
    scanAssetDir(ctx, doc)
  ]));
}

function parseFilename(config: string, path: string) {
  config = config.substring(0, config.length - extname(config).length);
  path = path.substring(0, path.length - extname(path).length);

  if (!permalink || permalink.rule !== config) {
    permalink = new Permalink(config, {
      segments: {
        year: /(\d{4})/,
        month: /(\d{2})/,
        day: /(\d{2})/,
        i_month: /(\d{1,2})/,
        i_day: /(\d{1,2})/,
        hash: /([0-9a-f]{12})/
      }
    });
  }

  const data = permalink.parse(path);

  if (data) {
    if (data.title !== undefined) {
      return data;
    }
    return Object.assign(data, {
      title: slugize(path)
    });
  }

  return {
    title: slugize(path)
  };
}

function scanAssetDir(ctx: Hexo, post) {
  if (!ctx.config.post_asset_folder) return;

  const assetDir = post.asset_dir;
  const baseDir = ctx.base_dir;
  const sourceDir = ctx.config.source_dir;
  const baseDirLength = baseDir.length;
  const sourceDirLength = sourceDir.length;
  const PostAsset = ctx.model('PostAsset');

  return stat(assetDir).then(stats => {
    if (!stats.isDirectory()) return [];

    return listDir(assetDir);
  }).catch(err => {
    if (err && err.code === 'ENOENT') return [];
    throw err;
  }).filter(item => !isExcludedFile(item, ctx.config)).map(item => {
    const id = join(assetDir, item).substring(baseDirLength);
    const renderablePath = id.substring(sourceDirLength + 1);
    const asset = PostAsset.findById(id);

    if (shouldSkipAsset(ctx, post, asset)) return undefined;

    return PostAsset.save({
      _id: id,
      post: post._id,
      slug: item,
      modified: true,
      renderable: ctx.render.isRenderable(renderablePath) && !isMatch(renderablePath, ctx.config.skip_render)
    });
  });
}

function shouldSkipAsset(ctx: Hexo, post, asset) {
  if (!ctx._showDrafts()) {
    if (post.published === false && asset) {
      // delete existing draft assets if draft posts are hidden
      asset.remove();
    }
    if (post.published === false) {
      // skip draft assets if draft posts are hidden
      return true;
    }
  }

  return asset !== undefined; // skip already existing assets
}

function processAsset(ctx: Hexo, file: _File) {
  const PostAsset = ctx.model('PostAsset');
  const Post = ctx.model('Post');
  const id = file.source.substring(ctx.base_dir.length);
  const postAsset = PostAsset.findById(id);

  if (file.type === 'delete' || Post.length === 0) {
    if (postAsset) {
      return postAsset.remove();
    }
    return;
  }

  const savePostAsset = (post: PostSchema) => {
    return PostAsset.save({
      _id: id,
      slug: file.source.substring(post.asset_dir.length),
      post: post._id,
      modified: file.type !== 'skip',
      renderable: file.params.renderable
    });
  };

  if (postAsset) {
    // `postAsset.post` is `Post.id`.
    const post = Post.findById(postAsset.post);
    if (post != null && (post.published || ctx._showDrafts())) {
      return savePostAsset(post);
    }
  }

  const assetDir = id.slice(0, id.lastIndexOf(sep));
  const post = Post.findOne(p => p.asset_dir.endsWith(posix.join(assetDir, '/')));
  if (post != null && (post.published || ctx._showDrafts())) {
    return savePostAsset(post);
  }

  // NOTE: Probably, unreachable.
  if (postAsset) {
    return postAsset.remove();
  }
}

1	import { toDate, timezone, isExcludedFile, isTmpFile, isHiddenFile, isMatch } from './common';	1✔
2	import Promise from 'bluebird';	1✔
3	import { parse as yfm } from 'hexo-front-matter';	1✔
4	import { extname, join, posix, sep } from 'path';	1✔
5	import { stat, listDir } from 'hexo-fs';	1✔
6	import { slugize, Pattern, Permalink } from 'hexo-util';	1✔
7	import { magenta } from 'picocolors';	1✔
8	import type { _File } from '../../box';	1✔
9	import type Hexo from '../../hexo';	1✔
10	import type { Stats } from 'fs';	1✔
11	import { PostSchema } from '../../types';	1✔
12		1✔
13	const postDir = '_posts/';	1✔
14	const draftDir = '_drafts/';	1✔
15	let permalink;	1✔
16		1✔
17	const preservedKeys = {	1✔
18	title: true,	1✔
19	year: true,	1✔
20	month: true,	1✔
21	day: true,	1✔
22	i_month: true,	1✔
23	i_day: true,	1✔
24	hash: true	1✔
25	};	1✔
26		1✔
27	export = (ctx: Hexo) => {	1✔
28	return {	70✔
29	pattern: new Pattern(path => {	70✔
30	if (isTmpFile(path)) return;	42✔
31		40✔
32	let result;	40✔
33		40✔
34	if (path.startsWith(postDir)) {	42✔
35	result = {	13✔
36	published: true,	13✔
37	path: path.substring(postDir.length)	13✔
38	};	13✔
39	} else if (path.startsWith(draftDir)) {	42✔
40	result = {	2✔
41	published: false,	2✔
42	path: path.substring(draftDir.length)	2✔
43	};	2✔
44	}	2✔
45		40✔
46	if (!result \|\| isHiddenFile(result.path)) return;	42✔
47		11✔
48	// checks only if there is a renderer for the file type or if is included in skip_render	11✔
49	result.renderable = ctx.render.isRenderable(path) && !isMatch(path, ctx.config.skip_render);	42✔
50		42✔
51	// if post_asset_folder is set, restrict renderable files to default file extension	42✔
52	if (result.renderable && ctx.config.post_asset_folder) {	42✔
53	result.renderable = (extname(ctx.config.new_post_name) === extname(path));	3✔
54	}	3✔
55		11✔
56	return result;	11✔
57	}),	70✔
58		70✔
59	process: function postProcessor(file: _File) {	70✔
60	if (file.params.renderable) {	45✔
61	return processPost(ctx, file);	37✔
62	} else if (ctx.config.post_asset_folder) {	45✔
63	return processAsset(ctx, file);	7✔
64	}	7✔
65	}	45✔
66	};	70✔
67	};	70✔
68		1✔
69	function processPost(ctx: Hexo, file: _File) {	37✔
70	const Post = ctx.model('Post');	37✔
71	const { path } = file.params;	37✔
72	const doc = Post.findOne({source: file.path});	37✔
73	const { config } = ctx;	37✔
74	const { timezone: timezoneCfg } = config;	37✔
75	const updated_option = config.updated_option;	37✔
76	let categories, tags;	37✔
77		37✔
78	if (file.type === 'skip' && doc) {	37✔
79	return;	1✔
80	}	1✔
81		36✔
82	if (file.type === 'delete') {	37✔
83	if (doc) {	3✔
84	return doc.remove();	1✔
85	}	1✔
86		2✔
87	return;	2✔
88	}	2✔
89		33✔
90	return Promise.all([	33✔
91	file.stat(),	33✔
92	file.read()	33✔
93	]).spread((stats: Stats, content: string) => {	33✔
94	const data = yfm(content);	33✔
95	const info = parseFilename(config.new_post_name, path);	33✔
96	const keys = Object.keys(info);	33✔
97		33✔
98	data.source = file.path;	33✔
99	data.raw = content;	33✔
100	data.slug = info.title;	33✔
101		33✔
102	if (file.params.published) {	33✔
103	if (!Object.prototype.hasOwnProperty.call(data, 'published')) data.published = true;	32✔
104	} else {	33✔
105	data.published = false;	1✔
106	}	1✔
107		33✔
108	for (let i = 0, len = keys.length; i < len; i++) {	33✔
109	const key = keys[i];	43✔
110	if (!preservedKeys[key]) data[key] = info[key];	43✔
111	}	43✔
112		33✔
113	if (data.date) {	33✔
114	data.date = toDate(data.date);	6✔
115	} else if (info && info.year && (info.month \|\| info.i_month) && (info.day \|\| info.i_day)) {	33!
116	data.date = new Date(	3✔
117	info.year,	3✔
118	parseInt(info.month \|\| info.i_month, 10) - 1,	3!
119	parseInt(info.day \|\| info.i_day, 10)	3!
120	);	3✔
121	}	3✔
122		33✔
123	if (data.date) {	33✔
124	if (timezoneCfg) data.date = timezone(data.date, timezoneCfg);	8✔
125	} else {	33✔
126	data.date = stats.birthtime;	25✔
127	}	25✔
128		33✔
129	data.updated = toDate(data.updated);	33✔
130		33✔
131	if (data.updated) {	33✔
132	if (timezoneCfg) data.updated = timezone(data.updated, timezoneCfg);	3✔
133	} else if (updated_option === 'date') {	33✔
134	data.updated = data.date;	1✔
135	} else if (updated_option === 'empty') {	30✔
136	data.updated = undefined;	1✔
137	} else {	29✔
138	data.updated = stats.mtime;	28✔
139	}	28✔
140		33✔
141	if (data.category && !data.categories) {	33✔
142	data.categories = data.category;	1✔
143	data.category = undefined;	1✔
144	}	1✔
145		33✔
146	if (data.tag && !data.tags) {	33✔
147	data.tags = data.tag;	1✔
148	data.tag = undefined;	1✔
149	}	1✔
150		33✔
151	categories = data.categories \|\| [];	33✔
152	tags = data.tags \|\| [];	33✔
153		33✔
154	if (!Array.isArray(categories)) categories = [categories];	33✔
155	if (!Array.isArray(tags)) tags = [tags];	33✔
156		33✔
157	if (data.photo && !data.photos) {	33✔
158	data.photos = data.photo;	1✔
159	data.photo = undefined;	1✔
160	}	1✔
161		33✔
162	if (data.photos && !Array.isArray(data.photos)) {	33✔
163	data.photos = [data.photos];	1✔
164	}	1✔
165		33✔
166	if (data.permalink) {	33✔
167	data.__permalink = data.permalink;	1✔
168	data.permalink = undefined;	1✔
169	}	1✔
170		33✔
171	if (doc) {	33✔
172	if (file.type !== 'update') {	3✔
173	ctx.log.warn(`Trying to "create" ${magenta(file.path)}, but the file already exists!`);	2✔
174	}	2✔
175	return doc.replace(data);	3✔
176	}	3✔
177		30✔
178	return Post.insert(data);	30✔
179	}).then(doc => Promise.all([	33✔
180	doc.setCategories(categories),	33✔
181	doc.setTags(tags),	33✔
182	scanAssetDir(ctx, doc)	33✔
183	]));	33✔
184	}	33✔
185		1✔
186	function parseFilename(config: string, path: string) {	33✔
187	config = config.substring(0, config.length - extname(config).length);	33✔
188	path = path.substring(0, path.length - extname(path).length);	33✔
189		33✔
190	if (!permalink \|\| permalink.rule !== config) {	33✔
191	permalink = new Permalink(config, {	8✔
192	segments: {	8✔
193	year: /(\d{4})/,	8✔
194	month: /(\d{2})/,	8✔
195	day: /(\d{2})/,	8✔
196	i_month: /(\d{1,2})/,	8✔
197	i_day: /(\d{1,2})/,	8✔
198	hash: /([0-9a-f]{12})/	8✔
199	}	8✔
200	});	8✔
201	}	8✔
202		33✔
203	const data = permalink.parse(path);	33✔
204		33✔
205	if (data) {	33✔
206	if (data.title !== undefined) {	32✔
207	return data;	31✔
208	}	31✔
209	return Object.assign(data, {	1✔
210	title: slugize(path)	1✔
211	});	1✔
212	}	1✔
213		1✔
214	return {	1✔
215	title: slugize(path)	1✔
216	};	1✔
217	}	1✔
218		1✔
219	function scanAssetDir(ctx: Hexo, post) {	33✔
220	if (!ctx.config.post_asset_folder) return;	33✔
221		7✔
222	const assetDir = post.asset_dir;	7✔
223	const baseDir = ctx.base_dir;	7✔
224	const sourceDir = ctx.config.source_dir;	7✔
225	const baseDirLength = baseDir.length;	7✔
226	const sourceDirLength = sourceDir.length;	7✔
227	const PostAsset = ctx.model('PostAsset');	7✔
228		7✔
229	return stat(assetDir).then(stats => {	7✔
230	if (!stats.isDirectory()) return [];	7!
231		7✔
232	return listDir(assetDir);	7✔
233	}).catch(err => {	7✔
234	if (err && err.code === 'ENOENT') return [];	×
235	throw err;	×
236	}).filter(item => !isExcludedFile(item, ctx.config)).map(item => {	7✔
237	const id = join(assetDir, item).substring(baseDirLength);	8✔
238	const renderablePath = id.substring(sourceDirLength + 1);	8✔
239	const asset = PostAsset.findById(id);	8✔
240		8✔
241	if (shouldSkipAsset(ctx, post, asset)) return undefined;	8✔
242		5✔
243	return PostAsset.save({	5✔
244	_id: id,	5✔
245	post: post._id,	5✔
246	slug: item,	5✔
247	modified: true,	5✔
248	renderable: ctx.render.isRenderable(renderablePath) && !isMatch(renderablePath, ctx.config.skip_render)	8✔
249	});	8✔
250	});	7✔
251	}	7✔
252		1✔
253	function shouldSkipAsset(ctx: Hexo, post, asset) {	8✔
254	if (!ctx._showDrafts()) {	8✔
255	if (post.published === false && asset) {	7✔
256	// delete existing draft assets if draft posts are hidden	1✔
257	asset.remove();	1✔
258	}	1✔
259	if (post.published === false) {	7✔
260	// skip draft assets if draft posts are hidden	3✔
261	return true;	3✔
262	}	3✔
263	}	7✔
264		5✔
265	return asset !== undefined; // skip already existing assets	5✔
266	}	5✔
267		1✔
268	function processAsset(ctx: Hexo, file: _File) {	7✔
269	const PostAsset = ctx.model('PostAsset');	7✔
270	const Post = ctx.model('Post');	7✔
271	const id = file.source.substring(ctx.base_dir.length);	7✔
272	const postAsset = PostAsset.findById(id);	7✔
273		7✔
274	if (file.type === 'delete' \|\| Post.length === 0) {	7✔
275	if (postAsset) {	4✔
276	return postAsset.remove();	2✔
277	}	2✔
278	return;	2✔
279	}	2✔
280		3✔
281	const savePostAsset = (post: PostSchema) => {	3✔
282	return PostAsset.save({	3✔
283	_id: id,	3✔
284	slug: file.source.substring(post.asset_dir.length),	3✔
285	post: post._id,	3✔
286	modified: file.type !== 'skip',	3✔
287	renderable: file.params.renderable	3✔
288	});	3✔
289	};	3✔
290		3✔
291	if (postAsset) {	7✔
292	// `postAsset.post` is `Post.id`.	2✔
293	const post = Post.findById(postAsset.post);	2✔
294	if (post != null && (post.published \|\| ctx._showDrafts())) {	2!
295	return savePostAsset(post);	2✔
296	}	2✔
297	}	2✔
298		1✔
299	const assetDir = id.slice(0, id.lastIndexOf(sep));	1✔
300	const post = Post.findOne(p => p.asset_dir.endsWith(posix.join(assetDir, '/')));	1✔
301	if (post != null && (post.published \|\| ctx._showDrafts())) {	7!
302	return savePostAsset(post);	1✔
303	}	1✔
UNCOV 304		×
NEW 305	// NOTE: Probably, unreachable.	×
NEW 306	if (postAsset) {	×
NEW 307	return postAsset.remove();	×
UNCOV 308	}	×
309	}	7✔

hexojs / hexo / 8800035659

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous