23305017021

Committed 19 Mar 2026 04:20PM UTC coverage: 85.425% (-0.1%) from 85.545%

Build # 23305017021

Build Type

push

github

Committed by

nodebb-misty

Commit Message

chore: update changelog for v4.10.0

Coverage Stats

13445 of 18442 branches covered (72.9%)

28384 of 33227 relevant lines covered (85.42%)

3333.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.73

/src/posts/uploads.js

'use strict';

const nconf = require('nconf');
const fs = require('fs').promises;
const crypto = require('crypto');
const path = require('path');
const winston = require('winston');
const mime = require('mime');
const validator = require('validator');
const chalk = require('chalk');

const db = require('../database');
const image = require('../image');
const user = require('../user');
const topics = require('../topics');
const file = require('../file');
const meta = require('../meta');
const cron = require('../cron');

module.exports = function (Posts) {
        Posts.uploads = {};

        const md5 = filename => crypto.createHash('md5').update(filename).digest('hex');
        const pathPrefix = path.join(nconf.get('upload_path'));
        const searchRegex = /\/assets\/uploads(\/files\/[^\s")]+\.?[\w]*)/g;

        const _getFullPath = relativePath => path.join(pathPrefix, relativePath);
        const _filterValidPaths = async filePaths => (await Promise.all(filePaths.map(async (filePath) => {
                const fullPath = _getFullPath(filePath);
                return fullPath.startsWith(pathPrefix) && await file.exists(fullPath) ? filePath : false;
        }))).filter(Boolean);

        Posts.uploads.startJobs = async function () {
                const runJobs = nconf.get('runJobs');
                if (!runJobs) {
                        return;
                }

                await cron.addJob({
                        name: 'posts:uploads:cleanupOrphans',
                        cronTime: '0 2 * * 0',
                        onTick: async () => {
                                const orphans = await Posts.uploads.cleanOrphans();
                                if (orphans.length) {
                                        winston.info(`[posts/uploads] Deleting ${orphans.length} orphaned uploads...`);
                                        orphans.forEach((relPath) => {
                                                process.stdout.write(`${chalk.red('  - ')} ${relPath}`);
                                        });
                                }
                        },
                });
        };

        Posts.uploads.sync = async function (pid) {
                // Scans a post's content and updates sorted set of uploads

                const [postData, isMainPost] = await Promise.all([
                        Posts.getPostFields(pid, ['content', 'uploads']),
                        Posts.isMain(pid),
                ]);

                const content = postData.content || '';
                const currentUploads = postData.uploads || [];

                // Extract upload file paths from post content
                let match = searchRegex.exec(content);
                let uploads = new Set();
                while (match) {
                        uploads.add(match[1].replace('-resized', ''));
                        match = searchRegex.exec(content);
                }

                // Main posts can contain topic thumbs, which are also tracked by pid
                if (isMainPost) {
                        const tid = await Posts.getPostField(pid, 'tid');
                        let thumbs = await topics.thumbs.get(tid, { thumbsOnly: true });
                        thumbs = thumbs.map(thumb => thumb.path).filter(path => !validator.isURL(path, {
                                require_protocol: true,
                        }));
                        thumbs.forEach(t => uploads.add(t));
                }

                uploads = Array.from(uploads);

                // Create add/remove sets
                const add = uploads.filter(path => !currentUploads.includes(path));
                const remove = currentUploads.filter(path => !uploads.includes(path));
                await Posts.uploads.associate(pid, add);
                await Posts.uploads.dissociate(pid, remove);
        };

        Posts.uploads.list = async function (pids) {
                const isArray = Array.isArray(pids);
                if (isArray) {
                        const uploads = await Posts.getPostsFields(pids, ['uploads']);
                        return uploads.map(p => p.uploads || []);
                }

                const uploads = await Posts.getPostField(pids, 'uploads');
                return uploads;
        };

        Posts.uploads.listWithSizes = async function (pid) {
                const paths = await Posts.uploads.list(pid);
                const sizes = await db.getObjects(paths.map(path => `upload:${md5(path)}`)) || [];

                return sizes.map((sizeObj, idx) => ({
                        ...sizeObj,
                        name: paths[idx],
                }));
        };

        Posts.uploads.getOrphans = async () => {
                let files = await fs.readdir(_getFullPath('/files'));
                files = files.filter(filename => filename !== '.gitignore');

                // Exclude non-timestamped files (e.g. group covers; see gh#10783/gh#10705)
                const tsPrefix = /^\d{13}-/;
                files = files.filter(filename => tsPrefix.test(filename));

                files = await Promise.all(files.map(
                        async filename => (await Posts.uploads.isOrphan(`/files/${filename}`) ? `/files/${filename}` : null)
                ));
                files = files.filter(Boolean);

                return files;
        };

        Posts.uploads.cleanOrphans = async () => {
                const now = Date.now();
                const expiration = now - (1000 * 60 * 60 * 24 * meta.config.orphanExpiryDays);
                const days = meta.config.orphanExpiryDays;
                if (!days) {
                        return [];
                }

                let orphans = await Posts.uploads.getOrphans();

                orphans = await Promise.all(orphans.map(async (relPath) => {
                        const { mtimeMs } = await fs.stat(_getFullPath(relPath));
                        return mtimeMs < expiration ? relPath : null;
                }));
                orphans = orphans.filter(Boolean);

                await Promise.all(orphans.map(async (relPath) => {
                        await file.delete(_getFullPath(relPath));
                }));

                return orphans;
        };

        Posts.uploads.isOrphan = async function (filePath) {
                const length = await db.sortedSetCard(`upload:${md5(filePath)}:pids`);
                return length === 0;
        };

        Posts.uploads.getUsage = async function (filePaths) {
                // Given an array of file names, determines which pids they are used in
                if (!Array.isArray(filePaths)) {
                        filePaths = [filePaths];
                }

                // windows path => 'files\\1685368788211-1-profileimg.jpg'
                // linux path => files/1685368788211-1-profileimg.jpg
                // turn them into => '/files/1685368788211-1-profileimg.jpg'
                filePaths.forEach((file) => {
                        file.path = `/${file.path.split(path.sep).join(path.posix.sep)}`;
                });

                const keys = filePaths.map(fileObj => `upload:${md5(fileObj.path.replace('-resized', ''))}:pids`);
                return await Promise.all(keys.map(k => db.getSortedSetRange(k, 0, -1)));
        };

        Posts.uploads.associate = async function (pid, filePaths) {
                filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths;
                if (!filePaths.length) {
                        return;
                }
                filePaths = await _filterValidPaths(filePaths); // Only process files that exist and are within uploads directory
                const currentUploads = await Posts.uploads.list(pid);
                filePaths.forEach((path) => {
                        if (!currentUploads.includes(path)) {
                                currentUploads.push(path);
                        }
                });

                const now = Date.now();
                const bulkAdd = filePaths.map(path => [`upload:${md5(path)}:pids`, now, pid]);

                await Promise.all([
                        db.setObjectField(`post:${pid}`, 'uploads', JSON.stringify(currentUploads)),
                        db.sortedSetAddBulk(bulkAdd),
                        Posts.uploads.saveSize(filePaths),
                ]);
        };

        Posts.uploads.dissociate = async function (pid, filePaths) {
                filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths;
                if (!filePaths.length) {
                        return;
                }
                let currentUploads = await Posts.uploads.list(pid);
                currentUploads = currentUploads.filter(upload => !filePaths.includes(upload));
                const bulkRemove = filePaths.map(path => [`upload:${md5(path)}:pids`, pid]);
                const promises = [
                        db.setObjectField(`post:${pid}`, 'uploads', JSON.stringify(currentUploads)),
                        db.sortedSetRemoveBulk(bulkRemove),
                ];

                await Promise.all(promises);

                if (!meta.config.preserveOrphanedUploads) {
                        const deletePaths = (await Promise.all(
                                filePaths.map(async filePath => (await Posts.uploads.isOrphan(filePath) ? filePath : false))
                        )).filter(Boolean);

                        const uploaderUids = (await db.getObjectsFields(
                                deletePaths.map(path => `upload:${md5(path)}`, ['uid'])
                        )).map(o => (o ? o.uid || null : null));
                        await Promise.all(uploaderUids.map((uid, idx) => (
                                uid && isFinite(uid) ? user.deleteUpload(uid, uid, deletePaths[idx]) : null
                        )).filter(Boolean));
                        await Posts.uploads.deleteFromDisk(deletePaths);
                }
        };

        Posts.uploads.dissociateAll = async (pid) => {
                const current = await Posts.uploads.list(pid);
                await Posts.uploads.dissociate(pid, current);
        };

        Posts.uploads.deleteFromDisk = async (filePaths) => {
                if (typeof filePaths === 'string') {
                        filePaths = [filePaths];
                } else if (!Array.isArray(filePaths)) {
                        throw new Error(`[[error:wrong-parameter-type, filePaths, ${typeof filePaths}, array]]`);
                }

                filePaths = (await _filterValidPaths(filePaths)).map(_getFullPath);
                await Promise.all(filePaths.map(file.delete));
        };

        Posts.uploads.saveSize = async (filePaths) => {
                filePaths = filePaths.filter((fileName) => {
                        const type = mime.getType(fileName);
                        return type && type.match(/image./);
                });
                await Promise.all(filePaths.map(async (fileName) => {
                        try {
                                const size = await image.size(_getFullPath(fileName));
                                await db.setObject(`upload:${md5(fileName)}`, {
                                        width: size.width,
                                        height: size.height,
                                });
                        } catch (err) {
                                winston.error(`[posts/uploads] Error while saving post upload sizes (${fileName}): ${err.message}`);
                        }
                }));
        };
};

1	'use strict';
2
3	const nconf = require('nconf');	32✔
4	const fs = require('fs').promises;	32✔
5	const crypto = require('crypto');	32✔
6	const path = require('path');	32✔
7	const winston = require('winston');	32✔
8	const mime = require('mime');	32✔
9	const validator = require('validator');	32✔
10	const chalk = require('chalk');	32✔
11
12	const db = require('../database');	32✔
13	const image = require('../image');	32✔
14	const user = require('../user');	32✔
15	const topics = require('../topics');	32✔
16	const file = require('../file');	32✔
17	const meta = require('../meta');	32✔
18	const cron = require('../cron');	32✔
19
20	module.exports = function (Posts) {	32✔
21	Posts.uploads = {};	32✔
22
23	const md5 = filename => crypto.createHash('md5').update(filename).digest('hex');	372✔
24	const pathPrefix = path.join(nconf.get('upload_path'));	32✔
25	const searchRegex = /\/assets\/uploads(\/files\/[^\s")]+\.?[\w]*)/g;	32✔
26
27	const _getFullPath = relativePath => path.join(pathPrefix, relativePath);	380✔
28	const _filterValidPaths = async filePaths => (await Promise.all(filePaths.map(async (filePath) => {	176✔
29	const fullPath = _getFullPath(filePath);	192✔
30	return fullPath.startsWith(pathPrefix) && await file.exists(fullPath) ? filePath : false;	192✔
31	}))).filter(Boolean);
32
33	Posts.uploads.startJobs = async function () {	32✔
34	const runJobs = nconf.get('runJobs');	×
35	if (!runJobs) {	×
36	return;	×
37	}
38
39	await cron.addJob({	×
40	name: 'posts:uploads:cleanupOrphans',
41	cronTime: '0 2 * * 0',
42	onTick: async () => {
43	const orphans = await Posts.uploads.cleanOrphans();	×
44	if (orphans.length) {	×
45	winston.info(`[posts/uploads] Deleting ${orphans.length} orphaned uploads...`);	×
46	orphans.forEach((relPath) => {	×
47	process.stdout.write(`${chalk.red(' - ')} ${relPath}`);	×
48	});
49	}
50	},
51	});
52	};
53
54	Posts.uploads.sync = async function (pid) {	32✔
55	// Scans a post's content and updates sorted set of uploads
56
57	const [postData, isMainPost] = await Promise.all([	1,588✔
58	Posts.getPostFields(pid, ['content', 'uploads']),
59	Posts.isMain(pid),
60	]);
61
62	const content = postData.content \|\| '';	1,588✔
63	const currentUploads = postData.uploads \|\| [];	1,588!
64
65	// Extract upload file paths from post content
66	let match = searchRegex.exec(content);	1,588✔
67	let uploads = new Set();	1,588✔
68	while (match) {	1,588✔
69	uploads.add(match[1].replace('-resized', ''));	76✔
70	match = searchRegex.exec(content);	76✔
71	}
72
73	// Main posts can contain topic thumbs, which are also tracked by pid
74	if (isMainPost) {	1,588✔
75	const tid = await Posts.getPostField(pid, 'tid');	67✔
76	let thumbs = await topics.thumbs.get(tid, { thumbsOnly: true });	67✔
77	thumbs = thumbs.map(thumb => thumb.path).filter(path => !validator.isURL(path, {	67✔
78	require_protocol: true,
79	}));
80	thumbs.forEach(t => uploads.add(t));	67✔
81	}
82
83	uploads = Array.from(uploads);	1,588✔
84
85	// Create add/remove sets
86	const add = uploads.filter(path => !currentUploads.includes(path));	1,588✔
87	const remove = currentUploads.filter(path => !uploads.includes(path));	1,588✔
88	await Posts.uploads.associate(pid, add);	1,588✔
89	await Posts.uploads.dissociate(pid, remove);	1,588✔
90	};
91
92	Posts.uploads.list = async function (pids) {	32✔
93	const isArray = Array.isArray(pids);	2,064✔
94	if (isArray) {	2,064✔
95	const uploads = await Posts.getPostsFields(pids, ['uploads']);	1,636✔
96	return uploads.map(p => p.uploads \|\| []);	1,636!
97	}
98
99	const uploads = await Posts.getPostField(pids, 'uploads');	428✔
100	return uploads;	428✔
101	};
102
103	Posts.uploads.listWithSizes = async function (pid) {	32✔
104	const paths = await Posts.uploads.list(pid);	116✔
105	const sizes = await db.getObjects(paths.map(path => `upload:${md5(path)}`)) \|\| [];	116!
106
107	return sizes.map((sizeObj, idx) => ({	116✔
108	...sizeObj,
109	name: paths[idx],
110	}));
111	};
112
113	Posts.uploads.getOrphans = async () => {	32✔
114	let files = await fs.readdir(_getFullPath('/files'));	24✔
115	files = files.filter(filename => filename !== '.gitignore');	24✔
116
117	// Exclude non-timestamped files (e.g. group covers; see gh#10783/gh#10705)
118	const tsPrefix = /^\d{13}-/;	24✔
119	files = files.filter(filename => tsPrefix.test(filename));	24✔
120
121	files = await Promise.all(files.map(	24✔
122	async filename => (await Posts.uploads.isOrphan(`/files/${filename}`) ? `/files/${filename}` : null)	20!
123	));
124	files = files.filter(Boolean);	24✔
125
126	return files;	24✔
127	};
128
129	Posts.uploads.cleanOrphans = async () => {	32✔
130	const now = Date.now();	12✔
131	const expiration = now - (1000 * 60 * 60 * 24 * meta.config.orphanExpiryDays);	12✔
132	const days = meta.config.orphanExpiryDays;	12✔
133	if (!days) {	12✔
134	return [];	4✔
135	}
136
137	let orphans = await Posts.uploads.getOrphans();	8✔
138
139	orphans = await Promise.all(orphans.map(async (relPath) => {	8✔
140	const { mtimeMs } = await fs.stat(_getFullPath(relPath));	8✔
141	return mtimeMs < expiration ? relPath : null;	8✔
142	}));
143	orphans = orphans.filter(Boolean);	8✔
144
145	await Promise.all(orphans.map(async (relPath) => {	8✔
146	await file.delete(_getFullPath(relPath));	4✔
147	}));
148
149	return orphans;	8✔
150	};
151
152	Posts.uploads.isOrphan = async function (filePath) {	32✔
153	const length = await db.sortedSetCard(`upload:${md5(filePath)}:pids`);	112✔
154	return length === 0;	112✔
155	};
156
157	Posts.uploads.getUsage = async function (filePaths) {	32✔
158	// Given an array of file names, determines which pids they are used in
159	if (!Array.isArray(filePaths)) {	×
160	filePaths = [filePaths];	×
161	}
162
163	// windows path => 'files\\1685368788211-1-profileimg.jpg'
164	// linux path => files/1685368788211-1-profileimg.jpg
165	// turn them into => '/files/1685368788211-1-profileimg.jpg'
166	filePaths.forEach((file) => {	×
167	file.path = `/${file.path.split(path.sep).join(path.posix.sep)}`;	×
168	});
169
170	const keys = filePaths.map(fileObj => `upload:${md5(fileObj.path.replace('-resized', ''))}:pids`);	×
171	return await Promise.all(keys.map(k => db.getSortedSetRange(k, 0, -1)));	×
172	};
173
174	Posts.uploads.associate = async function (pid, filePaths) {	32✔
175	filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths;	1,644✔
176	if (!filePaths.length) {	1,644✔
177	return;	1,548✔
178	}
179	filePaths = await _filterValidPaths(filePaths); // Only process files that exist and are within uploads directory	96✔
180	const currentUploads = await Posts.uploads.list(pid);	96✔
181	filePaths.forEach((path) => {	96✔
182	if (!currentUploads.includes(path)) {	116✔
183	currentUploads.push(path);	104✔
184	}
185	});
186
187	const now = Date.now();	96✔
188	const bulkAdd = filePaths.map(path => [`upload:${md5(path)}:pids`, now, pid]);	116✔
189
190	await Promise.all([	96✔
191	db.setObjectField(`post:${pid}`, 'uploads', JSON.stringify(currentUploads)),
192	db.sortedSetAddBulk(bulkAdd),
193	Posts.uploads.saveSize(filePaths),
194	]);
195	};
196
197	Posts.uploads.dissociate = async function (pid, filePaths) {	32✔
198	filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths;	1,700✔
199	if (!filePaths.length) {	1,700✔
200	return;	1,632✔
201	}
202	let currentUploads = await Posts.uploads.list(pid);	68✔
203	currentUploads = currentUploads.filter(upload => !filePaths.includes(upload));	104✔
204	const bulkRemove = filePaths.map(path => [`upload:${md5(path)}:pids`, pid]);	88✔
205	const promises = [	68✔
206	db.setObjectField(`post:${pid}`, 'uploads', JSON.stringify(currentUploads)),
207	db.sortedSetRemoveBulk(bulkRemove),
208	];
209
210	await Promise.all(promises);	68✔
211
212	if (!meta.config.preserveOrphanedUploads) {	68✔
213	const deletePaths = (await Promise.all(	64✔
214	filePaths.map(async filePath => (await Posts.uploads.isOrphan(filePath) ? filePath : false))	80✔
215	)).filter(Boolean);
216
217	const uploaderUids = (await db.getObjectsFields(	64✔
218	deletePaths.map(path => `upload:${md5(path)}`, ['uid'])	48✔
219	)).map(o => (o ? o.uid \|\| null : null));	48!
220	await Promise.all(uploaderUids.map((uid, idx) => (	64✔
221	uid && isFinite(uid) ? user.deleteUpload(uid, uid, deletePaths[idx]) : null	48✔
222	)).filter(Boolean));
223	await Posts.uploads.deleteFromDisk(deletePaths);	64✔
224	}
225	};
226
227	Posts.uploads.dissociateAll = async (pid) => {	32✔
228	const current = await Posts.uploads.list(pid);	76✔
229	await Posts.uploads.dissociate(pid, current);	76✔
230	};
231
232	Posts.uploads.deleteFromDisk = async (filePaths) => {	32✔
233	if (typeof filePaths === 'string') {	84✔
234	filePaths = [filePaths];	4✔
235	} else if (!Array.isArray(filePaths)) {	80✔
236	throw new Error(`[[error:wrong-parameter-type, filePaths, ${typeof filePaths}, array]]`);	4✔
237	}
238
239	filePaths = (await _filterValidPaths(filePaths)).map(_getFullPath);	80✔
240	await Promise.all(filePaths.map(file.delete));	80✔
241	};
242
243	Posts.uploads.saveSize = async (filePaths) => {	32✔
244	filePaths = filePaths.filter((fileName) => {	96✔
245	const type = mime.getType(fileName);	116✔
246	return type && type.match(/image./);	116✔
247	});
248	await Promise.all(filePaths.map(async (fileName) => {	96✔
249	try {	100✔
250	const size = await image.size(_getFullPath(fileName));	100✔
251	await db.setObject(`upload:${md5(fileName)}`, {	8✔
252	width: size.width,
253	height: size.height,
254	});
255	} catch (err) {
256	winston.error(`[posts/uploads] Error while saving post upload sizes (${fileName}): ${err.message}`);	92✔
257	}
258	}));
259	};
260	};

NodeBB / NodeBB / 23305017021

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous