3de434bb-6cc9-425e-9358-4d0b28bc0962

Committed 01 Feb 2025 05:16PM UTC coverage: 69.645% (+0.2%) from 69.408%

Build # 3de434bb-6cc9-425e-9358-4d0b28bc0962

Build Type

Pull #3882

circleci

Committed by

sr258

Commit Message

test: fix integration tests

Pull Request Pull Request #3882: refactor: removed fs-extra and replaced it with native Node functiona…

Run Details

2455 of 4080 branches covered (60.17%)

Branch coverage included in aggregate %.

154 of 162 new or added lines in 15 files covered. (95.06%)

243 existing lines in 12 files now uncovered.

5862 of 7862 relevant lines covered (74.56%)

5743.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.29

/packages/h5p-server/src/implementation/fs/DirectoryTemporaryFileStorage.ts

import { createReadStream, createWriteStream, ReadStream } from 'fs';
import {
    access,
    mkdir,
    readdir,
    readFile,
    rm,
    rmdir,
    stat,
    writeFile
} from 'fs/promises';

import { getAllFiles } from 'get-all-files';
import path from 'path';
import promisepipe from 'promisepipe';

import {
    ITemporaryFile,
    ITemporaryFileStorage,
    IUser,
    IFileStats
} from '../../types';
import H5pError from '../../helpers/H5pError';
import { checkFilename, sanitizeFilename } from './filenameUtils';

/**
 * Stores temporary files in directories on the disk.
 * Manages access rights by creating one sub-directory for each user.
 * Manages expiration times by creating companion '.metadata' files for every
 * file stored.
 */
export default class DirectoryTemporaryFileStorage
    implements ITemporaryFileStorage
{
    /**
     * @param directory the directory in which the temporary files are stored.
     * Must be read- and write accessible
     */
    constructor(
        private directory: string,
        protected options?: {
            /**
             * These characters will be removed from files that are saved to S3.
             * There is a very strict default list that basically only leaves
             * alphanumeric filenames intact. Should you need more relaxed
             * settings you can specify them here.
             */
            invalidCharactersRegexp?: RegExp;
            /*
             * How long paths can be in the filesystem (Differs between Windows,
             * Linux and MacOS, so check out the limitation of your
             * system!)
             */
            maxPathLength?: number;
        }
    ) {
        mkdir(directory, { recursive: true });
        this.maxFileLength =
            (options?.maxPathLength ?? 255) - (directory.length + 1) - 40;
        // we subtract 40 for the contentId (12), the unique id attached to the
        // file (8), the .metadata suffix (9), userIds (8) and separators (3).
        if (this.maxFileLength < 20) {
            throw new Error(
                'The path of the temporary files directory is too long to add files to it. Put the directory into a different location.'
            );
        }
    }

    private maxFileLength: number;

    public async deleteFile(filename: string, ownerId: string): Promise<void> {
        checkFilename(filename);
        if (!ownerId) {
            throw new Error(
                'Invalid arguments for DirectoryTemporaryFileStorage.deleteFile: you must specify an ownerId'
            );
        }
        checkFilename(ownerId);
        const filePath = this.getAbsoluteFilePath(ownerId, filename);
        await rm(filePath);
        await rm(`${filePath}.metadata`);

        const userDirectoryPath = this.getAbsoluteUserDirectoryPath(ownerId);
        const fileDirectoryPath = path.dirname(filePath);
        if (userDirectoryPath !== fileDirectoryPath) {
            await this.deleteEmptyDirectory(fileDirectoryPath);
        }
        await this.deleteEmptyDirectory(userDirectoryPath);
    }

    public async fileExists(filename: string, user: IUser): Promise<boolean> {
        checkFilename(filename);
        checkFilename(user.id);
        const filePath = this.getAbsoluteFilePath(user.id, filename);
        try {
            await access(filePath);
            return true;
        } catch {
            return false;
        }
    }

    public async getFileStats(
        filename: string,
        user: IUser
    ): Promise<IFileStats> {
        if (!(await this.fileExists(filename, user))) {
            throw new H5pError(
                'storage-file-implementations:temporary-file-not-found',
                {
                    filename,
                    userId: user.id
                },
                404
            );
        }
        const filePath = this.getAbsoluteFilePath(user.id, filename);
        return stat(filePath);
    }

    public async getFileStream(
        filename: string,
        user: IUser,
        rangeStart?: number,
        rangeEnd?: number
    ): Promise<ReadStream> {
        checkFilename(filename);
        checkFilename(user.id);
        const filePath = this.getAbsoluteFilePath(user.id, filename);
        try {
            await access(filePath);
        } catch {
            throw new H5pError(
                'storage-file-implementations:temporary-file-not-found',
                { filename, userId: user.id },
                404
            );
        }
        return createReadStream(filePath, {
            start: rangeStart,
            end: rangeEnd
        });
    }

    public async listFiles(user?: IUser): Promise<ITemporaryFile[]> {
        if (user) {
            checkFilename(user.id);
        }
        const users = user ? [user.id] : await readdir(this.directory);
        return (
            await Promise.all(
                users.map(async (u) => {
                    const basePath = this.getAbsoluteUserDirectoryPath(u);
                    const basePathLength = basePath.length + 1;
                    const filesOfUser = await getAllFiles(basePath).toArray();
                    return Promise.all(
                        filesOfUser
                            .map((f) => f.substr(basePathLength))
                            .filter((f) => !f.endsWith('.metadata'))
                            .map((f) => this.getTemporaryFileInfo(f, u))
                    );
                })
            )
        ).reduce((prev, curr) => prev.concat(curr), []);
    }

    /**
     * Removes invalid characters from filenames and enforces other filename
     * rules required by the storage implementation (e.g. filename length
     * restrictions).
     * @param filename the filename to sanitize; this can be a relative path
     * (e.g. "images/image1.png")
     * @returns the clean filename
     */
    public sanitizeFilename = (filename: string): string => {
        return sanitizeFilename(
            filename,
            this.maxFileLength,
            this.options?.invalidCharactersRegexp
        );
    };

    public async saveFile(
        filename: string,
        dataStream: ReadStream,
        user: IUser,
        expirationTime: Date
    ): Promise<ITemporaryFile> {
        checkFilename(filename);
        checkFilename(user.id);

        await mkdir(this.getAbsoluteUserDirectoryPath(user.id), {
            recursive: true
        });
        const filePath = this.getAbsoluteFilePath(user.id, filename);
        await mkdir(path.dirname(filePath), { recursive: true });
        const writeStream = createWriteStream(filePath);
        await promisepipe(dataStream, writeStream);
        await writeFile(
            `${filePath}.metadata`,
            JSON.stringify({
                expiresAt: expirationTime.getTime()
            })
        );
        return {
            expiresAt: expirationTime,
            filename,
            ownedByUserId: user.id
        };
    }

    private async deleteEmptyDirectory(directory: string): Promise<void> {
        const files = await readdir(directory);
        if (files.length === 0) {
            await rmdir(directory);
        }
    }

    private getAbsoluteFilePath(userId: string, filename: string): string {
        return path.join(this.directory, userId, filename);
    }

    private getAbsoluteUserDirectoryPath(userId: string): string {
        return path.join(this.directory, userId);
    }

    private async getTemporaryFileInfo(
        filename: string,
        userId: string
    ): Promise<ITemporaryFile> {
        const metadata = JSON.parse(
            await readFile(
                `${this.getAbsoluteFilePath(userId, filename)}.metadata`,
                'utf-8'
            )
        );
        return {
            expiresAt: new Date(metadata.expiresAt),
            filename,
            ownedByUserId: userId
        };
    }
}

1	import { createReadStream, createWriteStream, ReadStream } from 'fs';	12✔
2	import {	12✔
3	access,
4	mkdir,
5	readdir,
6	readFile,
7	rm,
8	rmdir,
9	stat,
10	writeFile
11	} from 'fs/promises';
12
13	import { getAllFiles } from 'get-all-files';	12✔
14	import path from 'path';	12✔
15	import promisepipe from 'promisepipe';	12✔
16
17	import {
18	ITemporaryFile,
19	ITemporaryFileStorage,
20	IUser,
21	IFileStats
22	} from '../../types';
23	import H5pError from '../../helpers/H5pError';	12✔
24	import { checkFilename, sanitizeFilename } from './filenameUtils';	12✔
25
26	/**
27	* Stores temporary files in directories on the disk.
28	* Manages access rights by creating one sub-directory for each user.
29	* Manages expiration times by creating companion '.metadata' files for every
30	* file stored.
31	*/
32	export default class DirectoryTemporaryFileStorage	12✔
33	implements ITemporaryFileStorage
34	{
35	/**
36	* @param directory the directory in which the temporary files are stored.
37	* Must be read- and write accessible
38	*/
39	constructor(
40	private directory: string,	46✔
41	protected options?: {	46✔
42	/**
43	* These characters will be removed from files that are saved to S3.
44	* There is a very strict default list that basically only leaves
45	* alphanumeric filenames intact. Should you need more relaxed
46	* settings you can specify them here.
47	*/
48	invalidCharactersRegexp?: RegExp;
49	/*
50	* How long paths can be in the filesystem (Differs between Windows,
51	* Linux and MacOS, so check out the limitation of your
52	* system!)
53	*/
54	maxPathLength?: number;
55	}
56	) {
57	mkdir(directory, { recursive: true });	46✔
58	this.maxFileLength =	46✔
59	(options?.maxPathLength ?? 255) - (directory.length + 1) - 40;	46✔
60	// we subtract 40 for the contentId (12), the unique id attached to the
61	// file (8), the .metadata suffix (9), userIds (8) and separators (3).
62	if (this.maxFileLength < 20) {	46!
63	throw new Error(	×
64	'The path of the temporary files directory is too long to add files to it. Put the directory into a different location.'
65	);
66	}
67	}
68
69	private maxFileLength: number;
70
71	public async deleteFile(filename: string, ownerId: string): Promise<void> {
72	checkFilename(filename);	8✔
73	if (!ownerId) {	8!
74	throw new Error(	×
75	'Invalid arguments for DirectoryTemporaryFileStorage.deleteFile: you must specify an ownerId'
76	);
77	}
78	checkFilename(ownerId);	8✔
79	const filePath = this.getAbsoluteFilePath(ownerId, filename);	8✔
80	await rm(filePath);	8✔
81	await rm(`${filePath}.metadata`);	8✔
82
83	const userDirectoryPath = this.getAbsoluteUserDirectoryPath(ownerId);	8✔
84	const fileDirectoryPath = path.dirname(filePath);	8✔
85	if (userDirectoryPath !== fileDirectoryPath) {	8✔
86	await this.deleteEmptyDirectory(fileDirectoryPath);	6✔
87	}
88	await this.deleteEmptyDirectory(userDirectoryPath);	8✔
89	}
90
91	public async fileExists(filename: string, user: IUser): Promise<boolean> {
92	checkFilename(filename);	766✔
93	checkFilename(user.id);	766✔
94	const filePath = this.getAbsoluteFilePath(user.id, filename);	766✔
95	try {	766✔
96	await access(filePath);	766✔
97	return true;	376✔
98	} catch {
99	return false;	390✔
100	}
101	}
102
103	public async getFileStats(
104	filename: string,
105	user: IUser
106	): Promise<IFileStats> {
107	if (!(await this.fileExists(filename, user))) {	×
108	throw new H5pError(	×
109	'storage-file-implementations:temporary-file-not-found',
110	{
111	filename,
112	userId: user.id
113	},
114	404
115	);
116	}
117	const filePath = this.getAbsoluteFilePath(user.id, filename);	×
NEW 118	return stat(filePath);	×
119	}
120
121	public async getFileStream(
122	filename: string,
123	user: IUser,
124	rangeStart?: number,
125	rangeEnd?: number
126	): Promise<ReadStream> {
127	checkFilename(filename);	382✔
128	checkFilename(user.id);	382✔
129	const filePath = this.getAbsoluteFilePath(user.id, filename);	382✔
130	try {	382✔
131	await access(filePath);	382✔
132	} catch {
133	throw new H5pError(	8✔
134	'storage-file-implementations:temporary-file-not-found',
135	{ filename, userId: user.id },
136	404
137	);
138	}
139	return createReadStream(filePath, {	374✔
140	start: rangeStart,
141	end: rangeEnd
142	});
143	}
144
145	public async listFiles(user?: IUser): Promise<ITemporaryFile[]> {
146	if (user) {	4✔
147	checkFilename(user.id);	2✔
148	}
149	const users = user ? [user.id] : await readdir(this.directory);	4✔
150	return (	4✔
151	await Promise.all(
152	users.map(async (u) => {
153	const basePath = this.getAbsoluteUserDirectoryPath(u);	4✔
154	const basePathLength = basePath.length + 1;	4✔
155	const filesOfUser = await getAllFiles(basePath).toArray();	4✔
156	return Promise.all(	4✔
157	filesOfUser
158	.map((f) => f.substr(basePathLength))	20✔
159	.filter((f) => !f.endsWith('.metadata'))	20✔
160	.map((f) => this.getTemporaryFileInfo(f, u))	10✔
161	);
162	})
163	)
164	).reduce((prev, curr) => prev.concat(curr), []);	4✔
165	}
166
167	/**
168	* Removes invalid characters from filenames and enforces other filename
169	* rules required by the storage implementation (e.g. filename length
170	* restrictions).
171	* @param filename the filename to sanitize; this can be a relative path
172	* (e.g. "images/image1.png")
173	* @returns the clean filename
174	*/
175	public sanitizeFilename = (filename: string): string => {	46✔
176	return sanitizeFilename(	390✔
177	filename,
178	this.maxFileLength,
179	this.options?.invalidCharactersRegexp
180	);
181	};
182
183	public async saveFile(
184	filename: string,
185	dataStream: ReadStream,
186	user: IUser,
187	expirationTime: Date
188	): Promise<ITemporaryFile> {
189	checkFilename(filename);	390✔
190	checkFilename(user.id);	390✔
191
192	await mkdir(this.getAbsoluteUserDirectoryPath(user.id), {	390✔
193	recursive: true
194	});
195	const filePath = this.getAbsoluteFilePath(user.id, filename);	390✔
196	await mkdir(path.dirname(filePath), { recursive: true });	390✔
197	const writeStream = createWriteStream(filePath);	390✔
198	await promisepipe(dataStream, writeStream);	390✔
199	await writeFile(	390✔
200	`${filePath}.metadata`,
201	JSON.stringify({
202	expiresAt: expirationTime.getTime()
203	})
204	);
205	return {	390✔
206	expiresAt: expirationTime,
207	filename,
208	ownedByUserId: user.id
209	};
210	}
211
212	private async deleteEmptyDirectory(directory: string): Promise<void> {
213	const files = await readdir(directory);	14✔
214	if (files.length === 0) {	14✔
215	await rmdir(directory);	8✔
216	}
217	}
218
219	private getAbsoluteFilePath(userId: string, filename: string): string {
220	return path.join(this.directory, userId, filename);	1,556✔
221	}
222
223	private getAbsoluteUserDirectoryPath(userId: string): string {
224	return path.join(this.directory, userId);	402✔
225	}
226
227	private async getTemporaryFileInfo(
228	filename: string,
229	userId: string
230	): Promise<ITemporaryFile> {
231	const metadata = JSON.parse(	10✔
232	await readFile(
233	`${this.getAbsoluteFilePath(userId, filename)}.metadata`,
234	'utf-8'
235	)
236	);
237	return {	10✔
238	expiresAt: new Date(metadata.expiresAt),
239	filename,
240	ownedByUserId: userId
241	};
242	}
243	}

Lumieducation / H5P-Nodejs-library / 3de434bb-6cc9-425e-9358-4d0b28bc0962

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous