7712302002

Committed 30 Jan 2024 02:01PM UTC coverage: 86.587% (+0.08%) from 86.503%

Build # 7712302002

Build Type

Pull #516

github

Committed by

web-flow

Commit Message

Merge 64aa0221f into 2fad540ba

Pull Request Pull Request #516: feat: Support overwrite/skip folder download

Run Details

1132 of 1458 branches covered (0.0%)

Branch coverage included in aggregate %.

18 of 21 new or added lines in 1 file covered. (85.71%)

4 existing lines in 1 file now uncovered.

4181 of 4678 relevant lines covered (89.38%)

351.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.31

/src/commands/folders/download.js

'use strict';

const { flags } = require('@oclif/command');
const BoxCommand = require('../../box-command');
const fs = require('fs');
const mkdirp = require('mkdirp');
const path = require('path');
const BoxCLIError = require('../../cli-error');
const ora = require('ora');
const archiver = require('archiver');
const dateTime = require('date-fns');
const utils = require('../../util');

/**
 * Saves a file to disk
 *
 * @param {string} folderPath The folder path this item occurs at within the folder being downloaded
 * @param {Object} file The file record
 * @param {Readable} stream The stream of file contents
 * @returns {Promise<void>} A promise resolving when the file is completely written to disk
 * @throws BoxCLIError
 * @private
 */
function saveFileToDisk(folderPath, file, stream) {
        let output;
        try {
                output = fs.createWriteStream(path.join(folderPath, file.path));
                stream.pipe(output);
        } catch (ex) {
                throw new BoxCLIError(
                        `Error downloading file ${file.id} to ${file.path}`,
                        ex
                );
        }

        /* eslint-disable promise/avoid-new */
        // We need to await the end of the stream to avoid a race condition here
        return new Promise((resolve, reject) => {
                output.on('close', resolve);
                stream.on('error', reject);
        });
        /* eslint-enable promise/avoid-new */
}

class FoldersDownloadCommand extends BoxCommand {
        async run() {
                const { flags, args } = this.parse(FoldersDownloadCommand);

                this.outputPath = null;
                this.maxRecurDepth =
                        flags.hasOwnProperty('depth') && flags.depth >= 0
                                ? flags.depth
                                : Number.POSITIVE_INFINITY;
                this.overwrite = flags.overwrite;
                this.maxDepth = flags['max-depth'];

                let id = args.id;
                let outputFinalized = Promise.resolve();
                let rootItemPath = null;

                let destinationPath;
                if (flags.destination) {
                        await utils.checkDir(flags.destination, flags['create-path']);
                        destinationPath = flags.destination;
                } else {
                        destinationPath = this.settings.boxDownloadsFolderPath;
                }

                /* eslint-disable no-sync */
                if (!fs.existsSync(destinationPath)) {
                        throw new BoxCLIError('Destination path must exist');
                }

                const fsStat = fs.statSync(destinationPath);
                if (!fsStat.isDirectory()) {
                        throw new BoxCLIError('Destination path must be a directory');
                }
                /* eslint-enable no-sync */

                let spinner = ora('Starting download').start();

                if (flags.zip) {
                        let fileName = `folders-download-${id}-${dateTime.format(
                                new Date(),
                                'YYYY-MM-DDTHH_mm_ss_SSS'
                        )}.zip`;
                        this.outputPath = path.join(destinationPath, fileName);
                        outputFinalized = this._setupZip(this.outputPath);
                }

                try {
                        this.outputPath = destinationPath;
                        for await (let item of this._getItems(id, '')) {
                                if (item.type === 'folder' && !this.zip) {
                                        // Set output path to the top-level folder, which is the first item in the generator
                                        rootItemPath = rootItemPath || item.path;

                                        spinner.text = `Creating folder ${item.id} at ${item.path}`;
                                        try {
                                                await mkdirp(path.join(destinationPath, item.path));
                                        } catch (ex) {
                                                throw new BoxCLIError(
                                                        `Folder ${item.path} could not be created`,
                                                        ex
                                                );
                                        }
                                } else if (item.type === 'file') {
                                        spinner.text = `Downloading file ${item.id} to ${item.path}`;
                                        let stream = await this.client.files.getReadStream(item.id);

                                        if (this.zip) {
                                                this.zip.append(stream, { name: item.path });
                                        } else {
                                                // @TODO(2018-08-15): Improve performance by queueing async work and performing in parallel
                                                await saveFileToDisk(destinationPath, item, stream);
                                        }
                                }
                        }
                } catch (err) {
                        spinner.stop();
                        throw err;
                }

                if (this.zip) {
                        this.zip.finalize();
                }
                await outputFinalized;
                spinner.succeed(
                        `Downloaded folder ${id} to ${path.join(this.outputPath, rootItemPath)}`
                );
        }

        /**
         * Generator for items in the given folder.  Yields items starting with the top-level folder itself.
         * @param {string} folderId The ID of the folder to generate items for
         * @param {string} folderPath The relative path so far down the folder tree
         * @yields {Object} Item records
         * @returns {void}
         * @private
         */
        async* _getItems(folderId, folderPath) {
                let folder = await this.client.folders.get(folderId);
                folderPath = path.join(folderPath, folder.name);

                yield {
                        type: 'folder',
                        id: folderId,
                        name: folder.name,
                        path: folderPath,
                };

                let folderItems = folder.item_collection.entries;
                if (folder.item_collection.total_count > folderItems.length) {
                        let iterator = await this.client.folders.getItems(folderId, {
                                usemarker: true,
                                fields: 'type,id,name',
                        });
                        folderItems = { [Symbol.asyncIterator]: () => iterator };
                }
                for await (let item of folderItems) {
                        if (
                                item.type === 'folder' &&
                                folderPath.split(path.sep).length <= this.maxRecurDepth
                        ) {
                                // We only recurse this folder by one of the following conditions:
                                // 1. The overwrite flag is true. We will download all files and folders.
                                // 2. The maxDepth flag is set to 'max'. We will go through all folders at any depth.
                                // 3. The folder does not exist. We will go through all folders at any depth and download all files.
                                /* eslint-disable no-sync */
                                if (
                                        this.overwrite ||
                                        this.maxDepth === 'max' ||
                                        !fs.existsSync(path.join(this.outputPath, folderPath, item.name))
                                ) {
                                        /* eslint-enable no-sync */
                                        yield* this._getItems(item.id, folderPath);
                                }
                        } else if (item.type === 'file') {
                                // We only download file if overwrite is true or the file does not exist.
                                // Skip downloading if overwrite is false and the file exists.
                                /* eslint-disable no-sync */
                                if (
                                        this.overwrite ||
                                        !fs.existsSync(path.join(this.outputPath, folderPath, item.name))
                                ) {
                                        /* eslint-enable no-sync */
                                        yield {
                                                type: 'file',
                                                id: item.id,
                                                name: item.name,
                                                path: path.join(folderPath, item.name),
                                        };
                                }
                        }
                }
        }

        /**
         * Sets up a zip archive writing to the given destination.
         *
         * @param {string} destinationPath The path where the .zip file should be written
         * @returns {Promise<void>} A promise resolving when the archive is finalized and written to disk
         * @throws BoxCLIError
         * @private
         */
        _setupZip(destinationPath) {
                // Set up archive stream
                this.zip = archiver('zip', {
                        zlib: { level: 9 }, // Use the best available compression
                });

                let output;
                try {
                        output = fs.createWriteStream(destinationPath);
                } catch (ex) {
                        throw new BoxCLIError(
                                `Could not write to destination path ${destinationPath}`,
                                ex
                        );
                }

                this.zip.on('error', (err) => {
                        throw new BoxCLIError('Error writing to zip file', err);
                });

                this.zip.pipe(output);

                /* eslint-disable promise/avoid-new */
                // We need to await the end of the stream to avoid a race condition here
                return new Promise((resolve, reject) => {
                        output.on('close', resolve);
                        output.on('error', reject);
                });
                /* eslint-enable promise/avoid-new */
        }
}

FoldersDownloadCommand.description = 'Download a folder';
FoldersDownloadCommand.examples = ['box folders:download 22222'];

FoldersDownloadCommand.flags = {
        ...BoxCommand.flags,
        destination: flags.string({
                description: 'The destination folder to download the Box folder into',
                parse: utils.parsePath,
        }),
        zip: flags.boolean({
                description: 'Download the folder into a single .zip archive',
        }),
        depth: flags.integer({
                description:
                        'Number of levels deep to recurse when downloading the folder tree',
        }),
        'create-path': flags.boolean({
                description:
                        'Recursively creates a path to a directory if it does not exist',
                allowNo: true,
                default: true,
        }),
        'max-depth': flags.enum({
                description:
                        'Maximum depth to verify if files and folders conflict, only used with --no-overwrite',
                options: ['root', 'max'],
                default: 'max',
        }),
        overwrite: flags.boolean({
                description: '[default: true] Overwrite the folder if it already exists.',
                allowNo: true,
                default: true,
        }),
};

FoldersDownloadCommand.args = [
        {
                name: 'id',
                required: true,
                hidden: false,
                description: 'ID of the folder to download',
        },
];

module.exports = FoldersDownloadCommand;

1	'use strict';
2
3	const { flags } = require('@oclif/command');	9✔
4	const BoxCommand = require('../../box-command');	9✔
5	const fs = require('fs');	9✔
6	const mkdirp = require('mkdirp');	9✔
7	const path = require('path');	9✔
8	const BoxCLIError = require('../../cli-error');	9✔
9	const ora = require('ora');	9✔
10	const archiver = require('archiver');	9✔
11	const dateTime = require('date-fns');	9✔
12	const utils = require('../../util');	9✔
13
14	/**
15	* Saves a file to disk
16	*
17	* @param {string} folderPath The folder path this item occurs at within the folder being downloaded
18	* @param {Object} file The file record
19	* @param {Readable} stream The stream of file contents
20	* @returns {Promise<void>} A promise resolving when the file is completely written to disk
21	* @throws BoxCLIError
22	* @private
23	*/
24	function saveFileToDisk(folderPath, file, stream) {
25	let output;
26	try {	1,971✔
27	output = fs.createWriteStream(path.join(folderPath, file.path));	1,971✔
28	stream.pipe(output);	1,971✔
29	} catch (ex) {
NEW 30	throw new BoxCLIError(	×
31	`Error downloading file ${file.id} to ${file.path}`,
32	ex
33	);
34	}
35
36	/* eslint-disable promise/avoid-new */
37	// We need to await the end of the stream to avoid a race condition here
38	return new Promise((resolve, reject) => {	1,971✔
39	output.on('close', resolve);	1,971✔
40	stream.on('error', reject);	1,971✔
41	});
42	/* eslint-enable promise/avoid-new */
43	}
44
45	class FoldersDownloadCommand extends BoxCommand {
46	async run() {
47	const { flags, args } = this.parse(FoldersDownloadCommand);	99✔
48
49	this.outputPath = null;	99✔
50	this.maxRecurDepth =	99✔
51	flags.hasOwnProperty('depth') && flags.depth >= 0	207✔
52	? flags.depth
53	: Number.POSITIVE_INFINITY;
54	this.overwrite = flags.overwrite;	99✔
55	this.maxDepth = flags['max-depth'];	99✔
56
57	let id = args.id;	99✔
58	let outputFinalized = Promise.resolve();	99✔
59	let rootItemPath = null;	99✔
60
61	let destinationPath;
62	if (flags.destination) {	99✔
63	await utils.checkDir(flags.destination, flags['create-path']);	90✔
64	destinationPath = flags.destination;	81✔
65	} else {
66	destinationPath = this.settings.boxDownloadsFolderPath;	9✔
67	}
68
69	/* eslint-disable no-sync */
70	if (!fs.existsSync(destinationPath)) {	90!
UNCOV 71	throw new BoxCLIError('Destination path must exist');	×
72	}
73
74	const fsStat = fs.statSync(destinationPath);	90✔
75	if (!fsStat.isDirectory()) {	90!
UNCOV 76	throw new BoxCLIError('Destination path must be a directory');	×
77	}
78	/* eslint-enable no-sync */
79
80	let spinner = ora('Starting download').start();	90✔
81
82	if (flags.zip) {	90✔
83	let fileName = `folders-download-${id}-${dateTime.format(	9✔
84	new Date(),
85	'YYYY-MM-DDTHH_mm_ss_SSS'
86	)}.zip`;
87	this.outputPath = path.join(destinationPath, fileName);	9✔
88	outputFinalized = this._setupZip(this.outputPath);	9✔
89	}
90
91	try {	90✔
92	this.outputPath = destinationPath;	90✔
93	for await (let item of this._getItems(id, '')) {	90✔
94	if (item.type === 'folder' && !this.zip) {	2,151✔
95	// Set output path to the top-level folder, which is the first item in the generator
96	rootItemPath = rootItemPath \|\| item.path;	135✔
97
98	spinner.text = `Creating folder ${item.id} at ${item.path}`;	135✔
99	try {	135✔
100	await mkdirp(path.join(destinationPath, item.path));	135✔
101	} catch (ex) {
NEW 102	throw new BoxCLIError(	×
103	`Folder ${item.path} could not be created`,
104	ex
105	);
106	}
107	} else if (item.type === 'file') {	2,016✔
108	spinner.text = `Downloading file ${item.id} to ${item.path}`;	1,998✔
109	let stream = await this.client.files.getReadStream(item.id);	1,998✔
110
111	if (this.zip) {	1,998✔
112	this.zip.append(stream, { name: item.path });	27✔
113	} else {
114	// @TODO(2018-08-15): Improve performance by queueing async work and performing in parallel
115	await saveFileToDisk(destinationPath, item, stream);	1,971✔
116	}
117	}
118	}
119	} catch (err) {
120	spinner.stop();	×
121	throw err;	×
122	}
123
124	if (this.zip) {	90✔
125	this.zip.finalize();	9✔
126	}
127	await outputFinalized;	90✔
128	spinner.succeed(	90✔
129	`Downloaded folder ${id} to ${path.join(this.outputPath, rootItemPath)}`
130	);
131	}
132
133	/**
134	* Generator for items in the given folder. Yields items starting with the top-level folder itself.
135	* @param {string} folderId The ID of the folder to generate items for
136	* @param {string} folderPath The relative path so far down the folder tree
137	* @yields {Object} Item records
138	* @returns {void}
139	* @private
140	*/
141	async* _getItems(folderId, folderPath) {
142	let folder = await this.client.folders.get(folderId);	153✔
143	folderPath = path.join(folderPath, folder.name);	153✔
144
145	yield {	153✔
146	type: 'folder',
147	id: folderId,
148	name: folder.name,
149	path: folderPath,
150	};
151
152	let folderItems = folder.item_collection.entries;	153✔
153	if (folder.item_collection.total_count > folderItems.length) {	153✔
154	let iterator = await this.client.folders.getItems(folderId, {	9✔
155	usemarker: true,
156	fields: 'type,id,name',
157	});
158	folderItems = { [Symbol.asyncIterator]: () => iterator };	9✔
159	}
160	for await (let item of folderItems) {	153✔
161	if (	2,106✔
162	item.type === 'folder' &&	2,187✔
163	folderPath.split(path.sep).length <= this.maxRecurDepth
164	) {
165	// We only recurse this folder by one of the following conditions:
166	// 1. The overwrite flag is true. We will download all files and folders.
167	// 2. The maxDepth flag is set to 'max'. We will go through all folders at any depth.
168	// 3. The folder does not exist. We will go through all folders at any depth and download all files.
169	/* eslint-disable no-sync */
170	if (	72✔
171	this.overwrite \|\|	108✔
172	this.maxDepth === 'max' \|\|
173	!fs.existsSync(path.join(this.outputPath, folderPath, item.name))
174	) {
175	/* eslint-enable no-sync */
176	yield* this._getItems(item.id, folderPath);	63✔
177	}
178	} else if (item.type === 'file') {	2,034✔
179	// We only download file if overwrite is true or the file does not exist.
180	// Skip downloading if overwrite is false and the file exists.
181	/* eslint-disable no-sync */
182	if (	2,025✔
183	this.overwrite \|\|	2,097✔
184	!fs.existsSync(path.join(this.outputPath, folderPath, item.name))
185	) {
186	/* eslint-enable no-sync */
187	yield {	1,998✔
188	type: 'file',
189	id: item.id,
190	name: item.name,
191	path: path.join(folderPath, item.name),
192	};
193	}
194	}
195	}
196	}
197
198	/**
199	* Sets up a zip archive writing to the given destination.
200	*
201	* @param {string} destinationPath The path where the .zip file should be written
202	* @returns {Promise<void>} A promise resolving when the archive is finalized and written to disk
203	* @throws BoxCLIError
204	* @private
205	*/
206	_setupZip(destinationPath) {
207	// Set up archive stream
208	this.zip = archiver('zip', {	9✔
209	zlib: { level: 9 }, // Use the best available compression
210	});
211
212	let output;
213	try {	9✔
214	output = fs.createWriteStream(destinationPath);	9✔
215	} catch (ex) {
NEW UNCOV 216	throw new BoxCLIError(	×
217	`Could not write to destination path ${destinationPath}`,
218	ex
219	);
220	}
221
222	this.zip.on('error', (err) => {	9✔
UNCOV 223	throw new BoxCLIError('Error writing to zip file', err);	×
224	});
225
226	this.zip.pipe(output);	9✔
227
228	/* eslint-disable promise/avoid-new */
229	// We need to await the end of the stream to avoid a race condition here
230	return new Promise((resolve, reject) => {	9✔
231	output.on('close', resolve);	9✔
232	output.on('error', reject);	9✔
233	});
234	/* eslint-enable promise/avoid-new */
235	}
236	}
237
238	FoldersDownloadCommand.description = 'Download a folder';	9✔
239	FoldersDownloadCommand.examples = ['box folders:download 22222'];	9✔
240
241	FoldersDownloadCommand.flags = {	9✔
242	...BoxCommand.flags,
243	destination: flags.string({
244	description: 'The destination folder to download the Box folder into',
245	parse: utils.parsePath,
246	}),
247	zip: flags.boolean({
248	description: 'Download the folder into a single .zip archive',
249	}),
250	depth: flags.integer({
251	description:
252	'Number of levels deep to recurse when downloading the folder tree',
253	}),
254	'create-path': flags.boolean({
255	description:
256	'Recursively creates a path to a directory if it does not exist',
257	allowNo: true,
258	default: true,
259	}),
260	'max-depth': flags.enum({
261	description:
262	'Maximum depth to verify if files and folders conflict, only used with --no-overwrite',
263	options: ['root', 'max'],
264	default: 'max',
265	}),
266	overwrite: flags.boolean({
267	description: '[default: true] Overwrite the folder if it already exists.',
268	allowNo: true,
269	default: true,
270	}),
271	};
272
273	FoldersDownloadCommand.args = [	9✔
274	{
275	name: 'id',
276	required: true,
277	hidden: false,
278	description: 'ID of the folder to download',
279	},
280	];
281
282	module.exports = FoldersDownloadCommand;	9✔

box / boxcli / 7712302002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous