4088643824

Build Type

github

Committed by GitHub

Commit Message

Merge 3588a38ba into c1acf7dc8

Pull Request Pull Request #951: build(deps): bump http-cache-semantics from 4.1.0 to 4.1.1 in /packages/cli

Run Details

436 of 530 branches covered (82.26%)

Branch coverage included in aggregate %.

692 of 736 relevant lines covered (94.02%)

4.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.51

/packages/cli/src/utils/index.ts

import {
  exec,
  ExecOptions,
  ExecException
} from 'child_process';
import * as fs from 'fs-extra';
import * as CliProgress from 'cli-progress';
import * as path from 'path';
import * as bent from 'bent';
import * as os from 'os';
import * as url from 'url';
import { promisify } from 'util';
import { customAlphabet } from 'nanoid';
import * as constants from '../constants';
import * as extract from 'extract-zip';
import * as realOra from 'ora';
import * as prettyBytes from 'pretty-bytes';
import * as dateformat from 'dateformat';

export * as Script from './script';
export * as Plugin from './plugin';
export * as Cache from './cache';
export * as Framework from './framework';
export * as PredictDataset from './predict-dataset';
export * as PostPredict from './post-predict';
export * as ServePredict from './serve-predict';

const { pipeline } = require('stream');

const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 8);

export const pipelineAsync = promisify(pipeline);

export const makeWorkspace = async (): Promise<string> => {
  const workspace = path.resolve('./pipcook-output');
  await fs.mkdirp(workspace);
  return workspace;
};

/**
 * download the file and stored in specified directory
 * @param url: url of the file
 * @param fileName: full path of file that will be stored
 */
export async function download(url: string, fileName: string): Promise<void> {
  await fs.ensureFile(fileName);
  const stream = await bent(url)('') as NodeJS.ReadableStream;
  return pipelineAsync(stream, fs.createWriteStream(fileName));
}

/**
 * unzip compressed data
 * @param filePath: path of zip
 * @param targetPath: target full path
 */
export function unZipData(filePath: string, targetPath: string): Promise<void> {
  return extract(filePath, { dir: targetPath });
}

/**
 * Fit model directory, if `model/model/pipeline.json` exists, move the subdirectory `model` to top level.
 * @param filePath: path of zip
 */
export async function fitModelDir(modelDir: string): Promise<void> {
  if (await fs.pathExists(path.join(modelDir, constants.WorkspaceModelDir, constants.PipelineFileInModelDir))) {
    await fs.move(path.join(modelDir, constants.WorkspaceModelDir), `${modelDir}.tmp`);
    await fs.remove(modelDir);
    await fs.move(`${modelDir}.tmp`, modelDir);
  }
}

/**
 * generate id
 */
export function generateId(): string {
  return nanoid();
}

export enum DownloadProtocol { HTTP = 'http:', HTTPS = 'https:', FILE = 'file:' }

export function execAsync(cmd: string, opts?: ExecOptions): Promise<string> {
  return new Promise((resolve, reject): void => {
    exec(cmd, opts, (err: ExecException | null, stdout: string) => {
      if (err) {
        reject(err);
      } else {
        resolve(stdout);
      }
    });
  });
}

/**
 * download the file and stored in specified directory
 * @param url: url of the file
 * @param fileName: full path of file that will be stored
 */
export async function downloadWithProgress(url: string, fileName: string): Promise<void> {
  await fs.ensureFile(fileName);
  const bar = new CliProgress.SingleBar({
    format: '{bar} {percentage}% {value}/{total}',
    formatValue: (v, _, type): string => {
      if (type === 'value' || type === 'total') {
        return prettyBytes(v);
      } else {
        return v.toString();
      }
    }
  }, CliProgress.Presets.shades_classic);
  const file = fs.createWriteStream(fileName);
  let receivedBytes = 0;
  const downloadStream = (await bent(url)('')) as bent.NodeResponse;
  const totalBytes = downloadStream.headers['content-length'];
  bar.start(Number(totalBytes), 0);
  downloadStream.on('data', (chunk: any) => {
    receivedBytes += chunk.length;
    bar.update(receivedBytes);
  });
  try {
    await pipelineAsync(downloadStream, file);
    bar.stop();
  } catch (err) {
    fs.unlink(fileName);
    bar.stop();
    throw err;
  }
}

/**
 * Download the dataset from specific URL and extract to a generated path as the returned value.
 * @param resUrl the resource url, support http://, https://, file://.
 * @param targetDir the directory to save the files
 */
export async function downloadAndExtractTo(resUrl: string, targetDir: string): Promise<void> {
  const { protocol, pathname } = url.parse(resUrl);
  if (!protocol || !pathname) {
    throw new TypeError('invalid url');
  }
  const filename = path.basename(pathname);
  const extname = path.extname(filename);
  if (protocol === 'file:') {
    if (extname === '.zip') {
      await this.unZipData(pathname, targetDir);
    } else {
      await fs.copy(pathname, targetDir);
    }
  } else if (protocol === 'http:' || protocol === 'https:') {
    if (extname === '.zip') {
      const tmpPath = path.join(constants.PIPCOOK_TMPDIR, this.generateId());
      await this.downloadWithProgress(resUrl, tmpPath);
      this.logger.start('extracting');
      await this.unZipData(tmpPath, targetDir);
      await fs.remove(tmpPath);
    } else {
      await this.downloadWithProgress(resUrl, targetDir);
    }
  } else {
    throw new TypeError(`[${extname}] file format is not supported.`);
  }
}

export const mirrorUrl = (mirror: string, framework: string): string => {
  const pyVersion = 'py37';
  const nodeVersion = `node${process.versions.node.substr(0, process.versions.node.indexOf('.'))}`;
  return url.resolve(
    mirror,
    `${nodeVersion}-${pyVersion}/${encodeURIComponent(framework)}-${os.platform()}-${os.arch()}-v${process.versions.napi}.zip`
  );
};

export interface Logger {
  success(message: string): void;
  fail(message: string, exit?: boolean, code?: number): void;
  info(message: string): void;
  warn(message: string): void;
}

export class TtyLogger implements Logger {
  private spinner: realOra.Ora;

  constructor() {
    this.spinner = realOra({
      stream: process.stdout,
      prefixText: (): string => dateformat(new Date(), '[hh:MM:ss:l]')
    });
  }

  success(message: string): void {
    this.spinner.succeed(message);
  }

  fail(message: string, exit = true, code = 1): void {
    this.spinner.fail(message);
    if (exit) {
      process.exit(code);
    }
  }

  info(message: string): void {
    this.spinner.info(message);
  }

  warn(message: string): void {
    this.spinner.warn(message);
  }

  start(message: string): void {
    this.spinner.start(message);
  }
}

export class DefaultLogger implements Logger {
  success(message: string): void {
    console.log('[success]: ' + message);
  }

  fail(message: string, exit = true, code = 1): void {
    console.error('[fail]: ' + message);
    if (exit) {
      process.exit(code);
    }
  }

  info(message: string): void {
    console.log('[info]: ' + message);
  }

  warn(message: string): void {
    console.warn('[warn]: ' + message);
  }

  start(message: string): void {
    console.log('[start]: ' + message);
  }
}

const { rows, columns, isTTY } = process.stdout;
export const logger: Logger = isTTY && rows > 0 && columns > 0 ? new TtyLogger() : new DefaultLogger();

1	import {	11✔
2	exec,
3	ExecOptions,
4	ExecException
5	} from 'child_process';
6	import * as fs from 'fs-extra';	11✔
7	import * as CliProgress from 'cli-progress';	11✔
8	import * as path from 'path';	11✔
9	import * as bent from 'bent';	11✔
10	import * as os from 'os';	11✔
11	import * as url from 'url';	11✔
12	import { promisify } from 'util';	11✔
13	import { customAlphabet } from 'nanoid';	11✔
14	import * as constants from '../constants';	11✔
15	import * as extract from 'extract-zip';	11✔
16	import * as realOra from 'ora';	11✔
17	import * as prettyBytes from 'pretty-bytes';	11✔
18	import * as dateformat from 'dateformat';	11✔
19
20	export * as Script from './script';	11✔
21	export * as Plugin from './plugin';	11✔
22	export * as Cache from './cache';	11✔
23	export * as Framework from './framework';	11✔
24	export * as PredictDataset from './predict-dataset';	11✔
25	export * as PostPredict from './post-predict';	11✔
26	export * as ServePredict from './serve-predict';	11✔
27
28	const { pipeline } = require('stream');	11✔
29
30	const nanoid = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 8);	11✔
31
32	export const pipelineAsync = promisify(pipeline);	11✔
33
34	export const makeWorkspace = async (): Promise<string> => {	11✔
35	const workspace = path.resolve('./pipcook-output');	×
36	await fs.mkdirp(workspace);	×
37	return workspace;	×
38	};
39
40	/**
41	* download the file and stored in specified directory
42	* @param url: url of the file
43	* @param fileName: full path of file that will be stored
44	*/
45	export async function download(url: string, fileName: string): Promise<void> {	11✔
46	await fs.ensureFile(fileName);	4✔
47	const stream = await bent(url)('') as NodeJS.ReadableStream;	4✔
48	return pipelineAsync(stream, fs.createWriteStream(fileName));	2✔
49	}
50
51	/**
52	* unzip compressed data
53	* @param filePath: path of zip
54	* @param targetPath: target full path
55	*/
56	export function unZipData(filePath: string, targetPath: string): Promise<void> {	11✔
57	return extract(filePath, { dir: targetPath });	1✔
58	}
59
60	/**
61	* Fit model directory, if `model/model/pipeline.json` exists, move the subdirectory `model` to top level.
62	* @param filePath: path of zip
63	*/
64	export async function fitModelDir(modelDir: string): Promise<void> {	11✔
65	if (await fs.pathExists(path.join(modelDir, constants.WorkspaceModelDir, constants.PipelineFileInModelDir))) {	3!
66	await fs.move(path.join(modelDir, constants.WorkspaceModelDir), `${modelDir}.tmp`);	×
67	await fs.remove(modelDir);	×
68	await fs.move(`${modelDir}.tmp`, modelDir);	×
69	}
70	}
71
72	/**
73	* generate id
74	*/
75	export function generateId(): string {	11✔
76	return nanoid();	2✔
77	}
78
79	export enum DownloadProtocol { HTTP = 'http:', HTTPS = 'https:', FILE = 'file:' }	11✔
80
81	export function execAsync(cmd: string, opts?: ExecOptions): Promise<string> {	11✔
82	return new Promise((resolve, reject): void => {	2✔
83	exec(cmd, opts, (err: ExecException \| null, stdout: string) => {	2✔
84	if (err) {	2✔
85	reject(err);	1✔
86	} else {
87	resolve(stdout);	1✔
88	}
89	});
90	});
91	}
92
93	/**
94	* download the file and stored in specified directory
95	* @param url: url of the file
96	* @param fileName: full path of file that will be stored
97	*/
98	export async function downloadWithProgress(url: string, fileName: string): Promise<void> {	11✔
99	await fs.ensureFile(fileName);	1✔
100	const bar = new CliProgress.SingleBar({	1✔
101	format: '{bar} {percentage}% {value}/{total}',
102	formatValue: (v, _, type): string => {
103	if (type === 'value' \|\| type === 'total') {	×
104	return prettyBytes(v);	×
105	} else {
106	return v.toString();	×
107	}
108	}
109	}, CliProgress.Presets.shades_classic);
110	const file = fs.createWriteStream(fileName);	1✔
111	let receivedBytes = 0;	1✔
112	const downloadStream = (await bent(url)('')) as bent.NodeResponse;	1✔
113	const totalBytes = downloadStream.headers['content-length'];	1✔
114	bar.start(Number(totalBytes), 0);	1✔
115	downloadStream.on('data', (chunk: any) => {	1✔
116	receivedBytes += chunk.length;	4✔
117	bar.update(receivedBytes);	4✔
118	});
119	try {	1✔
120	await pipelineAsync(downloadStream, file);	1✔
121	bar.stop();	1✔
122	} catch (err) {
123	fs.unlink(fileName);	×
124	bar.stop();	×
125	throw err;	×
126	}
127	}
128
129	/**
130	* Download the dataset from specific URL and extract to a generated path as the returned value.
131	* @param resUrl the resource url, support http://, https://, file://.
132	* @param targetDir the directory to save the files
133	*/
134	export async function downloadAndExtractTo(resUrl: string, targetDir: string): Promise<void> {	11✔
135	const { protocol, pathname } = url.parse(resUrl);	6✔
136	if (!protocol \|\| !pathname) {	6✔
137	throw new TypeError('invalid url');	1✔
138	}
139	const filename = path.basename(pathname);	5✔
140	const extname = path.extname(filename);	5✔
141	if (protocol === 'file:') {	5✔
142	if (extname === '.zip') {	2✔
143	await this.unZipData(pathname, targetDir);	1✔
144	} else {
145	await fs.copy(pathname, targetDir);	1✔
146	}
147	} else if (protocol === 'http:' \|\| protocol === 'https:') {	3✔
148	if (extname === '.zip') {	2✔
149	const tmpPath = path.join(constants.PIPCOOK_TMPDIR, this.generateId());	1✔
150	await this.downloadWithProgress(resUrl, tmpPath);	1✔
151	this.logger.start('extracting');	1✔
152	await this.unZipData(tmpPath, targetDir);	1✔
153	await fs.remove(tmpPath);	1✔
154	} else {
155	await this.downloadWithProgress(resUrl, targetDir);	1✔
156	}
157	} else {
158	throw new TypeError(`[${extname}] file format is not supported.`);	1✔
159	}
160	}
161
162	export const mirrorUrl = (mirror: string, framework: string): string => {	11✔
163	const pyVersion = 'py37';	1✔
164	const nodeVersion = `node${process.versions.node.substr(0, process.versions.node.indexOf('.'))}`;	1✔
165	return url.resolve(	1✔
166	mirror,
167	`${nodeVersion}-${pyVersion}/${encodeURIComponent(framework)}-${os.platform()}-${os.arch()}-v${process.versions.napi}.zip`
168	);
169	};
170
171	export interface Logger {
172	success(message: string): void;
173	fail(message: string, exit?: boolean, code?: number): void;
174	info(message: string): void;
175	warn(message: string): void;
176	}
177
178	export class TtyLogger implements Logger {	11✔
179	private spinner: realOra.Ora;
180
181	constructor() {
182	this.spinner = realOra({	1✔
183	stream: process.stdout,
184	prefixText: (): string => dateformat(new Date(), '[hh:MM:ss:l]')	1✔
185	});
186	}
187
188	success(message: string): void {
189	this.spinner.succeed(message);	1✔
190	}
191
192	fail(message: string, exit = true, code = 1): void {	3✔
193	this.spinner.fail(message);	3✔
194	if (exit) {	3✔
195	process.exit(code);	2✔
196	}
197	}
198
199	info(message: string): void {
200	this.spinner.info(message);	1✔
201	}
202
203	warn(message: string): void {
204	this.spinner.warn(message);	1✔
205	}
206
207	start(message: string): void {
208	this.spinner.start(message);	1✔
209	}
210	}
211
212	export class DefaultLogger implements Logger {	11✔
213	success(message: string): void {
214	console.log('[success]: ' + message);	9✔
215	}
216
217	fail(message: string, exit = true, code = 1): void {	3✔
218	console.error('[fail]: ' + message);	3✔
219	if (exit) {	3✔
220	process.exit(code);	2✔
221	}
222	}
223
224	info(message: string): void {
225	console.log('[info]: ' + message);	39✔
226	}
227
228	warn(message: string): void {
229	console.warn('[warn]: ' + message);	2✔
230	}
231
232	start(message: string): void {
233	console.log('[start]: ' + message);	2✔
234	}
235	}
236
237	const { rows, columns, isTTY } = process.stdout;	11✔
238	export const logger: Logger = isTTY && rows > 0 && columns > 0 ? new TtyLogger() : new DefaultLogger();	11✔

alibaba / pipcook / 4088643824

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous