10317771079

Committed 09 Aug 2024 10:30AM UTC coverage: 82.67% (+0.001%) from 82.669%

Build # 10317771079

Build Type

Pull #810

github

Committed by

web-flow

Commit Message

Merge 1ca001cff into 76ca756bb

Pull Request Pull Request #810: fix: import relative

Run Details

4404 of 4626 branches covered (95.2%)

90 of 102 new or added lines in 6 files covered. (88.24%)

28 existing lines in 4 files now uncovered.

29329 of 35477 relevant lines covered (82.67%)

1241.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.11

/apps/nestjs-backend/src/features/import/open-api/import.class.ts

import { existsSync } from 'fs';
import { join } from 'path';
import { BadRequestException } from '@nestjs/common';
import { getUniqName, FieldType } from '@teable/core';
import type { IValidateTypes, IAnalyzeVo } from '@teable/openapi';
import { SUPPORTEDTYPE, importTypeMap } from '@teable/openapi';
import { zip, toString, intersection, chunk as chunkArray } from 'lodash';
import fetch from 'node-fetch';
import sizeof from 'object-sizeof';
import Papa from 'papaparse';
import * as XLSX from 'xlsx';
import type { ZodType } from 'zod';
import z from 'zod';
import { exceptionParse } from '../../../utils/exception-parse';
import { toLineDelimitedStream } from './delimiter-stream';

const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {
  [FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {
    if (typeof value === 'boolean') {
      return true;
    }
    if (
      typeof value === 'string' &&
      (value.toLowerCase() === 'false' || value.toLowerCase() === 'true')
    ) {
      return true;
    }
    return false;
  }),
  [FieldType.Date]: z.coerce.date(),
  [FieldType.Number]: z.coerce.number(),
  [FieldType.LongText]: z
    .string()
    .refine((value) => z.string().safeParse(value) && /\n/.test(value)),
  [FieldType.SingleLineText]: z.string(),
};

export interface IImportConstructorParams {
  url: string;
  type: SUPPORTEDTYPE;
  maxRowCount?: number;
  fileName?: string;
}

interface IParseResult {
  [x: string]: unknown[][];
}

export abstract class Importer {
  public static DEFAULT_ERROR_MESSAGE = 'unknown error';

  public static CHUNK_SIZE = 1024 * 1024 * 0.2;

  public static MAX_CHUNK_LENGTH = 500;

  public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;

  // order make sence
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.LongText,
    FieldType.SingleLineText,
  ];

  constructor(public config: IImportConstructorParams) {}

  abstract parse(
    ...args: [
      options?: unknown,
      chunk?: (
        chunk: Record<string, unknown[][]>,
        onFinished?: () => void,
        onError?: (errorMsg: string) => void
      ) => Promise<void>,
    ]
  ): Promise<IParseResult>;

  private setFileNameFromHeader(fileName: string) {
    this.config.fileName = fileName;
  }

  getConfig() {
    return this.config;
  }

  async getFile() {
    const { url, type } = this.config;
    const { body: stream, headers } = await fetch(url);

    const supportType = importTypeMap[type].accept.split(',');

    const fileFormat = headers
      .get('content-type')
      ?.split(';')
      ?.map((item: string) => item.trim());

    // if (!fileFormat?.length) {
    //   throw new BadRequestException(
    //     `Input url is not a standard document service without right content-type`
    //   );
    // }

    if (fileFormat?.length && !intersection(fileFormat, supportType).length) {
      throw new BadRequestException(
        `File format is not supported, only ${supportType.join(',')} are supported,`
      );
    }

    const contentDisposition = headers.get('content-disposition');
    let fileName = 'Import Table.csv';

    if (contentDisposition) {
      const fileNameMatch =
        contentDisposition.match(/filename\*=UTF-8''([^;]+)/) ||
        contentDisposition.match(/filename="?([^"]+)"?/);
      if (fileNameMatch) {
        fileName = fileNameMatch[1];
      }
    }

    const finalFileName = fileName.split('.').shift() as string;

    this.setFileNameFromHeader(decodeURIComponent(finalFileName));

    return { stream, fileName: finalFileName };
  }

  async genColumns() {
    const supportTypes = Importer.SUPPORTEDTYPE;
    const parseResult = await this.parse();
    const { fileName, type } = this.config;
    const result: IAnalyzeVo['worksheets'] = {};

    for (const [sheetName, cols] of Object.entries(parseResult)) {
      const zipColumnInfo = zip(...cols);
      const existNames: string[] = [];
      const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {
        let isColumnEmpty = true;
        let validatingFieldTypes = [...supportTypes];
        for (let i = 0; i < column.length; i++) {
          if (validatingFieldTypes.length <= 1) {
            break;
          }

          // ignore empty value and first row causing first row as header
          if (column[i] === '' || column[i] == null || i === 0) {
            continue;
          }

          // when the whole columns aren't empty should flag
          isColumnEmpty = false;

          // when one of column's value validates long text, then break;
          if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {
            validatingFieldTypes = [FieldType.LongText];
            break;
          }

          const matchTypes = validatingFieldTypes.filter((type) => {
            const schema = validateZodSchemaMap[type];
            return schema.safeParse(column[i]).success;
          });

          validatingFieldTypes = matchTypes;
        }

        // empty columns should be default type
        validatingFieldTypes = !isColumnEmpty
          ? validatingFieldTypes
          : [Importer.DEFAULT_COLUMN_TYPE];

        const name = getUniqName(toString(column?.[0]).trim() || `Field ${index}`, existNames);

        existNames.push(name);

        return {
          type: validatingFieldTypes[0] || Importer.DEFAULT_COLUMN_TYPE,
          name: name.toString(),
        };
      });

      result[sheetName] = {
        name: type === SUPPORTEDTYPE.EXCEL ? sheetName : fileName ? fileName : sheetName,
        columns: calculatedColumnHeaders,
      };
    }

    return {
      worksheets: result,
    };
  }
}

export class CsvImporter extends Importer {
  public static readonly CHECK_LINES = 5000;
  public static readonly DEFAULT_SHEETKEY = 'Import Table';

  parse(): Promise<IParseResult>;
  parse(
    options: Papa.ParseConfig & { skipFirstNLines: number; key: string },
    chunk: (chunk: Record<string, unknown[][]>) => Promise<void>,
    onFinished?: () => void,
    onError?: (errorMsg: string) => void
  ): Promise<void>;
  async parse(
    ...args: [
      options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },
      chunkCb?: (chunk: Record<string, unknown[][]>) => Promise<void>,
      onFinished?: () => void,
      onError?: (errorMsg: string) => void,
    ]
  ): Promise<unknown> {
    const [options, chunkCb, onFinished, onError] = args;
    const { stream } = await this.getFile();

    // chunk parse
    if (options && chunkCb) {
      return new Promise((resolve, reject) => {
        let isFirst = true;
        let recordBuffer: unknown[][] = [];
        let isAbort = false;
        let totalRowCount = 0;

        Papa.parse(toLineDelimitedStream(stream), {
          download: false,
          dynamicTyping: true,
          chunk: (chunk, parser) => {
            (async () => {
              const newChunk = [...chunk.data] as unknown[][];
              if (isFirst && options.skipFirstNLines) {
                newChunk.splice(0, 1);
                isFirst = false;
              }

              recordBuffer.push(...newChunk);
              totalRowCount += newChunk.length;

              if (this.config.maxRowCount && totalRowCount > this.config.maxRowCount) {
                isAbort = true;
                recordBuffer = [];
                onError?.('please upgrade your plan to import more records');
                parser.abort();
              }

              if (
                recordBuffer.length >= Importer.MAX_CHUNK_LENGTH ||
                sizeof(recordBuffer) > Importer.CHUNK_SIZE
              ) {
                parser.pause();
                try {
                  await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer });
                } catch (e) {
                  isAbort = true;
                  recordBuffer = [];
                  const error = exceptionParse(e as Error);
                  onError?.(error?.message || Importer.DEFAULT_ERROR_MESSAGE);
                  parser.abort();
                }
                recordBuffer = [];
                parser.resume();
              }
            })();
          },
          complete: () => {
            (async () => {
              try {
                recordBuffer.length &&
                  (await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer }));
              } catch (e) {
                isAbort = true;
                recordBuffer = [];
                const error = exceptionParse(e as Error);
                onError?.(error?.message || Importer.DEFAULT_ERROR_MESSAGE);
              }
              !isAbort && onFinished?.();
              resolve({});
            })();
          },
          error: (e) => {
            onError?.(e?.message || Importer.DEFAULT_ERROR_MESSAGE);
            reject(e);
          },
        });
      });
    } else {
      return new Promise((resolve, reject) => {
        Papa.parse(stream, {
          download: false,
          dynamicTyping: true,
          preview: CsvImporter.CHECK_LINES,
          complete: (result) => {
            resolve({
              [CsvImporter.DEFAULT_SHEETKEY]: result.data,
            });
          },
          error: (err) => {
            reject(err);
          },
        });
      });
    }
  }
}

export class ExcelImporter extends Importer {
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.SingleLineText,
    FieldType.LongText,
  ];

  parse(): Promise<IParseResult>;
  parse(
    options: { skipFirstNLines: number; key: string },
    chunk: (chunk: Record<string, unknown[][]>) => Promise<void>,
    onFinished?: () => void,
    onError?: (errorMsg: string) => void
  ): Promise<void>;

  async parse(
    options?: { skipFirstNLines: number; key: string },
    chunk?: (chunk: Record<string, unknown[][]>) => Promise<void>,
    onFinished?: () => void,
    onError?: (errorMsg: string) => void
  ): Promise<unknown> {
    const { stream: fileSteam } = await this.getFile();

    const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>
      new Promise((res, rej) => {
        const buffers: Buffer[] = [];
        stream.on('data', function (data) {
          buffers.push(data);
        });
        stream.on('end', function () {
          const buf = Buffer.concat(buffers);
          const workbook = XLSX.read(buf, { dense: true });
          const result: IParseResult = {};
          Object.keys(workbook.Sheets).forEach((name) => {
            result[name] = workbook.Sheets[name]['!data']?.map((item) =>
              item.map((v) => v.w)
            ) as unknown[][];
          });
          res(result);
        });
        stream.on('error', (e) => {
          onError?.(e?.message || Importer.DEFAULT_ERROR_MESSAGE);
          rej(e);
        });
      });

    const parseResult = await asyncRs(fileSteam);

    if (options && chunk) {
      const { skipFirstNLines, key } = options;
      const chunks = parseResult[key];
      const parseResults = chunkArray(chunks, Importer.MAX_CHUNK_LENGTH);

      if (this.config.maxRowCount && chunks.length > this.config.maxRowCount) {
        onError?.('Please upgrade your plan to import more records');
        return;
      }

      for (let i = 0; i < parseResults.length; i++) {
        const currentChunk = parseResults[i];
        if (i === 0 && skipFirstNLines) {
          currentChunk.splice(0, 1);
        }
        try {
          await chunk({ [key]: currentChunk });
        } catch (e) {
          onError?.((e as Error)?.message || Importer.DEFAULT_ERROR_MESSAGE);
        }
      }
      onFinished?.();
    }

    return parseResult;
  }
}

export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {
  switch (type) {
    case SUPPORTEDTYPE.CSV:
      return new CsvImporter(config);
    case SUPPORTEDTYPE.EXCEL:
      return new ExcelImporter(config);
    default:
      throw new Error('not support');
  }
};

export const getWorkerPath = (fileName: string) => {
  // there are two possible paths for worker
  const workerPath = join(__dirname, 'worker', `${fileName}.js`);
  const workerPath2 = join(process.cwd(), 'dist', 'worker', `${fileName}.js`);

  if (existsSync(workerPath)) {
    return workerPath;
  } else {
    return workerPath2;
  }
};

1	import { existsSync } from 'fs';	2✔
2	import { join } from 'path';	2✔
3	import { BadRequestException } from '@nestjs/common';	2✔
4	import { getUniqName, FieldType } from '@teable/core';	2✔
5	import type { IValidateTypes, IAnalyzeVo } from '@teable/openapi';	2✔
6	import { SUPPORTEDTYPE, importTypeMap } from '@teable/openapi';	2✔
7	import { zip, toString, intersection, chunk as chunkArray } from 'lodash';	2✔
8	import fetch from 'node-fetch';	2✔
9	import sizeof from 'object-sizeof';	2✔
10	import Papa from 'papaparse';	2✔
11	import * as XLSX from 'xlsx';	2✔
12	import type { ZodType } from 'zod';	2✔
13	import z from 'zod';	2✔
14	import { exceptionParse } from '../../../utils/exception-parse';	2✔
15	import { toLineDelimitedStream } from './delimiter-stream';	2✔
16		2✔
17	const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {	2✔
18	[FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {	2✔
19	if (typeof value === 'boolean') {	44✔
20	return true;	12✔
21	}	12✔
22	if (	32✔
23	typeof value === 'string' &&	32✔
24	(value.toLowerCase() === 'false' \|\| value.toLowerCase() === 'true')	32✔
25	) {	44✔
26	return true;	8✔
27	}	8✔
28	return false;	24✔
29	}),	24✔
30	[FieldType.Date]: z.coerce.date(),	2✔
31	[FieldType.Number]: z.coerce.number(),	2✔
32	[FieldType.LongText]: z	2✔
33	.string()	2✔
34	.refine((value) => z.string().safeParse(value) && /\n/.test(value)),	2✔
35	[FieldType.SingleLineText]: z.string(),	2✔
36	};	2✔
37		2✔
38	export interface IImportConstructorParams {	2✔
39	url: string;	2✔
40	type: SUPPORTEDTYPE;	2✔
41	maxRowCount?: number;	2✔
42	fileName?: string;	2✔
43	}	2✔
44		2✔
45	interface IParseResult {	2✔
46	[x: string]: unknown[][];	2✔
47	}	2✔
48		2✔
49	export abstract class Importer {	2✔
50	public static DEFAULT_ERROR_MESSAGE = 'unknown error';	20✔
51		20✔
52	public static CHUNK_SIZE = 1024 * 1024 * 0.2;	20✔
53		20✔
54	public static MAX_CHUNK_LENGTH = 500;	20✔
55		20✔
56	public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;	20✔
57		20✔
58	// order make sence	20✔
59	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	20✔
60	FieldType.Checkbox,	74✔
61	FieldType.Number,	74✔
62	FieldType.Date,	74✔
63	FieldType.LongText,	74✔
64	FieldType.SingleLineText,	74✔
65	];	74✔
66		20✔
67	constructor(public config: IImportConstructorParams) {}	20✔
68		20✔
69	abstract parse(	20✔
70	...args: [	20✔
71	options?: unknown,	20✔
72	chunk?: (	20✔
73	chunk: Record<string, unknown[][]>,	20✔
74	onFinished?: () => void,	20✔
75	onError?: (errorMsg: string) => void	20✔
76	) => Promise<void>,	20✔
77	]	20✔
78	): Promise<IParseResult>;	20✔
79		20✔
80	private setFileNameFromHeader(fileName: string) {	20✔
81	this.config.fileName = fileName;	10✔
82	}	10✔
83		20✔
84	getConfig() {	20✔
85	return this.config;	8✔
86	}	8✔
87		20✔
88	async getFile() {	20✔
89	const { url, type } = this.config;	12✔
90	const { body: stream, headers } = await fetch(url);	12✔
91		12✔
92	const supportType = importTypeMap[type].accept.split(',');	12✔
93		12✔
94	const fileFormat = headers	12✔
95	.get('content-type')	12✔
96	?.split(';')	12✔
97	?.map((item: string) => item.trim());	12✔
98		12✔
99	// if (!fileFormat?.length) {	12✔
100	// throw new BadRequestException(	12✔
101	// `Input url is not a standard document service without right content-type`	12✔
102	// );	12✔
103	// }	12✔
104		12✔
105	if (fileFormat?.length && !intersection(fileFormat, supportType).length) {	12✔
106	throw new BadRequestException(	2✔
107	`File format is not supported, only ${supportType.join(',')} are supported,`	2✔
108	);	2✔
109	}	2✔
110		10✔
111	const contentDisposition = headers.get('content-disposition');	10✔
112	let fileName = 'Import Table.csv';	10✔
113		10✔
114	if (contentDisposition) {	10✔
115	const fileNameMatch =	10✔
116	contentDisposition.match(/filename\*=UTF-8''([^;]+)/) \|\|	10!
117	contentDisposition.match(/filename="?([^"]+)"?/);	×
118	if (fileNameMatch) {	10✔
119	fileName = fileNameMatch[1];	10✔
120	}	10✔
121	}	10✔
122		10✔
123	const finalFileName = fileName.split('.').shift() as string;	10✔
124		10✔
125	this.setFileNameFromHeader(decodeURIComponent(finalFileName));	10✔
126		10✔
127	return { stream, fileName: finalFileName };	10✔
128	}	10✔
129		20✔
130	async genColumns() {	20✔
131	const supportTypes = Importer.SUPPORTEDTYPE;	12✔
132	const parseResult = await this.parse();	12✔
133	const { fileName, type } = this.config;	10✔
134	const result: IAnalyzeVo['worksheets'] = {};	10✔
135		10✔
136	for (const [sheetName, cols] of Object.entries(parseResult)) {	10✔
137	const zipColumnInfo = zip(...cols);	10✔
138	const existNames: string[] = [];	10✔
139	const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {	10✔
140	let isColumnEmpty = true;	60✔
141	let validatingFieldTypes = [...supportTypes];	60✔
142	for (let i = 0; i < column.length; i++) {	60✔
143	if (validatingFieldTypes.length <= 1) {	170✔
144	break;	10✔
145	}	10✔
146		160✔
147	// ignore empty value and first row causing first row as header	160✔
148	if (column[i] === '' \|\| column[i] == null \|\| i === 0) {	170✔
149	continue;	80✔
150	}	80✔
151		80✔
152	// when the whole columns aren't empty should flag	80✔
153	isColumnEmpty = false;	80✔
154		80✔
155	// when one of column's value validates long text, then break;	80✔
156	if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {	170✔
157	validatingFieldTypes = [FieldType.LongText];	10✔
158	break;	10✔
159	}	10✔
160		70✔
161	const matchTypes = validatingFieldTypes.filter((type) => {	70✔
162	const schema = validateZodSchemaMap[type];	270✔
163	return schema.safeParse(column[i]).success;	270✔
164	});	270✔
165		70✔
166	validatingFieldTypes = matchTypes;	70✔
167	}	70✔
168		170✔
169	// empty columns should be default type	170✔
170	validatingFieldTypes = !isColumnEmpty	170✔
171	? validatingFieldTypes	50✔
172	: [Importer.DEFAULT_COLUMN_TYPE];	10✔
173		60✔
174	const name = getUniqName(toString(column?.[0]).trim() \|\| `Field ${index}`, existNames);	60!
175		60✔
176	existNames.push(name);	60✔
177		60✔
178	return {	60✔
179	type: validatingFieldTypes[0] \|\| Importer.DEFAULT_COLUMN_TYPE,	60!
180	name: name.toString(),	60✔
181	};	60✔
182	});	60✔
183		10✔
184	result[sheetName] = {	10✔
185	name: type === SUPPORTEDTYPE.EXCEL ? sheetName : fileName ? fileName : sheetName,	10✔
186	columns: calculatedColumnHeaders,	10✔
187	};	10✔
188	}	10✔
189		10✔
190	return {	10✔
191	worksheets: result,	10✔
192	};	10✔
193	}	10✔
194	}	20✔
195		2✔
196	export class CsvImporter extends Importer {	2✔
197	public static readonly CHECK_LINES = 5000;	2✔
198	public static readonly DEFAULT_SHEETKEY = 'Import Table';	2✔
199		2✔
200	parse(): Promise<IParseResult>;	2✔
201	parse(	2✔
202	options: Papa.ParseConfig & { skipFirstNLines: number; key: string },	2✔
203	chunk: (chunk: Record<string, unknown[][]>) => Promise<void>,	2✔
204	onFinished?: () => void,	2✔
205	onError?: (errorMsg: string) => void	2✔
206	): Promise<void>;	2✔
207	async parse(	2✔
208	...args: [	8✔
209	options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },	8✔
210	chunkCb?: (chunk: Record<string, unknown[][]>) => Promise<void>,	8✔
211	onFinished?: () => void,	8✔
212	onError?: (errorMsg: string) => void,	8✔
213	]	8✔
214	): Promise<unknown> {	8✔
215	const [options, chunkCb, onFinished, onError] = args;	8✔
216	const { stream } = await this.getFile();	8✔
217		6✔
218	// chunk parse	6✔
219	if (options && chunkCb) {	8!
220	return new Promise((resolve, reject) => {	×
221	let isFirst = true;	×
222	let recordBuffer: unknown[][] = [];	×
223	let isAbort = false;	×
224	let totalRowCount = 0;	×
225		×
NEW 226	Papa.parse(toLineDelimitedStream(stream), {	×
227	download: false,	×
228	dynamicTyping: true,	×
229	chunk: (chunk, parser) => {	×
230	(async () => {	×
231	const newChunk = [...chunk.data] as unknown[][];	×
232	if (isFirst && options.skipFirstNLines) {	×
233	newChunk.splice(0, 1);	×
234	isFirst = false;	×
235	}	×
236		×
237	recordBuffer.push(...newChunk);	×
238	totalRowCount += newChunk.length;	×
239		×
240	if (this.config.maxRowCount && totalRowCount > this.config.maxRowCount) {	×
241	isAbort = true;	×
242	recordBuffer = [];	×
243	onError?.('please upgrade your plan to import more records');	×
244	parser.abort();	×
245	}	×
246		×
247	if (	×
248	recordBuffer.length >= Importer.MAX_CHUNK_LENGTH \|\|	×
249	sizeof(recordBuffer) > Importer.CHUNK_SIZE	×
250	) {	×
251	parser.pause();	×
252	try {	×
253	await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer });	×
254	} catch (e) {	×
255	isAbort = true;	×
256	recordBuffer = [];	×
257	const error = exceptionParse(e as Error);	×
258	onError?.(error?.message \|\| Importer.DEFAULT_ERROR_MESSAGE);	×
259	parser.abort();	×
260	}	×
261	recordBuffer = [];	×
262	parser.resume();	×
263	}	×
264	})();	×
265	},	×
266	complete: () => {	×
267	(async () => {	×
268	try {	×
269	recordBuffer.length &&	×
270	(await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer }));	×
271	} catch (e) {	×
272	isAbort = true;	×
273	recordBuffer = [];	×
274	const error = exceptionParse(e as Error);	×
275	onError?.(error?.message \|\| Importer.DEFAULT_ERROR_MESSAGE);	×
276	}	×
277	!isAbort && onFinished?.();	×
278	resolve({});	×
279	})();	×
280	},	×
281	error: (e) => {	×
282	onError?.(e?.message \|\| Importer.DEFAULT_ERROR_MESSAGE);	×
283	reject(e);	×
284	},	×
285	});	×
286	});	×
287	} else {	8✔
288	return new Promise((resolve, reject) => {	6✔
289	Papa.parse(stream, {	6✔
290	download: false,	6✔
291	dynamicTyping: true,	6✔
292	preview: CsvImporter.CHECK_LINES,	6✔
293	complete: (result) => {	6✔
294	resolve({	6✔
295	[CsvImporter.DEFAULT_SHEETKEY]: result.data,	6✔
296	});	6✔
297	},	6✔
298	error: (err) => {	6✔
UNCOV 299	reject(err);	×
300	},	×
301	});	6✔
302	});	6✔
303	}	6✔
304	}	8✔
305	}	2✔
306		2✔
307	export class ExcelImporter extends Importer {	2✔
308	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	2✔
309	FieldType.Checkbox,	74✔
310	FieldType.Number,	74✔
311	FieldType.Date,	74✔
312	FieldType.SingleLineText,	74✔
313	FieldType.LongText,	74✔
314	];	74✔
315		2✔
316	parse(): Promise<IParseResult>;	2✔
317	parse(	2✔
318	options: { skipFirstNLines: number; key: string },	2✔
319	chunk: (chunk: Record<string, unknown[][]>) => Promise<void>,	2✔
320	onFinished?: () => void,	2✔
321	onError?: (errorMsg: string) => void	2✔
322	): Promise<void>;	2✔
323		2✔
324	async parse(	2✔
325	options?: { skipFirstNLines: number; key: string },	4✔
326	chunk?: (chunk: Record<string, unknown[][]>) => Promise<void>,	4✔
327	onFinished?: () => void,	4✔
328	onError?: (errorMsg: string) => void	4✔
329	): Promise<unknown> {	4✔
330	const { stream: fileSteam } = await this.getFile();	4✔
331		4✔
332	const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>	4✔
333	new Promise((res, rej) => {	4✔
334	const buffers: Buffer[] = [];	4✔
335	stream.on('data', function (data) {	4✔
336	buffers.push(data);	4✔
337	});	4✔
338	stream.on('end', function () {	4✔
339	const buf = Buffer.concat(buffers);	4✔
340	const workbook = XLSX.read(buf, { dense: true });	4✔
341	const result: IParseResult = {};	4✔
342	Object.keys(workbook.Sheets).forEach((name) => {	4✔
343	result[name] = workbook.Sheets[name]['!data']?.map((item) =>	4✔
344	item.map((v) => v.w)	12✔
345	) as unknown[][];	4✔
346	});	4✔
347	res(result);	4✔
348	});	4✔
349	stream.on('error', (e) => {	4✔
UNCOV 350	onError?.(e?.message \|\| Importer.DEFAULT_ERROR_MESSAGE);	×
351	rej(e);	×
352	});	×
353	});	4✔
354		4✔
355	const parseResult = await asyncRs(fileSteam);	4✔
356		4✔
357	if (options && chunk) {	4!
UNCOV 358	const { skipFirstNLines, key } = options;	×
359	const chunks = parseResult[key];	×
360	const parseResults = chunkArray(chunks, Importer.MAX_CHUNK_LENGTH);	×
361		×
362	if (this.config.maxRowCount && chunks.length > this.config.maxRowCount) {	×
363	onError?.('Please upgrade your plan to import more records');	×
364	return;	×
365	}	×
366		×
367	for (let i = 0; i < parseResults.length; i++) {	×
368	const currentChunk = parseResults[i];	×
369	if (i === 0 && skipFirstNLines) {	×
370	currentChunk.splice(0, 1);	×
371	}	×
372	try {	×
373	await chunk({ [key]: currentChunk });	×
374	} catch (e) {	×
375	onError?.((e as Error)?.message \|\| Importer.DEFAULT_ERROR_MESSAGE);	×
376	}	×
377	}	×
378	onFinished?.();	×
379	}	×
380		4✔
381	return parseResult;	4✔
382	}	4✔
383	}	2✔
384		2✔
385	export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {	2✔
386	switch (type) {	20✔
387	case SUPPORTEDTYPE.CSV:	20✔
388	return new CsvImporter(config);	14✔
389	case SUPPORTEDTYPE.EXCEL:	20✔
390	return new ExcelImporter(config);	6✔
391	default:	20!
UNCOV 392	throw new Error('not support');	×
393	}	20✔
394	};	20✔
395		2✔
396	export const getWorkerPath = (fileName: string) => {	2✔
397	// there are two possible paths for worker	8✔
398	const workerPath = join(__dirname, 'worker', `${fileName}.js`);	8✔
399	const workerPath2 = join(process.cwd(), 'dist', 'worker', `${fileName}.js`);	8✔
400		8✔
401	if (existsSync(workerPath)) {	8!
UNCOV 402	return workerPath;	×
403	} else {	8✔
404	return workerPath2;	8✔
405	}	8✔
406	};	8✔

teableio / teable / 10317771079

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous