8387093605

Committed 22 Mar 2024 07:48AM CUT coverage: 28.027% (-0.2%) from 28.222%

Build # 8387093605

Build Type

Pull #484

github

Committed by

web-flow

Commit Message

Merge 174ef76f7 into a06c6afb1

Pull Request Pull Request #484: feat: support increment import

Run Details

2099 of 3218 branches covered (65.23%)

24 of 703 new or added lines in 18 files covered. (3.41%)

49 existing lines in 6 files now uncovered.

25815 of 92109 relevant lines covered (28.03%)

5.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/apps/nestjs-backend/src/features/import/open-api/import.class.ts

import { BadRequestException } from '@nestjs/common';
import type { IValidateTypes, IAnalyzeVo } from '@teable/core';
import { getUniqName, FieldType, SUPPORTEDTYPE, importTypeMap } from '@teable/core';
import { zip, toString, intersection } from 'lodash';
import fetch from 'node-fetch';
import Papa from 'papaparse';
import * as XLSX from 'xlsx';
import type { ZodType } from 'zod';
import z from 'zod';

const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {
  [FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {
    if (typeof value === 'boolean') {
      return true;
    }
    if (
      typeof value === 'string' &&
      (value.toLowerCase() === 'false' || value.toLowerCase() === 'true')
    ) {
      return true;
    }
    return false;
  }),
  [FieldType.Date]: z.coerce.date(),
  [FieldType.Number]: z.coerce.number(),
  [FieldType.LongText]: z
    .string()
    .refine((value) => z.string().safeParse(value) && /\n/.test(value)),
  [FieldType.SingleLineText]: z.string(),
};

interface IImportConstructorParams {
  url: string;
  type: SUPPORTEDTYPE;
}

interface IParseResult {
  [x: string]: unknown[][];
}

export abstract class Importer {
  public static CHUNK_SIZE = 1024 * 1024 * 1;

  public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;

  constructor(public config: IImportConstructorParams) {}

  abstract parse(
    ...args: [options?: unknown, cb?: (chunk: Record<string, unknown[][]>) => Promise<void>]
  ): Promise<IParseResult>;

  abstract getSupportedFieldTypes(): IValidateTypes[];

  async getFile() {
    const { url, type } = this.config;
    const { body: stream, headers } = await fetch(url);

    const supportType = importTypeMap[type].accept.split(',');

    const fileFormat = headers
      .get('content-type')
      ?.split(';')
      ?.map((item: string) => item.trim());

    // if (!fileFormat?.length) {
    //   throw new BadRequestException(
    //     `Input url is not a standard document service without right content-type`
    //   );
    // }

    if (fileFormat?.length && !intersection(fileFormat, supportType).length) {
      throw new BadRequestException(
        `File format is not supported, only ${supportType.join(',')} are supported,`
      );
    }

    return stream;
  }

  async genColumns() {
    const supportTypes = this.getSupportedFieldTypes();
    const parseResult = await this.parse();
    const result: IAnalyzeVo['worksheets'] = {};

    for (const [sheetName, cols] of Object.entries(parseResult)) {
      const zipColumnInfo = zip(...cols);
      const existNames: string[] = [];
      const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {
        let isColumnEmpty = true;
        let validatingFieldTypes = [...supportTypes];
        for (let i = 0; i < column.length; i++) {
          if (validatingFieldTypes.length <= 1) {
            break;
          }

          // ignore empty value and first row causing first row as header
          if (column[i] === '' || column[i] == null || i === 0) {
            continue;
          }

          // when the whole columns aren't empty should flag
          isColumnEmpty = false;

          // when one of column's value validates long text, then break;
          if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {
            validatingFieldTypes = [FieldType.LongText];
            break;
          }

          const matchTypes = validatingFieldTypes.filter((type) => {
            const schema = validateZodSchemaMap[type];
            return schema.safeParse(column[i]).success;
          });

          validatingFieldTypes = matchTypes;
        }

        // empty columns should be default type
        validatingFieldTypes = !isColumnEmpty
          ? validatingFieldTypes
          : [Importer.DEFAULT_COLUMN_TYPE];

        const name = getUniqName(toString(column?.[0]).trim() || `Field ${index}`, existNames);

        existNames.push(name);

        return {
          type: validatingFieldTypes[0] || Importer.DEFAULT_COLUMN_TYPE,
          name: name.toString(),
        };
      });

      result[sheetName] = {
        name: sheetName,
        columns: calculatedColumnHeaders,
      };
    }

    return {
      worksheets: result,
    };
  }
}

export class CsvImporter extends Importer {
  public static readonly CHECK_LINES = 5000;
  public static readonly DEFAULT_SHEETKEY = 'Import Table';
  // order make sence
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.LongText,
    FieldType.SingleLineText,
  ];
  getSupportedFieldTypes() {
    return CsvImporter.SUPPORTEDTYPE;
  }

  parse(): Promise<IParseResult>;
  parse(
    options: Papa.ParseConfig & { skipFirstNLines: number; key: string },
    cb: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<void>;
  async parse(
    ...args: [
      options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },
      cb?: (chunk: Record<string, unknown[][]>) => Promise<void>,
    ]
  ): Promise<unknown> {
    const [options, cb] = args;
    const stream = await this.getFile();

    // chunk parse
    if (options && cb) {
      return new Promise((resolve, reject) => {
        let isFirst = true;
        Papa.parse(stream, {
          download: false,
          dynamicTyping: true,
          chunkSize: Importer.CHUNK_SIZE,
          chunk: (chunk, parser) => {
            (async () => {
              const newChunk = [...chunk.data] as unknown[][];
              if (isFirst && options.skipFirstNLines) {
                newChunk.splice(0, 1);
                isFirst = false;
              }
              parser.pause();
              await cb({ [CsvImporter.DEFAULT_SHEETKEY]: newChunk });
              parser.resume();
            })();
          },
          complete: () => {
            resolve({});
          },
          error: (err) => {
            reject(err);
          },
        });
      });
    } else {
      return new Promise((resolve, reject) => {
        Papa.parse(stream, {
          download: false,
          dynamicTyping: true,
          preview: CsvImporter.CHECK_LINES,
          complete: (result) => {
            resolve({
              [CsvImporter.DEFAULT_SHEETKEY]: result.data,
            });
          },
          error: (err) => {
            reject(err);
          },
        });
      });
    }
  }
}

export class ExcelImporter extends Importer {
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.SingleLineText,
    FieldType.LongText,
  ];

  parse(): Promise<IParseResult>;
  parse(
    options: { skipFirstNLines: number; key: string },
    cb: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<void>;

  async parse(
    options?: { skipFirstNLines: number; key: string },
    cb?: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<unknown> {
    const fileSteam = await this.getFile();

    const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>
      new Promise((res, rej) => {
        const buffers: Buffer[] = [];
        stream.on('data', function (data) {
          buffers.push(data);
        });
        stream.on('end', function () {
          const buf = Buffer.concat(buffers);
          const workbook = XLSX.read(buf, { dense: true });
          const result: IParseResult = {};
          Object.keys(workbook.Sheets).forEach((name) => {
            result[name] = workbook.Sheets[name]['!data']?.map((item) =>
              item.map((v) => v.w)
            ) as unknown[][];
          });
          res(result);
        });
        stream.on('error', (e) => {
          rej(e);
        });
      });

    const parseResult = await asyncRs(fileSteam);

    if (options && cb) {
      const { skipFirstNLines, key } = options;
      if (skipFirstNLines) {
        parseResult[key].splice(0, 1);
      }
      return await cb(parseResult);
    }

    return parseResult;
  }
  getSupportedFieldTypes() {
    return CsvImporter.SUPPORTEDTYPE;
  }
}

export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {
  switch (type) {
    case SUPPORTEDTYPE.CSV:
      return new CsvImporter(config);
    case SUPPORTEDTYPE.EXCEL:
      return new ExcelImporter(config);
    default:
      throw new Error('not support');
  }
};

1	import { BadRequestException } from '@nestjs/common';	×
2	import type { IValidateTypes, IAnalyzeVo } from '@teable/core';	×
3	import { getUniqName, FieldType, SUPPORTEDTYPE, importTypeMap } from '@teable/core';	×
4	import { zip, toString, intersection } from 'lodash';	×
5	import fetch from 'node-fetch';	×
6	import Papa from 'papaparse';	×
7	import * as XLSX from 'xlsx';	×
8	import type { ZodType } from 'zod';	×
9	import z from 'zod';	×
10		×
11	const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {	×
12	[FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {	×
13	if (typeof value === 'boolean') {	×
14	return true;	×
15	}	×
16	if (	×
17	typeof value === 'string' &&	×
18	(value.toLowerCase() === 'false' \|\| value.toLowerCase() === 'true')	×
19	) {	×
20	return true;	×
21	}	×
22	return false;	×
23	}),	×
24	[FieldType.Date]: z.coerce.date(),	×
25	[FieldType.Number]: z.coerce.number(),	×
26	[FieldType.LongText]: z	×
27	.string()	×
28	.refine((value) => z.string().safeParse(value) && /\n/.test(value)),	×
29	[FieldType.SingleLineText]: z.string(),	×
30	};	×
31		×
32	interface IImportConstructorParams {	×
33	url: string;	×
34	type: SUPPORTEDTYPE;	×
35	}	×
36		×
37	interface IParseResult {	×
38	[x: string]: unknown[][];	×
39	}	×
40		×
41	export abstract class Importer {	×
42	public static CHUNK_SIZE = 1024 * 1024 * 1;	×
43		×
44	public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;	×
45		×
46	constructor(public config: IImportConstructorParams) {}	×
47		×
48	abstract parse(	×
49	...args: [options?: unknown, cb?: (chunk: Record<string, unknown[][]>) => Promise<void>]	×
50	): Promise<IParseResult>;	×
51		×
52	abstract getSupportedFieldTypes(): IValidateTypes[];	×
53		×
54	async getFile() {	×
55	const { url, type } = this.config;	×
56	const { body: stream, headers } = await fetch(url);	×
57		×
58	const supportType = importTypeMap[type].accept.split(',');	×
59		×
60	const fileFormat = headers	×
61	.get('content-type')	×
62	?.split(';')	×
63	?.map((item: string) => item.trim());	×
64		×
65	// if (!fileFormat?.length) {	×
66	// throw new BadRequestException(	×
67	// `Input url is not a standard document service without right content-type`	×
68	// );	×
69	// }	×
70		×
71	if (fileFormat?.length && !intersection(fileFormat, supportType).length) {	×
72	throw new BadRequestException(	×
73	`File format is not supported, only ${supportType.join(',')} are supported,`	×
74	);	×
75	}	×
76		×
77	return stream;	×
78	}	×
79		×
80	async genColumns() {	×
81	const supportTypes = this.getSupportedFieldTypes();	×
82	const parseResult = await this.parse();	×
83	const result: IAnalyzeVo['worksheets'] = {};	×
84		×
85	for (const [sheetName, cols] of Object.entries(parseResult)) {	×
86	const zipColumnInfo = zip(...cols);	×
87	const existNames: string[] = [];	×
88	const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {	×
89	let isColumnEmpty = true;	×
90	let validatingFieldTypes = [...supportTypes];	×
91	for (let i = 0; i < column.length; i++) {	×
92	if (validatingFieldTypes.length <= 1) {	×
93	break;	×
94	}	×
95		×
96	// ignore empty value and first row causing first row as header	×
97	if (column[i] === '' \|\| column[i] == null \|\| i === 0) {	×
98	continue;	×
99	}	×
100		×
101	// when the whole columns aren't empty should flag	×
102	isColumnEmpty = false;	×
103		×
104	// when one of column's value validates long text, then break;	×
105	if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {	×
106	validatingFieldTypes = [FieldType.LongText];	×
107	break;	×
108	}	×
109		×
110	const matchTypes = validatingFieldTypes.filter((type) => {	×
111	const schema = validateZodSchemaMap[type];	×
112	return schema.safeParse(column[i]).success;	×
113	});	×
114		×
115	validatingFieldTypes = matchTypes;	×
116	}	×
117		×
118	// empty columns should be default type	×
119	validatingFieldTypes = !isColumnEmpty	×
120	? validatingFieldTypes	×
121	: [Importer.DEFAULT_COLUMN_TYPE];	×
122		×
NEW 123	const name = getUniqName(toString(column?.[0]).trim() \|\| `Field ${index}`, existNames);	×
124		×
125	existNames.push(name);	×
126		×
127	return {	×
128	type: validatingFieldTypes[0] \|\| Importer.DEFAULT_COLUMN_TYPE,	×
129	name: name.toString(),	×
130	};	×
131	});	×
132		×
133	result[sheetName] = {	×
134	name: sheetName,	×
135	columns: calculatedColumnHeaders,	×
136	};	×
137	}	×
138		×
139	return {	×
140	worksheets: result,	×
141	};	×
142	}	×
143	}	×
144		×
145	export class CsvImporter extends Importer {	×
146	public static readonly CHECK_LINES = 5000;	×
147	public static readonly DEFAULT_SHEETKEY = 'Import Table';	×
148	// order make sence	×
149	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	×
150	FieldType.Checkbox,	×
151	FieldType.Number,	×
152	FieldType.Date,	×
153	FieldType.LongText,	×
154	FieldType.SingleLineText,	×
155	];	×
156	getSupportedFieldTypes() {	×
157	return CsvImporter.SUPPORTEDTYPE;	×
158	}	×
159		×
160	parse(): Promise<IParseResult>;	×
161	parse(	×
162	options: Papa.ParseConfig & { skipFirstNLines: number; key: string },	×
163	cb: (chunk: Record<string, unknown[][]>) => Promise<void>	×
164	): Promise<void>;	×
165	async parse(	×
166	...args: [	×
167	options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },	×
168	cb?: (chunk: Record<string, unknown[][]>) => Promise<void>,	×
169	]	×
170	): Promise<unknown> {	×
171	const [options, cb] = args;	×
172	const stream = await this.getFile();	×
173		×
174	// chunk parse	×
175	if (options && cb) {	×
176	return new Promise((resolve, reject) => {	×
177	let isFirst = true;	×
178	Papa.parse(stream, {	×
179	download: false,	×
180	dynamicTyping: true,	×
181	chunkSize: Importer.CHUNK_SIZE,	×
182	chunk: (chunk, parser) => {	×
183	(async () => {	×
184	const newChunk = [...chunk.data] as unknown[][];	×
185	if (isFirst && options.skipFirstNLines) {	×
186	newChunk.splice(0, 1);	×
187	isFirst = false;	×
188	}	×
189	parser.pause();	×
190	await cb({ [CsvImporter.DEFAULT_SHEETKEY]: newChunk });	×
191	parser.resume();	×
192	})();	×
193	},	×
194	complete: () => {	×
195	resolve({});	×
196	},	×
197	error: (err) => {	×
198	reject(err);	×
199	},	×
200	});	×
201	});	×
202	} else {	×
203	return new Promise((resolve, reject) => {	×
204	Papa.parse(stream, {	×
205	download: false,	×
206	dynamicTyping: true,	×
207	preview: CsvImporter.CHECK_LINES,	×
208	complete: (result) => {	×
209	resolve({	×
210	[CsvImporter.DEFAULT_SHEETKEY]: result.data,	×
211	});	×
212	},	×
213	error: (err) => {	×
214	reject(err);	×
215	},	×
216	});	×
217	});	×
218	}	×
219	}	×
220	}	×
221		×
222	export class ExcelImporter extends Importer {	×
223	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	×
224	FieldType.Checkbox,	×
225	FieldType.Number,	×
226	FieldType.Date,	×
227	FieldType.SingleLineText,	×
228	FieldType.LongText,	×
229	];	×
230		×
231	parse(): Promise<IParseResult>;	×
232	parse(	×
233	options: { skipFirstNLines: number; key: string },	×
234	cb: (chunk: Record<string, unknown[][]>) => Promise<void>	×
235	): Promise<void>;	×
236		×
237	async parse(	×
238	options?: { skipFirstNLines: number; key: string },	×
239	cb?: (chunk: Record<string, unknown[][]>) => Promise<void>	×
240	): Promise<unknown> {	×
241	const fileSteam = await this.getFile();	×
242		×
243	const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>	×
244	new Promise((res, rej) => {	×
245	const buffers: Buffer[] = [];	×
246	stream.on('data', function (data) {	×
247	buffers.push(data);	×
248	});	×
249	stream.on('end', function () {	×
250	const buf = Buffer.concat(buffers);	×
251	const workbook = XLSX.read(buf, { dense: true });	×
252	const result: IParseResult = {};	×
253	Object.keys(workbook.Sheets).forEach((name) => {	×
254	result[name] = workbook.Sheets[name]['!data']?.map((item) =>	×
255	item.map((v) => v.w)	×
256	) as unknown[][];	×
257	});	×
258	res(result);	×
259	});	×
260	stream.on('error', (e) => {	×
261	rej(e);	×
262	});	×
263	});	×
264		×
265	const parseResult = await asyncRs(fileSteam);	×
266		×
267	if (options && cb) {	×
268	const { skipFirstNLines, key } = options;	×
269	if (skipFirstNLines) {	×
270	parseResult[key].splice(0, 1);	×
271	}	×
272	return await cb(parseResult);	×
273	}	×
274		×
275	return parseResult;	×
276	}	×
277	getSupportedFieldTypes() {	×
278	return CsvImporter.SUPPORTEDTYPE;	×
279	}	×
280	}	×
281		×
282	export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {	×
283	switch (type) {	×
284	case SUPPORTEDTYPE.CSV:	×
285	return new CsvImporter(config);	×
286	case SUPPORTEDTYPE.EXCEL:	×
287	return new ExcelImporter(config);	×
288	default:	×
289	throw new Error('not support');	×
290	}	×
291	};	×

teableio / teable / 8387093605

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous