8388153768

Committed 22 Mar 2024 09:25AM CUT coverage: 28.048% (-0.2%) from 28.208%

Build # 8388153768

Build Type

Pull #484

github

Committed by

web-flow

Commit Message

Merge 27e748d45 into a06c6afb1

Pull Request Pull Request #484: feat: support increment import

Run Details

2099 of 3215 branches covered (65.29%)

24 of 738 new or added lines in 21 files covered. (3.25%)

4 existing lines in 3 files now uncovered.

25814 of 92035 relevant lines covered (28.05%)

5.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/apps/nestjs-backend/src/features/import/open-api/import.class.ts

import { BadRequestException } from '@nestjs/common';
import { getUniqName, FieldType } from '@teable/core';
import { SUPPORTEDTYPE, importTypeMap } from '@teable/openapi';
import type { IValidateTypes, IAnalyzeVo } from '@teable/openapi';
import { zip, toString, intersection } from 'lodash';
import fetch from 'node-fetch';
import Papa from 'papaparse';
import * as XLSX from 'xlsx';
import type { ZodType } from 'zod';
import z from 'zod';

const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {
  [FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {
    if (typeof value === 'boolean') {
      return true;
    }
    if (
      typeof value === 'string' &&
      (value.toLowerCase() === 'false' || value.toLowerCase() === 'true')
    ) {
      return true;
    }
    return false;
  }),
  [FieldType.Date]: z.coerce.date(),
  [FieldType.Number]: z.coerce.number(),
  [FieldType.LongText]: z
    .string()
    .refine((value) => z.string().safeParse(value) && /\n/.test(value)),
  [FieldType.SingleLineText]: z.string(),
};

interface IImportConstructorParams {
  url: string;
  type: SUPPORTEDTYPE;
}

interface IParseResult {
  [x: string]: unknown[][];
}

export abstract class Importer {
  public static CHUNK_SIZE = 1024 * 1024 * 1;

  public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;

  constructor(public config: IImportConstructorParams) {}

  abstract parse(
    ...args: [options?: unknown, cb?: (chunk: Record<string, unknown[][]>) => Promise<void>]
  ): Promise<IParseResult>;

  abstract getSupportedFieldTypes(): IValidateTypes[];

  async getFile() {
    const { url, type } = this.config;
    const { body: stream, headers } = await fetch(url);

    const supportType = importTypeMap[type].accept.split(',');

    const fileFormat = headers
      .get('content-type')
      ?.split(';')
      ?.map((item: string) => item.trim());

    // if (!fileFormat?.length) {
    //   throw new BadRequestException(
    //     `Input url is not a standard document service without right content-type`
    //   );
    // }

    if (fileFormat?.length && !intersection(fileFormat, supportType).length) {
      throw new BadRequestException(
        `File format is not supported, only ${supportType.join(',')} are supported,`
      );
    }

    return stream;
  }

  async genColumns() {
    const supportTypes = this.getSupportedFieldTypes();
    const parseResult = await this.parse();
    const result: IAnalyzeVo['worksheets'] = {};

    for (const [sheetName, cols] of Object.entries(parseResult)) {
      const zipColumnInfo = zip(...cols);
      const existNames: string[] = [];
      const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {
        let isColumnEmpty = true;
        let validatingFieldTypes = [...supportTypes];
        for (let i = 0; i < column.length; i++) {
          if (validatingFieldTypes.length <= 1) {
            break;
          }

          // ignore empty value and first row causing first row as header
          if (column[i] === '' || column[i] == null || i === 0) {
            continue;
          }

          // when the whole columns aren't empty should flag
          isColumnEmpty = false;

          // when one of column's value validates long text, then break;
          if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {
            validatingFieldTypes = [FieldType.LongText];
            break;
          }

          const matchTypes = validatingFieldTypes.filter((type) => {
            const schema = validateZodSchemaMap[type];
            return schema.safeParse(column[i]).success;
          });

          validatingFieldTypes = matchTypes;
        }

        // empty columns should be default type
        validatingFieldTypes = !isColumnEmpty
          ? validatingFieldTypes
          : [Importer.DEFAULT_COLUMN_TYPE];

        const name = getUniqName(toString(column?.[0]).trim() || `Field ${index}`, existNames);

        existNames.push(name);

        return {
          type: validatingFieldTypes[0] || Importer.DEFAULT_COLUMN_TYPE,
          name: name.toString(),
        };
      });

      result[sheetName] = {
        name: sheetName,
        columns: calculatedColumnHeaders,
      };
    }

    return {
      worksheets: result,
    };
  }
}

export class CsvImporter extends Importer {
  public static readonly CHECK_LINES = 5000;
  public static readonly DEFAULT_SHEETKEY = 'Import Table';
  // order make sence
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.LongText,
    FieldType.SingleLineText,
  ];
  getSupportedFieldTypes() {
    return CsvImporter.SUPPORTEDTYPE;
  }

  parse(): Promise<IParseResult>;
  parse(
    options: Papa.ParseConfig & { skipFirstNLines: number; key: string },
    cb: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<void>;
  async parse(
    ...args: [
      options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },
      cb?: (chunk: Record<string, unknown[][]>) => Promise<void>,
    ]
  ): Promise<unknown> {
    const [options, cb] = args;
    const stream = await this.getFile();

    // chunk parse
    if (options && cb) {
      return new Promise((resolve, reject) => {
        let isFirst = true;
        Papa.parse(stream, {
          download: false,
          dynamicTyping: true,
          chunkSize: Importer.CHUNK_SIZE,
          chunk: (chunk, parser) => {
            (async () => {
              const newChunk = [...chunk.data] as unknown[][];
              if (isFirst && options.skipFirstNLines) {
                newChunk.splice(0, 1);
                isFirst = false;
              }
              parser.pause();
              await cb({ [CsvImporter.DEFAULT_SHEETKEY]: newChunk });
              parser.resume();
            })();
          },
          complete: () => {
            resolve({});
          },
          error: (err) => {
            reject(err);
          },
        });
      });
    } else {
      return new Promise((resolve, reject) => {
        Papa.parse(stream, {
          download: false,
          dynamicTyping: true,
          preview: CsvImporter.CHECK_LINES,
          complete: (result) => {
            resolve({
              [CsvImporter.DEFAULT_SHEETKEY]: result.data,
            });
          },
          error: (err) => {
            reject(err);
          },
        });
      });
    }
  }
}

export class ExcelImporter extends Importer {
  public static readonly SUPPORTEDTYPE: IValidateTypes[] = [
    FieldType.Checkbox,
    FieldType.Number,
    FieldType.Date,
    FieldType.SingleLineText,
    FieldType.LongText,
  ];

  parse(): Promise<IParseResult>;
  parse(
    options: { skipFirstNLines: number; key: string },
    cb: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<void>;

  async parse(
    options?: { skipFirstNLines: number; key: string },
    cb?: (chunk: Record<string, unknown[][]>) => Promise<void>
  ): Promise<unknown> {
    const fileSteam = await this.getFile();

    const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>
      new Promise((res, rej) => {
        const buffers: Buffer[] = [];
        stream.on('data', function (data) {
          buffers.push(data);
        });
        stream.on('end', function () {
          const buf = Buffer.concat(buffers);
          const workbook = XLSX.read(buf, { dense: true });
          const result: IParseResult = {};
          Object.keys(workbook.Sheets).forEach((name) => {
            result[name] = workbook.Sheets[name]['!data']?.map((item) =>
              item.map((v) => v.w)
            ) as unknown[][];
          });
          res(result);
        });
        stream.on('error', (e) => {
          rej(e);
        });
      });

    const parseResult = await asyncRs(fileSteam);

    if (options && cb) {
      const { skipFirstNLines, key } = options;
      if (skipFirstNLines) {
        parseResult[key].splice(0, 1);
      }
      return await cb(parseResult);
    }

    return parseResult;
  }
  getSupportedFieldTypes() {
    return CsvImporter.SUPPORTEDTYPE;
  }
}

export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {
  switch (type) {
    case SUPPORTEDTYPE.CSV:
      return new CsvImporter(config);
    case SUPPORTEDTYPE.EXCEL:
      return new ExcelImporter(config);
    default:
      throw new Error('not support');
  }
};

1	import { BadRequestException } from '@nestjs/common';	×
NEW 2	import { getUniqName, FieldType } from '@teable/core';	×
NEW 3	import { SUPPORTEDTYPE, importTypeMap } from '@teable/openapi';	×
NEW 4	import type { IValidateTypes, IAnalyzeVo } from '@teable/openapi';	×
5	import { zip, toString, intersection } from 'lodash';	×
6	import fetch from 'node-fetch';	×
7	import Papa from 'papaparse';	×
8	import * as XLSX from 'xlsx';	×
9	import type { ZodType } from 'zod';	×
10	import z from 'zod';	×
11		×
12	const validateZodSchemaMap: Record<IValidateTypes, ZodType> = {	×
13	[FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine((value: unknown) => {	×
14	if (typeof value === 'boolean') {	×
15	return true;	×
16	}	×
17	if (	×
18	typeof value === 'string' &&	×
19	(value.toLowerCase() === 'false' \|\| value.toLowerCase() === 'true')	×
20	) {	×
21	return true;	×
22	}	×
23	return false;	×
24	}),	×
25	[FieldType.Date]: z.coerce.date(),	×
26	[FieldType.Number]: z.coerce.number(),	×
27	[FieldType.LongText]: z	×
28	.string()	×
29	.refine((value) => z.string().safeParse(value) && /\n/.test(value)),	×
30	[FieldType.SingleLineText]: z.string(),	×
31	};	×
32		×
33	interface IImportConstructorParams {	×
34	url: string;	×
35	type: SUPPORTEDTYPE;	×
36	}	×
37		×
38	interface IParseResult {	×
39	[x: string]: unknown[][];	×
40	}	×
41		×
42	export abstract class Importer {	×
43	public static CHUNK_SIZE = 1024 * 1024 * 1;	×
44		×
45	public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText;	×
46		×
47	constructor(public config: IImportConstructorParams) {}	×
48		×
49	abstract parse(	×
50	...args: [options?: unknown, cb?: (chunk: Record<string, unknown[][]>) => Promise<void>]	×
51	): Promise<IParseResult>;	×
52		×
53	abstract getSupportedFieldTypes(): IValidateTypes[];	×
54		×
55	async getFile() {	×
56	const { url, type } = this.config;	×
57	const { body: stream, headers } = await fetch(url);	×
58		×
59	const supportType = importTypeMap[type].accept.split(',');	×
60		×
61	const fileFormat = headers	×
62	.get('content-type')	×
63	?.split(';')	×
64	?.map((item: string) => item.trim());	×
65		×
66	// if (!fileFormat?.length) {	×
67	// throw new BadRequestException(	×
68	// `Input url is not a standard document service without right content-type`	×
69	// );	×
70	// }	×
71		×
72	if (fileFormat?.length && !intersection(fileFormat, supportType).length) {	×
73	throw new BadRequestException(	×
74	`File format is not supported, only ${supportType.join(',')} are supported,`	×
75	);	×
76	}	×
77		×
78	return stream;	×
79	}	×
80		×
81	async genColumns() {	×
82	const supportTypes = this.getSupportedFieldTypes();	×
83	const parseResult = await this.parse();	×
84	const result: IAnalyzeVo['worksheets'] = {};	×
85		×
86	for (const [sheetName, cols] of Object.entries(parseResult)) {	×
87	const zipColumnInfo = zip(...cols);	×
88	const existNames: string[] = [];	×
89	const calculatedColumnHeaders = zipColumnInfo.map((column, index) => {	×
90	let isColumnEmpty = true;	×
91	let validatingFieldTypes = [...supportTypes];	×
92	for (let i = 0; i < column.length; i++) {	×
93	if (validatingFieldTypes.length <= 1) {	×
94	break;	×
95	}	×
96		×
97	// ignore empty value and first row causing first row as header	×
98	if (column[i] === '' \|\| column[i] == null \|\| i === 0) {	×
99	continue;	×
100	}	×
101		×
102	// when the whole columns aren't empty should flag	×
103	isColumnEmpty = false;	×
104		×
105	// when one of column's value validates long text, then break;	×
106	if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) {	×
107	validatingFieldTypes = [FieldType.LongText];	×
108	break;	×
109	}	×
110		×
111	const matchTypes = validatingFieldTypes.filter((type) => {	×
112	const schema = validateZodSchemaMap[type];	×
113	return schema.safeParse(column[i]).success;	×
114	});	×
115		×
116	validatingFieldTypes = matchTypes;	×
117	}	×
118		×
119	// empty columns should be default type	×
120	validatingFieldTypes = !isColumnEmpty	×
121	? validatingFieldTypes	×
122	: [Importer.DEFAULT_COLUMN_TYPE];	×
123		×
NEW 124	const name = getUniqName(toString(column?.[0]).trim() \|\| `Field ${index}`, existNames);	×
125		×
126	existNames.push(name);	×
127		×
128	return {	×
129	type: validatingFieldTypes[0] \|\| Importer.DEFAULT_COLUMN_TYPE,	×
130	name: name.toString(),	×
131	};	×
132	});	×
133		×
134	result[sheetName] = {	×
135	name: sheetName,	×
136	columns: calculatedColumnHeaders,	×
137	};	×
138	}	×
139		×
140	return {	×
141	worksheets: result,	×
142	};	×
143	}	×
144	}	×
145		×
146	export class CsvImporter extends Importer {	×
147	public static readonly CHECK_LINES = 5000;	×
148	public static readonly DEFAULT_SHEETKEY = 'Import Table';	×
149	// order make sence	×
150	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	×
151	FieldType.Checkbox,	×
152	FieldType.Number,	×
153	FieldType.Date,	×
154	FieldType.LongText,	×
155	FieldType.SingleLineText,	×
156	];	×
157	getSupportedFieldTypes() {	×
158	return CsvImporter.SUPPORTEDTYPE;	×
159	}	×
160		×
161	parse(): Promise<IParseResult>;	×
162	parse(	×
163	options: Papa.ParseConfig & { skipFirstNLines: number; key: string },	×
164	cb: (chunk: Record<string, unknown[][]>) => Promise<void>	×
165	): Promise<void>;	×
166	async parse(	×
167	...args: [	×
168	options?: Papa.ParseConfig & { skipFirstNLines: number; key: string },	×
169	cb?: (chunk: Record<string, unknown[][]>) => Promise<void>,	×
170	]	×
171	): Promise<unknown> {	×
172	const [options, cb] = args;	×
173	const stream = await this.getFile();	×
174		×
175	// chunk parse	×
176	if (options && cb) {	×
177	return new Promise((resolve, reject) => {	×
178	let isFirst = true;	×
179	Papa.parse(stream, {	×
180	download: false,	×
181	dynamicTyping: true,	×
182	chunkSize: Importer.CHUNK_SIZE,	×
183	chunk: (chunk, parser) => {	×
184	(async () => {	×
185	const newChunk = [...chunk.data] as unknown[][];	×
186	if (isFirst && options.skipFirstNLines) {	×
187	newChunk.splice(0, 1);	×
188	isFirst = false;	×
189	}	×
190	parser.pause();	×
191	await cb({ [CsvImporter.DEFAULT_SHEETKEY]: newChunk });	×
192	parser.resume();	×
193	})();	×
194	},	×
195	complete: () => {	×
196	resolve({});	×
197	},	×
198	error: (err) => {	×
199	reject(err);	×
200	},	×
201	});	×
202	});	×
203	} else {	×
204	return new Promise((resolve, reject) => {	×
205	Papa.parse(stream, {	×
206	download: false,	×
207	dynamicTyping: true,	×
208	preview: CsvImporter.CHECK_LINES,	×
209	complete: (result) => {	×
210	resolve({	×
211	[CsvImporter.DEFAULT_SHEETKEY]: result.data,	×
212	});	×
213	},	×
214	error: (err) => {	×
215	reject(err);	×
216	},	×
217	});	×
218	});	×
219	}	×
220	}	×
221	}	×
222		×
223	export class ExcelImporter extends Importer {	×
224	public static readonly SUPPORTEDTYPE: IValidateTypes[] = [	×
225	FieldType.Checkbox,	×
226	FieldType.Number,	×
227	FieldType.Date,	×
228	FieldType.SingleLineText,	×
229	FieldType.LongText,	×
230	];	×
231		×
232	parse(): Promise<IParseResult>;	×
233	parse(	×
234	options: { skipFirstNLines: number; key: string },	×
235	cb: (chunk: Record<string, unknown[][]>) => Promise<void>	×
236	): Promise<void>;	×
237		×
238	async parse(	×
239	options?: { skipFirstNLines: number; key: string },	×
240	cb?: (chunk: Record<string, unknown[][]>) => Promise<void>	×
241	): Promise<unknown> {	×
242	const fileSteam = await this.getFile();	×
243		×
244	const asyncRs = async (stream: NodeJS.ReadableStream): Promise<IParseResult> =>	×
245	new Promise((res, rej) => {	×
246	const buffers: Buffer[] = [];	×
247	stream.on('data', function (data) {	×
248	buffers.push(data);	×
249	});	×
250	stream.on('end', function () {	×
251	const buf = Buffer.concat(buffers);	×
252	const workbook = XLSX.read(buf, { dense: true });	×
253	const result: IParseResult = {};	×
254	Object.keys(workbook.Sheets).forEach((name) => {	×
255	result[name] = workbook.Sheets[name]['!data']?.map((item) =>	×
256	item.map((v) => v.w)	×
257	) as unknown[][];	×
258	});	×
259	res(result);	×
260	});	×
261	stream.on('error', (e) => {	×
262	rej(e);	×
263	});	×
264	});	×
265		×
266	const parseResult = await asyncRs(fileSteam);	×
267		×
268	if (options && cb) {	×
269	const { skipFirstNLines, key } = options;	×
270	if (skipFirstNLines) {	×
271	parseResult[key].splice(0, 1);	×
272	}	×
273	return await cb(parseResult);	×
274	}	×
275		×
276	return parseResult;	×
277	}	×
278	getSupportedFieldTypes() {	×
279	return CsvImporter.SUPPORTEDTYPE;	×
280	}	×
281	}	×
282		×
283	export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => {	×
284	switch (type) {	×
285	case SUPPORTEDTYPE.CSV:	×
286	return new CsvImporter(config);	×
287	case SUPPORTEDTYPE.EXCEL:	×
288	return new ExcelImporter(config);	×
289	default:	×
290	throw new Error('not support');	×
291	}	×
292	};	×

teableio / teable / 8388153768

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous