8477114564

Committed 29 Mar 2024 04:16AM CUT coverage: 82.458% (+60.6%) from 21.83%

Build # 8477114564

Build Type

push

github

Committed by

web-flow

Commit Message

feat: support increment import (#484)

* fix: should not set content-length in `axios` header

* chore: remove sdk `BaseSelect` customer filter

* feat: support increment import

* test: add inplace import e2e

fix: sqlite conflict error in import e2e

* fix: `BaseSelect` filter ignore case sensitive

* fix: gen uniq name compatible pure number string

* chore: unify import fn name

* feat: add inplace import error tips

* chore: move import types to `@teable/openapi` from `@teable/core`

* feat: reset selected import field when shift sheet when inplace import mul-workbook in excel

* chore: adjust inplace import params description in `@teable/openapi`

* chore: lint error

* feat: inplace import auth validate

* test: compatible import e2e datecompare

Run Details

3937 of 4142 branches covered (95.05%)

66 of 70 new or added lines in 3 files covered. (94.29%)

26136 of 31696 relevant lines covered (82.46%)

1210.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.71

/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts

import { Injectable, Logger } from '@nestjs/common';
import { FieldKeyType } from '@teable/core';
import type { IAnalyzeRo, IImportOptionRo, IInplaceImportOptionRo } from '@teable/openapi';
import { RecordOpenApiService } from '../../record/open-api/record-open-api.service';
import { DEFAULT_VIEWS } from '../../table/constant';
import { TableOpenApiService } from '../../table/open-api/table-open-api.service';
import { importerFactory } from './import.class';

@Injectable()
export class ImportOpenApiService {
  private logger = new Logger(ImportOpenApiService.name);
  constructor(
    private readonly tableOpenApiService: TableOpenApiService,
    private readonly recordOpenApiService: RecordOpenApiService
  ) {}

  async analyze(analyzeRo: IAnalyzeRo) {
    const { attachmentUrl, fileType } = analyzeRo;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    return await importer.genColumns();
  }

  async createTableFromImport(baseId: string, importRo: IImportOptionRo) {
    const { attachmentUrl, fileType, worksheets } = importRo;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    const tableResult = [];

    for (const [sheetKey, value] of Object.entries(worksheets)) {
      const { importData, useFirstRowAsHeader, columns: columnInfo, name } = value;
      const fieldsRo = columnInfo.map((col, index) => {
        return {
          ...col,
          isPrimary: index === 0 ? true : null,
        };
      });

      // create table with column
      const table = await this.tableOpenApiService.createTable(baseId, {
        name: name,
        fields: fieldsRo,
        views: DEFAULT_VIEWS,
        records: [],
      });

      tableResult.push(table);

      const { fields } = table;

      if (importData) {
        importer.parse(
          {
            skipFirstNLines: useFirstRowAsHeader ? 1 : 0,
            key: sheetKey,
          },
          async (result) => {
            const currentResult = result[sheetKey];
            // fill data
            const records = currentResult.map((row) => {
              const res: { fields: Record<string, unknown> } = {
                fields: {},
              };
              columnInfo.forEach((col, index) => {
                res.fields[fields[index].id] = row[col.sourceColumnIndex];
              });
              return res;
            });
            if (records.length === 0) {
              return;
            }
            try {
              await this.recordOpenApiService.multipleCreateRecords(table.id, {
                fieldKeyType: FieldKeyType.Id,
                typecast: true,
                records,
              });
            } catch (e) {
              this.logger.error((e as Error)?.message, 'Import: Records');
            }
          }
        );
      }
    }
    return tableResult;
  }

  async inplaceImportTable(tableId: string, inplaceImportRo: IInplaceImportOptionRo) {
    const { attachmentUrl, fileType, insertConfig } = inplaceImportRo;

    const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    importer.parse(
      {
        skipFirstNLines: excludeFirstRow ? 1 : 0,
        key: sourceWorkSheetKey,
      },
      async (result) => {
        const currentResult = result[sourceWorkSheetKey];
        if (currentResult.length === 0) {
          return;
        }
        // fill data
        const records = currentResult.map((row) => {
          const res: { fields: Record<string, unknown> } = {
            fields: {},
          };
          for (const [key, value] of Object.entries(sourceColumnMap)) {
            if (value !== null) {
              res.fields[key] = row[value];
            }
          }
          return res;
        });
        try {
          await this.recordOpenApiService.multipleCreateRecords(tableId, {
            fieldKeyType: FieldKeyType.Id,
            typecast: true,
            records,
          });
        } catch (e) {
          this.logger.error((e as Error)?.message, 'Import: Records');
        }
      }
    );
  }
}

1	import { Injectable, Logger } from '@nestjs/common';	2✔
2	import { FieldKeyType } from '@teable/core';	2✔
3	import type { IAnalyzeRo, IImportOptionRo, IInplaceImportOptionRo } from '@teable/openapi';	2✔
4	import { RecordOpenApiService } from '../../record/open-api/record-open-api.service';	2✔
5	import { DEFAULT_VIEWS } from '../../table/constant';	2✔
6	import { TableOpenApiService } from '../../table/open-api/table-open-api.service';	2✔
7	import { importerFactory } from './import.class';	2✔
8		2✔
9	@Injectable()	2✔
10	export class ImportOpenApiService {	2✔
11	private logger = new Logger(ImportOpenApiService.name);	66✔
12	constructor(	66✔
13	private readonly tableOpenApiService: TableOpenApiService,	66✔
14	private readonly recordOpenApiService: RecordOpenApiService	66✔
15	) {}	66✔
16		66✔
17	async analyze(analyzeRo: IAnalyzeRo) {	66✔
18	const { attachmentUrl, fileType } = analyzeRo;	12✔
19		12✔
20	const importer = importerFactory(fileType, {	12✔
21	url: attachmentUrl,	12✔
22	type: fileType,	12✔
23	});	12✔
24		12✔
25	return await importer.genColumns();	12✔
26	}	12✔
27		66✔
28	async createTableFromImport(baseId: string, importRo: IImportOptionRo) {	66✔
29	const { attachmentUrl, fileType, worksheets } = importRo;	6✔
30		6✔
31	const importer = importerFactory(fileType, {	6✔
32	url: attachmentUrl,	6✔
33	type: fileType,	6✔
34	});	6✔
35		6✔
36	const tableResult = [];	6✔
37		6✔
38	for (const [sheetKey, value] of Object.entries(worksheets)) {	6✔
39	const { importData, useFirstRowAsHeader, columns: columnInfo, name } = value;	6✔
40	const fieldsRo = columnInfo.map((col, index) => {	6✔
41	return {	36✔
42	...col,	36✔
43	isPrimary: index === 0 ? true : null,	36✔
44	};	36✔
45	});	36✔
46		6✔
47	// create table with column	6✔
48	const table = await this.tableOpenApiService.createTable(baseId, {	6✔
49	name: name,	6✔
50	fields: fieldsRo,	6✔
51	views: DEFAULT_VIEWS,	6✔
52	records: [],	6✔
53	});	6✔
54		6✔
55	tableResult.push(table);	6✔
56		6✔
57	const { fields } = table;	6✔
58		6✔
59	if (importData) {	6✔
60	importer.parse(	6✔
61	{	6✔
62	skipFirstNLines: useFirstRowAsHeader ? 1 : 0,	6!
63	key: sheetKey,	6✔
64	},	6✔
65	async (result) => {	6✔
66	const currentResult = result[sheetKey];	10✔
67	// fill data	10✔
68	const records = currentResult.map((row) => {	10✔
69	const res: { fields: Record<string, unknown> } = {	12✔
70	fields: {},	12✔
71	};	12✔
72	columnInfo.forEach((col, index) => {	12✔
73	res.fields[fields[index].id] = row[col.sourceColumnIndex];	72✔
74	});	72✔
75	return res;	12✔
76	});	12✔
77	if (records.length === 0) {	10!
78	return;	×
79	}	×
80	try {	10✔
81	await this.recordOpenApiService.multipleCreateRecords(table.id, {	10✔
82	fieldKeyType: FieldKeyType.Id,	10✔
83	typecast: true,	10✔
84	records,	10✔
85	});	10✔
86	} catch (e) {	10!
87	this.logger.error((e as Error)?.message, 'Import: Records');	2✔
88	}	2✔
89	}	10✔
90	);	6✔
91	}	6✔
92	}	6✔
93	return tableResult;	6✔
94	}	6✔
95		66✔
96	async inplaceImportTable(tableId: string, inplaceImportRo: IInplaceImportOptionRo) {	66✔
97	const { attachmentUrl, fileType, insertConfig } = inplaceImportRo;	2✔
98		2✔
99	const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig;	2✔
100		2✔
101	const importer = importerFactory(fileType, {	2✔
102	url: attachmentUrl,	2✔
103	type: fileType,	2✔
104	});	2✔
105		2✔
106	importer.parse(	2✔
107	{	2✔
108	skipFirstNLines: excludeFirstRow ? 1 : 0,	2!
109	key: sourceWorkSheetKey,	2✔
110	},	2✔
111	async (result) => {	2✔
112	const currentResult = result[sourceWorkSheetKey];	4✔
113	if (currentResult.length === 0) {	4!
NEW 114	return;	×
NEW 115	}	×
116	// fill data	4✔
117	const records = currentResult.map((row) => {	4✔
118	const res: { fields: Record<string, unknown> } = {	4✔
119	fields: {},	4✔
120	};	4✔
121	for (const [key, value] of Object.entries(sourceColumnMap)) {	4✔
122	if (value !== null) {	24✔
123	res.fields[key] = row[value];	24✔
124	}	24✔
125	}	24✔
126	return res;	4✔
127	});	4✔
128	try {	4✔
129	await this.recordOpenApiService.multipleCreateRecords(tableId, {	4✔
130	fieldKeyType: FieldKeyType.Id,	4✔
131	typecast: true,	4✔
132	records,	4✔
133	});	4✔
134	} catch (e) {	4!
NEW 135	this.logger.error((e as Error)?.message, 'Import: Records');	×
NEW 136	}	×
137	}	4✔
138	);	2✔
139	}	2✔
140	}	66✔

teableio / teable / 8477114564

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous