8387093605

Committed 22 Mar 2024 07:48AM CUT coverage: 28.027% (-0.2%) from 28.222%

Build # 8387093605

Build Type

Pull #484

github

Committed by

web-flow

Commit Message

Merge 174ef76f7 into a06c6afb1

Pull Request Pull Request #484: feat: support increment import

Run Details

2099 of 3218 branches covered (65.23%)

24 of 703 new or added lines in 18 files covered. (3.41%)

49 existing lines in 6 files now uncovered.

25815 of 92109 relevant lines covered (28.03%)

5.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts

import { Injectable, Logger } from '@nestjs/common';
import { FieldKeyType } from '@teable/core';
import type { IAnalyzeRo, IImportOptionRo, IInplaceImportOptionRo } from '@teable/core';
import { RecordOpenApiService } from '../../record/open-api/record-open-api.service';
import { DEFAULT_VIEWS } from '../../table/constant';
import { TableOpenApiService } from '../../table/open-api/table-open-api.service';
import { importerFactory } from './import.class';

@Injectable()
export class ImportOpenApiService {
  private logger = new Logger(ImportOpenApiService.name);
  constructor(
    private readonly tableOpenApiService: TableOpenApiService,
    private readonly recordOpenApiService: RecordOpenApiService
  ) {}

  async analyze(analyzeRo: IAnalyzeRo) {
    const { attachmentUrl, fileType } = analyzeRo;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    return await importer.genColumns();
  }

  async createTableFromImport(baseId: string, importRo: IImportOptionRo) {
    const { attachmentUrl, fileType, worksheets } = importRo;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    const tableResult = [];

    for (const [sheetKey, value] of Object.entries(worksheets)) {
      const { importData, useFirstRowAsHeader, columns: columnInfo, name } = value;
      const fieldsRo = columnInfo.map((col, index) => {
        return {
          ...col,
          isPrimary: index === 0 ? true : null,
        };
      });

      // create table with column
      const table = await this.tableOpenApiService.createTable(baseId, {
        name: name,
        fields: fieldsRo,
        views: DEFAULT_VIEWS,
        records: [],
      });

      tableResult.push(table);

      const { fields } = table;

      if (importData) {
        importer.parse(
          {
            skipFirstNLines: useFirstRowAsHeader ? 1 : 0,
            key: sheetKey,
          },
          async (result) => {
            const currentResult = result[sheetKey];
            // fill data
            const records = currentResult.map((row) => {
              const res: { fields: Record<string, unknown> } = {
                fields: {},
              };
              columnInfo.forEach((col, index) => {
                res.fields[fields[index].id] = row[col.sourceColumnIndex];
              });
              return res;
            });
            if (records.length === 0) {
              return;
            }
            try {
              await this.recordOpenApiService.multipleCreateRecords(table.id, {
                fieldKeyType: FieldKeyType.Id,
                typecast: true,
                records,
              });
            } catch (e) {
              this.logger.error((e as Error)?.message, 'Import: Records');
            }
          }
        );
      }
    }
    return tableResult;
  }

  async inplaceImportTable(tableId: string, inplaceImportRo: IInplaceImportOptionRo) {
    const { attachmentUrl, fileType, insertConfig } = inplaceImportRo;

    const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig;

    const importer = importerFactory(fileType, {
      url: attachmentUrl,
      type: fileType,
    });

    importer.parse(
      {
        skipFirstNLines: excludeFirstRow ? 1 : 0,
        key: sourceWorkSheetKey,
      },
      async (result) => {
        const currentResult = result[sourceWorkSheetKey];
        if (currentResult.length === 0) {
          return;
        }
        // fill data
        const records = currentResult.map((row) => {
          const res: { fields: Record<string, unknown> } = {
            fields: {},
          };
          for (const [key, value] of Object.entries(sourceColumnMap)) {
            if (value !== null) {
              res.fields[key] = row[value];
            }
          }
          return res;
        });
        try {
          await this.recordOpenApiService.multipleCreateRecords(tableId, {
            fieldKeyType: FieldKeyType.Id,
            typecast: true,
            records,
          });
        } catch (e) {
          this.logger.error((e as Error)?.message, 'Import: Records');
        }
      }
    );
  }
}

1	import { Injectable, Logger } from '@nestjs/common';	×
2	import { FieldKeyType } from '@teable/core';	×
NEW 3	import type { IAnalyzeRo, IImportOptionRo, IInplaceImportOptionRo } from '@teable/core';	×
4	import { RecordOpenApiService } from '../../record/open-api/record-open-api.service';	×
5	import { DEFAULT_VIEWS } from '../../table/constant';	×
6	import { TableOpenApiService } from '../../table/open-api/table-open-api.service';	×
7	import { importerFactory } from './import.class';	×
8		×
9	@Injectable()	×
10	export class ImportOpenApiService {	×
11	private logger = new Logger(ImportOpenApiService.name);	×
12	constructor(	×
13	private readonly tableOpenApiService: TableOpenApiService,	×
14	private readonly recordOpenApiService: RecordOpenApiService	×
15	) {}	×
16		×
17	async analyze(analyzeRo: IAnalyzeRo) {	×
18	const { attachmentUrl, fileType } = analyzeRo;	×
19		×
20	const importer = importerFactory(fileType, {	×
21	url: attachmentUrl,	×
22	type: fileType,	×
23	});	×
24		×
25	return await importer.genColumns();	×
26	}	×
27		×
28	async createTableFromImport(baseId: string, importRo: IImportOptionRo) {	×
29	const { attachmentUrl, fileType, worksheets } = importRo;	×
30		×
31	const importer = importerFactory(fileType, {	×
32	url: attachmentUrl,	×
33	type: fileType,	×
34	});	×
35		×
36	const tableResult = [];	×
37		×
38	for (const [sheetKey, value] of Object.entries(worksheets)) {	×
39	const { importData, useFirstRowAsHeader, columns: columnInfo, name } = value;	×
40	const fieldsRo = columnInfo.map((col, index) => {	×
41	return {	×
42	...col,	×
43	isPrimary: index === 0 ? true : null,	×
44	};	×
45	});	×
46		×
47	// create table with column	×
48	const table = await this.tableOpenApiService.createTable(baseId, {	×
49	name: name,	×
50	fields: fieldsRo,	×
51	views: DEFAULT_VIEWS,	×
52	records: [],	×
53	});	×
54		×
55	tableResult.push(table);	×
56		×
57	const { fields } = table;	×
58		×
59	if (importData) {	×
60	importer.parse(	×
61	{	×
62	skipFirstNLines: useFirstRowAsHeader ? 1 : 0,	×
63	key: sheetKey,	×
64	},	×
65	async (result) => {	×
66	const currentResult = result[sheetKey];	×
67	// fill data	×
68	const records = currentResult.map((row) => {	×
69	const res: { fields: Record<string, unknown> } = {	×
70	fields: {},	×
71	};	×
72	columnInfo.forEach((col, index) => {	×
73	res.fields[fields[index].id] = row[col.sourceColumnIndex];	×
74	});	×
75	return res;	×
76	});	×
77	if (records.length === 0) {	×
78	return;	×
79	}	×
80	try {	×
81	await this.recordOpenApiService.multipleCreateRecords(table.id, {	×
82	fieldKeyType: FieldKeyType.Id,	×
83	typecast: true,	×
84	records,	×
85	});	×
86	} catch (e) {	×
87	this.logger.error((e as Error)?.message, 'Import: Records');	×
88	}	×
89	}	×
90	);	×
91	}	×
92	}	×
93	return tableResult;	×
94	}	×
NEW 95		×
NEW 96	async inplaceImportTable(tableId: string, inplaceImportRo: IInplaceImportOptionRo) {	×
NEW 97	const { attachmentUrl, fileType, insertConfig } = inplaceImportRo;	×
NEW 98		×
NEW 99	const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig;	×
NEW 100		×
NEW 101	const importer = importerFactory(fileType, {	×
NEW 102	url: attachmentUrl,	×
NEW 103	type: fileType,	×
NEW 104	});	×
NEW 105		×
NEW 106	importer.parse(	×
NEW 107	{	×
NEW 108	skipFirstNLines: excludeFirstRow ? 1 : 0,	×
NEW 109	key: sourceWorkSheetKey,	×
NEW 110	},	×
NEW 111	async (result) => {	×
NEW 112	const currentResult = result[sourceWorkSheetKey];	×
NEW 113	if (currentResult.length === 0) {	×
NEW 114	return;	×
NEW 115	}	×
NEW 116	// fill data	×
NEW 117	const records = currentResult.map((row) => {	×
NEW 118	const res: { fields: Record<string, unknown> } = {	×
NEW 119	fields: {},	×
NEW 120	};	×
NEW 121	for (const [key, value] of Object.entries(sourceColumnMap)) {	×
NEW 122	if (value !== null) {	×
NEW 123	res.fields[key] = row[value];	×
NEW 124	}	×
NEW 125	}	×
NEW 126	return res;	×
NEW 127	});	×
NEW 128	try {	×
NEW 129	await this.recordOpenApiService.multipleCreateRecords(tableId, {	×
NEW 130	fieldKeyType: FieldKeyType.Id,	×
NEW 131	typecast: true,	×
NEW 132	records,	×
NEW 133	});	×
NEW 134	} catch (e) {	×
NEW 135	this.logger.error((e as Error)?.message, 'Import: Records');	×
NEW 136	}	×
NEW 137	}	×
NEW 138	);	×
NEW 139	}	×
UNCOV 140	}	×

teableio / teable / 8387093605

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous