11719263572

Committed 07 Nov 2024 08:28AM UTC coverage: 84.482% (-0.03%) from 84.512%

Build # 11719263572

Build Type

Pull #1065

github

Committed by

web-flow

Commit Message

Merge 039b08ae7 into e93dc6562

Pull Request Pull Request #1065: fix: missing signed url in attachment cell op

Run Details

5921 of 6211 branches covered (95.33%)

196 of 250 new or added lines in 7 files covered. (78.4%)

10 existing lines in 2 files now uncovered.

38876 of 46017 relevant lines covered (84.48%)

1716.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

6.61

/apps/nestjs-backend/src/features/attachments/plugins/minio.ts

/* eslint-disable @typescript-eslint/naming-convention */
import type { Readable as ReadableStream } from 'node:stream';
import { join, resolve } from 'path';
import { BadRequestException, Injectable } from '@nestjs/common';
import { getRandomString } from '@teable/core';
import * as fse from 'fs-extra';
import * as minio from 'minio';
import sharp from 'sharp';
import { IStorageConfig, StorageConfig } from '../../../configs/storage';
import { second } from '../../../utils/second';
import StorageAdapter from './adapter';
import type { IPresignParams, IPresignRes, IRespHeaders } from './types';

@Injectable()
export class MinioStorage implements StorageAdapter {
  minioClient: minio.Client;
  minioClientPrivateNetwork: minio.Client;

  constructor(@StorageConfig() readonly config: IStorageConfig) {
    const { endPoint, internalEndPoint, internalPort, port, useSSL, accessKey, secretKey } =
      this.config.minio;
    this.minioClient = new minio.Client({
      endPoint: endPoint!,
      port: port!,
      useSSL: useSSL!,
      accessKey: accessKey!,
      secretKey: secretKey!,
    });
    this.minioClientPrivateNetwork = internalEndPoint
      ? new minio.Client({
          endPoint: internalEndPoint,
          port: internalPort,
          useSSL: false,
          accessKey: accessKey!,
          secretKey: secretKey!,
        })
      : this.minioClient;
    fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR);
  }

  async presigned(
    bucket: string,
    dir: string,
    presignedParams: IPresignParams
  ): Promise<IPresignRes> {
    const { tokenExpireIn, uploadMethod } = this.config;
    const { expiresIn, contentLength, contentType, hash, internal } = presignedParams;
    const token = getRandomString(12);
    const filename = hash ?? token;
    const path = join(dir, filename);
    const requestHeaders = {
      'Content-Type': contentType,
      'Content-Length': contentLength,
      'response-cache-control': 'max-age=31536000, immutable',
    };
    try {
      const client = internal ? this.minioClientPrivateNetwork : this.minioClient;
      const url = await client.presignedUrl(
        uploadMethod,
        bucket,
        path,
        expiresIn ?? second(tokenExpireIn),
        requestHeaders
      );
      return {
        url,
        path,
        token,
        uploadMethod,
        requestHeaders,
      };
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
    } catch (e: any) {
      throw new BadRequestException(`Minio presigned error${e?.message ? `: ${e.message}` : ''}`);
    }
  }

  private async getShape(bucket: string, objectName: string) {
    try {
      const stream = await this.minioClientPrivateNetwork.getObject(bucket, objectName);
      const metaReader = sharp();
      const sharpReader = stream.pipe(metaReader);
      const { width, height } = await sharpReader.metadata();

      return {
        width,
        height,
      };
    } catch (e) {
      return {};
    }
  }

  async getObjectMeta(bucket: string, path: string, _token: string) {
    const objectName = path;
    const {
      metaData,
      size,
      etag: hash,
    } = await this.minioClientPrivateNetwork.statObject(bucket, objectName);
    const mimetype = metaData['content-type'] as string;
    const url = `/${bucket}/${objectName}`;
    if (!mimetype?.startsWith('image/')) {
      return {
        hash,
        size,
        mimetype,
        url,
      };
    }
    const sharpMeta = await this.getShape(bucket, objectName);
    return {
      ...sharpMeta,
      hash,
      size,
      mimetype,
      url,
    };
  }

  async getPreviewUrl(
    bucket: string,
    path: string,
    expiresIn: number = second(this.config.urlExpireIn),
    respHeaders?: IRespHeaders
  ) {
    const { 'Content-Disposition': contentDisposition, ...headers } = respHeaders ?? {};
    return this.minioClient.presignedGetObject(bucket, path, expiresIn, {
      ...headers,
      'response-content-disposition': contentDisposition,
    });
  }

  async uploadFileWidthPath(
    bucket: string,
    path: string,
    filePath: string,
    metadata: Record<string, string | number>
  ) {
    const { etag: hash } = await this.minioClientPrivateNetwork.fPutObject(
      bucket,
      path,
      filePath,
      metadata
    );
    return {
      hash,
      path,
    };
  }

  async uploadFile(
    bucket: string,
    path: string,
    stream: Buffer | ReadableStream,
    metadata: Record<string, string | number>
  ) {
    const { etag: hash } = await this.minioClientPrivateNetwork.putObject(
      bucket,
      path,
      stream,
      undefined,
      metadata
    );
    return {
      hash,
      path,
    };
  }

  // minio file exists
  private async fileExists(bucket: string, path: string) {
    try {
      await this.minioClientPrivateNetwork.statObject(bucket, path);
      return true;
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
    } catch (err: any) {
      if (err.code === 'NoSuchKey' || err.code === 'NotFound') {
        return false;
      }
      throw err;
    }
  }

  async cropImage(
    bucket: string,
    path: string,
    width?: number,
    height?: number,
    _newPath?: string
  ) {
    const newPath = _newPath || `${path}_${width ?? 0}_${height ?? 0}`;
    const resizedImagePath = resolve(
      StorageAdapter.TEMPORARY_DIR,
      encodeURIComponent(join(bucket, newPath))
    );
    if (await this.fileExists(bucket, newPath)) {
      return newPath;
    }

    const objectName = path;
    const { metaData } = await this.minioClientPrivateNetwork.statObject(bucket, objectName);
    const mimetype = metaData['content-type'] as string;
    if (!mimetype?.startsWith('image/')) {
      throw new BadRequestException('Invalid image');
    }
    const sourceFilePath = resolve(StorageAdapter.TEMPORARY_DIR, encodeURIComponent(path));
    const writeStream = fse.createWriteStream(sourceFilePath);
    const stream = await this.minioClientPrivateNetwork.getObject(bucket, objectName);
    // stream save in sourceFilePath
    stream.pipe(writeStream);
    const metaReader = sharp(sourceFilePath, { failOn: 'none', unlimited: true }).resize(
      width,
      height
    );
    await metaReader.toFile(resizedImagePath);
    // delete source file
    fse.removeSync(sourceFilePath);

    const upload = await this.uploadFileWidthPath(bucket, newPath, resizedImagePath, {
      'Content-Type': mimetype,
    });
    // delete resized image
    fse.removeSync(resizedImagePath);
    return upload.path;
  }
}

1	/* eslint-disable @typescript-eslint/naming-convention */	4✔
2	import type { Readable as ReadableStream } from 'node:stream';	4✔
3	import { join, resolve } from 'path';	4✔
4	import { BadRequestException, Injectable } from '@nestjs/common';	4✔
5	import { getRandomString } from '@teable/core';	4✔
6	import * as fse from 'fs-extra';	4✔
7	import * as minio from 'minio';	4✔
8	import sharp from 'sharp';	4✔
9	import { IStorageConfig, StorageConfig } from '../../../configs/storage';	4✔
10	import { second } from '../../../utils/second';	4✔
11	import StorageAdapter from './adapter';	4✔
12	import type { IPresignParams, IPresignRes, IRespHeaders } from './types';	4✔
13		4✔
14	@Injectable()	4✔
15	export class MinioStorage implements StorageAdapter {	4!
16	minioClient: minio.Client;	×
17	minioClientPrivateNetwork: minio.Client;	×
18		×
19	constructor(@StorageConfig() readonly config: IStorageConfig) {	×
20	const { endPoint, internalEndPoint, internalPort, port, useSSL, accessKey, secretKey } =	×
21	this.config.minio;	×
22	this.minioClient = new minio.Client({	×
23	endPoint: endPoint!,	×
24	port: port!,	×
25	useSSL: useSSL!,	×
26	accessKey: accessKey!,	×
27	secretKey: secretKey!,	×
28	});	×
29	this.minioClientPrivateNetwork = internalEndPoint	×
30	? new minio.Client({	×
31	endPoint: internalEndPoint,	×
32	port: internalPort,	×
33	useSSL: false,	×
34	accessKey: accessKey!,	×
35	secretKey: secretKey!,	×
36	})	×
37	: this.minioClient;	×
38	fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR);	×
39	}	×
40		×
41	async presigned(	×
42	bucket: string,	×
43	dir: string,	×
44	presignedParams: IPresignParams	×
45	): Promise<IPresignRes> {	×
46	const { tokenExpireIn, uploadMethod } = this.config;	×
47	const { expiresIn, contentLength, contentType, hash, internal } = presignedParams;	×
48	const token = getRandomString(12);	×
49	const filename = hash ?? token;	×
50	const path = join(dir, filename);	×
51	const requestHeaders = {	×
52	'Content-Type': contentType,	×
53	'Content-Length': contentLength,	×
54	'response-cache-control': 'max-age=31536000, immutable',	×
55	};	×
56	try {	×
57	const client = internal ? this.minioClientPrivateNetwork : this.minioClient;	×
58	const url = await client.presignedUrl(	×
59	uploadMethod,	×
60	bucket,	×
61	path,	×
62	expiresIn ?? second(tokenExpireIn),	×
63	requestHeaders	×
64	);	×
65	return {	×
66	url,	×
67	path,	×
68	token,	×
69	uploadMethod,	×
70	requestHeaders,	×
71	};	×
72	// eslint-disable-next-line @typescript-eslint/no-explicit-any	×
73	} catch (e: any) {	×
74	throw new BadRequestException(`Minio presigned error${e?.message ? `: ${e.message}` : ''}`);	×
75	}	×
76	}	×
77		×
78	private async getShape(bucket: string, objectName: string) {	×
79	try {	×
80	const stream = await this.minioClientPrivateNetwork.getObject(bucket, objectName);	×
81	const metaReader = sharp();	×
82	const sharpReader = stream.pipe(metaReader);	×
83	const { width, height } = await sharpReader.metadata();	×
84		×
85	return {	×
86	width,	×
87	height,	×
88	};	×
89	} catch (e) {	×
90	return {};	×
91	}	×
92	}	×
93		×
94	async getObjectMeta(bucket: string, path: string, _token: string) {	×
95	const objectName = path;	×
96	const {	×
97	metaData,	×
98	size,	×
99	etag: hash,	×
100	} = await this.minioClientPrivateNetwork.statObject(bucket, objectName);	×
101	const mimetype = metaData['content-type'] as string;	×
102	const url = `/${bucket}/${objectName}`;	×
103	if (!mimetype?.startsWith('image/')) {	×
104	return {	×
105	hash,	×
106	size,	×
107	mimetype,	×
108	url,	×
109	};	×
110	}	×
111	const sharpMeta = await this.getShape(bucket, objectName);	×
112	return {	×
113	...sharpMeta,	×
114	hash,	×
115	size,	×
116	mimetype,	×
117	url,	×
118	};	×
119	}	×
120		×
121	async getPreviewUrl(	×
122	bucket: string,	×
123	path: string,	×
124	expiresIn: number = second(this.config.urlExpireIn),	×
125	respHeaders?: IRespHeaders	×
126	) {	×
127	const { 'Content-Disposition': contentDisposition, ...headers } = respHeaders ?? {};	×
128	return this.minioClient.presignedGetObject(bucket, path, expiresIn, {	×
129	...headers,	×
130	'response-content-disposition': contentDisposition,	×
131	});	×
132	}	×
133		×
134	async uploadFileWidthPath(	×
135	bucket: string,	×
136	path: string,	×
137	filePath: string,	×
138	metadata: Record<string, string \| number>	×
139	) {	×
140	const { etag: hash } = await this.minioClientPrivateNetwork.fPutObject(	×
141	bucket,	×
142	path,	×
143	filePath,	×
144	metadata	×
145	);	×
146	return {	×
147	hash,	×
148	path,	×
149	};	×
150	}	×
151		×
152	async uploadFile(	×
153	bucket: string,	×
154	path: string,	×
155	stream: Buffer \| ReadableStream,	×
156	metadata: Record<string, string \| number>	×
157	) {	×
158	const { etag: hash } = await this.minioClientPrivateNetwork.putObject(	×
159	bucket,	×
160	path,	×
161	stream,	×
162	undefined,	×
163	metadata	×
164	);	×
165	return {	×
166	hash,	×
167	path,	×
168	};	×
169	}	×
170		×
171	// minio file exists	×
172	private async fileExists(bucket: string, path: string) {	×
173	try {	×
174	await this.minioClientPrivateNetwork.statObject(bucket, path);	×
175	return true;	×
176	// eslint-disable-next-line @typescript-eslint/no-explicit-any	×
177	} catch (err: any) {	×
178	if (err.code === 'NoSuchKey' \|\| err.code === 'NotFound') {	×
179	return false;	×
180	}	×
181	throw err;	×
182	}	×
183	}	×
184		×
185	async cropImage(	×
186	bucket: string,	×
187	path: string,	×
188	width?: number,	×
189	height?: number,	×
190	_newPath?: string	×
191	) {	×
192	const newPath = _newPath \|\| `${path}_${width ?? 0}_${height ?? 0}`;	×
193	const resizedImagePath = resolve(	×
194	StorageAdapter.TEMPORARY_DIR,	×
195	encodeURIComponent(join(bucket, newPath))	×
196	);	×
197	if (await this.fileExists(bucket, newPath)) {	×
198	return newPath;	×
199	}	×
200		×
201	const objectName = path;	×
202	const { metaData } = await this.minioClientPrivateNetwork.statObject(bucket, objectName);	×
203	const mimetype = metaData['content-type'] as string;	×
204	if (!mimetype?.startsWith('image/')) {	×
205	throw new BadRequestException('Invalid image');	×
206	}	×
NEW 207	const sourceFilePath = resolve(StorageAdapter.TEMPORARY_DIR, encodeURIComponent(path));	×
NEW 208	const writeStream = fse.createWriteStream(sourceFilePath);	×
209	const stream = await this.minioClientPrivateNetwork.getObject(bucket, objectName);	×
NEW 210	// stream save in sourceFilePath	×
NEW 211	stream.pipe(writeStream);	×
NEW 212	const metaReader = sharp(sourceFilePath, { failOn: 'none', unlimited: true }).resize(	×
NEW 213	width,	×
NEW 214	height	×
NEW 215	);	×
NEW 216	await metaReader.toFile(resizedImagePath);	×
NEW 217	// delete source file	×
NEW 218	fse.removeSync(sourceFilePath);	×
NEW 219		×
220	const upload = await this.uploadFileWidthPath(bucket, newPath, resizedImagePath, {	×
221	'Content-Type': mimetype,	×
222	});	×
223	// delete resized image	×
224	fse.removeSync(resizedImagePath);	×
225	return upload.path;	×
226	}	×
227	}	×

teableio / teable / 11719263572

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous