16133292422

Committed 08 Jul 2025 03:33AM UTC coverage: 81.106% (+0.4%) from 80.687%

Build # 16133292422

Build Type

Pull #1648

github

Committed by

web-flow

Commit Message

Merge 2e1e22d51 into eccac0705

Pull Request Pull Request #1648: refactor: replace Knex with Prisma for readonly database connection

Run Details

8122 of 8654 branches covered (93.85%)

24 of 26 new or added lines in 1 file covered. (92.31%)

90 existing lines in 6 files now uncovered.

38716 of 47735 relevant lines covered (81.11%)

2570.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.4

/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts

import { BadRequestException, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { IDsn } from '@teable/core';
import { DriverClient, parseDsn } from '@teable/core';
import type { Prisma } from '@teable/db-main-prisma';
import { PrismaService, PrismaClient } from '@teable/db-main-prisma';
import { Knex } from 'knex';
import { InjectModel } from 'nest-knexjs';
import { BASE_READ_ONLY_ROLE_PREFIX, BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME } from './const';
import { checkTableAccess, validateRoleOperations } from './utils';

@Injectable()
export class BaseSqlExecutorService {
  private db?: PrismaClient;
  private readonly dsn: IDsn;
  readonly driver: DriverClient;
  private hasPgReadAllDataRole?: boolean;

  constructor(
    private readonly prismaService: PrismaService,
    private readonly configService: ConfigService,
    @InjectModel('CUSTOM_KNEX') private readonly knex: Knex
  ) {
    this.dsn = parseDsn(this.getDatabaseUrl());
    this.driver = this.dsn.driver as DriverClient;
  }

  private getDatabaseUrl() {
    return this.configService.getOrThrow<string>('PRISMA_DATABASE_URL');
  }

  private getDisablePreSqlExecutorCheck() {
    return this.configService.get<string>('DISABLE_PRE_SQL_EXECUTOR_CHECK') === 'true';
  }

  private async getReadOnlyDatabaseConnectionConfig(): Promise<string | undefined> {
    if (this.driver === DriverClient.Sqlite) {
      return;
    }
    if (!this.hasPgReadAllDataRole) {
      return;
    }
    const isExistReadOnlyRole = await this.roleExits(BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME);
    if (!isExistReadOnlyRole) {
      await this.prismaService.$tx(async (prisma) => {
        await prisma.$executeRawUnsafe(
          this.knex
            .raw(
              `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION`,
              [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME, this.dsn.pass]
            )
            .toQuery()
        );
        await prisma.$executeRawUnsafe(
          this.knex
            .raw(`GRANT pg_read_all_data TO ??`, [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME])
            .toQuery()
        );
      });
    }
    return `postgresql://${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME}:${this.dsn.pass}@${this.dsn.host}:${this.dsn.port}/${this.dsn.db}${
      this.dsn.params
        ? `?${Object.entries(this.dsn.params)
            .map(([key, value]) => `${key}=${value}`)
            .join('&')}`
        : ''
    }`;
  }

  async onModuleInit() {
    if (this.driver !== DriverClient.Pg) {
      return;
    }
    if (this.getDisablePreSqlExecutorCheck()) {
      return;
    }
    // if pg_read_all_data role not exist, no need to create read only role
    this.hasPgReadAllDataRole = await this.roleExits('pg_read_all_data');
    if (!this.hasPgReadAllDataRole) {
      return;
    }
    this.db = await this.createConnection();
  }

  async onModuleDestroy() {
    await this.db?.$disconnect();
  }

  private async createConnection(): Promise<PrismaClient | undefined> {
    if (this.db) {
      return this.db;
    }
    const connectionConfig = await this.getReadOnlyDatabaseConnectionConfig();
    if (!connectionConfig) {
      return;
    }
    const connection = new PrismaClient({
      datasources: {
        db: {
          url: connectionConfig,
        },
      },
    });
    await connection.$connect();

    // validate connection
    try {
      await connection.$queryRawUnsafe('SELECT 1');
      return connection;
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
    } catch (error: any) {
      await connection.$disconnect();
      throw new Error(`database connection failed: ${error.message}`);
    }
  }

  private getReadOnlyRoleName(baseId: string) {
    return `${BASE_READ_ONLY_ROLE_PREFIX}${baseId}`;
  }

  async createReadOnlyRole(baseId: string) {
    const roleName = this.getReadOnlyRoleName(baseId);
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex
          .raw(
            `CREATE ROLE ?? WITH NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION`,
            [roleName]
          )
          .toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex
          .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [
            baseId,
            roleName,
          ])
          .toQuery()
      );
  }

  async dropReadOnlyRole(baseId: string) {
    const roleName = this.getReadOnlyRoleName(baseId);
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex.raw(`REVOKE USAGE ON SCHEMA ?? FROM ??`, [baseId, roleName]).toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex
          .raw(`REVOKE SELECT ON ALL TABLES IN SCHEMA ?? FROM ??`, [baseId, roleName])
          .toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex
          .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [
            baseId,
            roleName,
          ])
          .toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(this.knex.raw(`DROP ROLE IF EXISTS ??`, [roleName]).toQuery());
  }

  async grantReadOnlyRole(baseId: string) {
    const roleName = this.getReadOnlyRoleName(baseId);
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()
      );
    await this.prismaService
      .txClient()
      .$executeRawUnsafe(
        this.knex
          .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [
            baseId,
            roleName,
          ])
          .toQuery()
      );
  }

  private async roleExits(role: string): Promise<boolean> {
    const roleExists = await this.prismaService.$queryRaw<
      { count: bigint }[]
    >`SELECT count(*) FROM pg_roles WHERE rolname=${role}`;
    return Boolean(roleExists[0].count);
  }

  private async roleCheckAndCreate(baseId: string) {
    if (this.driver !== DriverClient.Pg) {
      return;
    }
    const roleName = this.getReadOnlyRoleName(baseId);
    if (!(await this.roleExits(roleName))) {
      await this.createReadOnlyRole(baseId);
    }
  }

  private async setRole(prisma: Prisma.TransactionClient, baseId: string) {
    const roleName = this.getReadOnlyRoleName(baseId);
    await prisma.$executeRawUnsafe(this.knex.raw(`SET ROLE ??`, [roleName]).toQuery());
  }

  private async resetRole(prisma: Prisma.TransactionClient) {
    await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery());
  }

  private async readonlyExecuteSql(sql: string) {
    return this.db?.$queryRawUnsafe(sql);
  }

  /**
   * check sql is safe
   * 1. role operations validation
   * 2. parse sql to valid table names
   * 3. read only role check table access
   */
  private async safeCheckSql(
    baseId: string,
    sql: string,
    opts?: { projectionTableDbNames?: string[]; projectionTableIds?: string[] }
  ) {
    const { projectionTableDbNames = [] } = opts ?? {};
    // 1. role operations keywords validation, only pg support
    if (this.driver == DriverClient.Pg) {
      validateRoleOperations(sql);
    }
    let tableNames = projectionTableDbNames;
    if (!projectionTableDbNames.length) {
      const tables = await this.prismaService.tableMeta.findMany({
        where: {
          baseId,
        },
        select: {
          dbTableName: true,
        },
      });
      tableNames = tables.map((table) => table.dbTableName);
    }
    // 2. parse sql to valid table names
    checkTableAccess(sql, {
      tableNames,
      database: this.driver,
    });
    // 3. read only role check table access, only pg and pg version > 14 support
    await this.readonlyExecuteSql(sql);
  }

  async executeQuerySql<T = unknown>(
    baseId: string,
    sql: string,
    opts?: {
      projectionTableDbNames?: string[];
      projectionTableIds?: string[];
    }
  ) {
    await this.safeCheckSql(baseId, sql, opts);
    await this.roleCheckAndCreate(baseId);
    return this.prismaService.$tx(async (prisma) => {
      try {
        await this.setRole(prisma, baseId);
        return await prisma.$queryRawUnsafe<T>(sql);
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
      } catch (error: any) {
        throw new BadRequestException(error?.meta?.message || error?.message);
      } finally {
        await this.resetRole(prisma).catch((error) => {
          console.log('resetRole error', error);
        });
      }
    });
  }
}

1	import { BadRequestException, Injectable } from '@nestjs/common';	3✔
2	import { ConfigService } from '@nestjs/config';
3	import type { IDsn } from '@teable/core';
4	import { DriverClient, parseDsn } from '@teable/core';
5	import type { Prisma } from '@teable/db-main-prisma';
6	import { PrismaService, PrismaClient } from '@teable/db-main-prisma';
7	import { Knex } from 'knex';
8	import { InjectModel } from 'nest-knexjs';
9	import { BASE_READ_ONLY_ROLE_PREFIX, BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME } from './const';
10	import { checkTableAccess, validateRoleOperations } from './utils';
11
12	@Injectable()
13	export class BaseSqlExecutorService {	3✔
14	private db?: PrismaClient;	184✔
15	private readonly dsn: IDsn;	184✔
16	readonly driver: DriverClient;	184✔
17	private hasPgReadAllDataRole?: boolean;	184✔
18
19	constructor(	184✔
20	private readonly prismaService: PrismaService,	184✔
21	private readonly configService: ConfigService,	184✔
22	@InjectModel('CUSTOM_KNEX') private readonly knex: Knex	184✔
23	) {	184✔
24	this.dsn = parseDsn(this.getDatabaseUrl());	184✔
25	this.driver = this.dsn.driver as DriverClient;	184✔
26	}	184✔
27
28	private getDatabaseUrl() {	184✔
29	return this.configService.getOrThrow<string>('PRISMA_DATABASE_URL');	184✔
30	}	184✔
31
32	private getDisablePreSqlExecutorCheck() {	184✔
33	return this.configService.get<string>('DISABLE_PRE_SQL_EXECUTOR_CHECK') === 'true';	124✔
34	}	124✔
35
36	private async getReadOnlyDatabaseConnectionConfig(): Promise<string \| undefined> {	184✔
37	if (this.driver === DriverClient.Sqlite) {	124!
38	return;	×
39	}	×
40	if (!this.hasPgReadAllDataRole) {	124!
41	return;	×
42	}	×
43	const isExistReadOnlyRole = await this.roleExits(BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME);	124✔
44	if (!isExistReadOnlyRole) {	124✔
45	await this.prismaService.$tx(async (prisma) => {	2✔
46	await prisma.$executeRawUnsafe(	2✔
47	this.knex	2✔
48	.raw(	2✔
49	`CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION`,	2✔
50	[BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME, this.dsn.pass]	2✔
51	)
52	.toQuery()	2✔
53	);
54	await prisma.$executeRawUnsafe(	2✔
55	this.knex	2✔
56	.raw(`GRANT pg_read_all_data TO ??`, [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME])	2✔
57	.toQuery()	2✔
58	);
59	});	2✔
60	}	2✔
61	return `postgresql://${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME}:${this.dsn.pass}@${this.dsn.host}:${this.dsn.port}/${this.dsn.db}${	124✔
62	this.dsn.params	124✔
63	? `?${Object.entries(this.dsn.params)	124✔
64	.map(([key, value]) => `${key}=${value}`)	124✔
65	.join('&')}`	124✔
NEW 66	: ''	×
67	}`;	124✔
68	}	124✔
69
70	async onModuleInit() {	184✔
71	if (this.driver !== DriverClient.Pg) {	184!
72	return;	60✔
73	}	60!
74	if (this.getDisablePreSqlExecutorCheck()) {	124!
75	return;	×
76	}	×
77	// if pg_read_all_data role not exist, no need to create read only role	124✔
78	this.hasPgReadAllDataRole = await this.roleExits('pg_read_all_data');	124✔
79	if (!this.hasPgReadAllDataRole) {	124!
80	return;	×
81	}	×
82	this.db = await this.createConnection();	124✔
83	}	124✔
84
85	async onModuleDestroy() {	184✔
86	await this.db?.$disconnect();	181!
87	}	181✔
88
89	private async createConnection(): Promise<PrismaClient \| undefined> {	184✔
90	if (this.db) {	124!
91	return this.db;	×
92	}	×
93	const connectionConfig = await this.getReadOnlyDatabaseConnectionConfig();	124✔
94	if (!connectionConfig) {	124!
95	return;	×
96	}	×
97	const connection = new PrismaClient({	124✔
98	datasources: {	124✔
99	db: {	124✔
100	url: connectionConfig,	124✔
101	},	124✔
102	},	124✔
103	});	124✔
104	await connection.$connect();	124✔
105
106	// validate connection	124✔
107	try {	124✔
108	await connection.$queryRawUnsafe('SELECT 1');	124✔
109	return connection;	124✔
110	// eslint-disable-next-line @typescript-eslint/no-explicit-any	124✔
111	} catch (error: any) {	124!
NEW 112	await connection.$disconnect();	×
113	throw new Error(`database connection failed: ${error.message}`);	×
114	}	×
115	}	124✔
116
117	private getReadOnlyRoleName(baseId: string) {	184✔
118	return `${BASE_READ_ONLY_ROLE_PREFIX}${baseId}`;	12✔
119	}	12✔
120
121	async createReadOnlyRole(baseId: string) {	184✔
122	const roleName = this.getReadOnlyRoleName(baseId);	4✔
123	await this.prismaService	4✔
124	.txClient()	4✔
125	.$executeRawUnsafe(	4✔
126	this.knex	4✔
127	.raw(	4✔
128	`CREATE ROLE ?? WITH NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION`,	4✔
129	[roleName]	4✔
130	)
131	.toQuery()	4✔
132	);
133	await this.prismaService	4✔
134	.txClient()	4✔
135	.$executeRawUnsafe(	4✔
136	this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()	4✔
137	);
138	await this.prismaService	4✔
139	.txClient()	4✔
140	.$executeRawUnsafe(	4✔
141	this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()	4✔
142	);
143	await this.prismaService	4✔
144	.txClient()	4✔
145	.$executeRawUnsafe(	4✔
146	this.knex	4✔
147	.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [	4✔
148	baseId,	4✔
149	roleName,	4✔
150	])	4✔
151	.toQuery()	4✔
152	);
153	}	4✔
154
155	async dropReadOnlyRole(baseId: string) {	184✔
156	const roleName = this.getReadOnlyRoleName(baseId);	×
157	await this.prismaService	×
158	.txClient()	×
159	.$executeRawUnsafe(	×
160	this.knex.raw(`REVOKE USAGE ON SCHEMA ?? FROM ??`, [baseId, roleName]).toQuery()	×
161	);
162	await this.prismaService	×
163	.txClient()	×
164	.$executeRawUnsafe(	×
165	this.knex	×
166	.raw(`REVOKE SELECT ON ALL TABLES IN SCHEMA ?? FROM ??`, [baseId, roleName])	×
167	.toQuery()	×
168	);
169	await this.prismaService	×
170	.txClient()	×
171	.$executeRawUnsafe(	×
172	this.knex	×
173	.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [	×
174	baseId,	×
175	roleName,	×
176	])	×
177	.toQuery()	×
178	);
179	await this.prismaService	×
180	.txClient()	×
181	.$executeRawUnsafe(this.knex.raw(`DROP ROLE IF EXISTS ??`, [roleName]).toQuery());	×
182	}	×
183
184	async grantReadOnlyRole(baseId: string) {	184✔
185	const roleName = this.getReadOnlyRoleName(baseId);	×
186	await this.prismaService	×
187	.txClient()	×
188	.$executeRawUnsafe(	×
189	this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()	×
190	);
191	await this.prismaService	×
192	.txClient()	×
193	.$executeRawUnsafe(	×
194	this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery()	×
195	);
196	await this.prismaService	×
197	.txClient()	×
198	.$executeRawUnsafe(	×
199	this.knex	×
200	.raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [	×
201	baseId,	×
202	roleName,	×
203	])	×
204	.toQuery()	×
205	);
206	}	×
207
208	private async roleExits(role: string): Promise<boolean> {	184✔
209	const roleExists = await this.prismaService.$queryRaw<	252✔
210	{ count: bigint }[]
211	>`SELECT count(*) FROM pg_roles WHERE rolname=${role}`;	252✔
212	return Boolean(roleExists[0].count);	252✔
213	}	252✔
214
215	private async roleCheckAndCreate(baseId: string) {	184✔
216	if (this.driver !== DriverClient.Pg) {	4!
217	return;	×
218	}	×
219	const roleName = this.getReadOnlyRoleName(baseId);	4✔
220	if (!(await this.roleExits(roleName))) {	4✔
221	await this.createReadOnlyRole(baseId);	4✔
222	}	4✔
223	}	4✔
224
225	private async setRole(prisma: Prisma.TransactionClient, baseId: string) {	184✔
226	const roleName = this.getReadOnlyRoleName(baseId);	4✔
227	await prisma.$executeRawUnsafe(this.knex.raw(`SET ROLE ??`, [roleName]).toQuery());	4✔
228	}	4✔
229
230	private async resetRole(prisma: Prisma.TransactionClient) {	184✔
231	await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery());	4✔
232	}	2✔
233
234	private async readonlyExecuteSql(sql: string) {	184✔
235	return this.db?.$queryRawUnsafe(sql);	4✔
236	}	4✔
237
238	/**	184✔
239	* check sql is safe
240	* 1. role operations validation
241	* 2. parse sql to valid table names
242	* 3. read only role check table access
243	*/	184✔
244	private async safeCheckSql(	184✔
245	baseId: string,	4✔
246	sql: string,	4✔
247	opts?: { projectionTableDbNames?: string[]; projectionTableIds?: string[] }	4✔
248	) {	4✔
249	const { projectionTableDbNames = [] } = opts ?? {};	4✔
250	// 1. role operations keywords validation, only pg support	4✔
251	if (this.driver == DriverClient.Pg) {	4✔
252	validateRoleOperations(sql);	4✔
253	}	4✔
254	let tableNames = projectionTableDbNames;	4✔
255	if (!projectionTableDbNames.length) {	4✔
256	const tables = await this.prismaService.tableMeta.findMany({	2✔
257	where: {	2✔
258	baseId,	2✔
259	},	2✔
260	select: {	2✔
261	dbTableName: true,	2✔
262	},	2✔
263	});	2✔
264	tableNames = tables.map((table) => table.dbTableName);	2✔
265	}	2✔
266	// 2. parse sql to valid table names	4✔
267	checkTableAccess(sql, {	4✔
268	tableNames,	4✔
269	database: this.driver,	4✔
270	});	4✔
271	// 3. read only role check table access, only pg and pg version > 14 support	4✔
272	await this.readonlyExecuteSql(sql);	4✔
273	}	4✔
274
275	async executeQuerySql<T = unknown>(	184✔
276	baseId: string,	4✔
277	sql: string,	4✔
278	opts?: {	4✔
279	projectionTableDbNames?: string[];
280	projectionTableIds?: string[];
281	}	4✔
282	) {	4✔
283	await this.safeCheckSql(baseId, sql, opts);	4✔
284	await this.roleCheckAndCreate(baseId);	4✔
285	return this.prismaService.$tx(async (prisma) => {	4✔
286	try {	4✔
287	await this.setRole(prisma, baseId);	4✔
288	return await prisma.$queryRawUnsafe<T>(sql);	4✔
289	// eslint-disable-next-line @typescript-eslint/no-explicit-any	4✔
290	} catch (error: any) {	4✔
291	throw new BadRequestException(error?.meta?.message \|\| error?.message);	2!
292	} finally {	4!
293	await this.resetRole(prisma).catch((error) => {	4✔
294	console.log('resetRole error', error);	2✔
295	});	2✔
296	}	4✔
297	});	4✔
298	}	4✔
299	}	184✔

teableio / teable / 16133292422

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous