24509568695

Committed 16 Apr 2026 12:13PM UTC coverage: 56.692% (+0.03%) from 56.662%

Build # 24509568695

Build Type

push

github

Committed by

web-flow

Commit Message

feat(sql) New SQL module with SQLDataSource and DuckDB connector (#3389)

Coverage Stats

10064 of 19243 branches covered (52.3%)

Branch coverage included in aggregate %.

159 of 239 new or added lines in 4 files covered. (66.53%)

20686 of 34997 relevant lines covered (59.11%)

5007.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.06

/modules/sql/src/sql-utils.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import * as arrow from 'apache-arrow';
import type {ArrowTable, DataType, Field, ObjectRowTable, Schema} from '@loaders.gl/schema';
import {makeTableFromData, getDataTypeFromArray} from '@loaders.gl/schema-utils';

import type {SQLColumnInfo} from './sql-types';

/** Returns true when the current runtime is Node.js. */
export function isNodeRuntime(): boolean {
  return typeof process !== 'undefined' && Boolean(process.versions?.node);
}

/** Escapes a SQL string literal using single-quote doubling. */
export function escapeSqlString(value: string): string {
  return value.replace(/'/g, "''");
}

/** Wraps an identifier in double quotes, escaping embedded quotes. */
export function quoteSqlIdentifier(identifier: string): string {
  return `"${identifier.replace(/"/g, '""')}"`;
}

/** Converts object rows into a loaders.gl Arrow table. */
export function convertRowsToArrowTable(rows: Record<string, unknown>[]): ArrowTable {
  const table: ObjectRowTable =
    rows.length > 0
      ? makeTableFromData(rows)
      : {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: rows};
  const schema = deducePrimitiveAwareSchema(table);

  if (table.data.length === 0 || schema.fields.length === 0) {
    return {
      shape: 'arrow-table',
      schema,
      data: new arrow.Table(new arrow.Schema([]))
    };
  }

  return {
    shape: 'arrow-table',
    schema,
    data: arrow.tableFromJSON(table.data)
  };
}

/** Converts normalized SQL columns into a loaders.gl schema. */
export function convertSQLColumnsToSchema(columns: SQLColumnInfo[]): Schema {
  return {
    metadata: {},
    fields: columns
      .sort((left, right) => (left.ordinalPosition || 0) - (right.ordinalPosition || 0))
      .map(column => ({
        name: column.columnName,
        type: mapSQLTypeToDataType(column.sqlType),
        nullable: column.nullable,
        metadata: {
          sqlType: column.sqlType
        }
      }))
  };
}

function mapSQLTypeToDataType(sqlType: string): DataType {
  const normalizedType = sqlType.toUpperCase();
  if (
    normalizedType.includes('BIGINT') ||
    normalizedType === 'HUGEINT' ||
    normalizedType === 'LONG'
  ) {
    return 'int64';
  }
  if (
    normalizedType.includes('INT') ||
    normalizedType === 'INTEGER' ||
    normalizedType === 'SMALLINT' ||
    normalizedType === 'TINYINT'
  ) {
    return 'int32';
  }
  if (
    normalizedType.includes('DOUBLE') ||
    normalizedType.includes('FLOAT8') ||
    normalizedType.includes('REAL')
  ) {
    return 'float64';
  }
  if (
    normalizedType.includes('FLOAT') ||
    normalizedType.includes('DECIMAL') ||
    normalizedType.includes('NUMERIC')
  ) {
    return 'float64';
  }
  if (normalizedType.includes('BOOL')) {
    return 'bool';
  }
  if (normalizedType.includes('DATE')) {
    return 'date-day';
  }
  if (normalizedType.includes('TIME') || normalizedType.includes('TIMESTAMP')) {
    return 'timestamp-millisecond';
  }
  if (normalizedType.includes('BLOB') || normalizedType.includes('BINARY')) {
    return 'binary';
  }
  return 'utf8';
}

function deducePrimitiveAwareSchema(table: ObjectRowTable): Schema {
  return {
    ...table.schema!,
    fields: table.schema!.fields.map((field): Field => {
      if (field.type !== 'null') {
        return field;
      }

      const columnValues = table.data.map(row => row[field.name]);
      const inferredType = getDataTypeFromArray(columnValues);
      return {...field, type: inferredType.type, nullable: inferredType.nullable};
    })
  };
}

/** Returns a fully qualified SQL object name. */
export function getQualifiedTableName(options: {
  catalogName?: string;
  schemaName?: string;
  tableName: string;
}): string {
  const identifiers = [options.catalogName, options.schemaName, options.tableName].filter(
    (value): value is string => Boolean(value)
  );
  return identifiers.map(identifier => quoteSqlIdentifier(identifier)).join('.');
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import * as arrow from 'apache-arrow';
6	import type {ArrowTable, DataType, Field, ObjectRowTable, Schema} from '@loaders.gl/schema';
7	import {makeTableFromData, getDataTypeFromArray} from '@loaders.gl/schema-utils';
8
9	import type {SQLColumnInfo} from './sql-types';
10
11	/** Returns true when the current runtime is Node.js. */
12	export function isNodeRuntime(): boolean {
13	return typeof process !== 'undefined' && Boolean(process.versions?.node);	6✔
14	}
15
16	/** Escapes a SQL string literal using single-quote doubling. */
17	export function escapeSqlString(value: string): string {
18	return value.replace(/'/g, "''");	3✔
19	}
20
21	/** Wraps an identifier in double quotes, escaping embedded quotes. */
22	export function quoteSqlIdentifier(identifier: string): string {
23	return `"${identifier.replace(/"/g, '""')}"`;	4✔
24	}
25
26	/** Converts object rows into a loaders.gl Arrow table. */
27	export function convertRowsToArrowTable(rows: Record<string, unknown>[]): ArrowTable {
28	const table: ObjectRowTable =
29	rows.length > 0	5✔
30	? makeTableFromData(rows)
31	: {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: rows};
32	const schema = deducePrimitiveAwareSchema(table);	5✔
33
34	if (table.data.length === 0 \|\| schema.fields.length === 0) {	5✔
35	return {	1✔
36	shape: 'arrow-table',
37	schema,
38	data: new arrow.Table(new arrow.Schema([]))
39	};
40	}
41
42	return {	4✔
43	shape: 'arrow-table',
44	schema,
45	data: arrow.tableFromJSON(table.data)
46	};
47	}
48
49	/** Converts normalized SQL columns into a loaders.gl schema. */
50	export function convertSQLColumnsToSchema(columns: SQLColumnInfo[]): Schema {
51	return {	2✔
52	metadata: {},
53	fields: columns
54	.sort((left, right) => (left.ordinalPosition \|\| 0) - (right.ordinalPosition \|\| 0))	2!
55	.map(column => ({	4✔
56	name: column.columnName,
57	type: mapSQLTypeToDataType(column.sqlType),
58	nullable: column.nullable,
59	metadata: {
60	sqlType: column.sqlType
61	}
62	}))
63	};
64	}
65
66	function mapSQLTypeToDataType(sqlType: string): DataType {
67	const normalizedType = sqlType.toUpperCase();	4✔
68	if (	4✔
69	normalizedType.includes('BIGINT') \|\|	10✔
70	normalizedType === 'HUGEINT' \|\|
71	normalizedType === 'LONG'
72	) {
73	return 'int64';	1✔
74	}
75	if (	3✔
76	normalizedType.includes('INT') \|\|	9✔
77	normalizedType === 'INTEGER' \|\|
78	normalizedType === 'SMALLINT' \|\|
79	normalizedType === 'TINYINT'
80	) {
81	return 'int32';	1✔
82	}
83	if (	2!
84	normalizedType.includes('DOUBLE') \|\|	6✔
85	normalizedType.includes('FLOAT8') \|\|
86	normalizedType.includes('REAL')
87	) {
NEW 88	return 'float64';	×
89	}
90	if (	2!
91	normalizedType.includes('FLOAT') \|\|	6✔
92	normalizedType.includes('DECIMAL') \|\|
93	normalizedType.includes('NUMERIC')
94	) {
NEW 95	return 'float64';	×
96	}
97	if (normalizedType.includes('BOOL')) {	2!
NEW 98	return 'bool';	×
99	}
100	if (normalizedType.includes('DATE')) {	2!
NEW 101	return 'date-day';	×
102	}
103	if (normalizedType.includes('TIME') \|\| normalizedType.includes('TIMESTAMP')) {	2✔
104	return 'timestamp-millisecond';	1✔
105	}
106	if (normalizedType.includes('BLOB') \|\| normalizedType.includes('BINARY')) {	1!
107	return 'binary';	1✔
108	}
NEW 109	return 'utf8';	×
110	}
111
112	function deducePrimitiveAwareSchema(table: ObjectRowTable): Schema {
113	return {	5✔
114	...table.schema!,
115	fields: table.schema!.fields.map((field): Field => {
116	if (field.type !== 'null') {	5✔
117	return field;	3✔
118	}
119
120	const columnValues = table.data.map(row => row[field.name]);	4✔
121	const inferredType = getDataTypeFromArray(columnValues);	4✔
122	return {...field, type: inferredType.type, nullable: inferredType.nullable};	2✔
123	})
124	};
125	}
126
127	/** Returns a fully qualified SQL object name. */
128	export function getQualifiedTableName(options: {
129	catalogName?: string;
130	schemaName?: string;
131	tableName: string;
132	}): string {
133	const identifiers = [options.catalogName, options.schemaName, options.tableName].filter(	1✔
134	(value): value is string => Boolean(value)	3✔
135	);
136	return identifiers.map(identifier => quoteSqlIdentifier(identifier)).join('.');	3✔
137	}

visgl / loaders.gl / 24509568695

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous