25798238260

Committed 13 May 2026 12:10PM UTC coverage: 60.607% (+0.3%) from 60.27%

Build # 25798238260

Build Type

push

github

Committed by

web-flow

Commit Message

feat(json) GeoJSON -> geoarrow, schema, logging  (#3399)

Coverage Stats

13466 of 24516 branches covered (54.93%)

Branch coverage included in aggregate %.

448 of 541 new or added lines in 12 files covered. (82.81%)

1264 existing lines in 117 files now uncovered.

27516 of 43103 relevant lines covered (63.84%)

15056.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/modules/json/src/json-table-loader-with-parser.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  ArrayRowTable,
  ArrowTable,
  ArrowTableBatch,
  ObjectRowTable,
  Table,
  TableBatch
} from '@loaders.gl/schema';
import {convertTable, makeTableFromData} from '@loaders.gl/schema-utils';
import type {LoaderWithParser} from '@loaders.gl/loader-utils';
import FastStreamingJSONParser from './lib/json-parser/fast-streaming-json-parser';
import type {StreamingJSONParserFactory} from './lib/json-parser/streaming-json-parser-types';
import {parseJSONSync} from './lib/parsers/parse-json';
import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
import {
  convertRowTableToArrowTable,
  convertTableBatchesToArrow
} from './lib/parsers/convert-row-table-to-arrow';
import type {JSONBatch, JSONLoaderOptions, MetadataBatch} from './json-loader';
import {
  JSONTableLoader as JSONTableLoaderMetadata,
  type JSONTableLoaderOptions
} from './json-table-loader';

const {preload: _JSONTableLoaderPreload, ...JSONTableLoaderMetadataWithoutPreload} =
  JSONTableLoaderMetadata;

/**
 * Parser-bearing loader for JSON documents that must resolve to loaders.gl table output.
 */
export const JSONTableLoaderWithParser = {
  ...JSONTableLoaderMetadataWithoutPreload,
  parse,
  parseTextSync,
  parseInBatches
} as const satisfies LoaderWithParser<
  Table | ArrowTable,
  TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch,
  JSONTableLoaderOptions
>;

/**
 * Parses UTF-8 JSON bytes into row-table or Arrow-table output.
 *
 * @param arrayBuffer - UTF-8 encoded JSON payload.
 * @param options - JSON table loader options.
 * @returns Parsed table output.
 */
async function parse(arrayBuffer: ArrayBuffer, options?: JSONTableLoaderOptions) {
  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
}

/**
 * Parses JSON text into row-table or Arrow-table output.
 *
 * @param text - JSON text to parse.
 * @param options - JSON table loader options.
 * @returns Parsed table output.
 * @throws If the payload does not contain row-array data.
 */
function parseTextSync(text: string, options?: JSONTableLoaderOptions) {
  const jsonOptions = normalizeJSONTableLoaderOptions(options);
  validateJSONTableArrowOptions(jsonOptions);

  const json = parseJSONSync(text, getRawJSONOptions(jsonOptions));
  const table = getJSONRowTable(json);
  if (!table) {
    throw new Error(
      'JSONTableLoader: expected a JSON row array or an object containing a JSON row array'
    );
  }

  if (jsonOptions.json?.shape !== 'arrow-table') {
    return jsonOptions.json?.shape === 'array-row-table'
      ? convertTable(table, 'array-row-table')
      : convertTable(table, 'object-row-table');
  }

  return convertRowTableToArrowTable(table, {
    schema: jsonOptions.json.schema,
    arrowConversion: jsonOptions.json.arrowConversion,
    log: getJSONTableLoaderLog(jsonOptions)
  });
}

/**
 * Parses streamed JSON row arrays into table or Arrow-table batches.
 *
 * @param asyncIterator - UTF-8 JSON byte chunks.
 * @param options - JSON table loader options.
 * @returns Table, Arrow-table, metadata, and JSON parser batches.
 */
function parseInBatches(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: JSONTableLoaderOptions
): AsyncIterable<TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch> {
  const jsonOptions = normalizeJSONTableLoaderOptions(options);
  const parseOptions =
    jsonOptions.json?.backend === 'fast'
      ? {parserFactory: getFastStreamingJSONParserFactory()}
      : undefined;
  validateJSONTableArrowOptions(jsonOptions);

  const rowBatches = parseJSONInBatches(
    asyncIterator,
    getRowBatchJSONOptions(jsonOptions),
    parseOptions
  );

  if (jsonOptions.json?.shape !== 'arrow-table') {
    return rowBatches;
  }

  return convertTableBatchesToArrow(rowBatches, {
    schema: jsonOptions.json.schema,
    arrowConversion: jsonOptions.json.arrowConversion,
    log: getJSONTableLoaderLog(jsonOptions)
  });
}

/**
 * Applies JSON table loader default options for direct parser calls.
 *
 * @param options - User-supplied JSON table loader options.
 * @returns Options merged with loader defaults.
 */
function normalizeJSONTableLoaderOptions(options?: JSONTableLoaderOptions): JSONTableLoaderOptions {
  return {
    ...options,
    json: {
      ...JSONTableLoaderWithParser.options.json,
      ...options?.json
    }
  };
}

/**
 * Requests raw JSON from the shared document parser before table conversion.
 *
 * @param options - Normalized JSON table loader options.
 * @returns JSON document parser options that preserve the raw parsed payload.
 */
function getRawJSONOptions(options: JSONTableLoaderOptions): JSONLoaderOptions {
  return {
    ...options,
    json: {
      backend: options.json?.backend,
      shape: undefined,
      table: false,
      jsonpaths: options.json?.jsonpaths
    }
  };
}

/**
 * Requests row batches from the shared streaming JSON parser before optional Arrow conversion.
 *
 * @param options - Normalized JSON table loader options.
 * @returns JSON document parser options that emit row-table batches.
 */
function getRowBatchJSONOptions(options: JSONTableLoaderOptions): JSONLoaderOptions {
  return {
    ...options,
    json: {
      backend: options.json?.backend,
      shape: options.json?.shape === 'array-row-table' ? 'array-row-table' : 'object-row-table',
      table: true,
      jsonpaths: options.json?.jsonpaths
    }
  };
}

/**
 * Returns a row table when a JSON value contains row-array data.
 *
 * @param json - Parsed JSON value.
 * @returns A loaders.gl row table, or `null` when no row array is present.
 */
function getJSONRowTable(json: unknown): ArrayRowTable | ObjectRowTable | null {
  if (isRowTable(json)) {
    return json;
  }

  const rows = Array.isArray(json) ? json : getFirstArray(json);
  if (!rows) {
    return null;
  }

  if (rows.length === 0) {
    return {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: []};
  }

  const firstRow = rows[0];
  if (Array.isArray(firstRow)) {
    return makeTableFromData(rows as unknown[][]);
  }
  if (firstRow && typeof firstRow === 'object') {
    return makeTableFromData(rows as {[key: string]: unknown}[]);
  }

  return null;
}

/**
 * Checks whether a parsed JSON value is already a row-table wrapper.
 *
 * @param value - Parsed JSON value.
 * @returns `true` when `value` is a loaders.gl object-row or array-row table.
 */
function isRowTable(value: unknown): value is ArrayRowTable | ObjectRowTable {
  return Boolean(
    value &&
      typeof value === 'object' &&
      'shape' in value &&
      ((value as Table).shape === 'array-row-table' ||
        (value as Table).shape === 'object-row-table')
  );
}

/**
 * Finds the first nested row array within a parsed JSON object.
 *
 * @param json - Parsed JSON value.
 * @returns The first nested array, or `null` when none exists.
 */
function getFirstArray(json: unknown): unknown[] | null {
  if (Array.isArray(json)) {
    return json;
  }
  if (json && typeof json === 'object') {
    for (const value of Object.values(json)) {
      const array = getFirstArray(value);
      if (array) {
        return array;
      }
    }
  }
  return null;
}

/**
 * Returns a factory for the fast streaming JSON parser backend.
 *
 * @returns Streaming parser factory used by the `fast` backend.
 */
function getFastStreamingJSONParserFactory(): StreamingJSONParserFactory {
  return parserOptions => new FastStreamingJSONParser(parserOptions);
}

/**
 * Returns the loader log object from normalized or deprecated option locations.
 *
 * @param options - Normalized JSON table loader options.
 * @returns Logger object, when supplied by the caller.
 */
function getJSONTableLoaderLog(options: JSONTableLoaderOptions): any {
  return options.core?.log || options.log;
}

/**
 * Ensures Arrow-only options are used only when Arrow-table output is requested.
 *
 * @param options - Normalized JSON table loader options.
 * @throws If Arrow conversion options are supplied without `json.shape: 'arrow-table'`.
 */
function validateJSONTableArrowOptions(options: JSONTableLoaderOptions): void {
  const hasArrowOnlyOptions = Boolean(options.json?.schema || options.json?.arrowConversion);
  if (hasArrowOnlyOptions && options.json?.shape !== 'arrow-table') {
    throw new Error(
      'JSONTableLoader: json.schema and json.arrowConversion require json.shape to be "arrow-table"'
    );
  }
}

export type {JSONTableLoaderOptions} from './json-table-loader';

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	ArrayRowTable,
7	ArrowTable,
8	ArrowTableBatch,
9	ObjectRowTable,
10	Table,
11	TableBatch
12	} from '@loaders.gl/schema';
13	import {convertTable, makeTableFromData} from '@loaders.gl/schema-utils';
14	import type {LoaderWithParser} from '@loaders.gl/loader-utils';
15	import FastStreamingJSONParser from './lib/json-parser/fast-streaming-json-parser';
16	import type {StreamingJSONParserFactory} from './lib/json-parser/streaming-json-parser-types';
17	import {parseJSONSync} from './lib/parsers/parse-json';
18	import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
19	import {
20	convertRowTableToArrowTable,
21	convertTableBatchesToArrow
22	} from './lib/parsers/convert-row-table-to-arrow';
23	import type {JSONBatch, JSONLoaderOptions, MetadataBatch} from './json-loader';
24	import {
25	JSONTableLoader as JSONTableLoaderMetadata,
26	type JSONTableLoaderOptions
27	} from './json-table-loader';
28
29	const {preload: _JSONTableLoaderPreload, ...JSONTableLoaderMetadataWithoutPreload} =
30	JSONTableLoaderMetadata;	8✔
31
32	/**
33	* Parser-bearing loader for JSON documents that must resolve to loaders.gl table output.
34	*/
35	export const JSONTableLoaderWithParser = {	8✔
36	...JSONTableLoaderMetadataWithoutPreload,
37	parse,
38	parseTextSync,
39	parseInBatches
40	} as const satisfies LoaderWithParser<
41	Table \| ArrowTable,
42	TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch,
43	JSONTableLoaderOptions
44	>;
45
46	/**
47	* Parses UTF-8 JSON bytes into row-table or Arrow-table output.
48	*
49	* @param arrayBuffer - UTF-8 encoded JSON payload.
50	* @param options - JSON table loader options.
51	* @returns Parsed table output.
52	*/
53	async function parse(arrayBuffer: ArrayBuffer, options?: JSONTableLoaderOptions) {
NEW 54	return parseTextSync(new TextDecoder().decode(arrayBuffer), options);	×
55	}
56
57	/**
58	* Parses JSON text into row-table or Arrow-table output.
59	*
60	* @param text - JSON text to parse.
61	* @param options - JSON table loader options.
62	* @returns Parsed table output.
63	* @throws If the payload does not contain row-array data.
64	*/
65	function parseTextSync(text: string, options?: JSONTableLoaderOptions) {
66	const jsonOptions = normalizeJSONTableLoaderOptions(options);	52✔
67	validateJSONTableArrowOptions(jsonOptions);	52✔
68
69	const json = parseJSONSync(text, getRawJSONOptions(jsonOptions));	52✔
70	const table = getJSONRowTable(json);	52✔
71	if (!table) {	52✔
72	throw new Error(	2✔
73	'JSONTableLoader: expected a JSON row array or an object containing a JSON row array'
74	);
75	}
76
77	if (jsonOptions.json?.shape !== 'arrow-table') {	46✔
78	return jsonOptions.json?.shape === 'array-row-table'	4✔
79	? convertTable(table, 'array-row-table')
80	: convertTable(table, 'object-row-table');
81	}
82
83	return convertRowTableToArrowTable(table, {	42✔
84	schema: jsonOptions.json.schema,
85	arrowConversion: jsonOptions.json.arrowConversion,
86	log: getJSONTableLoaderLog(jsonOptions)
87	});
88	}
89
90	/**
91	* Parses streamed JSON row arrays into table or Arrow-table batches.
92	*
93	* @param asyncIterator - UTF-8 JSON byte chunks.
94	* @param options - JSON table loader options.
95	* @returns Table, Arrow-table, metadata, and JSON parser batches.
96	*/
97	function parseInBatches(
98	asyncIterator:
99	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
100	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
101	options?: JSONTableLoaderOptions
102	): AsyncIterable<TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch> {
103	const jsonOptions = normalizeJSONTableLoaderOptions(options);	8✔
104	const parseOptions =
105	jsonOptions.json?.backend === 'fast'	8✔
106	? {parserFactory: getFastStreamingJSONParserFactory()}
107	: undefined;
108	validateJSONTableArrowOptions(jsonOptions);	8✔
109
110	const rowBatches = parseJSONInBatches(	8✔
111	asyncIterator,
112	getRowBatchJSONOptions(jsonOptions),
113	parseOptions
114	);
115
116	if (jsonOptions.json?.shape !== 'arrow-table') {	8!
NEW 117	return rowBatches;	×
118	}
119
120	return convertTableBatchesToArrow(rowBatches, {	8✔
121	schema: jsonOptions.json.schema,
122	arrowConversion: jsonOptions.json.arrowConversion,
123	log: getJSONTableLoaderLog(jsonOptions)
124	});
125	}
126
127	/**
128	* Applies JSON table loader default options for direct parser calls.
129	*
130	* @param options - User-supplied JSON table loader options.
131	* @returns Options merged with loader defaults.
132	*/
133	function normalizeJSONTableLoaderOptions(options?: JSONTableLoaderOptions): JSONTableLoaderOptions {
134	return {	60✔
135	...options,
136	json: {
137	...JSONTableLoaderWithParser.options.json,
138	...options?.json
139	}
140	};
141	}
142
143	/**
144	* Requests raw JSON from the shared document parser before table conversion.
145	*
146	* @param options - Normalized JSON table loader options.
147	* @returns JSON document parser options that preserve the raw parsed payload.
148	*/
149	function getRawJSONOptions(options: JSONTableLoaderOptions): JSONLoaderOptions {
150	return {	48✔
151	...options,
152	json: {
153	backend: options.json?.backend,
154	shape: undefined,
155	table: false,
156	jsonpaths: options.json?.jsonpaths
157	}
158	};
159	}
160
161	/**
162	* Requests row batches from the shared streaming JSON parser before optional Arrow conversion.
163	*
164	* @param options - Normalized JSON table loader options.
165	* @returns JSON document parser options that emit row-table batches.
166	*/
167	function getRowBatchJSONOptions(options: JSONTableLoaderOptions): JSONLoaderOptions {
168	return {	8✔
169	...options,
170	json: {
171	backend: options.json?.backend,
172	shape: options.json?.shape === 'array-row-table' ? 'array-row-table' : 'object-row-table',	8!
173	table: true,
174	jsonpaths: options.json?.jsonpaths
175	}
176	};
177	}
178
179	/**
180	* Returns a row table when a JSON value contains row-array data.
181	*
182	* @param json - Parsed JSON value.
183	* @returns A loaders.gl row table, or `null` when no row array is present.
184	*/
185	function getJSONRowTable(json: unknown): ArrayRowTable \| ObjectRowTable \| null {
186	if (isRowTable(json)) {	48!
NEW 187	return json;	×
188	}
189
190	const rows = Array.isArray(json) ? json : getFirstArray(json);	48✔
191	if (!rows) {	48✔
192	return null;	2✔
193	}
194
195	if (rows.length === 0) {	46✔
196	return {shape: 'object-row-table', schema: {fields: [], metadata: {}}, data: []};	2✔
197	}
198
199	const firstRow = rows[0];	44✔
200	if (Array.isArray(firstRow)) {	44!
NEW 201	return makeTableFromData(rows as unknown[][]);	×
202	}
203	if (firstRow && typeof firstRow === 'object') {	44!
204	return makeTableFromData(rows as {[key: string]: unknown}[]);	44✔
205	}
206
NEW 207	return null;	×
208	}
209
210	/**
211	* Checks whether a parsed JSON value is already a row-table wrapper.
212	*
213	* @param value - Parsed JSON value.
214	* @returns `true` when `value` is a loaders.gl object-row or array-row table.
215	*/
216	function isRowTable(value: unknown): value is ArrayRowTable \| ObjectRowTable {
217	return Boolean(	48✔
218	value &&	144!
219	typeof value === 'object' &&
220	'shape' in value &&
221	((value as Table).shape === 'array-row-table' \|\|
222	(value as Table).shape === 'object-row-table')
223	);
224	}
225
226	/**
227	* Finds the first nested row array within a parsed JSON object.
228	*
229	* @param json - Parsed JSON value.
230	* @returns The first nested array, or `null` when none exists.
231	*/
232	function getFirstArray(json: unknown): unknown[] \| null {
233	if (Array.isArray(json)) {	38✔
234	return json;	12✔
235	}
236	if (json && typeof json === 'object') {	26✔
237	for (const value of Object.values(json)) {	18✔
238	const array = getFirstArray(value);	24✔
239	if (array) {	24✔
240	return array;	12✔
241	}
242	}
243	}
244	return null;	14✔
245	}
246
247	/**
248	* Returns a factory for the fast streaming JSON parser backend.
249	*
250	* @returns Streaming parser factory used by the `fast` backend.
251	*/
252	function getFastStreamingJSONParserFactory(): StreamingJSONParserFactory {
253	return parserOptions => new FastStreamingJSONParser(parserOptions);	2✔
254	}
255
256	/**
257	* Returns the loader log object from normalized or deprecated option locations.
258	*
259	* @param options - Normalized JSON table loader options.
260	* @returns Logger object, when supplied by the caller.
261	*/
262	function getJSONTableLoaderLog(options: JSONTableLoaderOptions): any {
263	return options.core?.log \|\| options.log;	50✔
264	}
265
266	/**
267	* Ensures Arrow-only options are used only when Arrow-table output is requested.
268	*
269	* @param options - Normalized JSON table loader options.
270	* @throws If Arrow conversion options are supplied without `json.shape: 'arrow-table'`.
271	*/
272	function validateJSONTableArrowOptions(options: JSONTableLoaderOptions): void {
273	const hasArrowOnlyOptions = Boolean(options.json?.schema \|\| options.json?.arrowConversion);	60✔
274	if (hasArrowOnlyOptions && options.json?.shape !== 'arrow-table') {	60✔
275	throw new Error(	4✔
276	'JSONTableLoader: json.schema and json.arrowConversion require json.shape to be "arrow-table"'
277	);
278	}
279	}
280
281	export type {JSONTableLoaderOptions} from './json-table-loader';

visgl / loaders.gl / 25798238260

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous