24907303489

Committed 24 Apr 2026 07:12PM UTC coverage: 59.423% (+0.09%) from 59.334%

Build # 24907303489

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Dynamic import loaders (#3405)

Coverage Stats

11252 of 20783 branches covered (54.14%)

Branch coverage included in aggregate %.

1164 of 1518 new or added lines in 244 files covered. (76.68%)

41 existing lines in 18 files now uncovered.

23432 of 37585 relevant lines covered (62.34%)

16317.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.21

/modules/json/src/json-loader-with-parser.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  ArrayRowTable,
  ArrowTable,
  ArrowTableBatch,
  Batch,
  ObjectRowTable,
  Table,
  TableBatch
} from '@loaders.gl/schema';
import {makeTableFromData} from '@loaders.gl/schema-utils';
import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
import {parseJSONSync} from './lib/parsers/parse-json';
import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
import {
  convertRowTableToArrowTable,
  convertTableBatchesToArrow
} from './lib/parsers/convert-row-table-to-arrow';
import {JSONLoader as JSONLoaderMetadata} from './json-loader';

const {preload: _JSONLoaderPreload, ...JSONLoaderMetadataWithoutPreload} = JSONLoaderMetadata;

/** Metadata batch emitted while streaming JSON. */
export type MetadataBatch = Batch & {
  shape: 'metadata';
};

/** Partial or final container object emitted while streaming JSON. */
export type JSONBatch = Batch & {
  shape: 'json';
  /** JSON data */
  container: any;
};

/** Options for parsing JSON documents and tabular selections. */
export type JSONLoaderOptions = LoaderOptions & {
  json?: {
    /** Selects row-table output or Apache Arrow output for tabular JSON. */
    shape?: 'object-row-table' | 'array-row-table' | 'arrow-table';
    /** Enables table extraction from non-streaming JSON. */
    table?: boolean;
    /** Selects one or more JSON arrays to stream. */
    jsonpaths?: string[];
  };
};

/** Loader for JSON documents, including tabular JSON and streaming table extraction. */
export const JSONLoaderWithParser = {
  ...JSONLoaderMetadataWithoutPreload,
  parse,
  parseTextSync,
  parseInBatches
} as const satisfies LoaderWithParser<
  Table | ArrowTable,
  TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch,
  JSONLoaderOptions
>;

async function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {
  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
}

function parseTextSync(text: string, options?: JSONLoaderOptions) {
  const jsonOptions = {...options, json: {...JSONLoaderWithParser.options.json, ...options?.json}};
  const json = parseJSONSync(text, jsonOptions as JSONLoaderOptions);
  if (jsonOptions.json?.shape !== 'arrow-table') {
    return json;
  }

  const table = getArrowCompatibleTable(json, jsonOptions as JSONLoaderOptions);
  return table ? convertRowTableToArrowTable(table) : json;
}

function parseInBatches(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: JSONLoaderOptions
): AsyncIterable<TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch> {
  const jsonOptions = {...options, json: {...JSONLoaderWithParser.options.json, ...options?.json}};
  const batches = parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);
  return jsonOptions.json?.shape === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;
}

/**
 * Returns a row table that can be converted to Arrow when the parsed JSON is tabular.
 *
 * @param json - Parsed JSON value or row table returned from the JSON parser.
 * @param options - Normalized JSON loader options.
 * @returns Row table when the parsed JSON is tabular, otherwise `null`.
 */
function getArrowCompatibleTable(
  json: unknown,
  options: JSONLoaderOptions
): ArrayRowTable | ObjectRowTable | null {
  if (isRowTable(json)) {
    return json;
  }

  if (Array.isArray(json)) {
    if (json.length === 0) {
      return {shape: 'array-row-table', schema: {fields: [], metadata: {}}, data: []};
    }

    const firstRow = json[0];
    if (Array.isArray(firstRow)) {
      return makeTableFromData(json as unknown[][]);
    }

    if (firstRow && typeof firstRow === 'object') {
      return makeTableFromData(json as {[key: string]: unknown}[]);
    }
  }

  if (options.json?.table && json && typeof json === 'object') {
    const firstArray = getFirstArray(json);
    if (firstArray?.length) {
      return Array.isArray(firstArray[0])
        ? makeTableFromData(firstArray as unknown[][])
        : makeTableFromData(firstArray as {[key: string]: unknown}[]);
    }
  }

  return null;
}

/**
 * Checks whether a parsed JSON value is already a row-table wrapper.
 *
 * @param value - Parsed JSON value.
 * @returns `true` when the value is an array-row or object-row table.
 */
function isRowTable(value: unknown): value is ArrayRowTable | ObjectRowTable {
  return Boolean(
    value &&
      typeof value === 'object' &&
      'shape' in value &&
      ((value as Table).shape === 'array-row-table' ||
        (value as Table).shape === 'object-row-table')
  );
}

/**
 * Finds the first nested array within a parsed JSON object.
 *
 * @param json - Parsed JSON object.
 * @returns The first nested array, if one exists.
 */
function getFirstArray(json: unknown): unknown[][] | {[key: string]: unknown}[] | null {
  if (Array.isArray(json)) {
    return json as unknown[][] | {[key: string]: unknown}[];
  }
  if (json && typeof json === 'object') {
    for (const value of Object.values(json)) {
      const array = getFirstArray(value);
      if (array) {
        return array;
      }
    }
  }
  return null;
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	ArrayRowTable,
7	ArrowTable,
8	ArrowTableBatch,
9	Batch,
10	ObjectRowTable,
11	Table,
12	TableBatch
13	} from '@loaders.gl/schema';
14	import {makeTableFromData} from '@loaders.gl/schema-utils';
15	import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
16	import {parseJSONSync} from './lib/parsers/parse-json';
17	import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
18	import {
19	convertRowTableToArrowTable,
20	convertTableBatchesToArrow
21	} from './lib/parsers/convert-row-table-to-arrow';
22	import {JSONLoader as JSONLoaderMetadata} from './json-loader';
23
24	const {preload: _JSONLoaderPreload, ...JSONLoaderMetadataWithoutPreload} = JSONLoaderMetadata;	7✔
25
26	/** Metadata batch emitted while streaming JSON. */
27	export type MetadataBatch = Batch & {
28	shape: 'metadata';
29	};
30
31	/** Partial or final container object emitted while streaming JSON. */
32	export type JSONBatch = Batch & {
33	shape: 'json';
34	/** JSON data */
35	container: any;
36	};
37
38	/** Options for parsing JSON documents and tabular selections. */
39	export type JSONLoaderOptions = LoaderOptions & {
40	json?: {
41	/** Selects row-table output or Apache Arrow output for tabular JSON. */
42	shape?: 'object-row-table' \| 'array-row-table' \| 'arrow-table';
43	/** Enables table extraction from non-streaming JSON. */
44	table?: boolean;
45	/** Selects one or more JSON arrays to stream. */
46	jsonpaths?: string[];
47	};
48	};
49
50	/** Loader for JSON documents, including tabular JSON and streaming table extraction. */
51	export const JSONLoaderWithParser = {	7✔
52	...JSONLoaderMetadataWithoutPreload,
53	parse,
54	parseTextSync,
55	parseInBatches
56	} as const satisfies LoaderWithParser<
57	Table \| ArrowTable,
58	TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch,
59	JSONLoaderOptions
60	>;
61
62	async function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {
63	return parseTextSync(new TextDecoder().decode(arrayBuffer), options);	3✔
64	}
65
66	function parseTextSync(text: string, options?: JSONLoaderOptions) {
67	const jsonOptions = {...options, json: {...JSONLoaderWithParser.options.json, ...options?.json}};	14✔
68	const json = parseJSONSync(text, jsonOptions as JSONLoaderOptions);	14✔
69	if (jsonOptions.json?.shape !== 'arrow-table') {	14✔
70	return json;	12✔
71	}
72
73	const table = getArrowCompatibleTable(json, jsonOptions as JSONLoaderOptions);	2✔
74	return table ? convertRowTableToArrowTable(table) : json;	2!
75	}
76
77	function parseInBatches(
78	asyncIterator:
79	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
80	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
81	options?: JSONLoaderOptions
82	): AsyncIterable<TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch> {
83	const jsonOptions = {...options, json: {...JSONLoaderWithParser.options.json, ...options?.json}};	18✔
84	const batches = parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);	18✔
85	return jsonOptions.json?.shape === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;	18✔
86	}
87
88	/**
89	* Returns a row table that can be converted to Arrow when the parsed JSON is tabular.
90	*
91	* @param json - Parsed JSON value or row table returned from the JSON parser.
92	* @param options - Normalized JSON loader options.
93	* @returns Row table when the parsed JSON is tabular, otherwise `null`.
94	*/
95	function getArrowCompatibleTable(
96	json: unknown,
97	options: JSONLoaderOptions
98	): ArrayRowTable \| ObjectRowTable \| null {
99	if (isRowTable(json)) {	2!
100	return json;	2✔
101	}
102
NEW 103	if (Array.isArray(json)) {	×
NEW 104	if (json.length === 0) {	×
NEW 105	return {shape: 'array-row-table', schema: {fields: [], metadata: {}}, data: []};	×
106	}
107
NEW 108	const firstRow = json[0];	×
NEW 109	if (Array.isArray(firstRow)) {	×
NEW 110	return makeTableFromData(json as unknown[][]);	×
111	}
112
NEW 113	if (firstRow && typeof firstRow === 'object') {	×
NEW 114	return makeTableFromData(json as {[key: string]: unknown}[]);	×
115	}
116	}
117
NEW 118	if (options.json?.table && json && typeof json === 'object') {	×
NEW 119	const firstArray = getFirstArray(json);	×
NEW 120	if (firstArray?.length) {	×
NEW 121	return Array.isArray(firstArray[0])	×
122	? makeTableFromData(firstArray as unknown[][])
123	: makeTableFromData(firstArray as {[key: string]: unknown}[]);
124	}
125	}
126
NEW 127	return null;	×
128	}
129
130	/**
131	* Checks whether a parsed JSON value is already a row-table wrapper.
132	*
133	* @param value - Parsed JSON value.
134	* @returns `true` when the value is an array-row or object-row table.
135	*/
136	function isRowTable(value: unknown): value is ArrayRowTable \| ObjectRowTable {
137	return Boolean(	2✔
138	value &&	10✔
139	typeof value === 'object' &&
140	'shape' in value &&
141	((value as Table).shape === 'array-row-table' \|\|
142	(value as Table).shape === 'object-row-table')
143	);
144	}
145
146	/**
147	* Finds the first nested array within a parsed JSON object.
148	*
149	* @param json - Parsed JSON object.
150	* @returns The first nested array, if one exists.
151	*/
152	function getFirstArray(json: unknown): unknown[][] \| {[key: string]: unknown}[] \| null {
NEW 153	if (Array.isArray(json)) {	×
NEW 154	return json as unknown[][] \| {[key: string]: unknown}[];	×
155	}
NEW 156	if (json && typeof json === 'object') {	×
NEW 157	for (const value of Object.values(json)) {	×
NEW 158	const array = getFirstArray(value);	×
NEW 159	if (array) {	×
NEW 160	return array;	×
161	}
162	}
163	}
NEW 164	return null;	×
165	}

visgl / loaders.gl / 24907303489

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous