24839896359

Committed 23 Apr 2026 02:06PM UTC coverage: 59.334% (-0.3%) from 59.627%

Build # 24839896359

Build Type

push

github

Committed by

web-flow

Commit Message

fix(json) Only emit batches when we have complete elements (#3400)

Coverage Stats

11234 of 20699 branches covered (54.27%)

Branch coverage included in aggregate %.

24 of 25 new or added lines in 1 file covered. (96.0%)

123 existing lines in 8 files now uncovered.

23043 of 37071 relevant lines covered (62.16%)

16510.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.62

/modules/json/src/json-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  ArrayRowTable,
  ArrowTable,
  ArrowTableBatch,
  Batch,
  ObjectRowTable,
  Table,
  TableBatch
} from '@loaders.gl/schema';
import {makeTableFromData} from '@loaders.gl/schema-utils';
import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
import {parseJSONSync} from './lib/parsers/parse-json';
import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
import {
  convertRowTableToArrowTable,
  convertTableBatchesToArrow
} from './lib/parsers/convert-row-table-to-arrow';

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

/** Metadata batch emitted while streaming JSON. */
export type MetadataBatch = Batch & {
  shape: 'metadata';
};

/** Partial or final container object emitted while streaming JSON. */
export type JSONBatch = Batch & {
  shape: 'json';
  /** JSON data */
  container: any;
};

/** Options for parsing JSON documents and tabular selections. */
export type JSONLoaderOptions = LoaderOptions & {
  json?: {
    /** Selects row-table output or Apache Arrow output for tabular JSON. */
    shape?: 'object-row-table' | 'array-row-table' | 'arrow-table';
    /** Enables table extraction from non-streaming JSON. */
    table?: boolean;
    /** Selects one or more JSON arrays to stream. */
    jsonpaths?: string[];
  };
};

/** Loader for JSON documents, including tabular JSON and streaming table extraction. */
export const JSONLoader = {
  dataType: null as unknown as Table | ArrowTable,
  batchType: null as unknown as TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch,

  name: 'JSON',
  id: 'json',
  module: 'json',
  version: VERSION,
  extensions: ['json', 'geojson'],
  mimeTypes: ['application/json'],
  category: 'table',
  text: true,
  options: {
    json: {
      shape: undefined,
      table: false,
      jsonpaths: []
      // batchSize: 'auto'
    }
  },
  parse,
  parseTextSync,
  parseInBatches
} as const satisfies LoaderWithParser<
  Table | ArrowTable,
  TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch,
  JSONLoaderOptions
>;

async function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {
  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
}

function parseTextSync(text: string, options?: JSONLoaderOptions) {
  const jsonOptions = {...options, json: {...JSONLoader.options.json, ...options?.json}};
  const json = parseJSONSync(text, jsonOptions as JSONLoaderOptions);
  if (jsonOptions.json?.shape !== 'arrow-table') {
    return json;
  }

  const table = getArrowCompatibleTable(json, jsonOptions as JSONLoaderOptions);
  return table ? convertRowTableToArrowTable(table) : json;
}

function parseInBatches(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: JSONLoaderOptions
): AsyncIterable<TableBatch | ArrowTableBatch | MetadataBatch | JSONBatch> {
  const jsonOptions = {...options, json: {...JSONLoader.options.json, ...options?.json}};
  const batches = parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);
  return jsonOptions.json?.shape === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;
}

/**
 * Returns a row table that can be converted to Arrow when the parsed JSON is tabular.
 *
 * @param json - Parsed JSON value or row table returned from the JSON parser.
 * @param options - Normalized JSON loader options.
 * @returns Row table when the parsed JSON is tabular, otherwise `null`.
 */
function getArrowCompatibleTable(
  json: unknown,
  options: JSONLoaderOptions
): ArrayRowTable | ObjectRowTable | null {
  if (isRowTable(json)) {
    return json;
  }

  if (Array.isArray(json)) {
    if (json.length === 0) {
      return {shape: 'array-row-table', schema: {fields: [], metadata: {}}, data: []};
    }

    const firstRow = json[0];
    if (Array.isArray(firstRow)) {
      return makeTableFromData(json as unknown[][]);
    }

    if (firstRow && typeof firstRow === 'object') {
      return makeTableFromData(json as {[key: string]: unknown}[]);
    }
  }

  if (options.json?.table && json && typeof json === 'object') {
    const firstArray = getFirstArray(json);
    if (firstArray?.length) {
      return Array.isArray(firstArray[0])
        ? makeTableFromData(firstArray as unknown[][])
        : makeTableFromData(firstArray as {[key: string]: unknown}[]);
    }
  }

  return null;
}

/**
 * Checks whether a parsed JSON value is already a row-table wrapper.
 *
 * @param value - Parsed JSON value.
 * @returns `true` when the value is an array-row or object-row table.
 */
function isRowTable(value: unknown): value is ArrayRowTable | ObjectRowTable {
  return Boolean(
    value &&
      typeof value === 'object' &&
      'shape' in value &&
      ((value as Table).shape === 'array-row-table' ||
        (value as Table).shape === 'object-row-table')
  );
}

/**
 * Finds the first nested array within a parsed JSON object.
 *
 * @param json - Parsed JSON object.
 * @returns The first nested array, if one exists.
 */
function getFirstArray(json: unknown): unknown[][] | {[key: string]: unknown}[] | null {
  if (Array.isArray(json)) {
    return json as unknown[][] | {[key: string]: unknown}[];
  }
  if (json && typeof json === 'object') {
    for (const value of Object.values(json)) {
      const array = getFirstArray(value);
      if (array) {
        return array;
      }
    }
  }
  return null;
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	ArrayRowTable,
7	ArrowTable,
8	ArrowTableBatch,
9	Batch,
10	ObjectRowTable,
11	Table,
12	TableBatch
13	} from '@loaders.gl/schema';
14	import {makeTableFromData} from '@loaders.gl/schema-utils';
15	import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
16	import {parseJSONSync} from './lib/parsers/parse-json';
17	import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
18	import {
19	convertRowTableToArrowTable,
20	convertTableBatchesToArrow
21	} from './lib/parsers/convert-row-table-to-arrow';
22
23	// __VERSION__ is injected by babel-plugin-version-inline
24	// @ts-ignore TS2304: Cannot find name '__VERSION__'.
25	const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';	17!
26
27	/** Metadata batch emitted while streaming JSON. */
28	export type MetadataBatch = Batch & {
29	shape: 'metadata';
30	};
31
32	/** Partial or final container object emitted while streaming JSON. */
33	export type JSONBatch = Batch & {
34	shape: 'json';
35	/** JSON data */
36	container: any;
37	};
38
39	/** Options for parsing JSON documents and tabular selections. */
40	export type JSONLoaderOptions = LoaderOptions & {
41	json?: {
42	/** Selects row-table output or Apache Arrow output for tabular JSON. */
43	shape?: 'object-row-table' \| 'array-row-table' \| 'arrow-table';
44	/** Enables table extraction from non-streaming JSON. */
45	table?: boolean;
46	/** Selects one or more JSON arrays to stream. */
47	jsonpaths?: string[];
48	};
49	};
50
51	/** Loader for JSON documents, including tabular JSON and streaming table extraction. */
52	export const JSONLoader = {	17✔
53	dataType: null as unknown as Table \| ArrowTable,
54	batchType: null as unknown as TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch,
55
56	name: 'JSON',
57	id: 'json',
58	module: 'json',
59	version: VERSION,
60	extensions: ['json', 'geojson'],
61	mimeTypes: ['application/json'],
62	category: 'table',
63	text: true,
64	options: {
65	json: {
66	shape: undefined,
67	table: false,
68	jsonpaths: []
69	// batchSize: 'auto'
70	}
71	},
72	parse,
73	parseTextSync,
74	parseInBatches
75	} as const satisfies LoaderWithParser<
76	Table \| ArrowTable,
77	TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch,
78	JSONLoaderOptions
79	>;
80
81	async function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {
82	return parseTextSync(new TextDecoder().decode(arrayBuffer), options);	3✔
83	}
84
85	function parseTextSync(text: string, options?: JSONLoaderOptions) {
86	const jsonOptions = {...options, json: {...JSONLoader.options.json, ...options?.json}};	14✔
87	const json = parseJSONSync(text, jsonOptions as JSONLoaderOptions);	14✔
88	if (jsonOptions.json?.shape !== 'arrow-table') {	14✔
89	return json;	12✔
90	}
91
92	const table = getArrowCompatibleTable(json, jsonOptions as JSONLoaderOptions);	2✔
93	return table ? convertRowTableToArrowTable(table) : json;	2!
94	}
95
96	function parseInBatches(
97	asyncIterator:
98	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
99	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
100	options?: JSONLoaderOptions
101	): AsyncIterable<TableBatch \| ArrowTableBatch \| MetadataBatch \| JSONBatch> {
102	const jsonOptions = {...options, json: {...JSONLoader.options.json, ...options?.json}};	18✔
103	const batches = parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);	18✔
104	return jsonOptions.json?.shape === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;	18✔
105	}
106
107	/**
108	* Returns a row table that can be converted to Arrow when the parsed JSON is tabular.
109	*
110	* @param json - Parsed JSON value or row table returned from the JSON parser.
111	* @param options - Normalized JSON loader options.
112	* @returns Row table when the parsed JSON is tabular, otherwise `null`.
113	*/
114	function getArrowCompatibleTable(
115	json: unknown,
116	options: JSONLoaderOptions
117	): ArrayRowTable \| ObjectRowTable \| null {
118	if (isRowTable(json)) {	2!
119	return json;	2✔
120	}
121
UNCOV 122	if (Array.isArray(json)) {	×
UNCOV 123	if (json.length === 0) {	×
UNCOV 124	return {shape: 'array-row-table', schema: {fields: [], metadata: {}}, data: []};	×
125	}
126
UNCOV 127	const firstRow = json[0];	×
UNCOV 128	if (Array.isArray(firstRow)) {	×
UNCOV 129	return makeTableFromData(json as unknown[][]);	×
130	}
131
UNCOV 132	if (firstRow && typeof firstRow === 'object') {	×
UNCOV 133	return makeTableFromData(json as {[key: string]: unknown}[]);	×
134	}
135	}
136
UNCOV 137	if (options.json?.table && json && typeof json === 'object') {	×
UNCOV 138	const firstArray = getFirstArray(json);	×
UNCOV 139	if (firstArray?.length) {	×
UNCOV 140	return Array.isArray(firstArray[0])	×
141	? makeTableFromData(firstArray as unknown[][])
142	: makeTableFromData(firstArray as {[key: string]: unknown}[]);
143	}
144	}
145
UNCOV 146	return null;	×
147	}
148
149	/**
150	* Checks whether a parsed JSON value is already a row-table wrapper.
151	*
152	* @param value - Parsed JSON value.
153	* @returns `true` when the value is an array-row or object-row table.
154	*/
155	function isRowTable(value: unknown): value is ArrayRowTable \| ObjectRowTable {
156	return Boolean(	2✔
157	value &&	10✔
158	typeof value === 'object' &&
159	'shape' in value &&
160	((value as Table).shape === 'array-row-table' \|\|
161	(value as Table).shape === 'object-row-table')
162	);
163	}
164
165	/**
166	* Finds the first nested array within a parsed JSON object.
167	*
168	* @param json - Parsed JSON object.
169	* @returns The first nested array, if one exists.
170	*/
171	function getFirstArray(json: unknown): unknown[][] \| {[key: string]: unknown}[] \| null {
UNCOV 172	if (Array.isArray(json)) {	×
UNCOV 173	return json as unknown[][] \| {[key: string]: unknown}[];	×
174	}
UNCOV 175	if (json && typeof json === 'object') {	×
UNCOV 176	for (const value of Object.values(json)) {	×
UNCOV 177	const array = getFirstArray(value);	×
UNCOV 178	if (array) {	×
UNCOV 179	return array;	×
180	}
181	}
182	}
UNCOV 183	return null;	×
184	}

visgl / loaders.gl / 24839896359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous