24839896359

Committed 23 Apr 2026 02:06PM UTC coverage: 59.334% (-0.3%) from 59.627%

Build # 24839896359

Build Type

push

github

Committed by

web-flow

Commit Message

fix(json) Only emit batches when we have complete elements (#3400)

Coverage Stats

11234 of 20699 branches covered (54.27%)

Branch coverage included in aggregate %.

24 of 25 new or added lines in 1 file covered. (96.0%)

123 existing lines in 8 files now uncovered.

23043 of 37071 relevant lines covered (62.16%)

16510.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.47

/modules/json/src/ndjson-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
import {
  ObjectRowTable,
  ArrayRowTable,
  ArrowTable,
  ArrowTableBatch,
  TableBatch
} from '@loaders.gl/schema';
import {parseNDJSONSync} from './lib/parsers/parse-ndjson';
import {parseNDJSONInBatches} from './lib/parsers/parse-ndjson-in-batches';
import {
  convertRowTableToArrowTable,
  convertTableBatchesToArrow
} from './lib/parsers/convert-row-table-to-arrow';

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

type NDJSONShape = 'array-row-table' | 'object-row-table' | 'arrow-table';

/** Options for parsing newline-delimited JSON data. */
export type NDJSONLoaderOptions = LoaderOptions & {
  ndjson?: {
    /** Selects row-table output or Apache Arrow output. */
    shape?: NDJSONShape;
  };
  json?: {
    /** Deprecated alias for `ndjson.shape`. */
    shape?: NDJSONShape;
  };
};

/** Loader for newline-delimited JSON row and Arrow tables. */
export const NDJSONLoader = {
  dataType: null as unknown as ArrayRowTable | ObjectRowTable | ArrowTable,
  batchType: null as unknown as TableBatch | ArrowTableBatch,

  name: 'NDJSON',
  id: 'ndjson',
  module: 'json',
  version: VERSION,
  extensions: ['ndjson', 'jsonl'],
  mimeTypes: [
    'application/x-ndjson',
    'application/jsonlines', // https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#cm-batch
    'application/json-seq'
  ],
  category: 'table',
  text: true,
  parse: async (arrayBuffer: ArrayBuffer, options?: NDJSONLoaderOptions) =>
    parseNDJSONText(new TextDecoder().decode(arrayBuffer), options),
  parseTextSync: (text: string, options?: NDJSONLoaderOptions) => parseNDJSONText(text, options),
  parseInBatches: (
    asyncIterator:
      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
      | Iterable<ArrayBufferLike | ArrayBufferView>,
    options?: NDJSONLoaderOptions
  ) => parseNDJSONInRequestedShape(asyncIterator, options),
  options: {
    ndjson: {
      shape: 'object-row-table'
    }
  }
} as const satisfies LoaderWithParser<
  ObjectRowTable | ArrayRowTable | ArrowTable,
  TableBatch | ArrowTableBatch,
  NDJSONLoaderOptions
>;

/**
 * Parses NDJSON text and optionally converts the result to Arrow.
 *
 * @param text - NDJSON text to parse.
 * @param options - Loader options including optional output shape.
 * @returns Row-table output by default, or an Arrow table when requested.
 */
function parseNDJSONText(
  text: string,
  options?: NDJSONLoaderOptions
): ObjectRowTable | ArrayRowTable | ArrowTable {
  const table = parseNDJSONSync(text);
  return getNDJSONShape(options) === 'arrow-table' ? convertRowTableToArrowTable(table) : table;
}

/**
 * Parses NDJSON batches and optionally converts data batches to Arrow.
 *
 * @param asyncIterator - NDJSON byte iterator.
 * @param options - Loader options including optional output shape.
 * @returns Batch iterator yielding row-table or Arrow batches.
 */
function parseNDJSONInRequestedShape(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: NDJSONLoaderOptions
): AsyncIterable<TableBatch | ArrowTableBatch> {
  const batches = parseNDJSONInBatches(asyncIterator, options);
  return getNDJSONShape(options) === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;
}

/** Returns the requested NDJSON output shape, including the deprecated JSON alias. */
function getNDJSONShape(options?: NDJSONLoaderOptions): NDJSONShape {
  return options?.ndjson?.shape || options?.json?.shape || NDJSONLoader.options.ndjson.shape;
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
6	import {
7	ObjectRowTable,
8	ArrayRowTable,
9	ArrowTable,
10	ArrowTableBatch,
11	TableBatch
12	} from '@loaders.gl/schema';
13	import {parseNDJSONSync} from './lib/parsers/parse-ndjson';
14	import {parseNDJSONInBatches} from './lib/parsers/parse-ndjson-in-batches';
15	import {
16	convertRowTableToArrowTable,
17	convertTableBatchesToArrow
18	} from './lib/parsers/convert-row-table-to-arrow';
19
20	// __VERSION__ is injected by babel-plugin-version-inline
21	// @ts-ignore TS2304: Cannot find name '__VERSION__'.
22	const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';	17!
23
24	type NDJSONShape = 'array-row-table' \| 'object-row-table' \| 'arrow-table';
25
26	/** Options for parsing newline-delimited JSON data. */
27	export type NDJSONLoaderOptions = LoaderOptions & {
28	ndjson?: {
29	/** Selects row-table output or Apache Arrow output. */
30	shape?: NDJSONShape;
31	};
32	json?: {
33	/** Deprecated alias for `ndjson.shape`. */
34	shape?: NDJSONShape;
35	};
36	};
37
38	/** Loader for newline-delimited JSON row and Arrow tables. */
39	export const NDJSONLoader = {	17✔
40	dataType: null as unknown as ArrayRowTable \| ObjectRowTable \| ArrowTable,
41	batchType: null as unknown as TableBatch \| ArrowTableBatch,
42
43	name: 'NDJSON',
44	id: 'ndjson',
45	module: 'json',
46	version: VERSION,
47	extensions: ['ndjson', 'jsonl'],
48	mimeTypes: [
49	'application/x-ndjson',
50	'application/jsonlines', // https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#cm-batch
51	'application/json-seq'
52	],
53	category: 'table',
54	text: true,
55	parse: async (arrayBuffer: ArrayBuffer, options?: NDJSONLoaderOptions) =>
UNCOV 56	parseNDJSONText(new TextDecoder().decode(arrayBuffer), options),	×
57	parseTextSync: (text: string, options?: NDJSONLoaderOptions) => parseNDJSONText(text, options),	22✔
58	parseInBatches: (
59	asyncIterator:
60	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
61	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
62	options?: NDJSONLoaderOptions
63	) => parseNDJSONInRequestedShape(asyncIterator, options),	18✔
64	options: {
65	ndjson: {
66	shape: 'object-row-table'
67	}
68	}
69	} as const satisfies LoaderWithParser<
70	ObjectRowTable \| ArrayRowTable \| ArrowTable,
71	TableBatch \| ArrowTableBatch,
72	NDJSONLoaderOptions
73	>;
74
75	/**
76	* Parses NDJSON text and optionally converts the result to Arrow.
77	*
78	* @param text - NDJSON text to parse.
79	* @param options - Loader options including optional output shape.
80	* @returns Row-table output by default, or an Arrow table when requested.
81	*/
82	function parseNDJSONText(
83	text: string,
84	options?: NDJSONLoaderOptions
85	): ObjectRowTable \| ArrayRowTable \| ArrowTable {
86	const table = parseNDJSONSync(text);	22✔
87	return getNDJSONShape(options) === 'arrow-table' ? convertRowTableToArrowTable(table) : table;	22✔
88	}
89
90	/**
91	* Parses NDJSON batches and optionally converts data batches to Arrow.
92	*
93	* @param asyncIterator - NDJSON byte iterator.
94	* @param options - Loader options including optional output shape.
95	* @returns Batch iterator yielding row-table or Arrow batches.
96	*/
97	function parseNDJSONInRequestedShape(
98	asyncIterator:
99	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
100	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
101	options?: NDJSONLoaderOptions
102	): AsyncIterable<TableBatch \| ArrowTableBatch> {
103	const batches = parseNDJSONInBatches(asyncIterator, options);	18✔
104	return getNDJSONShape(options) === 'arrow-table' ? convertTableBatchesToArrow(batches) : batches;	18✔
105	}
106
107	/** Returns the requested NDJSON output shape, including the deprecated JSON alias. */
108	function getNDJSONShape(options?: NDJSONLoaderOptions): NDJSONShape {
109	return options?.ndjson?.shape \|\| options?.json?.shape \|\| NDJSONLoader.options.ndjson.shape;	36✔
110	}

visgl / loaders.gl / 24839896359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous