24108422669

Committed 07 Apr 2026 10:56PM UTC coverage: 35.134% (-0.3%) from 35.411%

Build # 24108422669

Build Type

push

github

Committed by

web-flow

Commit Message

feat(csv) CSVArrowLoader (#3345)

Coverage Stats

1225 of 2058 branches covered (59.52%)

Branch coverage included in aggregate %.

568 of 2529 new or added lines in 12 files covered. (22.46%)

2 existing lines in 2 files now uncovered.

39940 of 115107 relevant lines covered (34.7%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.13

/modules/csv/src/csv-arrow-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {LoaderWithParser} from '@loaders.gl/loader-utils';
import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';

import type {CSVTypedArrowLoaderOptions} from './csv-typed-arrow-loader';
import {CSVTypedArrowLoader} from './csv-typed-arrow-loader';

/** Options for parsing CSV input into Apache Arrow tables. */
export type CSVArrowLoaderOptions = CSVTypedArrowLoaderOptions;

/**
 * CSV loader that returns Apache Arrow tables.
 *
 * The default `csv.dynamicTyping: false` path emits Arrow Utf8 columns and uses
 * the byte-oriented parser when the supplied options are supported. Set
 * `csv.dynamicTyping: true` to opt into typed Arrow columns.
 */
export const CSVArrowLoader = {
  ...CSVTypedArrowLoader,

  options: {
    ...CSVTypedArrowLoader.options,
    csv: {
      ...CSVTypedArrowLoader.options.csv,
      dynamicTyping: false,
      skipEmptyLines: false
    }
  },

  parse: async (arrayBuffer: ArrayBuffer, options?: CSVArrowLoaderOptions) =>
    CSVTypedArrowLoader.parse(arrayBuffer, createCSVArrowLoaderOptions(options)),

  parseText: (text: string, options?: CSVArrowLoaderOptions) =>
    CSVTypedArrowLoader.parseText(text, createCSVArrowLoaderOptions(options)),

  parseInBatches: (
    asyncIterator:
      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
      | Iterable<ArrayBufferLike | ArrayBufferView>,
    options?: CSVArrowLoaderOptions
  ) => CSVTypedArrowLoader.parseInBatches(asyncIterator, createCSVArrowLoaderOptions(options))
} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, CSVArrowLoaderOptions>;

/** Applies CSVArrowLoader defaults before delegating to internal Arrow CSV parsing helpers. */
function createCSVArrowLoaderOptions(options?: CSVArrowLoaderOptions): CSVArrowLoaderOptions {
  const skipEmptyLinesIsExplicit =
    (options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLinesIsExplicit')
      ? Boolean(options.csv.skipEmptyLinesIsExplicit)
      : undefined) ?? Boolean(options?.csv && options.csv.skipEmptyLines === true);
  return {
    ...options,
    csv: {
      ...CSVArrowLoader.options.csv,
      ...options?.csv,
      skipEmptyLinesIsExplicit
    }
  };
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {LoaderWithParser} from '@loaders.gl/loader-utils';	1✔
6	import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';	1✔
7		1✔
8	import type {CSVTypedArrowLoaderOptions} from './csv-typed-arrow-loader';	1✔
9	import {CSVTypedArrowLoader} from './csv-typed-arrow-loader';	1✔
10		1✔
11	/** Options for parsing CSV input into Apache Arrow tables. */	1✔
12	export type CSVArrowLoaderOptions = CSVTypedArrowLoaderOptions;	1✔
13		1✔
14	/**	1✔
15	* CSV loader that returns Apache Arrow tables.	1✔
16	*	1✔
17	* The default `csv.dynamicTyping: false` path emits Arrow Utf8 columns and uses	1✔
18	* the byte-oriented parser when the supplied options are supported. Set	1✔
19	* `csv.dynamicTyping: true` to opt into typed Arrow columns.	1✔
20	*/	1✔
21	export const CSVArrowLoader = {	1✔
22	...CSVTypedArrowLoader,	1✔
23		1✔
24	options: {	1✔
25	...CSVTypedArrowLoader.options,	1✔
26	csv: {	1✔
27	...CSVTypedArrowLoader.options.csv,	1✔
28	dynamicTyping: false,	1✔
29	skipEmptyLines: false	1✔
30	}	1✔
31	},	1✔
32		1✔
33	parse: async (arrayBuffer: ArrayBuffer, options?: CSVArrowLoaderOptions) =>	1✔
NEW 34	CSVTypedArrowLoader.parse(arrayBuffer, createCSVArrowLoaderOptions(options)),	×
35		1✔
36	parseText: (text: string, options?: CSVArrowLoaderOptions) =>	1✔
NEW 37	CSVTypedArrowLoader.parseText(text, createCSVArrowLoaderOptions(options)),	×
38		1✔
39	parseInBatches: (	1✔
NEW 40	asyncIterator:	×
NEW 41	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>	×
NEW 42	\| Iterable<ArrayBufferLike \| ArrayBufferView>,	×
NEW 43	options?: CSVArrowLoaderOptions	×
NEW 44	) => CSVTypedArrowLoader.parseInBatches(asyncIterator, createCSVArrowLoaderOptions(options))	×
45	} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, CSVArrowLoaderOptions>;	1✔
46		1✔
47	/** Applies CSVArrowLoader defaults before delegating to internal Arrow CSV parsing helpers. */	1✔
NEW 48	function createCSVArrowLoaderOptions(options?: CSVArrowLoaderOptions): CSVArrowLoaderOptions {	×
NEW 49	const skipEmptyLinesIsExplicit =	×
NEW 50	(options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLinesIsExplicit')	×
NEW 51	? Boolean(options.csv.skipEmptyLinesIsExplicit)	×
NEW 52	: undefined) ?? Boolean(options?.csv && options.csv.skipEmptyLines === true);	×
NEW 53	return {	×
NEW 54	...options,	×
NEW 55	csv: {	×
NEW 56	...CSVArrowLoader.options.csv,	×
NEW 57	...options?.csv,	×
NEW 58	skipEmptyLinesIsExplicit	×
NEW 59	}	×
NEW 60	};	×
UNCOV 61	}	×

visgl / loaders.gl / 24108422669

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous