24108422669

Committed 07 Apr 2026 10:56PM UTC coverage: 35.134% (-0.3%) from 35.411%

Build # 24108422669

Build Type

push

github

Committed by

web-flow

Commit Message

feat(csv) CSVArrowLoader (#3345)

Coverage Stats

1225 of 2058 branches covered (59.52%)

Branch coverage included in aggregate %.

568 of 2529 new or added lines in 12 files covered. (22.46%)

2 existing lines in 2 files now uncovered.

39940 of 115107 relevant lines covered (34.7%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.52

/modules/csv/src/csv2-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {LoaderWithParser} from '@loaders.gl/loader-utils';
import type {
  ArrowTable,
  ArrowTableBatch,
  ArrayRowTable,
  ObjectRowTable,
  TableBatch
} from '@loaders.gl/schema';
import * as arrow from 'apache-arrow';

import type {CSVArrowLoaderOptions} from './csv-arrow-loader';
import {CSVArrowLoader} from './csv-arrow-loader';
import type {CSVLoaderOptions} from './csv-loader';
import {CSVLoader} from './csv-loader';

/** Row table shapes produced by the internal CSV2 loader. */
type CSV2Table = ObjectRowTable | ArrayRowTable;

/** Supported row materialization shape names. */
type CSV2Shape = NonNullable<NonNullable<CSVLoaderOptions['csv']>['shape']>;

/** Minimal Arrow column interface used during JS row materialization. */
type CSVArrowColumn = {get: (index: number) => unknown};

/** CSV loader that parses through CSVArrowLoader and materializes JS row tables. */
export const CSV2Loader = {
  ...CSVLoader,

  id: 'csv2',
  name: 'CSV2',
  dataType: null as unknown as CSV2Table,
  batchType: null as unknown as TableBatch,

  parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => {
    const arrowTable = await CSVArrowLoader.parse(arrayBuffer, createCSVArrowOptions(options));
    return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));
  },

  parseText: async (text: string, options?: CSVLoaderOptions) => {
    const arrowTable = await CSVArrowLoader.parseText(text, createCSVArrowOptions(options));
    return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));
  },

  parseInBatches: (
    asyncIterator:
      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
      | Iterable<ArrayBufferLike | ArrayBufferView>,
    options?: CSVLoaderOptions
  ) =>
    makeCSV2BatchIterator(
      CSVArrowLoader.parseInBatches(asyncIterator, createCSVArrowOptions(options)),
      options
    )
} as const satisfies LoaderWithParser<CSV2Table, TableBatch, CSVLoaderOptions>;

/** Converts Arrow CSV batches to the configured JS row table shape. */
async function* makeCSV2BatchIterator(
  arrowBatchIterator: AsyncIterable<ArrowTableBatch>,
  options?: CSVLoaderOptions
): AsyncIterable<TableBatch> {
  const shape = getCSV2Shape(options);
  for await (const arrowBatch of arrowBatchIterator) {
    const table = convertCSVArrowTableToRowTable(
      {
        shape: 'arrow-table',
        schema: arrowBatch.schema,
        data: arrowBatch.data
      },
      shape
    );
    yield {
      ...table,
      batchType: arrowBatch.batchType,
      length: arrowBatch.length
    };
  }
}

/** Returns the requested CSV2 row shape, using CSVLoader's default shape. */
function getCSV2Shape(options?: CSVLoaderOptions): CSV2Shape {
  return options?.csv?.shape || CSV2Loader.options.csv.shape || 'object-row-table';
}

/** Creates CSVArrowLoader options by removing row-shape-only CSV options. */
function createCSVArrowOptions(options?: CSVLoaderOptions): CSVArrowLoaderOptions {
  const csvOptions = {...CSV2Loader.options.csv, ...options?.csv};
  const arrowCSVOptions = {...csvOptions};
  delete (arrowCSVOptions as {shape?: CSV2Shape}).shape;
  return {
    ...options,
    csv: {
      ...arrowCSVOptions,
      skipEmptyLinesIsExplicit:
        options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLines')
    }
  };
}

/** Converts an Arrow CSV table to a loaders.gl JS row table without generic row wrappers. */
function convertCSVArrowTableToRowTable(arrowTable: ArrowTable, shape: CSV2Shape): CSV2Table {
  switch (shape) {
    case 'array-row-table':
      return convertCSVArrowTableToArrayRowTable(arrowTable);
    case 'object-row-table':
      return convertCSVArrowTableToObjectRowTable(arrowTable);
    default:
      throw new Error(shape);
  }
}

/** Converts an Arrow CSV table to an array-row table. */
function convertCSVArrowTableToArrayRowTable(arrowTable: ArrowTable): ArrayRowTable {
  const arrowData = arrowTable.data;
  const columnCount = arrowData.numCols;
  const columns = getArrowColumns(arrowTable);
  const rows = new Array<unknown[]>(arrowData.numRows);

  for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {
    const row = new Array<unknown>(columnCount);
    for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {
      row[columnIndex] = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));
    }
    rows[rowIndex] = row;
  }

  return {
    shape: 'array-row-table',
    schema: arrowTable.schema,
    data: rows
  };
}

/** Converts an Arrow CSV table to an object-row table. */
function convertCSVArrowTableToObjectRowTable(arrowTable: ArrowTable): ObjectRowTable {
  const arrowData = arrowTable.data;
  const fields = arrowData.schema.fields;
  const columns = getArrowColumns(arrowTable);
  const rows = new Array<{[key: string]: unknown}>(arrowData.numRows);

  for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {
    const row: {[key: string]: unknown} = {};
    for (let columnIndex = 0; columnIndex < fields.length; columnIndex++) {
      const cellValue = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));
      if (cellValue === null && fields[columnIndex].type instanceof arrow.List) {
        continue;
      }
      row[fields[columnIndex].name] = cellValue;
    }
    rows[rowIndex] = row;
  }

  return {
    shape: 'object-row-table',
    schema: arrowTable.schema,
    data: rows
  };
}

/** Returns Arrow columns once so row materialization does not repeatedly look them up by name. */
function getArrowColumns(arrowTable: ArrowTable): Array<CSVArrowColumn | null> {
  const columns: Array<CSVArrowColumn | null> = new Array(arrowTable.data.numCols);
  for (let columnIndex = 0; columnIndex < arrowTable.data.numCols; columnIndex++) {
    columns[columnIndex] = arrowTable.data.getChildAt(columnIndex) || null;
  }
  return columns;
}

/** Materializes nested Arrow vector cell values into plain JS arrays. */
function getCSVArrowCellValue(value: unknown): unknown {
  if (value && typeof value === 'object' && Symbol.iterator in value) {
    return Array.from(value as Iterable<unknown>);
  }
  return value;
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {LoaderWithParser} from '@loaders.gl/loader-utils';	1✔
6	import type {	1✔
7	ArrowTable,	1✔
8	ArrowTableBatch,	1✔
9	ArrayRowTable,	1✔
10	ObjectRowTable,	1✔
11	TableBatch	1✔
12	} from '@loaders.gl/schema';	1✔
13	import * as arrow from 'apache-arrow';	1✔
14		1✔
15	import type {CSVArrowLoaderOptions} from './csv-arrow-loader';	1✔
16	import {CSVArrowLoader} from './csv-arrow-loader';	1✔
17	import type {CSVLoaderOptions} from './csv-loader';	1✔
18	import {CSVLoader} from './csv-loader';	1✔
19		1✔
20	/** Row table shapes produced by the internal CSV2 loader. */	1✔
21	type CSV2Table = ObjectRowTable \| ArrayRowTable;	1✔
22		1✔
23	/** Supported row materialization shape names. */	1✔
24	type CSV2Shape = NonNullable<NonNullable<CSVLoaderOptions['csv']>['shape']>;	1✔
25		1✔
26	/** Minimal Arrow column interface used during JS row materialization. */	1✔
27	type CSVArrowColumn = {get: (index: number) => unknown};	1✔
28		1✔
29	/** CSV loader that parses through CSVArrowLoader and materializes JS row tables. */	1✔
30	export const CSV2Loader = {	1✔
31	...CSVLoader,	1✔
32		1✔
33	id: 'csv2',	1✔
34	name: 'CSV2',	1✔
35	dataType: null as unknown as CSV2Table,	1✔
36	batchType: null as unknown as TableBatch,	1✔
37		1✔
38	parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => {	1✔
NEW 39	const arrowTable = await CSVArrowLoader.parse(arrayBuffer, createCSVArrowOptions(options));	×
NEW 40	return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));	×
NEW 41	},	×
42		1✔
43	parseText: async (text: string, options?: CSVLoaderOptions) => {	1✔
NEW 44	const arrowTable = await CSVArrowLoader.parseText(text, createCSVArrowOptions(options));	×
NEW 45	return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));	×
NEW 46	},	×
47		1✔
48	parseInBatches: (	1✔
NEW 49	asyncIterator:	×
NEW 50	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>	×
NEW 51	\| Iterable<ArrayBufferLike \| ArrayBufferView>,	×
NEW 52	options?: CSVLoaderOptions	×
NEW 53	) =>	×
NEW 54	makeCSV2BatchIterator(	×
NEW 55	CSVArrowLoader.parseInBatches(asyncIterator, createCSVArrowOptions(options)),	×
NEW 56	options	×
NEW 57	)	×
58	} as const satisfies LoaderWithParser<CSV2Table, TableBatch, CSVLoaderOptions>;	1✔
59		1✔
60	/** Converts Arrow CSV batches to the configured JS row table shape. */	1✔
NEW 61	async function* makeCSV2BatchIterator(	×
NEW 62	arrowBatchIterator: AsyncIterable<ArrowTableBatch>,	×
NEW 63	options?: CSVLoaderOptions	×
NEW 64	): AsyncIterable<TableBatch> {	×
NEW 65	const shape = getCSV2Shape(options);	×
NEW 66	for await (const arrowBatch of arrowBatchIterator) {	×
NEW 67	const table = convertCSVArrowTableToRowTable(	×
NEW 68	{	×
NEW 69	shape: 'arrow-table',	×
NEW 70	schema: arrowBatch.schema,	×
NEW 71	data: arrowBatch.data	×
NEW 72	},	×
NEW 73	shape	×
NEW 74	);	×
NEW 75	yield {	×
NEW 76	...table,	×
NEW 77	batchType: arrowBatch.batchType,	×
NEW 78	length: arrowBatch.length	×
NEW 79	};	×
NEW 80	}	×
NEW 81	}	×
82		1✔
83	/** Returns the requested CSV2 row shape, using CSVLoader's default shape. */	1✔
NEW 84	function getCSV2Shape(options?: CSVLoaderOptions): CSV2Shape {	×
NEW 85	return options?.csv?.shape \|\| CSV2Loader.options.csv.shape \|\| 'object-row-table';	×
NEW 86	}	×
87		1✔
88	/** Creates CSVArrowLoader options by removing row-shape-only CSV options. */	1✔
NEW 89	function createCSVArrowOptions(options?: CSVLoaderOptions): CSVArrowLoaderOptions {	×
NEW 90	const csvOptions = {...CSV2Loader.options.csv, ...options?.csv};	×
NEW 91	const arrowCSVOptions = {...csvOptions};	×
NEW 92	delete (arrowCSVOptions as {shape?: CSV2Shape}).shape;	×
NEW 93	return {	×
NEW 94	...options,	×
NEW 95	csv: {	×
NEW 96	...arrowCSVOptions,	×
NEW 97	skipEmptyLinesIsExplicit:	×
NEW 98	options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLines')	×
NEW 99	}	×
NEW 100	};	×
NEW 101	}	×
102		1✔
103	/** Converts an Arrow CSV table to a loaders.gl JS row table without generic row wrappers. */	1✔
NEW 104	function convertCSVArrowTableToRowTable(arrowTable: ArrowTable, shape: CSV2Shape): CSV2Table {	×
NEW 105	switch (shape) {	×
NEW 106	case 'array-row-table':	×
NEW 107	return convertCSVArrowTableToArrayRowTable(arrowTable);	×
NEW 108	case 'object-row-table':	×
NEW 109	return convertCSVArrowTableToObjectRowTable(arrowTable);	×
NEW 110	default:	×
NEW 111	throw new Error(shape);	×
NEW 112	}	×
NEW 113	}	×
114		1✔
115	/** Converts an Arrow CSV table to an array-row table. */	1✔
NEW 116	function convertCSVArrowTableToArrayRowTable(arrowTable: ArrowTable): ArrayRowTable {	×
NEW 117	const arrowData = arrowTable.data;	×
NEW 118	const columnCount = arrowData.numCols;	×
NEW 119	const columns = getArrowColumns(arrowTable);	×
NEW 120	const rows = new Array<unknown[]>(arrowData.numRows);	×
NEW 121		×
NEW 122	for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {	×
NEW 123	const row = new Array<unknown>(columnCount);	×
NEW 124	for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {	×
NEW 125	row[columnIndex] = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));	×
NEW 126	}	×
NEW 127	rows[rowIndex] = row;	×
NEW 128	}	×
NEW 129		×
NEW 130	return {	×
NEW 131	shape: 'array-row-table',	×
NEW 132	schema: arrowTable.schema,	×
NEW 133	data: rows	×
NEW 134	};	×
NEW 135	}	×
136		1✔
137	/** Converts an Arrow CSV table to an object-row table. */	1✔
NEW 138	function convertCSVArrowTableToObjectRowTable(arrowTable: ArrowTable): ObjectRowTable {	×
NEW 139	const arrowData = arrowTable.data;	×
NEW 140	const fields = arrowData.schema.fields;	×
NEW 141	const columns = getArrowColumns(arrowTable);	×
NEW 142	const rows = new Array<{[key: string]: unknown}>(arrowData.numRows);	×
NEW 143		×
NEW 144	for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {	×
NEW 145	const row: {[key: string]: unknown} = {};	×
NEW 146	for (let columnIndex = 0; columnIndex < fields.length; columnIndex++) {	×
NEW 147	const cellValue = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));	×
NEW 148	if (cellValue === null && fields[columnIndex].type instanceof arrow.List) {	×
NEW 149	continue;	×
NEW 150	}	×
NEW 151	row[fields[columnIndex].name] = cellValue;	×
NEW 152	}	×
NEW 153	rows[rowIndex] = row;	×
NEW 154	}	×
NEW 155		×
NEW 156	return {	×
NEW 157	shape: 'object-row-table',	×
NEW 158	schema: arrowTable.schema,	×
NEW 159	data: rows	×
NEW 160	};	×
NEW 161	}	×
162		1✔
163	/** Returns Arrow columns once so row materialization does not repeatedly look them up by name. */	1✔
NEW 164	function getArrowColumns(arrowTable: ArrowTable): Array<CSVArrowColumn \| null> {	×
NEW 165	const columns: Array<CSVArrowColumn \| null> = new Array(arrowTable.data.numCols);	×
NEW 166	for (let columnIndex = 0; columnIndex < arrowTable.data.numCols; columnIndex++) {	×
NEW 167	columns[columnIndex] = arrowTable.data.getChildAt(columnIndex) \|\| null;	×
NEW 168	}	×
NEW 169	return columns;	×
NEW 170	}	×
171		1✔
172	/** Materializes nested Arrow vector cell values into plain JS arrays. */	1✔
NEW 173	function getCSVArrowCellValue(value: unknown): unknown {	×
NEW 174	if (value && typeof value === 'object' && Symbol.iterator in value) {	×
NEW 175	return Array.from(value as Iterable<unknown>);	×
NEW 176	}	×
NEW 177	return value;	×
NEW 178	}	×

visgl / loaders.gl / 24108422669

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous