24839896359

Committed 23 Apr 2026 02:06PM UTC coverage: 59.334% (-0.3%) from 59.627%

Build # 24839896359

Build Type

push

github

Committed by

web-flow

Commit Message

fix(json) Only emit batches when we have complete elements (#3400)

Coverage Stats

11234 of 20699 branches covered (54.27%)

Branch coverage included in aggregate %.

24 of 25 new or added lines in 1 file covered. (96.0%)

123 existing lines in 8 files now uncovered.

23043 of 37071 relevant lines covered (62.16%)

16510.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.49

/modules/csv/src/csv-arrow-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {LoaderOptions} from '@loaders.gl/loader-utils';
import type {
  ArrayRowTable,
  ArrowTable,
  ArrowTableBatch,
  ObjectRowTable,
  Schema,
  TableBatch
} from '@loaders.gl/schema';
import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
import * as arrow from 'apache-arrow';

import type {CSVLoaderOptions} from './csv-loader';
import {CSVLoader} from './csv-loader';
import {CSVFormat} from './csv-format';
import {DEFAULT_CSV_OPTIONS} from './lib/csv-default-options';
import {
  parseRawArrowCSVInBatches,
  parseRawArrowCSVTable,
  parseRawArrowCSVText
} from './lib/parsers/parse-csv-to-arrow';
import type {CSVRawArrowParseOptions} from './lib/parsers/parse-csv-to-arrow';

/** CSV options accepted by Arrow-shaped CSV parsing helpers. */
type CSVArrowOptions = Omit<NonNullable<CSVLoaderOptions['csv']>, 'shape'> & {
  /** @internal Whether the caller explicitly supplied `skipEmptyLines`. */
  skipEmptyLinesIsExplicit?: boolean;
};

/** Cell value after Papa-style dynamic typing has been applied. */
type DynamicColumnValue = string | number | boolean | Date | null;

/** Arrow data types inferred by the typed Arrow conversion pass. */
type TypedColumnDataType = 'utf8' | 'float64' | 'bool' | 'date-millisecond';

/** Result of converting a raw Utf8 Arrow table to typed Arrow columns. */
type TypedArrowConversionResult = {
  typedArrowTable: ArrowTable;
  typedColumnDataTypes: TypedColumnDataType[];
};

const FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
const ISO_DATE =
  /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

/** Default CSV options for Arrow-shaped CSV parsing. */
const CSV_ARROW_DEFAULT_OPTIONS: CSVArrowOptions = {
  optimizeMemoryUsage: DEFAULT_CSV_OPTIONS.optimizeMemoryUsage,
  header: DEFAULT_CSV_OPTIONS.header,
  columnPrefix: DEFAULT_CSV_OPTIONS.columnPrefix,
  quoteChar: DEFAULT_CSV_OPTIONS.quoteChar,
  escapeChar: DEFAULT_CSV_OPTIONS.escapeChar,
  dynamicTyping: false,
  comments: DEFAULT_CSV_OPTIONS.comments,
  skipEmptyLines: false,
  detectGeometryColumns: DEFAULT_CSV_OPTIONS.detectGeometryColumns,
  delimitersToGuess: DEFAULT_CSV_OPTIONS.delimitersToGuess
};

/** Options for parsing CSV input into Apache Arrow tables. */
export type CSVArrowParseOptions = LoaderOptions & {
  csv?: CSVArrowOptions;
};

/** Compatibility options for the deprecated CSVArrowLoader wrapper. */
export type CSVArrowLoaderOptions = CSVArrowParseOptions;

/** Applies Arrow-shaped CSV defaults before delegating to Arrow CSV parsing helpers. */
function normalizeCSVArrowOptions(options?: CSVArrowParseOptions): CSVArrowParseOptions {
  const skipEmptyLinesIsExplicit =
    (options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLinesIsExplicit')
      ? Boolean(options.csv.skipEmptyLinesIsExplicit)
      : undefined) ?? Boolean(options?.csv && options.csv.skipEmptyLines === true);

  return {
    ...options,
    csv: {
      ...CSV_ARROW_DEFAULT_OPTIONS,
      ...options?.csv,
      skipEmptyLinesIsExplicit
    }
  };
}

/** Parses ArrayBuffer CSV input into an Arrow table. */
export async function parseCSVArrayBufferAsArrow(
  arrayBuffer: ArrayBuffer,
  options?: CSVArrowParseOptions
): Promise<ArrowTable> {
  const normalizedOptions = normalizeCSVArrowOptions(options);
  const csvOptions = createCSVArrowOptions(normalizedOptions);
  if (csvOptions.detectGeometryColumns) {
    const rowTable = await CSVLoader.parse(arrayBuffer, {
      ...normalizedOptions,
      csv: {
        ...normalizedOptions.csv,
        shape: 'object-row-table',
        dynamicTyping: csvOptions.dynamicTyping
      }
    });
    return convertCSVRowTableToArrowTable(rowTable as ObjectRowTable);
  }
  const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);

  const rawArrowTable = await parseRawArrowCSVTable(arrayBuffer, rawArrowCSVOptions);

  if (!shouldApplyDynamicTyping(csvOptions)) {
    return rawArrowTable;
  }

  return convertRawArrowTableToTypedArrowTable(rawArrowTable).typedArrowTable;
}

/** Parses string CSV input into an Arrow table. */
export async function parseCSVTextAsArrow(
  csvText: string,
  options?: CSVArrowParseOptions
): Promise<ArrowTable> {
  const normalizedOptions = normalizeCSVArrowOptions(options);
  const csvOptions = createCSVArrowOptions(normalizedOptions);
  if (csvOptions.detectGeometryColumns) {
    const rowTable = await CSVLoader.parseText(csvText, {
      ...normalizedOptions,
      csv: {
        ...normalizedOptions.csv,
        shape: 'object-row-table',
        dynamicTyping: csvOptions.dynamicTyping
      }
    });
    return convertCSVRowTableToArrowTable(rowTable as ObjectRowTable);
  }
  const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);

  const rawArrowTable = await parseRawArrowCSVText(csvText, rawArrowCSVOptions);

  if (!shouldApplyDynamicTyping(csvOptions)) {
    return rawArrowTable;
  }

  return convertRawArrowTableToTypedArrowTable(rawArrowTable).typedArrowTable;
}

/** Parses batch CSV input into Arrow table batches. */
export function parseCSVInArrowBatches(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: CSVArrowParseOptions
): AsyncIterable<ArrowTableBatch> {
  const normalizedOptions = normalizeCSVArrowOptions(options);
  const csvOptions = createCSVArrowOptions(normalizedOptions);
  if (csvOptions.detectGeometryColumns) {
    return convertCSVRowBatchesToArrowBatches(
      CSVLoader.parseInBatches(asyncIterator, {
        ...normalizedOptions,
        csv: {
          ...normalizedOptions.csv,
          shape: 'object-row-table',
          dynamicTyping: csvOptions.dynamicTyping
        }
      })
    );
  }
  const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);

  const rawArrowBatchIterator = parseRawArrowCSVInBatches(asyncIterator, rawArrowCSVOptions);

  return makeTypedArrowBatchIterator(rawArrowBatchIterator, csvOptions);
}

/**
 * Compatibility wrapper that keeps the legacy CSVArrowLoader export available
 * while `CSVLoader` adopts `csv.shape: 'arrow-table'`.
 */
export const CSVArrowLoader = {
  ...CSVFormat,
  dataType: null as unknown as ArrowTable,
  batchType: null as unknown as ArrowTableBatch,
  version: VERSION,
  options: {
    csv: {
      ...CSV_ARROW_DEFAULT_OPTIONS,
      shape: 'arrow-table'
    }
  },
  parse: async (arrayBuffer: ArrayBuffer, options?: CSVArrowLoaderOptions) =>
    parseCSVArrayBufferAsArrow(arrayBuffer, options),
  parseText: (text: string, options?: CSVArrowLoaderOptions) => parseCSVTextAsArrow(text, options),
  parseInBatches: (
    asyncIterator:
      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
      | Iterable<ArrayBufferLike | ArrayBufferView>,
    options?: CSVArrowLoaderOptions
  ) => parseCSVInArrowBatches(asyncIterator, options)
} as const;

/** Converts CSV row-table output to an Arrow table using the supplied CSV schema. */
function convertCSVRowTableToArrowTable(table: ObjectRowTable | ArrayRowTable): ArrowTable {
  const arrowTableBuilder = new ArrowTableBuilder(table.schema!);
  for (const row of table.data) {
    if (table.shape === 'object-row-table') {
      arrowTableBuilder.addObjectRow(row as {[columnName: string]: unknown});
    } else {
      arrowTableBuilder.addArrayRow(row as unknown[]);
    }
  }
  return arrowTableBuilder.finishTable();
}

/** Converts CSV row batches to Arrow batches while preserving the CSV-derived schema. */
async function* convertCSVRowBatchesToArrowBatches(
  rowBatchIterator: AsyncIterable<TableBatch>
): AsyncIterable<ArrowTableBatch> {
  for await (const rowBatch of rowBatchIterator) {
    if (
      (rowBatch.shape !== 'array-row-table' && rowBatch.shape !== 'object-row-table') ||
      !rowBatch.schema
    ) {
      continue;
    }

    const arrowTableBuilder = new ArrowTableBuilder(rowBatch.schema);
    for (const row of rowBatch.data) {
      if (rowBatch.shape === 'object-row-table') {
        arrowTableBuilder.addObjectRow(row as {[columnName: string]: unknown});
      } else {
        arrowTableBuilder.addArrayRow(row as unknown[]);
      }
    }
    const arrowTable = arrowTableBuilder.finishTable();
    yield {
      ...rowBatch,
      shape: 'arrow-table',
      schema: rowBatch.schema,
      data: arrowTable.data,
      length: arrowTable.data.numRows
    };
  }
}

/** Converts an async iterator of raw Utf8 Arrow batches to typed Arrow batches. */
async function* makeTypedArrowBatchIterator(
  rawArrowBatchIterator: AsyncIterable<ArrowTableBatch>,
  csvOptions: CSVArrowOptions
): AsyncIterable<ArrowTableBatch> {
  let frozenColumnDataTypes: TypedColumnDataType[] | null = null;

  for await (const rawArrowBatch of rawArrowBatchIterator) {
    if (!shouldApplyDynamicTyping(csvOptions)) {
      yield rawArrowBatch;
      continue;
    }

    const rawArrowTable: ArrowTable = {
      shape: 'arrow-table',
      schema: rawArrowBatch.schema,
      data: rawArrowBatch.data
    };

    const conversionResult = convertRawArrowTableToTypedArrowTable(rawArrowTable, {
      frozenColumnDataTypes
    });

    if (!frozenColumnDataTypes && conversionResult.typedColumnDataTypes.length > 0) {
      frozenColumnDataTypes = conversionResult.typedColumnDataTypes;
    }

    yield {
      ...rawArrowBatch,
      schema: conversionResult.typedArrowTable.schema,
      data: conversionResult.typedArrowTable.data,
      length: conversionResult.typedArrowTable.data.numRows
    };
  }
}

/** Merges caller options with Arrow CSV defaults. */
function createCSVArrowOptions(options?: CSVArrowParseOptions): CSVArrowOptions {
  return {
    ...CSV_ARROW_DEFAULT_OPTIONS,
    ...options?.csv
  };
}

/** Creates raw Arrow options by stripping the typed conversion flag. */
function createRawArrowCSVOptions(options?: CSVArrowParseOptions): CSVRawArrowParseOptions {
  const csvOptions = createCSVArrowOptions(options);
  const {dynamicTyping, ...rawArrowCSVOptions} = csvOptions;

  return {
    ...options,
    csv: {
      ...rawArrowCSVOptions,
      dynamicTyping
    }
  };
}

/** Returns whether typed Arrow conversion should be applied. */
function shouldApplyDynamicTyping(csvOptions: CSVArrowOptions): boolean {
  return csvOptions.dynamicTyping !== false;
}

/** Converts an Arrow table of Utf8 columns to inferred typed Arrow columns. */
function convertRawArrowTableToTypedArrowTable(
  rawArrowTable: ArrowTable,
  options?: {frozenColumnDataTypes?: TypedColumnDataType[] | null}
): TypedArrowConversionResult {
  const rawArrowSchemaFields = rawArrowTable.data.schema.fields;
  const rowCount = rawArrowTable.data.numRows;

  if (rawArrowSchemaFields.length === 0) {
    return {
      typedArrowTable: {
        shape: 'arrow-table',
        schema: {
          fields: [],
          metadata: {
            ...rawArrowTable.schema?.metadata,
            'loaders.gl#format': 'csv',
            'loaders.gl#loader': 'CSVLoader'
          }
        },
        data: rawArrowTable.data
      },
      typedColumnDataTypes: []
    };
  }

  const typedSchemaFields: Schema['fields'] = [];
  const typedColumnValues: unknown[][] = [];
  const typedColumnDataTypes: TypedColumnDataType[] = [];

  for (let columnIndex = 0; columnIndex < rawArrowSchemaFields.length; columnIndex++) {
    const rawArrowSchemaField = rawArrowSchemaFields[columnIndex];
    const rawArrowColumn = rawArrowTable.data.getChildAt(columnIndex);

    if (rawArrowSchemaField.type instanceof arrow.List) {
      typedSchemaFields.push(
        rawArrowTable.schema?.fields[columnIndex] || {
          name: rawArrowSchemaField.name,
          type: 'utf8',
          nullable: true
        }
      );
      typedColumnDataTypes.push('utf8');
      typedColumnValues.push(
        rawArrowColumn
          ? readRawArrowListValues(rawArrowColumn, rowCount)
          : new Array(rowCount).fill(null)
      );
      continue;
    }

    const rawStringValues: (string | null)[] = [];
    for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
      const rawArrowValue = rawArrowColumn?.get(rowIndex);
      rawStringValues.push(readRawArrowStringValue(rawArrowValue));
    }

    const dynamicValues = rawStringValues.map(rawStringValue =>
      parseValueWithDynamicTyping(rawStringValue)
    );

    const typedColumnDataType =
      options?.frozenColumnDataTypes?.[columnIndex] ?? deduceTypedColumnDataType(dynamicValues);

    typedSchemaFields.push({
      name: rawArrowSchemaField.name,
      type: typedColumnDataType,
      nullable: true
    });

    typedColumnDataTypes.push(typedColumnDataType);
    typedColumnValues.push(
      convertDynamicValuesToTypedColumnValues(dynamicValues, typedColumnDataType)
    );
  }

  const typedSchema: Schema = {
    fields: typedSchemaFields,
    metadata: {
      ...rawArrowTable.schema?.metadata,
      'loaders.gl#format': 'csv',
      'loaders.gl#loader': 'CSVLoader'
    }
  };

  const typedArrowTableBuilder = new ArrowTableBuilder(typedSchema);
  for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
    const rowValues = typedColumnValues.map(typedColumnValue => typedColumnValue[rowIndex]);
    typedArrowTableBuilder.addArrayRow(rowValues);
  }

  return {
    typedArrowTable: typedArrowTableBuilder.finishTable(),
    typedColumnDataTypes
  };
}

/** Reads an Arrow list column back to nullable JS arrays for table rebuilding. */
function readRawArrowListValues(rawArrowColumn: arrow.Vector, rowCount: number): unknown[] {
  const values: unknown[] = [];
  for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
    const rawArrowValue = rawArrowColumn.get(rowIndex);
    values.push(
      rawArrowValue === null || rawArrowValue === undefined ? null : Array.from(rawArrowValue)
    );
  }
  return values;
}

/** Converts an Arrow cell value to a nullable string value. */
function readRawArrowStringValue(rawArrowValue: unknown): string | null {
  if (rawArrowValue === null || rawArrowValue === undefined) {
    return null;
  }

  return String(rawArrowValue);
}

/** Applies Papa-compatible dynamic typing to one nullable CSV string value. */
function parseValueWithDynamicTyping(rawStringValue: string | null): DynamicColumnValue {
  if (rawStringValue === null) {
    return null;
  }

  if (rawStringValue === 'true' || rawStringValue === 'TRUE') {
    return true;
  }

  if (rawStringValue === 'false' || rawStringValue === 'FALSE') {
    return false;
  }

  if (FLOAT.test(rawStringValue)) {
    return Number.parseFloat(rawStringValue);
  }

  if (ISO_DATE.test(rawStringValue)) {
    return new Date(rawStringValue);
  }

  if (rawStringValue === '') {
    return null;
  }

  return rawStringValue;
}

/** Deduces the narrowest supported Arrow type for one column. */
function deduceTypedColumnDataType(dynamicValues: DynamicColumnValue[]): TypedColumnDataType {
  let inferredColumnDataType: TypedColumnDataType | null = null;

  for (const dynamicValue of dynamicValues) {
    if (dynamicValue === null) {
      continue;
    }

    const currentValueDataType = getTypedColumnDataType(dynamicValue);

    if (currentValueDataType === 'utf8') {
      return 'utf8';
    }

    if (inferredColumnDataType === null) {
      inferredColumnDataType = currentValueDataType;
      continue;
    }

    if (inferredColumnDataType !== currentValueDataType) {
      return 'utf8';
    }
  }

  return inferredColumnDataType ?? 'utf8';
}

/** Returns the typed Arrow column type for a non-null dynamically typed value. */
function getTypedColumnDataType(
  dynamicValue: Exclude<DynamicColumnValue, null>
): TypedColumnDataType {
  if (typeof dynamicValue === 'boolean') {
    return 'bool';
  }

  if (typeof dynamicValue === 'number') {
    return 'float64';
  }

  if (dynamicValue instanceof Date) {
    return 'date-millisecond';
  }

  return 'utf8';
}

/** Coerces dynamically typed values to values compatible with the selected Arrow type. */
function convertDynamicValuesToTypedColumnValues(
  dynamicValues: DynamicColumnValue[],
  typedColumnDataType: TypedColumnDataType
): DynamicColumnValue[] {
  switch (typedColumnDataType) {
    case 'bool':
      return dynamicValues.map(dynamicValue =>
        typeof dynamicValue === 'boolean' ? dynamicValue : null
      );
    case 'float64':
      return dynamicValues.map(dynamicValue =>
        typeof dynamicValue === 'number' ? dynamicValue : null
      );
    case 'date-millisecond':
      return dynamicValues.map(dynamicValue =>
        dynamicValue instanceof Date ? dynamicValue : null
      );
    case 'utf8':
    default:
      return dynamicValues.map(dynamicValue =>
        dynamicValue === null ? null : String(dynamicValue)
      );
  }
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {LoaderOptions} from '@loaders.gl/loader-utils';
6	import type {
7	ArrayRowTable,
8	ArrowTable,
9	ArrowTableBatch,
10	ObjectRowTable,
11	Schema,
12	TableBatch
13	} from '@loaders.gl/schema';
14	import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
15	import * as arrow from 'apache-arrow';
16
17	import type {CSVLoaderOptions} from './csv-loader';
18	import {CSVLoader} from './csv-loader';
19	import {CSVFormat} from './csv-format';
20	import {DEFAULT_CSV_OPTIONS} from './lib/csv-default-options';
21	import {
22	parseRawArrowCSVInBatches,
23	parseRawArrowCSVTable,
24	parseRawArrowCSVText
25	} from './lib/parsers/parse-csv-to-arrow';
26	import type {CSVRawArrowParseOptions} from './lib/parsers/parse-csv-to-arrow';
27
28	/** CSV options accepted by Arrow-shaped CSV parsing helpers. */
29	type CSVArrowOptions = Omit<NonNullable<CSVLoaderOptions['csv']>, 'shape'> & {
30	/** @internal Whether the caller explicitly supplied `skipEmptyLines`. */
31	skipEmptyLinesIsExplicit?: boolean;
32	};
33
34	/** Cell value after Papa-style dynamic typing has been applied. */
35	type DynamicColumnValue = string \| number \| boolean \| Date \| null;
36
37	/** Arrow data types inferred by the typed Arrow conversion pass. */
38	type TypedColumnDataType = 'utf8' \| 'float64' \| 'bool' \| 'date-millisecond';
39
40	/** Result of converting a raw Utf8 Arrow table to typed Arrow columns. */
41	type TypedArrowConversionResult = {
42	typedArrowTable: ArrowTable;
43	typedColumnDataTypes: TypedColumnDataType[];
44	};
45
46	const FLOAT = /^\s-?(\d\.?\d+\|\d+\.?\d)(e[-+]?\d+)?\s$/i;	14✔
47	const ISO_DATE =
48	/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d\|Z))\|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d\|Z))\|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d\|Z))/;	14✔
49
50	// __VERSION__ is injected by babel-plugin-version-inline
51	// @ts-ignore TS2304: Cannot find name '__VERSION__'.
52	const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';	14!
53
54	/** Default CSV options for Arrow-shaped CSV parsing. */
55	const CSV_ARROW_DEFAULT_OPTIONS: CSVArrowOptions = {	14✔
56	optimizeMemoryUsage: DEFAULT_CSV_OPTIONS.optimizeMemoryUsage,
57	header: DEFAULT_CSV_OPTIONS.header,
58	columnPrefix: DEFAULT_CSV_OPTIONS.columnPrefix,
59	quoteChar: DEFAULT_CSV_OPTIONS.quoteChar,
60	escapeChar: DEFAULT_CSV_OPTIONS.escapeChar,
61	dynamicTyping: false,
62	comments: DEFAULT_CSV_OPTIONS.comments,
63	skipEmptyLines: false,
64	detectGeometryColumns: DEFAULT_CSV_OPTIONS.detectGeometryColumns,
65	delimitersToGuess: DEFAULT_CSV_OPTIONS.delimitersToGuess
66	};
67
68	/** Options for parsing CSV input into Apache Arrow tables. */
69	export type CSVArrowParseOptions = LoaderOptions & {
70	csv?: CSVArrowOptions;
71	};
72
73	/** Compatibility options for the deprecated CSVArrowLoader wrapper. */
74	export type CSVArrowLoaderOptions = CSVArrowParseOptions;
75
76	/** Applies Arrow-shaped CSV defaults before delegating to Arrow CSV parsing helpers. */
77	function normalizeCSVArrowOptions(options?: CSVArrowParseOptions): CSVArrowParseOptions {
78	const skipEmptyLinesIsExplicit =	84✔
79	(options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLinesIsExplicit')	252!
80	? Boolean(options.csv.skipEmptyLinesIsExplicit)
81	: undefined) ?? Boolean(options?.csv && options.csv.skipEmptyLines === true);	168✔
82
83	return {	84✔
84	...options,
85	csv: {
86	...CSV_ARROW_DEFAULT_OPTIONS,
87	...options?.csv,
88	skipEmptyLinesIsExplicit
89	}
90	};
91	}
92
93	/** Parses ArrayBuffer CSV input into an Arrow table. */
94	export async function parseCSVArrayBufferAsArrow(
95	arrayBuffer: ArrayBuffer,
96	options?: CSVArrowParseOptions
97	): Promise<ArrowTable> {
98	const normalizedOptions = normalizeCSVArrowOptions(options);	10✔
99	const csvOptions = createCSVArrowOptions(normalizedOptions);	10✔
100	if (csvOptions.detectGeometryColumns) {	10!
UNCOV 101	const rowTable = await CSVLoader.parse(arrayBuffer, {	×
102	...normalizedOptions,
103	csv: {
104	...normalizedOptions.csv,
105	shape: 'object-row-table',
106	dynamicTyping: csvOptions.dynamicTyping
107	}
108	});
UNCOV 109	return convertCSVRowTableToArrowTable(rowTable as ObjectRowTable);	×
110	}
111	const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);	10✔
112
113	const rawArrowTable = await parseRawArrowCSVTable(arrayBuffer, rawArrowCSVOptions);	10✔
114
115	if (!shouldApplyDynamicTyping(csvOptions)) {	10!
116	return rawArrowTable;	10✔
117	}
118
UNCOV 119	return convertRawArrowTableToTypedArrowTable(rawArrowTable).typedArrowTable;	×
120	}
121
122	/** Parses string CSV input into an Arrow table. */
123	export async function parseCSVTextAsArrow(
124	csvText: string,
125	options?: CSVArrowParseOptions
126	): Promise<ArrowTable> {
127	const normalizedOptions = normalizeCSVArrowOptions(options);	48✔
128	const csvOptions = createCSVArrowOptions(normalizedOptions);	48✔
129	if (csvOptions.detectGeometryColumns) {	48!
UNCOV 130	const rowTable = await CSVLoader.parseText(csvText, {	×
131	...normalizedOptions,
132	csv: {
133	...normalizedOptions.csv,
134	shape: 'object-row-table',
135	dynamicTyping: csvOptions.dynamicTyping
136	}
137	});
UNCOV 138	return convertCSVRowTableToArrowTable(rowTable as ObjectRowTable);	×
139	}
140	const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);	48✔
141
142	const rawArrowTable = await parseRawArrowCSVText(csvText, rawArrowCSVOptions);	48✔
143
144	if (!shouldApplyDynamicTyping(csvOptions)) {	48✔
145	return rawArrowTable;	28✔
146	}
147
148	return convertRawArrowTableToTypedArrowTable(rawArrowTable).typedArrowTable;	20✔
149	}
150
151	/** Parses batch CSV input into Arrow table batches. */
152	export function parseCSVInArrowBatches(
153	asyncIterator:
154	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
155	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
156	options?: CSVArrowParseOptions
157	): AsyncIterable<ArrowTableBatch> {
158	const normalizedOptions = normalizeCSVArrowOptions(options);	26✔
159	const csvOptions = createCSVArrowOptions(normalizedOptions);	26✔
160	if (csvOptions.detectGeometryColumns) {	26!
UNCOV 161	return convertCSVRowBatchesToArrowBatches(	×
162	CSVLoader.parseInBatches(asyncIterator, {
163	...normalizedOptions,
164	csv: {
165	...normalizedOptions.csv,
166	shape: 'object-row-table',
167	dynamicTyping: csvOptions.dynamicTyping
168	}
169	})
170	);
171	}
172	const rawArrowCSVOptions = createRawArrowCSVOptions(normalizedOptions);	26✔
173
174	const rawArrowBatchIterator = parseRawArrowCSVInBatches(asyncIterator, rawArrowCSVOptions);	26✔
175
176	return makeTypedArrowBatchIterator(rawArrowBatchIterator, csvOptions);	26✔
177	}
178
179	/**
180	* Compatibility wrapper that keeps the legacy CSVArrowLoader export available
181	* while `CSVLoader` adopts `csv.shape: 'arrow-table'`.
182	*/
183	export const CSVArrowLoader = {	14✔
184	...CSVFormat,
185	dataType: null as unknown as ArrowTable,
186	batchType: null as unknown as ArrowTableBatch,
187	version: VERSION,
188	options: {
189	csv: {
190	...CSV_ARROW_DEFAULT_OPTIONS,
191	shape: 'arrow-table'
192	}
193	},
194	parse: async (arrayBuffer: ArrayBuffer, options?: CSVArrowLoaderOptions) =>
195	parseCSVArrayBufferAsArrow(arrayBuffer, options),	10✔
196	parseText: (text: string, options?: CSVArrowLoaderOptions) => parseCSVTextAsArrow(text, options),	46✔
197	parseInBatches: (
198	asyncIterator:
199	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
200	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
201	options?: CSVArrowLoaderOptions
202	) => parseCSVInArrowBatches(asyncIterator, options)	24✔
203	} as const;
204
205	/** Converts CSV row-table output to an Arrow table using the supplied CSV schema. */
206	function convertCSVRowTableToArrowTable(table: ObjectRowTable \| ArrayRowTable): ArrowTable {
UNCOV 207	const arrowTableBuilder = new ArrowTableBuilder(table.schema!);	×
UNCOV 208	for (const row of table.data) {	×
209	if (table.shape === 'object-row-table') {	×
210	arrowTableBuilder.addObjectRow(row as {[columnName: string]: unknown});	×
211	} else {
UNCOV 212	arrowTableBuilder.addArrayRow(row as unknown[]);	×
213	}
214	}
UNCOV 215	return arrowTableBuilder.finishTable();	×
216	}
217
218	/** Converts CSV row batches to Arrow batches while preserving the CSV-derived schema. */
219	async function* convertCSVRowBatchesToArrowBatches(
220	rowBatchIterator: AsyncIterable<TableBatch>
221	): AsyncIterable<ArrowTableBatch> {
222	for await (const rowBatch of rowBatchIterator) {	×
UNCOV 223	if (	×
224	(rowBatch.shape !== 'array-row-table' && rowBatch.shape !== 'object-row-table') \|\|	×
225	!rowBatch.schema
226	) {
UNCOV 227	continue;	×
228	}
229
UNCOV 230	const arrowTableBuilder = new ArrowTableBuilder(rowBatch.schema);	×
UNCOV 231	for (const row of rowBatch.data) {	×
UNCOV 232	if (rowBatch.shape === 'object-row-table') {	×
UNCOV 233	arrowTableBuilder.addObjectRow(row as {[columnName: string]: unknown});	×
234	} else {
UNCOV 235	arrowTableBuilder.addArrayRow(row as unknown[]);	×
236	}
237	}
UNCOV 238	const arrowTable = arrowTableBuilder.finishTable();	×
UNCOV 239	yield {	×
240	...rowBatch,
241	shape: 'arrow-table',
242	schema: rowBatch.schema,
243	data: arrowTable.data,
244	length: arrowTable.data.numRows
245	};
246	}
247	}
248
249	/** Converts an async iterator of raw Utf8 Arrow batches to typed Arrow batches. */
250	async function* makeTypedArrowBatchIterator(
251	rawArrowBatchIterator: AsyncIterable<ArrowTableBatch>,
252	csvOptions: CSVArrowOptions
253	): AsyncIterable<ArrowTableBatch> {
254	let frozenColumnDataTypes: TypedColumnDataType[] \| null = null;	26✔
255
256	for await (const rawArrowBatch of rawArrowBatchIterator) {	26✔
257	if (!shouldApplyDynamicTyping(csvOptions)) {	48✔
258	yield rawArrowBatch;	26✔
259	continue;	26✔
260	}
261
262	const rawArrowTable: ArrowTable = {	22✔
263	shape: 'arrow-table',
264	schema: rawArrowBatch.schema,
265	data: rawArrowBatch.data
266	};
267
268	const conversionResult = convertRawArrowTableToTypedArrowTable(rawArrowTable, {	22✔
269	frozenColumnDataTypes
270	});
271
272	if (!frozenColumnDataTypes && conversionResult.typedColumnDataTypes.length > 0) {	22✔
273	frozenColumnDataTypes = conversionResult.typedColumnDataTypes;	16✔
274	}
275
276	yield {	22✔
277	...rawArrowBatch,
278	schema: conversionResult.typedArrowTable.schema,
279	data: conversionResult.typedArrowTable.data,
280	length: conversionResult.typedArrowTable.data.numRows
281	};
282	}
283	}
284
285	/** Merges caller options with Arrow CSV defaults. */
286	function createCSVArrowOptions(options?: CSVArrowParseOptions): CSVArrowOptions {
287	return {	168✔
288	...CSV_ARROW_DEFAULT_OPTIONS,
289	...options?.csv
290	};
291	}
292
293	/** Creates raw Arrow options by stripping the typed conversion flag. */
294	function createRawArrowCSVOptions(options?: CSVArrowParseOptions): CSVRawArrowParseOptions {
295	const csvOptions = createCSVArrowOptions(options);	84✔
296	const {dynamicTyping, ...rawArrowCSVOptions} = csvOptions;	84✔
297
298	return {	84✔
299	...options,
300	csv: {
301	...rawArrowCSVOptions,
302	dynamicTyping
303	}
304	};
305	}
306
307	/** Returns whether typed Arrow conversion should be applied. */
308	function shouldApplyDynamicTyping(csvOptions: CSVArrowOptions): boolean {
309	return csvOptions.dynamicTyping !== false;	106✔
310	}
311
312	/** Converts an Arrow table of Utf8 columns to inferred typed Arrow columns. */
313	function convertRawArrowTableToTypedArrowTable(
314	rawArrowTable: ArrowTable,
315	options?: {frozenColumnDataTypes?: TypedColumnDataType[] \| null}
316	): TypedArrowConversionResult {
317	const rawArrowSchemaFields = rawArrowTable.data.schema.fields;	42✔
318	const rowCount = rawArrowTable.data.numRows;	42✔
319
320	if (rawArrowSchemaFields.length === 0) {	42!
UNCOV 321	return {	×
322	typedArrowTable: {
323	shape: 'arrow-table',
324	schema: {
325	fields: [],
326	metadata: {
327	...rawArrowTable.schema?.metadata,
328	'loaders.gl#format': 'csv',
329	'loaders.gl#loader': 'CSVLoader'
330	}
331	},
332	data: rawArrowTable.data
333	},
334	typedColumnDataTypes: []
335	};
336	}
337
338	const typedSchemaFields: Schema['fields'] = [];	42✔
339	const typedColumnValues: unknown[][] = [];	42✔
340	const typedColumnDataTypes: TypedColumnDataType[] = [];	42✔
341
342	for (let columnIndex = 0; columnIndex < rawArrowSchemaFields.length; columnIndex++) {	42✔
343	const rawArrowSchemaField = rawArrowSchemaFields[columnIndex];	188✔
344	const rawArrowColumn = rawArrowTable.data.getChildAt(columnIndex);	188✔
345
346	if (rawArrowSchemaField.type instanceof arrow.List) {	188✔
347	typedSchemaFields.push(	4✔
348	rawArrowTable.schema?.fields[columnIndex] \|\| {	4!
349	name: rawArrowSchemaField.name,
350	type: 'utf8',
351	nullable: true
352	}
353	);
354	typedColumnDataTypes.push('utf8');	4✔
355	typedColumnValues.push(	4✔
356	rawArrowColumn	4!
357	? readRawArrowListValues(rawArrowColumn, rowCount)
358	: new Array(rowCount).fill(null)
359	);
360	continue;	4✔
361	}
362
363	const rawStringValues: (string \| null)[] = [];	184✔
364	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {	184✔
365	const rawArrowValue = rawArrowColumn?.get(rowIndex);	23,518✔
366	rawStringValues.push(readRawArrowStringValue(rawArrowValue));	23,518✔
367	}
368
369	const dynamicValues = rawStringValues.map(rawStringValue =>	184✔
370	parseValueWithDynamicTyping(rawStringValue)	23,518✔
371	);
372
373	const typedColumnDataType =
374	options?.frozenColumnDataTypes?.[columnIndex] ?? deduceTypedColumnDataType(dynamicValues);	184✔
375
376	typedSchemaFields.push({	188✔
377	name: rawArrowSchemaField.name,
378	type: typedColumnDataType,
379	nullable: true
380	});
381
382	typedColumnDataTypes.push(typedColumnDataType);	188✔
383	typedColumnValues.push(	188✔
384	convertDynamicValuesToTypedColumnValues(dynamicValues, typedColumnDataType)
385	);
386	}
387
388	const typedSchema: Schema = {	188✔
389	fields: typedSchemaFields,
390	metadata: {
391	...rawArrowTable.schema?.metadata,
392	'loaders.gl#format': 'csv',
393	'loaders.gl#loader': 'CSVLoader'
394	}
395	};
396
397	const typedArrowTableBuilder = new ArrowTableBuilder(typedSchema);	42✔
398	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {	42✔
399	const rowValues = typedColumnValues.map(typedColumnValue => typedColumnValue[rowIndex]);	23,534✔
400	typedArrowTableBuilder.addArrayRow(rowValues);	3,086✔
401	}
402
403	return {	3,086✔
404	typedArrowTable: typedArrowTableBuilder.finishTable(),
405	typedColumnDataTypes
406	};
407	}
408
409	/** Reads an Arrow list column back to nullable JS arrays for table rebuilding. */
410	function readRawArrowListValues(rawArrowColumn: arrow.Vector, rowCount: number): unknown[] {
411	const values: unknown[] = [];	4✔
412	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {	4✔
413	const rawArrowValue = rawArrowColumn.get(rowIndex);	16✔
414	values.push(	16✔
415	rawArrowValue === null \|\| rawArrowValue === undefined ? null : Array.from(rawArrowValue)	40✔
416	);
417	}
418	return values;	4✔
419	}
420
421	/** Converts an Arrow cell value to a nullable string value. */
422	function readRawArrowStringValue(rawArrowValue: unknown): string \| null {
423	if (rawArrowValue === null \|\| rawArrowValue === undefined) {	23,518✔
424	return null;	24✔
425	}
426
427	return String(rawArrowValue);	23,494✔
428	}
429
430	/** Applies Papa-compatible dynamic typing to one nullable CSV string value. */
431	function parseValueWithDynamicTyping(rawStringValue: string \| null): DynamicColumnValue {
432	if (rawStringValue === null) {	23,518✔
433	return null;	24✔
434	}
435
436	if (rawStringValue === 'true' \|\| rawStringValue === 'TRUE') {	23,494!
UNCOV 437	return true;	×
438	}
439
440	if (rawStringValue === 'false' \|\| rawStringValue === 'FALSE') {	23,494!
UNCOV 441	return false;	×
442	}
443
444	if (FLOAT.test(rawStringValue)) {	23,494✔
445	return Number.parseFloat(rawStringValue);	8,480✔
446	}
447
448	if (ISO_DATE.test(rawStringValue)) {	15,014!
UNCOV 449	return new Date(rawStringValue);	×
450	}
451
452	if (rawStringValue === '') {	15,014✔
453	return null;	130✔
454	}
455
456	return rawStringValue;	14,884✔
457	}
458
459	/** Deduces the narrowest supported Arrow type for one column. */
460	function deduceTypedColumnDataType(dynamicValues: DynamicColumnValue[]): TypedColumnDataType {
461	let inferredColumnDataType: TypedColumnDataType \| null = null;	170✔
462
463	for (const dynamicValue of dynamicValues) {	170✔
464	if (dynamicValue === null) {	8,324✔
465	continue;	108✔
466	}
467
468	const currentValueDataType = getTypedColumnDataType(dynamicValue);	8,216✔
469
470	if (currentValueDataType === 'utf8') {	8,216✔
471	return 'utf8';	96✔
472	}
473
474	if (inferredColumnDataType === null) {	8,120✔
475	inferredColumnDataType = currentValueDataType;	64✔
476	continue;	64✔
477	}
478
479	if (inferredColumnDataType !== currentValueDataType) {	8,056!
UNCOV 480	return 'utf8';	×
481	}
482	}
483
484	return inferredColumnDataType ?? 'utf8';	74✔
485	}
486
487	/** Returns the typed Arrow column type for a non-null dynamically typed value. */
488	function getTypedColumnDataType(
489	dynamicValue: Exclude<DynamicColumnValue, null>
490	): TypedColumnDataType {
491	if (typeof dynamicValue === 'boolean') {	8,216!
UNCOV 492	return 'bool';	×
493	}
494
495	if (typeof dynamicValue === 'number') {	8,216✔
496	return 'float64';	8,120✔
497	}
498
499	if (dynamicValue instanceof Date) {	96!
500	return 'date-millisecond';	×
501	}
502
503	return 'utf8';	96✔
504	}
505
506	/** Coerces dynamically typed values to values compatible with the selected Arrow type. */
507	function convertDynamicValuesToTypedColumnValues(
508	dynamicValues: DynamicColumnValue[],
509	typedColumnDataType: TypedColumnDataType
510	): DynamicColumnValue[] {
511	switch (typedColumnDataType) {	184!
512	case 'bool':
UNCOV 513	return dynamicValues.map(dynamicValue =>	×
UNCOV 514	typeof dynamicValue === 'boolean' ? dynamicValue : null	×
515	);
516	case 'float64':
517	return dynamicValues.map(dynamicValue =>	78✔
518	typeof dynamicValue === 'number' ? dynamicValue : null	8,490✔
519	);
520	case 'date-millisecond':
UNCOV 521	return dynamicValues.map(dynamicValue =>	×
UNCOV 522	dynamicValue instanceof Date ? dynamicValue : null	×
523	);
524	case 'utf8':
525	default:
526	return dynamicValues.map(dynamicValue =>	106✔
527	dynamicValue === null ? null : String(dynamicValue)	15,028✔
528	);
529	}
530	}

visgl / loaders.gl / 24839896359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous