• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

visgl / loaders.gl / 24108422669

07 Apr 2026 10:56PM UTC coverage: 35.134% (-0.3%) from 35.411%
24108422669

push

github

web-flow
feat(csv) CSVArrowLoader (#3345)

1225 of 2058 branches covered (59.52%)

Branch coverage included in aggregate %.

568 of 2529 new or added lines in 12 files covered. (22.46%)

2 existing lines in 2 files now uncovered.

39940 of 115107 relevant lines covered (34.7%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.52
/modules/csv/src/csv2-loader.ts
1
// loaders.gl
1✔
2
// SPDX-License-Identifier: MIT
1✔
3
// Copyright (c) vis.gl contributors
1✔
4

1✔
5
import type {LoaderWithParser} from '@loaders.gl/loader-utils';
1✔
6
import type {
1✔
7
  ArrowTable,
1✔
8
  ArrowTableBatch,
1✔
9
  ArrayRowTable,
1✔
10
  ObjectRowTable,
1✔
11
  TableBatch
1✔
12
} from '@loaders.gl/schema';
1✔
13
import * as arrow from 'apache-arrow';
1✔
14

1✔
15
import type {CSVArrowLoaderOptions} from './csv-arrow-loader';
1✔
16
import {CSVArrowLoader} from './csv-arrow-loader';
1✔
17
import type {CSVLoaderOptions} from './csv-loader';
1✔
18
import {CSVLoader} from './csv-loader';
1✔
19

1✔
20
/** Row table shapes produced by the internal CSV2 loader. */
1✔
21
type CSV2Table = ObjectRowTable | ArrayRowTable;
1✔
22

1✔
23
/** Supported row materialization shape names. */
1✔
24
type CSV2Shape = NonNullable<NonNullable<CSVLoaderOptions['csv']>['shape']>;
1✔
25

1✔
26
/** Minimal Arrow column interface used during JS row materialization. */
1✔
27
type CSVArrowColumn = {get: (index: number) => unknown};
1✔
28

1✔
29
/** CSV loader that parses through CSVArrowLoader and materializes JS row tables. */
1✔
30
export const CSV2Loader = {
1✔
31
  ...CSVLoader,
1✔
32

1✔
33
  id: 'csv2',
1✔
34
  name: 'CSV2',
1✔
35
  dataType: null as unknown as CSV2Table,
1✔
36
  batchType: null as unknown as TableBatch,
1✔
37

1✔
38
  parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => {
1✔
NEW
39
    const arrowTable = await CSVArrowLoader.parse(arrayBuffer, createCSVArrowOptions(options));
×
NEW
40
    return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));
×
NEW
41
  },
×
42

1✔
43
  parseText: async (text: string, options?: CSVLoaderOptions) => {
1✔
NEW
44
    const arrowTable = await CSVArrowLoader.parseText(text, createCSVArrowOptions(options));
×
NEW
45
    return convertCSVArrowTableToRowTable(arrowTable, getCSV2Shape(options));
×
NEW
46
  },
×
47

1✔
48
  parseInBatches: (
1✔
NEW
49
    asyncIterator:
×
NEW
50
      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
×
NEW
51
      | Iterable<ArrayBufferLike | ArrayBufferView>,
×
NEW
52
    options?: CSVLoaderOptions
×
NEW
53
  ) =>
×
NEW
54
    makeCSV2BatchIterator(
×
NEW
55
      CSVArrowLoader.parseInBatches(asyncIterator, createCSVArrowOptions(options)),
×
NEW
56
      options
×
NEW
57
    )
×
58
} as const satisfies LoaderWithParser<CSV2Table, TableBatch, CSVLoaderOptions>;
1✔
59

1✔
60
/** Converts Arrow CSV batches to the configured JS row table shape. */
1✔
NEW
61
async function* makeCSV2BatchIterator(
×
NEW
62
  arrowBatchIterator: AsyncIterable<ArrowTableBatch>,
×
NEW
63
  options?: CSVLoaderOptions
×
NEW
64
): AsyncIterable<TableBatch> {
×
NEW
65
  const shape = getCSV2Shape(options);
×
NEW
66
  for await (const arrowBatch of arrowBatchIterator) {
×
NEW
67
    const table = convertCSVArrowTableToRowTable(
×
NEW
68
      {
×
NEW
69
        shape: 'arrow-table',
×
NEW
70
        schema: arrowBatch.schema,
×
NEW
71
        data: arrowBatch.data
×
NEW
72
      },
×
NEW
73
      shape
×
NEW
74
    );
×
NEW
75
    yield {
×
NEW
76
      ...table,
×
NEW
77
      batchType: arrowBatch.batchType,
×
NEW
78
      length: arrowBatch.length
×
NEW
79
    };
×
NEW
80
  }
×
NEW
81
}
×
82

1✔
83
/** Returns the requested CSV2 row shape, using CSVLoader's default shape. */
1✔
NEW
84
function getCSV2Shape(options?: CSVLoaderOptions): CSV2Shape {
×
NEW
85
  return options?.csv?.shape || CSV2Loader.options.csv.shape || 'object-row-table';
×
NEW
86
}
×
87

1✔
88
/** Creates CSVArrowLoader options by removing row-shape-only CSV options. */
1✔
NEW
89
function createCSVArrowOptions(options?: CSVLoaderOptions): CSVArrowLoaderOptions {
×
NEW
90
  const csvOptions = {...CSV2Loader.options.csv, ...options?.csv};
×
NEW
91
  const arrowCSVOptions = {...csvOptions};
×
NEW
92
  delete (arrowCSVOptions as {shape?: CSV2Shape}).shape;
×
NEW
93
  return {
×
NEW
94
    ...options,
×
NEW
95
    csv: {
×
NEW
96
      ...arrowCSVOptions,
×
NEW
97
      skipEmptyLinesIsExplicit:
×
NEW
98
        options?.csv && Object.prototype.hasOwnProperty.call(options.csv, 'skipEmptyLines')
×
NEW
99
    }
×
NEW
100
  };
×
NEW
101
}
×
102

1✔
103
/** Converts an Arrow CSV table to a loaders.gl JS row table without generic row wrappers. */
1✔
NEW
104
function convertCSVArrowTableToRowTable(arrowTable: ArrowTable, shape: CSV2Shape): CSV2Table {
×
NEW
105
  switch (shape) {
×
NEW
106
    case 'array-row-table':
×
NEW
107
      return convertCSVArrowTableToArrayRowTable(arrowTable);
×
NEW
108
    case 'object-row-table':
×
NEW
109
      return convertCSVArrowTableToObjectRowTable(arrowTable);
×
NEW
110
    default:
×
NEW
111
      throw new Error(shape);
×
NEW
112
  }
×
NEW
113
}
×
114

1✔
115
/** Converts an Arrow CSV table to an array-row table. */
1✔
NEW
116
function convertCSVArrowTableToArrayRowTable(arrowTable: ArrowTable): ArrayRowTable {
×
NEW
117
  const arrowData = arrowTable.data;
×
NEW
118
  const columnCount = arrowData.numCols;
×
NEW
119
  const columns = getArrowColumns(arrowTable);
×
NEW
120
  const rows = new Array<unknown[]>(arrowData.numRows);
×
NEW
121

×
NEW
122
  for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {
×
NEW
123
    const row = new Array<unknown>(columnCount);
×
NEW
124
    for (let columnIndex = 0; columnIndex < columnCount; columnIndex++) {
×
NEW
125
      row[columnIndex] = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));
×
NEW
126
    }
×
NEW
127
    rows[rowIndex] = row;
×
NEW
128
  }
×
NEW
129

×
NEW
130
  return {
×
NEW
131
    shape: 'array-row-table',
×
NEW
132
    schema: arrowTable.schema,
×
NEW
133
    data: rows
×
NEW
134
  };
×
NEW
135
}
×
136

1✔
137
/** Converts an Arrow CSV table to an object-row table. */
1✔
NEW
138
function convertCSVArrowTableToObjectRowTable(arrowTable: ArrowTable): ObjectRowTable {
×
NEW
139
  const arrowData = arrowTable.data;
×
NEW
140
  const fields = arrowData.schema.fields;
×
NEW
141
  const columns = getArrowColumns(arrowTable);
×
NEW
142
  const rows = new Array<{[key: string]: unknown}>(arrowData.numRows);
×
NEW
143

×
NEW
144
  for (let rowIndex = 0; rowIndex < arrowData.numRows; rowIndex++) {
×
NEW
145
    const row: {[key: string]: unknown} = {};
×
NEW
146
    for (let columnIndex = 0; columnIndex < fields.length; columnIndex++) {
×
NEW
147
      const cellValue = getCSVArrowCellValue(columns[columnIndex]?.get(rowIndex));
×
NEW
148
      if (cellValue === null && fields[columnIndex].type instanceof arrow.List) {
×
NEW
149
        continue;
×
NEW
150
      }
×
NEW
151
      row[fields[columnIndex].name] = cellValue;
×
NEW
152
    }
×
NEW
153
    rows[rowIndex] = row;
×
NEW
154
  }
×
NEW
155

×
NEW
156
  return {
×
NEW
157
    shape: 'object-row-table',
×
NEW
158
    schema: arrowTable.schema,
×
NEW
159
    data: rows
×
NEW
160
  };
×
NEW
161
}
×
162

1✔
163
/** Returns Arrow columns once so row materialization does not repeatedly look them up by name. */
1✔
NEW
164
function getArrowColumns(arrowTable: ArrowTable): Array<CSVArrowColumn | null> {
×
NEW
165
  const columns: Array<CSVArrowColumn | null> = new Array(arrowTable.data.numCols);
×
NEW
166
  for (let columnIndex = 0; columnIndex < arrowTable.data.numCols; columnIndex++) {
×
NEW
167
    columns[columnIndex] = arrowTable.data.getChildAt(columnIndex) || null;
×
NEW
168
  }
×
NEW
169
  return columns;
×
NEW
170
}
×
171

1✔
172
/** Materializes nested Arrow vector cell values into plain JS arrays. */
1✔
NEW
173
function getCSVArrowCellValue(value: unknown): unknown {
×
NEW
174
  if (value && typeof value === 'object' && Symbol.iterator in value) {
×
NEW
175
    return Array.from(value as Iterable<unknown>);
×
NEW
176
  }
×
NEW
177
  return value;
×
NEW
178
}
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc