19670167673

Committed 25 Nov 2025 12:52PM UTC coverage: 61.76% (-0.007%) from 61.767%

Build # 19670167673

Build Type

push

github

Committed by

web-flow

Commit Message

fix: Allow passing arrow tables to ArrowDataContainer (#3242)

* fix: copy geometry when geometry is of binary format (#3236)

* fix: copy geometry when geometry is of binary format

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* nit

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

* fix: Allow passing arrow tables to ArrowDataContainer

Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

* only add arrowTable prop to CREATE_TABLE_TASK when set

Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>
Co-authored-by: Igor Dykhta <igorDykhta@users.noreply.github.com>

Run Details

6349 of 12189 branches covered (52.09%)

Branch coverage included in aggregate %.

2 of 5 new or added lines in 2 files covered. (40.0%)

48 existing lines in 2 files now uncovered.

13043 of 19210 relevant lines covered (67.9%)

81.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/utils/src/arrow-data-container.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import { ALL_FIELD_TYPES } from '@kepler.gl/constants';
import { ProtoDatasetField } from '@kepler.gl/types';
import * as arrow from 'apache-arrow';
import { console as globalConsole } from 'global/window';
import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';

import { DataContainerInterface, RangeOptions } from './data-container-interface';
import { DataRow, SharedRowOptions } from './data-row';

type ArrowDataContainerInput = {
  cols: arrow.Vector[];
  fields?: ProtoDatasetField[];
  arrowTable?: arrow.Table;
};


/**
 * @param dataContainer
 * @param sharedRow
 */
function* rowsIterator(dataContainer: DataContainerInterface, sharedRow: SharedRowOptions) {
  const numRows = dataContainer.numRows();
  for (let rowIndex = 0; rowIndex < numRows; ++rowIndex) {
    yield dataContainer.row(rowIndex, sharedRow);
  }
}

/**
 * @param dataContainer
 * @param columnIndex
 */
function* columnIterator(dataContainer: DataContainerInterface, columnIndex: number) {
  const numRows = dataContainer.numRows();
  for (let rowIndex = 0; rowIndex < numRows; ++rowIndex) {
    yield dataContainer.valueAt(rowIndex, columnIndex);
  }
}

/**
 * A data container where all data is stored in raw Arrow table
 */
export class ArrowDataContainer implements DataContainerInterface {
  _cols: arrow.Vector[];
  _numColumns: number;
  _numRows: number;
  _fields: ProtoDatasetField[];
  _numChunks: number;
  // cache column data to make valueAt() faster
  // _colData: any[][];

  /** An arrow table recreated from vectors */
  _arrowTable: arrow.Table;

  constructor(data: ArrowDataContainerInput) {
    if (!data.cols) {
      throw Error('ArrowDataContainer: no columns provided');
    }

    if (!Array.isArray(data.cols)) {
      throw Error("ArrowDataContainer: columns object isn't an array");
    }

    this._cols = data.cols;
    this._numColumns = data.cols.length;
    this._numRows = data.cols[0].length;
    this._fields = data.fields || [];
    this._numChunks = data.cols[0].data.length;
    // this._colData = data.cols.map(c => c.toArray());

    this._arrowTable = data.arrowTable || this._createTable();
  }

  /**
   * Restores internal Arrow table from vectors.
   * TODO: consider using original arrow table, as it could contain extra metadata, not passed to the fields.
   */
  private _createTable() {
    const creaOpts = {};
    this._fields.map((field, index) => {
      creaOpts[field.name] = this._cols[index];
    });
    return new arrow.Table(creaOpts);
  }

  getTable() {
    return this._arrowTable;
  }

  update(updateData: arrow.Vector<any>[]) {
    this._cols = updateData;
    this._numColumns = this._cols.length;
    this._numRows = this._cols[0].length;
    this._numChunks = this._cols[0].data.length;

    this._arrowTable = this._createTable();

    // cache column data to make valueAt() faster
    // this._colData = this._cols.map(c => c.toArray());
  }

  numChunks(): number {
    return this._numChunks;
  }

  numRows(): number {
    return this._numRows;
  }

  numColumns(): number {
    return this._numColumns;
  }

  valueAt(rowIndex: number, columnIndex: number): any {
    // return this._colData[columnIndex][rowIndex];
    return this._cols[columnIndex].get(rowIndex);
  }

  row(rowIndex: number, sharedRow?: SharedRowOptions): DataRow {
    const tSharedRow = DataRow.createSharedRow(sharedRow);
    if (tSharedRow) {
      tSharedRow.setSource(this, rowIndex);
      return tSharedRow;
    }

    return new DataRow(this, rowIndex);
  }

  rowAsArray(rowIndex: number): any[] {
    // return this._colData.map(col => col[rowIndex]);
    return this._cols.map(col => col.get(rowIndex));
  }

  rows(sharedRow: SharedRowOptions) {
    const tSharedRow = DataRow.createSharedRow(sharedRow);
    return rowsIterator(this, tSharedRow);
  }

  column(columnIndex: number) {
    return columnIterator(this, columnIndex);
  }

  getColumn(columnIndex: number): arrow.Vector {
    return this._cols[columnIndex];
  }

  getField(columnIndex: number): ProtoDatasetField {
    return this._fields[columnIndex];
  }

  flattenData(): any[][] {
    const data: any[][] = [];
    for (let i = 0; i < this._numRows; ++i) {
      data.push(this.rowAsArray(i));
    }
    return data;
  }

  getPlainIndex(): number[] {
    return [...Array(this._numRows).keys()];
  }

  map<T>(
    func: (row: DataRow, index: number) => T,
    sharedRow?: SharedRowOptions,
    options: RangeOptions = {}
  ): T[] {
    const tSharedRow = DataRow.createSharedRow(sharedRow);

    const {start = 0, end = this.numRows()} = options;
    const endRow = Math.min(this.numRows(), end);

    const out: T[] = [];
    for (let rowIndex = start; rowIndex < endRow; ++rowIndex) {
      const row = this.row(rowIndex, tSharedRow);
      out.push(func(row, rowIndex));
    }
    return out;
  }

  mapIndex<T>(func: ({index}, dc: DataContainerInterface) => T, options: RangeOptions = {}): T[] {
    const {start = 0, end = this.numRows()} = options;
    const endRow = Math.min(this.numRows(), end);

    const out: T[] = [];
    for (let rowIndex = start; rowIndex < endRow; ++rowIndex) {
      out.push(func({index: rowIndex}, this));
    }
    return out;
  }

  find(
    func: (row: DataRow, index: number) => boolean,
    sharedRow?: SharedRowOptions
  ): DataRow | undefined {
    const tSharedRow = DataRow.createSharedRow(sharedRow);

    for (let rowIndex = 0; rowIndex < this._numRows; ++rowIndex) {
      const row = this.row(rowIndex, tSharedRow);
      if (func(row, rowIndex)) {
        return row;
      }
    }
    return undefined;
  }

  reduce<T>(
    func: (acc: T, row: DataRow, index: number) => T,
    initialValue: T,
    sharedRow?: SharedRowOptions
  ): T {
    const tSharedRow = DataRow.createSharedRow(sharedRow);

    for (let rowIndex = 0; rowIndex < this._numRows; ++rowIndex) {
      const row = this.row(rowIndex, tSharedRow);
      initialValue = func(initialValue, row, rowIndex);
    }
    return initialValue;
  }
}

/**
 * Convert arrow data type to kepler.gl field types
 *
 * @param arrowType the arrow data type
 * @returns corresponding type in `ALL_FIELD_TYPES`
 */
export function arrowDataTypeToFieldType(arrowType: arrow.DataType): string {
  // Note: this function doesn't return ALL_FIELD_TYPES.geojson or ALL_FIELD_TYPES.array, which
  // should be further detected by caller
  if (arrow.DataType.isDate(arrowType)) {
    return ALL_FIELD_TYPES.date;
  } else if (arrow.DataType.isTimestamp(arrowType) || arrow.DataType.isTime(arrowType)) {
    return ALL_FIELD_TYPES.timestamp;
  } else if (arrow.DataType.isFloat(arrowType)) {
    return ALL_FIELD_TYPES.real;
  } else if (arrow.DataType.isInt(arrowType)) {
    return ALL_FIELD_TYPES.integer;
  } else if (arrow.DataType.isBool(arrowType)) {
    return ALL_FIELD_TYPES.boolean;
  } else if (arrow.DataType.isUtf8(arrowType) || arrow.DataType.isNull(arrowType)) {
    return ALL_FIELD_TYPES.string;
  } else if (
    arrow.DataType.isBinary(arrowType) ||
    arrow.DataType.isDictionary(arrowType) ||
    arrow.DataType.isFixedSizeBinary(arrowType) ||
    arrow.DataType.isFixedSizeList(arrowType) ||
    arrow.DataType.isList(arrowType) ||
    arrow.DataType.isMap(arrowType) ||
    arrow.DataType.isStruct(arrowType)
  ) {
    return ALL_FIELD_TYPES.object;
  }
  globalConsole.warn(`Unsupported arrow type: ${arrowType}`);
  return ALL_FIELD_TYPES.string;
}

/**
 * Convert arrow data type to analyzer type
 *
 * @param arrowType the arrow data type
 * @returns corresponding type in `AnalyzerDATA_TYPES`
 */
export function arrowDataTypeToAnalyzerDataType(
  arrowType: arrow.DataType
): typeof AnalyzerDATA_TYPES {
  if (arrow.DataType.isDate(arrowType)) {
    return AnalyzerDATA_TYPES.DATE;
  } else if (arrow.DataType.isTimestamp(arrowType) || arrow.DataType.isTime(arrowType)) {
    return AnalyzerDATA_TYPES.DATETIME;
  } else if (arrow.DataType.isFloat(arrowType)) {
    return AnalyzerDATA_TYPES.FLOAT;
  } else if (arrow.DataType.isInt(arrowType)) {
    return AnalyzerDATA_TYPES.INT;
  } else if (arrow.DataType.isBool(arrowType)) {
    return AnalyzerDATA_TYPES.BOOLEAN;
  } else if (arrow.DataType.isUtf8(arrowType) || arrow.DataType.isNull(arrowType)) {
    return AnalyzerDATA_TYPES.STRING;
  } else if (
    arrow.DataType.isBinary(arrowType) ||
    arrow.DataType.isDictionary(arrowType) ||
    arrow.DataType.isFixedSizeBinary(arrowType) ||
    arrow.DataType.isFixedSizeList(arrowType) ||
    arrow.DataType.isList(arrowType) ||
    arrow.DataType.isMap(arrowType) ||
    arrow.DataType.isStruct(arrowType)
  ) {
    return AnalyzerDATA_TYPES.OBJECT;
  }
  globalConsole.warn(`Unsupported arrow type: ${arrowType}`);
  return AnalyzerDATA_TYPES.STRING;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import { ALL_FIELD_TYPES } from '@kepler.gl/constants';
5	import { ProtoDatasetField } from '@kepler.gl/types';
6	import * as arrow from 'apache-arrow';
7	import { console as globalConsole } from 'global/window';
8	import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';
9
10	import { DataContainerInterface, RangeOptions } from './data-container-interface';
11	import { DataRow, SharedRowOptions } from './data-row';
12
13	type ArrowDataContainerInput = {
14	cols: arrow.Vector[];
15	fields?: ProtoDatasetField[];
16	arrowTable?: arrow.Table;
17	};
18
19
20	/**
21	* @param dataContainer
22	* @param sharedRow
23	*/
UNCOV 24	function* rowsIterator(dataContainer: DataContainerInterface, sharedRow: SharedRowOptions) {	×
25	const numRows = dataContainer.numRows();	×
26	for (let rowIndex = 0; rowIndex < numRows; ++rowIndex) {	×
27	yield dataContainer.row(rowIndex, sharedRow);
28	}
29	}
30
31	/**
32	* @param dataContainer
33	* @param columnIndex
34	*/
UNCOV 35	function* columnIterator(dataContainer: DataContainerInterface, columnIndex: number) {	×
36	const numRows = dataContainer.numRows();	×
37	for (let rowIndex = 0; rowIndex < numRows; ++rowIndex) {	×
38	yield dataContainer.valueAt(rowIndex, columnIndex);
39	}
40	}
41
42	/**
43	* A data container where all data is stored in raw Arrow table
44	*/
45	export class ArrowDataContainer implements DataContainerInterface {
46	_cols: arrow.Vector[];
47	_numColumns: number;
48	_numRows: number;
49	_fields: ProtoDatasetField[];
50	_numChunks: number;
51	// cache column data to make valueAt() faster
52	// _colData: any[][];
53
54	/** An arrow table recreated from vectors */
55	_arrowTable: arrow.Table;
56
UNCOV 57	constructor(data: ArrowDataContainerInput) {	×
58	if (!data.cols) {	×
59	throw Error('ArrowDataContainer: no columns provided');
60	}
UNCOV 61		×
62	if (!Array.isArray(data.cols)) {	×
63	throw Error("ArrowDataContainer: columns object isn't an array");
64	}
UNCOV 65		×
66	this._cols = data.cols;	×
67	this._numColumns = data.cols.length;	×
68	this._numRows = data.cols[0].length;	×
69	this._fields = data.fields \|\| [];	×
70	this._numChunks = data.cols[0].data.length;
71	// this._colData = data.cols.map(c => c.toArray());
UNCOV 72		×
73	this._arrowTable = data.arrowTable \|\| this._createTable();
74	}
75
76	/**
77	* Restores internal Arrow table from vectors.
78	* TODO: consider using original arrow table, as it could contain extra metadata, not passed to the fields.
79	*/
UNCOV 80	private _createTable() {	×
81	const creaOpts = {};	×
82	this._fields.map((field, index) => {	×
83	creaOpts[field.name] = this._cols[index];
UNCOV 84	});	×
85	return new arrow.Table(creaOpts);
86	}
87
UNCOV 88	getTable() {	×
89	return this._arrowTable;
90	}
91
UNCOV 92	update(updateData: arrow.Vector<any>[]) {	×
93	this._cols = updateData;	×
94	this._numColumns = this._cols.length;	×
95	this._numRows = this._cols[0].length;	×
96	this._numChunks = this._cols[0].data.length;
UNCOV 97		×
98	this._arrowTable = this._createTable();
99
100	// cache column data to make valueAt() faster
101	// this._colData = this._cols.map(c => c.toArray());
102	}
103
UNCOV 104	numChunks(): number {	×
105	return this._numChunks;
106	}
107
UNCOV 108	numRows(): number {	×
109	return this._numRows;
110	}
111
UNCOV 112	numColumns(): number {	×
113	return this._numColumns;
114	}
115
116	valueAt(rowIndex: number, columnIndex: number): any {
UNCOV 117	// return this._colData[columnIndex][rowIndex];	×
118	return this._cols[columnIndex].get(rowIndex);
119	}
120
UNCOV 121	row(rowIndex: number, sharedRow?: SharedRowOptions): DataRow {	×
122	const tSharedRow = DataRow.createSharedRow(sharedRow);	×
123	if (tSharedRow) {	×
124	tSharedRow.setSource(this, rowIndex);	×
125	return tSharedRow;
126	}
UNCOV 127		×
128	return new DataRow(this, rowIndex);
129	}
130
131	rowAsArray(rowIndex: number): any[] {
UNCOV 132	// return this._colData.map(col => col[rowIndex]);	×
133	return this._cols.map(col => col.get(rowIndex));
134	}
135
UNCOV 136	rows(sharedRow: SharedRowOptions) {	×
137	const tSharedRow = DataRow.createSharedRow(sharedRow);	×
138	return rowsIterator(this, tSharedRow);
139	}
140
UNCOV 141	column(columnIndex: number) {	×
142	return columnIterator(this, columnIndex);
143	}
144
UNCOV 145	getColumn(columnIndex: number): arrow.Vector {	×
146	return this._cols[columnIndex];
147	}
148
UNCOV 149	getField(columnIndex: number): ProtoDatasetField {	×
150	return this._fields[columnIndex];
151	}
152
UNCOV 153	flattenData(): any[][] {	×
154	const data: any[][] = [];	×
155	for (let i = 0; i < this._numRows; ++i) {	×
156	data.push(this.rowAsArray(i));
UNCOV 157	}	×
158	return data;
159	}
160
UNCOV 161	getPlainIndex(): number[] {	×
162	return [...Array(this._numRows).keys()];
163	}
164
165	map<T>(
166	func: (row: DataRow, index: number) => T,
167	sharedRow?: SharedRowOptions,	×
168	options: RangeOptions = {}
UNCOV 169	): T[] {	×
170	const tSharedRow = DataRow.createSharedRow(sharedRow);
UNCOV 171		×
172	const {start = 0, end = this.numRows()} = options;	×
173	const endRow = Math.min(this.numRows(), end);
UNCOV 174		×
175	const out: T[] = [];	×
176	for (let rowIndex = start; rowIndex < endRow; ++rowIndex) {	×
177	const row = this.row(rowIndex, tSharedRow);	×
178	out.push(func(row, rowIndex));
UNCOV 179	}	×
180	return out;
181	}
182		×
UNCOV 183	mapIndex<T>(func: ({index}, dc: DataContainerInterface) => T, options: RangeOptions = {}): T[] {	×
184	const {start = 0, end = this.numRows()} = options;	×
185	const endRow = Math.min(this.numRows(), end);
UNCOV 186		×
187	const out: T[] = [];	×
188	for (let rowIndex = start; rowIndex < endRow; ++rowIndex) {	×
189	out.push(func({index: rowIndex}, this));
UNCOV 190	}	×
191	return out;
192	}
193
194	find(
195	func: (row: DataRow, index: number) => boolean,
196	sharedRow?: SharedRowOptions
UNCOV 197	): DataRow \| undefined {	×
198	const tSharedRow = DataRow.createSharedRow(sharedRow);
UNCOV 199		×
200	for (let rowIndex = 0; rowIndex < this._numRows; ++rowIndex) {	×
201	const row = this.row(rowIndex, tSharedRow);	×
202	if (func(row, rowIndex)) {	×
203	return row;
204	}
UNCOV 205	}	×
206	return undefined;
207	}
208
209	reduce<T>(
210	func: (acc: T, row: DataRow, index: number) => T,
211	initialValue: T,
212	sharedRow?: SharedRowOptions
UNCOV 213	): T {	×
214	const tSharedRow = DataRow.createSharedRow(sharedRow);
UNCOV 215		×
216	for (let rowIndex = 0; rowIndex < this._numRows; ++rowIndex) {	×
217	const row = this.row(rowIndex, tSharedRow);	×
218	initialValue = func(initialValue, row, rowIndex);
UNCOV 219	}	×
220	return initialValue;
221	}
222	}
223
224	/**
225	* Convert arrow data type to kepler.gl field types
226	*
227	* @param arrowType the arrow data type
228	* @returns corresponding type in `ALL_FIELD_TYPES`
229	*/
230	export function arrowDataTypeToFieldType(arrowType: arrow.DataType): string {
231	// Note: this function doesn't return ALL_FIELD_TYPES.geojson or ALL_FIELD_TYPES.array, which
UNCOV 232	// should be further detected by caller	×
233	if (arrow.DataType.isDate(arrowType)) {	×
234	return ALL_FIELD_TYPES.date;	×
235	} else if (arrow.DataType.isTimestamp(arrowType) \|\| arrow.DataType.isTime(arrowType)) {	×
236	return ALL_FIELD_TYPES.timestamp;	×
237	} else if (arrow.DataType.isFloat(arrowType)) {	×
238	return ALL_FIELD_TYPES.real;	×
239	} else if (arrow.DataType.isInt(arrowType)) {	×
240	return ALL_FIELD_TYPES.integer;	×
241	} else if (arrow.DataType.isBool(arrowType)) {	×
242	return ALL_FIELD_TYPES.boolean;	×
243	} else if (arrow.DataType.isUtf8(arrowType) \|\| arrow.DataType.isNull(arrowType)) {	×
244	return ALL_FIELD_TYPES.string;	×
245	} else if (	×
246	arrow.DataType.isBinary(arrowType) \|\|
247	arrow.DataType.isDictionary(arrowType) \|\|
248	arrow.DataType.isFixedSizeBinary(arrowType) \|\|
249	arrow.DataType.isFixedSizeList(arrowType) \|\|
250	arrow.DataType.isList(arrowType) \|\|
251	arrow.DataType.isMap(arrowType) \|\|
252	arrow.DataType.isStruct(arrowType)
UNCOV 253	) {	×
254	return ALL_FIELD_TYPES.object;
UNCOV 255	}	×
256	globalConsole.warn(`Unsupported arrow type: ${arrowType}`);	×
257	return ALL_FIELD_TYPES.string;
258	}
259
260	/**
261	* Convert arrow data type to analyzer type
262	*
263	* @param arrowType the arrow data type
264	* @returns corresponding type in `AnalyzerDATA_TYPES`
265	*/
266	export function arrowDataTypeToAnalyzerDataType(
267	arrowType: arrow.DataType
UNCOV 268	): typeof AnalyzerDATA_TYPES {	×
269	if (arrow.DataType.isDate(arrowType)) {	×
270	return AnalyzerDATA_TYPES.DATE;	×
271	} else if (arrow.DataType.isTimestamp(arrowType) \|\| arrow.DataType.isTime(arrowType)) {	×
272	return AnalyzerDATA_TYPES.DATETIME;	×
273	} else if (arrow.DataType.isFloat(arrowType)) {	×
274	return AnalyzerDATA_TYPES.FLOAT;	×
275	} else if (arrow.DataType.isInt(arrowType)) {	×
276	return AnalyzerDATA_TYPES.INT;	×
277	} else if (arrow.DataType.isBool(arrowType)) {	×
278	return AnalyzerDATA_TYPES.BOOLEAN;	×
279	} else if (arrow.DataType.isUtf8(arrowType) \|\| arrow.DataType.isNull(arrowType)) {	×
280	return AnalyzerDATA_TYPES.STRING;	×
281	} else if (	×
282	arrow.DataType.isBinary(arrowType) \|\|
283	arrow.DataType.isDictionary(arrowType) \|\|
284	arrow.DataType.isFixedSizeBinary(arrowType) \|\|
285	arrow.DataType.isFixedSizeList(arrowType) \|\|
286	arrow.DataType.isList(arrowType) \|\|
287	arrow.DataType.isMap(arrowType) \|\|
288	arrow.DataType.isStruct(arrowType)
UNCOV 289	) {	×
290	return AnalyzerDATA_TYPES.OBJECT;
UNCOV 291	}	×
292	globalConsole.warn(`Unsupported arrow type: ${arrowType}`);	×
293	return AnalyzerDATA_TYPES.STRING;
294	}

keplergl / kepler.gl / 19670167673

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous