25990278810

Committed 17 May 2026 12:01PM UTC coverage: 75.092% (+0.2%) from 74.881%

Build # 25990278810

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Columnar GPU-data stack (#2616)

Coverage Stats

6711 of 10084 branches covered (66.55%)

Branch coverage included in aggregate %.

625 of 865 new or added lines in 22 files covered. (72.25%)

1 existing line in 1 file now uncovered.

14631 of 18337 relevant lines covered (79.79%)

792.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.77

/modules/arrow/src/arrow/arrow-vector-utils.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import * as arrow from 'apache-arrow';
import {getArrowVectorBufferSource} from './arrow-gpu-data';
import {isNumericArrowType, type AttributeArrowType, type NumericArrowType} from './arrow-types';

type IntegerTypedArray =
  | Int8Array
  | Int16Array
  | Int32Array
  | Uint8Array
  | Uint16Array
  | Uint32Array;

export type ArrowVectorRowMapping = IntegerTypedArray | arrow.Vector<arrow.Int>;

const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: T;
  length: number;
  data: T['TArray'];
}) => arrow.Data<T>;

const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: arrow.FixedSizeList<T>;
  length: number;
  nullCount: number;
  nullBitmap: null;
  child: arrow.Data<T>;
}) => arrow.Data<arrow.FixedSizeList<T>>;

/**
 * Gather scalar or FixedSizeList Arrow rows into a new contiguous vector.
 *
 * This is useful when compact table data needs to be expanded into vertex-aligned data.
 */
export function expandArrowVector<T extends AttributeArrowType>(
  vector: arrow.Vector<T>,
  rowMapping: ArrowVectorRowMapping
): arrow.Vector<T> {
  const {numericType, rowStride} = getExpansionVectorInfo(vector);
  const sourceValues = getArrowVectorBufferSource(vector);
  const rowIndices = getRowIndices(rowMapping);
  const expandedValues = createTypedArrayLike(
    sourceValues,
    rowIndices.length * rowStride
  ) as NumericArrowType['TArray'];

  for (let targetRowIndex = 0; targetRowIndex < rowIndices.length; targetRowIndex++) {
    const sourceRowIndex = Number(rowIndices[targetRowIndex]);
    validateRowIndex(sourceRowIndex, vector.length);

    const sourceOffset = sourceRowIndex * rowStride;
    const targetOffset = targetRowIndex * rowStride;
    expandedValues.set(
      sourceValues.subarray(sourceOffset, sourceOffset + rowStride) as never,
      targetOffset
    );
  }

  if (arrow.DataType.isFixedSizeList(vector.type)) {
    return makeFixedSizeListVector(
      numericType,
      rowStride as 1 | 2 | 3 | 4,
      expandedValues
    ) as unknown as arrow.Vector<T>;
  }

  const data = makeNumericData({
    type: numericType,
    length: rowIndices.length,
    data: expandedValues as never
  });
  return arrow.makeVector(data) as arrow.Vector<T>;
}

function makeFixedSizeListVector<T extends NumericArrowType>(
  numericType: T,
  rowStride: 1 | 2 | 3 | 4,
  values: T['TArray']
): arrow.Vector<arrow.FixedSizeList<T>> {
  const childData = makeNumericData({
    type: numericType,
    length: values.length,
    data: values
  });
  const listType = new arrow.FixedSizeList(rowStride, new arrow.Field('value', numericType));
  const listData = makeFixedSizeListData({
    type: listType,
    length: values.length / rowStride,
    nullCount: 0,
    nullBitmap: null,
    child: childData
  });

  return arrow.makeVector(listData);
}

function getExpansionVectorInfo<T extends AttributeArrowType>(
  vector: arrow.Vector<T>
): {numericType: NumericArrowType; rowStride: number} {
  if (arrow.DataType.isFixedSizeList(vector.type)) {
    const numericType = vector.type.children[0].type;
    if (!isSupportedExpansionNumericType(numericType)) {
      throw new Error(`expandArrowVector does not support Arrow type ${numericType}`);
    }
    if (vector.type.listSize < 1 || vector.type.listSize > 4) {
      throw new Error('expandArrowVector only supports FixedSizeList sizes between 1 and 4');
    }
    return {numericType, rowStride: vector.type.listSize};
  }

  if (!isSupportedExpansionNumericType(vector.type)) {
    throw new Error(`expandArrowVector does not support Arrow type ${vector.type}`);
  }
  return {numericType: vector.type, rowStride: 1};
}

function isSupportedExpansionNumericType(type: arrow.DataType): type is NumericArrowType {
  if (!isNumericArrowType(type)) {
    return false;
  }
  if (arrow.DataType.isInt(type)) {
    return type.bitWidth <= 32;
  }
  return type.precision !== arrow.Precision.DOUBLE;
}

function getRowIndices(rowMapping: ArrowVectorRowMapping): IntegerTypedArray {
  if (isIntegerTypedArray(rowMapping)) {
    return rowMapping;
  }

  if (!arrow.DataType.isInt(rowMapping.type) || rowMapping.type.bitWidth > 32) {
    throw new Error('expandArrowVector row mapping must use 8, 16, or 32-bit integers');
  }

  return getArrowVectorBufferSource(rowMapping) as IntegerTypedArray;
}

function isIntegerTypedArray(value: unknown): value is IntegerTypedArray {
  return (
    value instanceof Int8Array ||
    value instanceof Int16Array ||
    value instanceof Int32Array ||
    value instanceof Uint8Array ||
    value instanceof Uint16Array ||
    value instanceof Uint32Array
  );
}

function validateRowIndex(rowIndex: number, rowCount: number): void {
  if (!Number.isInteger(rowIndex)) {
    throw new Error('expandArrowVector row mapping must contain integers');
  }
  if (rowIndex < 0) {
    throw new Error('expandArrowVector row mapping cannot contain negative indices');
  }
  if (rowIndex >= rowCount) {
    throw new Error(
      `expandArrowVector row mapping index ${rowIndex} is outside vector length ${rowCount}`
    );
  }
}

function createTypedArrayLike(
  source: NumericArrowType['TArray'],
  length: number
): NumericArrowType['TArray'] {
  const TypedArrayConstructor = source.constructor as {
    new (typedArrayLength: number): NumericArrowType['TArray'];
  };
  return new TypedArrayConstructor(length);
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import * as arrow from 'apache-arrow';
6	import {getArrowVectorBufferSource} from './arrow-gpu-data';
7	import {isNumericArrowType, type AttributeArrowType, type NumericArrowType} from './arrow-types';
8
9	type IntegerTypedArray =
10	\| Int8Array
11	\| Int16Array
12	\| Int32Array
13	\| Uint8Array
14	\| Uint16Array
15	\| Uint32Array;
16
17	export type ArrowVectorRowMapping = IntegerTypedArray \| arrow.Vector<arrow.Int>;
18
19	const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {	13✔
20	type: T;
21	length: number;
22	data: T['TArray'];
23	}) => arrow.Data<T>;
24
25	const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {	13✔
26	type: arrow.FixedSizeList<T>;
27	length: number;
28	nullCount: number;
29	nullBitmap: null;
30	child: arrow.Data<T>;
31	}) => arrow.Data<arrow.FixedSizeList<T>>;
32
33	/**
34	* Gather scalar or FixedSizeList Arrow rows into a new contiguous vector.
35	*
36	* This is useful when compact table data needs to be expanded into vertex-aligned data.
37	*/
38	export function expandArrowVector<T extends AttributeArrowType>(
39	vector: arrow.Vector<T>,
40	rowMapping: ArrowVectorRowMapping
41	): arrow.Vector<T> {
42	const {numericType, rowStride} = getExpansionVectorInfo(vector);	7✔
43	const sourceValues = getArrowVectorBufferSource(vector);	7✔
44	const rowIndices = getRowIndices(rowMapping);	7✔
45	const expandedValues = createTypedArrayLike(	7✔
46	sourceValues,
47	rowIndices.length * rowStride
48	) as NumericArrowType['TArray'];
49
50	for (let targetRowIndex = 0; targetRowIndex < rowIndices.length; targetRowIndex++) {	7✔
51	const sourceRowIndex = Number(rowIndices[targetRowIndex]);	12✔
52	validateRowIndex(sourceRowIndex, vector.length);	12✔
53
54	const sourceOffset = sourceRowIndex * rowStride;	12✔
55	const targetOffset = targetRowIndex * rowStride;	12✔
56	expandedValues.set(	12✔
57	sourceValues.subarray(sourceOffset, sourceOffset + rowStride) as never,
58	targetOffset
59	);
60	}
61
62	if (arrow.DataType.isFixedSizeList(vector.type)) {	3✔
63	return makeFixedSizeListVector(	2✔
64	numericType,
65	rowStride as 1 \| 2 \| 3 \| 4,
66	expandedValues
67	) as unknown as arrow.Vector<T>;
68	}
69
70	const data = makeNumericData({	1✔
71	type: numericType,
72	length: rowIndices.length,
73	data: expandedValues as never
74	});
75	return arrow.makeVector(data) as arrow.Vector<T>;	1✔
76	}
77
78	function makeFixedSizeListVector<T extends NumericArrowType>(
79	numericType: T,
80	rowStride: 1 \| 2 \| 3 \| 4,
81	values: T['TArray']
82	): arrow.Vector<arrow.FixedSizeList<T>> {
83	const childData = makeNumericData({	2✔
84	type: numericType,
85	length: values.length,
86	data: values
87	});
88	const listType = new arrow.FixedSizeList(rowStride, new arrow.Field('value', numericType));	2✔
89	const listData = makeFixedSizeListData({	2✔
90	type: listType,
91	length: values.length / rowStride,
92	nullCount: 0,
93	nullBitmap: null,
94	child: childData
95	});
96
97	return arrow.makeVector(listData);	2✔
98	}
99
100	function getExpansionVectorInfo<T extends AttributeArrowType>(
101	vector: arrow.Vector<T>
102	): {numericType: NumericArrowType; rowStride: number} {
103	if (arrow.DataType.isFixedSizeList(vector.type)) {	7✔
104	const numericType = vector.type.children[0].type;	2✔
105	if (!isSupportedExpansionNumericType(numericType)) {	2!
NEW 106	throw new Error(`expandArrowVector does not support Arrow type ${numericType}`);	×
107	}
108	if (vector.type.listSize < 1 \|\| vector.type.listSize > 4) {	2!
NEW 109	throw new Error('expandArrowVector only supports FixedSizeList sizes between 1 and 4');	×
110	}
111	return {numericType, rowStride: vector.type.listSize};	2✔
112	}
113
114	if (!isSupportedExpansionNumericType(vector.type)) {	5✔
115	throw new Error(`expandArrowVector does not support Arrow type ${vector.type}`);	1✔
116	}
117	return {numericType: vector.type, rowStride: 1};	4✔
118	}
119
120	function isSupportedExpansionNumericType(type: arrow.DataType): type is NumericArrowType {
121	if (!isNumericArrowType(type)) {	7✔
122	return false;	1✔
123	}
124	if (arrow.DataType.isInt(type)) {	6✔
125	return type.bitWidth <= 32;	2✔
126	}
127	return type.precision !== arrow.Precision.DOUBLE;	4✔
128	}
129
130	function getRowIndices(rowMapping: ArrowVectorRowMapping): IntegerTypedArray {
131	if (isIntegerTypedArray(rowMapping)) {	6✔
132	return rowMapping;	4✔
133	}
134
135	if (!arrow.DataType.isInt(rowMapping.type) \|\| rowMapping.type.bitWidth > 32) {	2✔
136	throw new Error('expandArrowVector row mapping must use 8, 16, or 32-bit integers');	1✔
137	}
138
139	return getArrowVectorBufferSource(rowMapping) as IntegerTypedArray;	1✔
140	}
141
142	function isIntegerTypedArray(value: unknown): value is IntegerTypedArray {
143	return (	6✔
144	value instanceof Int8Array \|\|	32✔
145	value instanceof Int16Array \|\|
146	value instanceof Int32Array \|\|
147	value instanceof Uint8Array \|\|
148	value instanceof Uint16Array \|\|
149	value instanceof Uint32Array
150	);
151	}
152
153	function validateRowIndex(rowIndex: number, rowCount: number): void {
154	if (!Number.isInteger(rowIndex)) {	12!
NEW 155	throw new Error('expandArrowVector row mapping must contain integers');	×
156	}
157	if (rowIndex < 0) {	12✔
158	throw new Error('expandArrowVector row mapping cannot contain negative indices');	1✔
159	}
160	if (rowIndex >= rowCount) {	11✔
161	throw new Error(	1✔
162	`expandArrowVector row mapping index ${rowIndex} is outside vector length ${rowCount}`
163	);
164	}
165	}
166
167	function createTypedArrayLike(
168	source: NumericArrowType['TArray'],
169	length: number
170	): NumericArrowType['TArray'] {
171	const TypedArrayConstructor = source.constructor as {	5✔
172	new (typedArrayLength: number): NumericArrowType['TArray'];
173	};
174	return new TypedArrayConstructor(length);	5✔
175	}

visgl / luma.gl / 25990278810

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous