26365832078

Committed 24 May 2026 03:53PM UTC coverage: 74.573% (+0.06%) from 74.51%

Build # 26365832078

Build Type

push

github

Committed by

web-flow

Commit Message

chore: Simplify arrow examples (#2636)

Coverage Stats

8463 of 12834 branches covered (65.94%)

Branch coverage included in aggregate %.

281 of 357 new or added lines in 3 files covered. (78.71%)

733 existing lines in 26 files now uncovered.

17709 of 22262 relevant lines covered (79.55%)

4350.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.91

/modules/arrow/src/arrow/arrow-gpu-appendable.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {BufferLayout, ShaderLayout} from '@luma.gl/core';
import {
  Data,
  DataType,
  Dictionary,
  Field,
  Int16,
  Int32,
  Int8,
  RecordBatch,
  Schema,
  Uint16,
  Uint32,
  Uint8,
  Utf8,
  util
} from 'apache-arrow';
import {findArrowFieldByPath, getArrowDataByPath, getArrowSchemaPaths} from './arrow-paths';
import {getArrowVertexFormat} from './arrow-shader-layout';
import {
  getSignedShaderType,
  isInstanceArrowType,
  isNumericArrowType,
  type ArrowColumnInfo,
  type AttributeArrowType,
  type NumericArrowType,
  type VariableLengthAttributeArrowType
} from './arrow-types';
import {validateArrowGPUDataDirectUpload} from './arrow-gpu-data';

/** One shader-selected Arrow column that can append directly into GPU storage. */
export type AppendableGPUColumn = {
  attributeName: string;
  arrowPath: string;
  field: Field;
  bufferLayout?: BufferLayout;
  storageBinding?: boolean;
};

/** Validated Arrow data selected for one appendable GPU column. */
export type AppendableGPUColumnData = {
  column: AppendableGPUColumn;
  data: Data<AttributeArrowType | Utf8 | ArrowUtf8Dictionary>;
};

type ArrowUtf8DictionaryIndexType = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32;
type ArrowUtf8Dictionary = Dictionary<Utf8, ArrowUtf8DictionaryIndexType>;

/** Resolves shader-selected Arrow columns for appendable GPU storage. */
export function getAppendableGPUColumns(props: {
  schema: Schema;
  shaderLayout: ShaderLayout;
  arrowPaths?: Record<string, string>;
  allowWebGLOnlyFormats?: boolean;
}): AppendableGPUColumn[] {
  const schemaPaths = new Set(getArrowSchemaPaths(props.schema));
  const columns: AppendableGPUColumn[] = [];

  for (const attribute of props.shaderLayout.attributes) {
    const hasExplicitPath = Boolean(
      props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, attribute.name)
    );
    const arrowPath = props.arrowPaths?.[attribute.name] || attribute.name;
    if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {
      continue;
    }

    const field = findArrowFieldByPath(props.schema, arrowPath);
    if (!field) {
      throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);
    }
    if (!isInstanceArrowType(field.type)) {
      throw new Error(`Arrow column "${arrowPath}" is not compatible with shader attributes`);
    }

    const format = getArrowVertexFormat(getArrowColumnInfoFromType(field.type), attribute.type, {
      allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
    });
    columns.push({
      attributeName: attribute.name,
      arrowPath,
      field,
      bufferLayout: {
        name: attribute.name,
        format,
        ...(attribute.stepMode ? {stepMode: attribute.stepMode} : {})
      }
    });
  }

  const selectedNames = new Set(columns.map(column => column.attributeName));
  for (const binding of props.shaderLayout.bindings) {
    if (
      (binding.type !== 'storage' && binding.type !== 'read-only-storage') ||
      'format' in binding
    ) {
      continue;
    }
    if (selectedNames.has(binding.name)) {
      throw new Error(
        `Appendable Arrow shader input "${binding.name}" cannot be both an attribute and a storage binding`
      );
    }
    const hasExplicitPath = Boolean(
      props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, binding.name)
    );
    const arrowPath = props.arrowPaths?.[binding.name] || binding.name;
    if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {
      continue;
    }
    const field = findArrowFieldByPath(props.schema, arrowPath);
    if (!field) {
      throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);
    }
    if (
      !isInstanceArrowType(field.type) &&
      !DataType.isUtf8(field.type) &&
      !isArrowUtf8DictionaryType(field.type)
    ) {
      throw new Error(`Arrow column "${arrowPath}" is not compatible with appendable GPU storage`);
    }
    columns.push({
      attributeName: binding.name,
      arrowPath,
      field,
      storageBinding: true
    });
    selectedNames.add(binding.name);
  }

  return columns;
}

/** Validates append compatibility and extracts one direct-upload Arrow data chunk per selected column. */
export function getAppendableGPUColumnData(
  recordBatch: RecordBatch,
  columns: AppendableGPUColumn[],
  ownerName: string
): AppendableGPUColumnData[] {
  for (const column of columns) {
    const sourceField = findArrowFieldByPath(recordBatch.schema, column.arrowPath);
    if (!sourceField || !util.compareTypes(sourceField.type, column.field.type)) {
      throw new Error(`${ownerName} column "${column.arrowPath}" does not match the source schema`);
    }
  }

  return columns.map(column => {
    const data = getArrowDataByPath(recordBatch, column.arrowPath) as Data<
      AttributeArrowType | Utf8 | ArrowUtf8Dictionary
    >;
    if (data.length !== recordBatch.numRows) {
      throw new Error(`${ownerName} column "${column.arrowPath}" row count mismatch`);
    }
    if (!isArrowUtf8DictionaryType(data.type)) {
      validateArrowGPUDataDirectUpload(
        column.attributeName,
        data as Data<AttributeArrowType | Utf8 | VariableLengthAttributeArrowType>
      );
    }
    return {column, data};
  });
}

/** Derives Arrow column characteristics needed to map one selected column to a vertex format. */
export function getArrowColumnInfoFromType(type: AttributeArrowType): ArrowColumnInfo {
  let numericType = type as NumericArrowType;
  let components: 1 | 2 | 3 | 4 = 1;
  if (DataType.isFixedSizeList(type)) {
    numericType = type.children[0].type as NumericArrowType;
    if (type.listSize < 1 || type.listSize > 4) {
      throw new Error('Attribute column fixed list size must be between 1 and 4');
    }
    components = type.listSize as 1 | 2 | 3 | 4;
  }
  if (!isNumericArrowType(numericType)) {
    throw new Error('Attribute column must be numeric or fixed list of numeric');
  }
  return {
    signedDataType: getSignedShaderType(numericType, components),
    components,
    stepMode: 'instance',
    values: [],
    offsets: []
  };
}

function isArrowUtf8DictionaryType(type: DataType): type is ArrowUtf8Dictionary {
  return (
    DataType.isDictionary(type) &&
    type.dictionary instanceof Utf8 &&
    DataType.isInt(type.indices) &&
    type.indices.bitWidth <= 32
  );
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {BufferLayout, ShaderLayout} from '@luma.gl/core';
6	import {
7	Data,
8	DataType,
9	Dictionary,
10	Field,
11	Int16,
12	Int32,
13	Int8,
14	RecordBatch,
15	Schema,
16	Uint16,
17	Uint32,
18	Uint8,
19	Utf8,
20	util
21	} from 'apache-arrow';
22	import {findArrowFieldByPath, getArrowDataByPath, getArrowSchemaPaths} from './arrow-paths';
23	import {getArrowVertexFormat} from './arrow-shader-layout';
24	import {
25	getSignedShaderType,
26	isInstanceArrowType,
27	isNumericArrowType,
28	type ArrowColumnInfo,
29	type AttributeArrowType,
30	type NumericArrowType,
31	type VariableLengthAttributeArrowType
32	} from './arrow-types';
33	import {validateArrowGPUDataDirectUpload} from './arrow-gpu-data';
34
35	/** One shader-selected Arrow column that can append directly into GPU storage. */
36	export type AppendableGPUColumn = {
37	attributeName: string;
38	arrowPath: string;
39	field: Field;
40	bufferLayout?: BufferLayout;
41	storageBinding?: boolean;
42	};
43
44	/** Validated Arrow data selected for one appendable GPU column. */
45	export type AppendableGPUColumnData = {
46	column: AppendableGPUColumn;
47	data: Data<AttributeArrowType \| Utf8 \| ArrowUtf8Dictionary>;
48	};
49
50	type ArrowUtf8DictionaryIndexType = Int8 \| Int16 \| Int32 \| Uint8 \| Uint16 \| Uint32;
51	type ArrowUtf8Dictionary = Dictionary<Utf8, ArrowUtf8DictionaryIndexType>;
52
53	/** Resolves shader-selected Arrow columns for appendable GPU storage. */
54	export function getAppendableGPUColumns(props: {
55	schema: Schema;
56	shaderLayout: ShaderLayout;
57	arrowPaths?: Record<string, string>;
58	allowWebGLOnlyFormats?: boolean;
59	}): AppendableGPUColumn[] {
60	const schemaPaths = new Set(getArrowSchemaPaths(props.schema));	3✔
61	const columns: AppendableGPUColumn[] = [];	3✔
62
63	for (const attribute of props.shaderLayout.attributes) {	3✔
64	const hasExplicitPath = Boolean(	5✔
65	props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, attribute.name)	5!
66	);
67	const arrowPath = props.arrowPaths?.[attribute.name] \|\| attribute.name;	5✔
68	if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {	5!
UNCOV 69	continue;	×
70	}
71
72	const field = findArrowFieldByPath(props.schema, arrowPath);	5✔
73	if (!field) {	5!
UNCOV 74	throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);	×
75	}
76	if (!isInstanceArrowType(field.type)) {	5!
UNCOV 77	throw new Error(`Arrow column "${arrowPath}" is not compatible with shader attributes`);	×
78	}
79
80	const format = getArrowVertexFormat(getArrowColumnInfoFromType(field.type), attribute.type, {	5✔
81	allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
82	});
83	columns.push({	5✔
84	attributeName: attribute.name,
85	arrowPath,
86	field,
87	bufferLayout: {
88	name: attribute.name,
89	format,
90	...(attribute.stepMode ? {stepMode: attribute.stepMode} : {})	5✔
91	}
92	});
93	}
94
95	const selectedNames = new Set(columns.map(column => column.attributeName));	5✔
96	for (const binding of props.shaderLayout.bindings) {	3✔
97	if (	1!
98	(binding.type !== 'storage' && binding.type !== 'read-only-storage') \|\|	3✔
99	'format' in binding
100	) {
UNCOV 101	continue;	×
102	}
103	if (selectedNames.has(binding.name)) {	1!
104	throw new Error(	×
105	`Appendable Arrow shader input "${binding.name}" cannot be both an attribute and a storage binding`
106	);
107	}
108	const hasExplicitPath = Boolean(	1✔
109	props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, binding.name)	1!
110	);
111	const arrowPath = props.arrowPaths?.[binding.name] \|\| binding.name;	1✔
112	if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {	1!
UNCOV 113	continue;	×
114	}
115	const field = findArrowFieldByPath(props.schema, arrowPath);	1✔
116	if (!field) {	1!
UNCOV 117	throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);	×
118	}
119	if (	1!
120	!isInstanceArrowType(field.type) &&	2!
121	!DataType.isUtf8(field.type) &&
122	!isArrowUtf8DictionaryType(field.type)
123	) {
UNCOV 124	throw new Error(`Arrow column "${arrowPath}" is not compatible with appendable GPU storage`);	×
125	}
126	columns.push({	1✔
127	attributeName: binding.name,
128	arrowPath,
129	field,
130	storageBinding: true
131	});
132	selectedNames.add(binding.name);	1✔
133	}
134
135	return columns;	3✔
136	}
137
138	/** Validates append compatibility and extracts one direct-upload Arrow data chunk per selected column. */
139	export function getAppendableGPUColumnData(
140	recordBatch: RecordBatch,
141	columns: AppendableGPUColumn[],
142	ownerName: string
143	): AppendableGPUColumnData[] {
144	for (const column of columns) {	5✔
145	const sourceField = findArrowFieldByPath(recordBatch.schema, column.arrowPath);	10✔
146	if (!sourceField \|\| !util.compareTypes(sourceField.type, column.field.type)) {	10!
147	throw new Error(`${ownerName} column "${column.arrowPath}" does not match the source schema`);	×
148	}
149	}
150
151	return columns.map(column => {	5✔
152	const data = getArrowDataByPath(recordBatch, column.arrowPath) as Data<	10✔
153	AttributeArrowType \| Utf8 \| ArrowUtf8Dictionary
154	>;
155	if (data.length !== recordBatch.numRows) {	10!
UNCOV 156	throw new Error(`${ownerName} column "${column.arrowPath}" row count mismatch`);	×
157	}
158	if (!isArrowUtf8DictionaryType(data.type)) {	10!
159	validateArrowGPUDataDirectUpload(	10✔
160	column.attributeName,
161	data as Data<AttributeArrowType \| Utf8 \| VariableLengthAttributeArrowType>
162	);
163	}
164	return {column, data};	10✔
165	});
166	}
167
168	/** Derives Arrow column characteristics needed to map one selected column to a vertex format. */
169	export function getArrowColumnInfoFromType(type: AttributeArrowType): ArrowColumnInfo {
170	let numericType = type as NumericArrowType;	5✔
171	let components: 1 \| 2 \| 3 \| 4 = 1;	5✔
172	if (DataType.isFixedSizeList(type)) {	5!
173	numericType = type.children[0].type as NumericArrowType;	5✔
174	if (type.listSize < 1 \|\| type.listSize > 4) {	5!
UNCOV 175	throw new Error('Attribute column fixed list size must be between 1 and 4');	×
176	}
177	components = type.listSize as 1 \| 2 \| 3 \| 4;	5✔
178	}
179	if (!isNumericArrowType(numericType)) {	5!
UNCOV 180	throw new Error('Attribute column must be numeric or fixed list of numeric');	×
181	}
182	return {	5✔
183	signedDataType: getSignedShaderType(numericType, components),
184	components,
185	stepMode: 'instance',
186	values: [],
187	offsets: []
188	};
189	}
190
191	function isArrowUtf8DictionaryType(type: DataType): type is ArrowUtf8Dictionary {
192	return (	10✔
193	DataType.isDictionary(type) &&	10!
194	type.dictionary instanceof Utf8 &&
195	DataType.isInt(type.indices) &&
196	type.indices.bitWidth <= 32
197	);
198	}

visgl / luma.gl / 26365832078

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous