Build has been canceled!

26033117099

Committed 18 May 2026 12:20PM UTC coverage: 74.769% (-0.1%) from 74.887%

Build # 26033117099

Build Type

push

github

Committed by

web-flow

Commit Message

feat(arrow) Streaming ArrowTextLayer (#2620)

Coverage Stats

6882 of 10396 branches covered (66.2%)

Branch coverage included in aggregate %.

194 of 250 new or added lines in 9 files covered. (77.6%)

13 existing lines in 7 files now uncovered.

15038 of 18921 relevant lines covered (79.48%)

914.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.27

/modules/arrow/src/arrow/arrow-gpu-appendable.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {BufferLayout, ShaderLayout} from '@luma.gl/core';
import * as arrow from 'apache-arrow';
import {findArrowFieldByPath, getArrowDataByPath, getArrowSchemaPaths} from './arrow-paths';
import {getArrowVertexFormat} from './arrow-shader-layout';
import {
  getSignedShaderType,
  isInstanceArrowType,
  isNumericArrowType,
  type ArrowColumnInfo,
  type AttributeArrowType,
  type NumericArrowType
} from './arrow-types';
import {validateArrowGPUDataDirectUpload} from './arrow-gpu-data';

/** One shader-selected Arrow column that can append directly into GPU storage. */
export type AppendableGPUColumn = {
  attributeName: string;
  arrowPath: string;
  field: arrow.Field;
  bufferLayout?: BufferLayout;
  storageBinding?: boolean;
};

/** Validated Arrow data selected for one appendable GPU column. */
export type AppendableGPUColumnData = {
  column: AppendableGPUColumn;
  data: arrow.Data<AttributeArrowType | arrow.Utf8>;
};

/** Resolves shader-selected Arrow columns for appendable GPU storage. */
export function getAppendableGPUColumns(props: {
  schema: arrow.Schema;
  shaderLayout: ShaderLayout;
  arrowPaths?: Record<string, string>;
  allowWebGLOnlyFormats?: boolean;
}): AppendableGPUColumn[] {
  const schemaPaths = new Set(getArrowSchemaPaths(props.schema));
  const columns: AppendableGPUColumn[] = [];

  for (const attribute of props.shaderLayout.attributes) {
    const hasExplicitPath = Boolean(
      props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, attribute.name)
    );
    const arrowPath = props.arrowPaths?.[attribute.name] || attribute.name;
    if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {
      continue;
    }

    const field = findArrowFieldByPath(props.schema, arrowPath);
    if (!field) {
      throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);
    }
    if (!isInstanceArrowType(field.type)) {
      throw new Error(`Arrow column "${arrowPath}" is not compatible with shader attributes`);
    }

    const format = getArrowVertexFormat(getArrowColumnInfoFromType(field.type), attribute.type, {
      allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
    });
    columns.push({
      attributeName: attribute.name,
      arrowPath,
      field,
      bufferLayout: {
        name: attribute.name,
        format,
        ...(attribute.stepMode ? {stepMode: attribute.stepMode} : {})
      }
    });
  }

  const selectedNames = new Set(columns.map(column => column.attributeName));
  for (const binding of props.shaderLayout.bindings) {
    if (
      (binding.type !== 'storage' && binding.type !== 'read-only-storage') ||
      'format' in binding
    ) {
      continue;
    }
    if (selectedNames.has(binding.name)) {
      throw new Error(
        `Appendable Arrow shader input "${binding.name}" cannot be both an attribute and a storage binding`
      );
    }
    const hasExplicitPath = Boolean(
      props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, binding.name)
    );
    const arrowPath = props.arrowPaths?.[binding.name] || binding.name;
    if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {
      continue;
    }
    const field = findArrowFieldByPath(props.schema, arrowPath);
    if (!field) {
      throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);
    }
    if (!isInstanceArrowType(field.type) && !arrow.DataType.isUtf8(field.type)) {
      throw new Error(`Arrow column "${arrowPath}" is not compatible with appendable GPU storage`);
    }
    columns.push({
      attributeName: binding.name,
      arrowPath,
      field,
      storageBinding: true
    });
    selectedNames.add(binding.name);
  }

  return columns;
}

/** Validates append compatibility and extracts one direct-upload Arrow data chunk per selected column. */
export function getAppendableGPUColumnData(
  recordBatch: arrow.RecordBatch,
  columns: AppendableGPUColumn[],
  ownerName: string
): AppendableGPUColumnData[] {
  for (const column of columns) {
    const sourceField = findArrowFieldByPath(recordBatch.schema, column.arrowPath);
    if (!sourceField || !arrow.util.compareTypes(sourceField.type, column.field.type)) {
      throw new Error(`${ownerName} column "${column.arrowPath}" does not match the source schema`);
    }
  }

  return columns.map(column => {
    const data = getArrowDataByPath(recordBatch, column.arrowPath) as arrow.Data<
      AttributeArrowType | arrow.Utf8
    >;
    if (data.length !== recordBatch.numRows) {
      throw new Error(`${ownerName} column "${column.arrowPath}" row count mismatch`);
    }
    validateArrowGPUDataDirectUpload(column.attributeName, data);
    return {column, data};
  });
}

/** Derives Arrow column characteristics needed to map one selected column to a vertex format. */
export function getArrowColumnInfoFromType(type: AttributeArrowType): ArrowColumnInfo {
  let numericType = type as NumericArrowType;
  let components: 1 | 2 | 3 | 4 = 1;
  if (arrow.DataType.isFixedSizeList(type)) {
    numericType = type.children[0].type as NumericArrowType;
    if (type.listSize < 1 || type.listSize > 4) {
      throw new Error('Attribute column fixed list size must be between 1 and 4');
    }
    components = type.listSize as 1 | 2 | 3 | 4;
  }
  if (!isNumericArrowType(numericType)) {
    throw new Error('Attribute column must be numeric or fixed list of numeric');
  }
  return {
    signedDataType: getSignedShaderType(numericType, components),
    components,
    stepMode: 'instance',
    values: [],
    offsets: []
  };
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {BufferLayout, ShaderLayout} from '@luma.gl/core';
6	import * as arrow from 'apache-arrow';
7	import {findArrowFieldByPath, getArrowDataByPath, getArrowSchemaPaths} from './arrow-paths';
8	import {getArrowVertexFormat} from './arrow-shader-layout';
9	import {
10	getSignedShaderType,
11	isInstanceArrowType,
12	isNumericArrowType,
13	type ArrowColumnInfo,
14	type AttributeArrowType,
15	type NumericArrowType
16	} from './arrow-types';
17	import {validateArrowGPUDataDirectUpload} from './arrow-gpu-data';
18
19	/** One shader-selected Arrow column that can append directly into GPU storage. */
20	export type AppendableGPUColumn = {
21	attributeName: string;
22	arrowPath: string;
23	field: arrow.Field;
24	bufferLayout?: BufferLayout;
25	storageBinding?: boolean;
26	};
27
28	/** Validated Arrow data selected for one appendable GPU column. */
29	export type AppendableGPUColumnData = {
30	column: AppendableGPUColumn;
31	data: arrow.Data<AttributeArrowType \| arrow.Utf8>;
32	};
33
34	/** Resolves shader-selected Arrow columns for appendable GPU storage. */
35	export function getAppendableGPUColumns(props: {
36	schema: arrow.Schema;
37	shaderLayout: ShaderLayout;
38	arrowPaths?: Record<string, string>;
39	allowWebGLOnlyFormats?: boolean;
40	}): AppendableGPUColumn[] {
41	const schemaPaths = new Set(getArrowSchemaPaths(props.schema));	3✔
42	const columns: AppendableGPUColumn[] = [];	3✔
43
44	for (const attribute of props.shaderLayout.attributes) {	3✔
45	const hasExplicitPath = Boolean(	5✔
46	props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, attribute.name)	5!
47	);
48	const arrowPath = props.arrowPaths?.[attribute.name] \|\| attribute.name;	5✔
49	if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {	5!
50	continue;	×
51	}
52
53	const field = findArrowFieldByPath(props.schema, arrowPath);	5✔
54	if (!field) {	5!
55	throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);	×
56	}
57	if (!isInstanceArrowType(field.type)) {	5!
58	throw new Error(`Arrow column "${arrowPath}" is not compatible with shader attributes`);	×
59	}
60
61	const format = getArrowVertexFormat(getArrowColumnInfoFromType(field.type), attribute.type, {	5✔
62	allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
63	});
64	columns.push({	5✔
65	attributeName: attribute.name,
66	arrowPath,
67	field,
68	bufferLayout: {
69	name: attribute.name,
70	format,
71	...(attribute.stepMode ? {stepMode: attribute.stepMode} : {})	5✔
72	}
73	});
74	}
75
76	const selectedNames = new Set(columns.map(column => column.attributeName));	5✔
77	for (const binding of props.shaderLayout.bindings) {	3✔
78	if (	1!
79	(binding.type !== 'storage' && binding.type !== 'read-only-storage') \|\|	3✔
80	'format' in binding
81	) {
NEW 82	continue;	×
83	}
84	if (selectedNames.has(binding.name)) {	1!
NEW 85	throw new Error(	×
86	`Appendable Arrow shader input "${binding.name}" cannot be both an attribute and a storage binding`
87	);
88	}
89	const hasExplicitPath = Boolean(	1✔
90	props.arrowPaths && Object.prototype.hasOwnProperty.call(props.arrowPaths, binding.name)	1!
91	);
92	const arrowPath = props.arrowPaths?.[binding.name] \|\| binding.name;	1✔
93	if (!hasExplicitPath && !schemaPaths.has(arrowPath)) {	1!
NEW 94	continue;	×
95	}
96	const field = findArrowFieldByPath(props.schema, arrowPath);	1✔
97	if (!field) {	1!
NEW 98	throw new Error(`Arrow table schema does not contain column "${arrowPath}"`);	×
99	}
100	if (!isInstanceArrowType(field.type) && !arrow.DataType.isUtf8(field.type)) {	1!
NEW 101	throw new Error(`Arrow column "${arrowPath}" is not compatible with appendable GPU storage`);	×
102	}
103	columns.push({	1✔
104	attributeName: binding.name,
105	arrowPath,
106	field,
107	storageBinding: true
108	});
109	selectedNames.add(binding.name);	1✔
110	}
111
112	return columns;	3✔
113	}
114
115	/** Validates append compatibility and extracts one direct-upload Arrow data chunk per selected column. */
116	export function getAppendableGPUColumnData(
117	recordBatch: arrow.RecordBatch,
118	columns: AppendableGPUColumn[],
119	ownerName: string
120	): AppendableGPUColumnData[] {
121	for (const column of columns) {	5✔
122	const sourceField = findArrowFieldByPath(recordBatch.schema, column.arrowPath);	10✔
123	if (!sourceField \|\| !arrow.util.compareTypes(sourceField.type, column.field.type)) {	10!
UNCOV 124	throw new Error(`${ownerName} column "${column.arrowPath}" does not match the source schema`);	×
125	}
126	}
127
128	return columns.map(column => {	5✔
129	const data = getArrowDataByPath(recordBatch, column.arrowPath) as arrow.Data<	10✔
130	AttributeArrowType \| arrow.Utf8
131	>;
132	if (data.length !== recordBatch.numRows) {	10!
133	throw new Error(`${ownerName} column "${column.arrowPath}" row count mismatch`);	×
134	}
135	validateArrowGPUDataDirectUpload(column.attributeName, data);	10✔
136	return {column, data};	10✔
137	});
138	}
139
140	/** Derives Arrow column characteristics needed to map one selected column to a vertex format. */
141	export function getArrowColumnInfoFromType(type: AttributeArrowType): ArrowColumnInfo {
142	let numericType = type as NumericArrowType;	5✔
143	let components: 1 \| 2 \| 3 \| 4 = 1;	5✔
144	if (arrow.DataType.isFixedSizeList(type)) {	5!
145	numericType = type.children[0].type as NumericArrowType;	5✔
146	if (type.listSize < 1 \|\| type.listSize > 4) {	5!
147	throw new Error('Attribute column fixed list size must be between 1 and 4');	×
148	}
149	components = type.listSize as 1 \| 2 \| 3 \| 4;	5✔
150	}
151	if (!isNumericArrowType(numericType)) {	5!
152	throw new Error('Attribute column must be numeric or fixed list of numeric');	×
153	}
154	return {	5✔
155	signedDataType: getSignedShaderType(numericType, components),
156	components,
157	stepMode: 'instance',
158	values: [],
159	offsets: []
160	};
161	}

visgl / luma.gl / 26033117099

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous