25990278810

Committed 17 May 2026 12:01PM UTC coverage: 75.092% (+0.2%) from 74.881%

Build # 25990278810

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Columnar GPU-data stack (#2616)

Coverage Stats

6711 of 10084 branches covered (66.55%)

Branch coverage included in aggregate %.

625 of 865 new or added lines in 22 files covered. (72.25%)

1 existing line in 1 file now uncovered.

14631 of 18337 relevant lines covered (79.79%)

792.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/modules/arrow/src/arrow/table-transform.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {type BufferLayout, Device, type RenderPassProps, type ShaderLayout} from '@luma.gl/core';
import {BufferTransform, type BufferTransformProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import type {ArrowVertexFormatOptions} from './arrow-shader-layout';
import type {GPUVectorProps} from './arrow-gpu-vector';
import {GPURecordBatch} from './arrow-gpu-record-batch';
import {GPUTable, type GPUTableProps} from './plain-gpu-table';

type TableTransformBufferMap = NonNullable<Parameters<BufferTransform['run']>[0]>['outputBuffers'];

/** Options supplied for one {@link TableTransform.runBatches} dispatch. */
export type TableTransformBatchOptions = RenderPassProps & {
  /** Output transform-feedback buffers for the current batch. */
  outputBuffers?:
    | TableTransformBufferMap
    | ((batch: GPURecordBatch, batchIndex: number) => TableTransformBufferMap);
};

/** Props for creating a WebGL transform backed by a GPU-resident Arrow table. */
export type TableTransformProps = BufferTransformProps &
  ArrowVertexFormatOptions & {
    /** Existing GPU table used as the source for transform input buffers. */
    table?: GPUTable;
    /** Arrow table convenience input converted into a {@link GPUTable}. */
    arrowTable?: arrow.Table;
    /** Maps shader attribute names to Arrow column paths. Defaults to using attribute names. */
    arrowPaths?: Record<string, string>;
    /** Buffer props applied when `arrowTable` is materialized into GPU vectors. */
    arrowBufferProps?: GPUVectorProps;
    /** Controls whether GPU table row count is assigned to vertexCount. */
    tableCount?: 'vertex' | 'none';
  };

type TableTransformState = {
  table: GPUTable;
  ownsTable: boolean;
  transformProps: BufferTransformProps;
  explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
  explicitBufferLayout: BufferLayout[];
  inferVertexCount: boolean;
};

/**
 * A WebGL transform-feedback program whose input buffers come from a {@link GPUTable}.
 *
 * `TableTransform` keeps the regular {@link BufferTransform} execution model,
 * while adding table construction and batch-by-batch execution helpers.
 */
export class TableTransform extends BufferTransform {
  /** GPU table backing the transform input attributes. */
  readonly table: GPUTable;
  private readonly ownsTable: boolean;
  private readonly explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
  private readonly explicitBufferLayout: BufferLayout[];
  private readonly inferVertexCount: boolean;
  private tableTransformDestroyed = false;

  constructor(device: Device, props: TableTransformProps) {
    const {
      table,
      ownsTable,
      transformProps,
      explicitAttributes,
      explicitBufferLayout,
      inferVertexCount
    } = getTableTransformState(device, props);

    try {
      super(device, transformProps);
    } catch (error) {
      if (ownsTable) {
        table.destroy();
      }
      throw error;
    }

    this.table = table;
    this.ownsTable = ownsTable;
    this.explicitAttributes = explicitAttributes;
    this.explicitBufferLayout = explicitBufferLayout;
    this.inferVertexCount = inferVertexCount;
  }

  /** Runs the transform once per preserved GPU record batch. */
  runBatches(options: TableTransformBatchOptions = {}): void {
    assertMatchingBufferLayouts(
      this.table.bufferLayout,
      this.explicitBufferLayout,
      this.model.bufferLayout,
      'TableTransform.runBatches() model buffer layout does not match its GPU table'
    );

    try {
      this.table.batches.forEach((batch, batchIndex) => {
        assertMatchingBufferLayouts(
          this.table.bufferLayout,
          [],
          batch.bufferLayout,
          'TableTransform.runBatches() requires every GPU batch to use the table buffer layout'
        );
        this.model.setAttributes({
          ...this.explicitAttributes,
          ...batch.attributes
        });
        if (this.inferVertexCount) {
          this.model.setVertexCount(batch.numRows);
        }

        const outputBuffers =
          typeof options.outputBuffers === 'function'
            ? options.outputBuffers(batch, batchIndex)
            : options.outputBuffers;
        const {outputBuffers: _ignoredOutputBuffers, ...renderPassProps} = options;
        super.run({...renderPassProps, outputBuffers});
      });
    } finally {
      this.model.setAttributes({
        ...this.explicitAttributes,
        ...this.table.attributes
      });
      if (this.inferVertexCount) {
        this.model.setVertexCount(this.table.numRows);
      }
    }
  }

  override destroy(): void {
    if (this.tableTransformDestroyed) {
      return;
    }
    super.destroy();
    if (this.ownsTable) {
      this.table.destroy();
    }
    this.tableTransformDestroyed = true;
  }
}

function getTableTransformState(device: Device, props: TableTransformProps): TableTransformState {
  const {
    table: explicitTable,
    arrowTable,
    arrowPaths,
    arrowBufferProps,
    tableCount = 'vertex',
    allowWebGLOnlyFormats,
    ...transformProps
  } = props;

  validateTableTransformSources({table: explicitTable, arrowTable});
  if (!transformProps.shaderLayout) {
    throw new Error('TableTransform requires shaderLayout');
  }

  const {table, ownsTable} = getInitialTable({
    device,
    table: explicitTable,
    arrowTable,
    shaderLayout: transformProps.shaderLayout,
    arrowPaths,
    arrowBufferProps,
    allowWebGLOnlyFormats
  });

  const explicitAttributes = transformProps.attributes || {};
  const explicitBufferLayout = transformProps.bufferLayout || [];
  const inferVertexCount = tableCount === 'vertex' && transformProps.vertexCount === undefined;

  try {
    assertNoDuplicateNames(
      Object.keys(explicitAttributes),
      Object.keys(table.attributes),
      'attribute'
    );
    assertNoDuplicateNames(
      getBufferLayoutNames(explicitBufferLayout),
      getBufferLayoutNames(table.bufferLayout),
      'buffer layout'
    );
  } catch (error) {
    if (ownsTable) {
      table.destroy();
    }
    throw error;
  }

  return {
    table,
    ownsTable,
    explicitAttributes,
    explicitBufferLayout,
    inferVertexCount,
    transformProps: {
      ...transformProps,
      attributes: {...explicitAttributes, ...table.attributes},
      bufferLayout: [...explicitBufferLayout, ...table.bufferLayout],
      ...(inferVertexCount ? {vertexCount: table.numRows} : {})
    }
  };
}

function getInitialTable(props: {
  device: Device;
  table?: GPUTable;
  arrowTable?: arrow.Table;
  shaderLayout: ShaderLayout;
  arrowPaths?: Record<string, string>;
  arrowBufferProps?: GPUVectorProps;
  allowWebGLOnlyFormats?: boolean;
}): {table: GPUTable; ownsTable: boolean} {
  if (props.table) {
    return {table: props.table, ownsTable: false};
  }

  return {
    table: new GPUTable(props.device, props.arrowTable!, {
      shaderLayout: props.shaderLayout,
      arrowPaths: props.arrowPaths,
      bufferProps: props.arrowBufferProps,
      allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
    } satisfies GPUTableProps),
    ownsTable: true
  };
}

function validateTableTransformSources(props: {table?: GPUTable; arrowTable?: arrow.Table}): void {
  const sourceCount = Number(Boolean(props.table)) + Number(Boolean(props.arrowTable));
  if (sourceCount > 1) {
    throw new Error('TableTransform requires only one of table or arrowTable');
  }
  if (sourceCount === 0) {
    throw new Error('TableTransform requires table or arrowTable');
  }
}

function getBufferLayoutNames(bufferLayout: BufferLayout[]): string[] {
  return bufferLayout.map(layout => layout.name);
}

function assertNoDuplicateNames(
  explicitNames: string[],
  tableNames: string[],
  nameType: string
): void {
  const explicitNameSet = new Set(explicitNames);
  for (const tableName of tableNames) {
    if (explicitNameSet.has(tableName)) {
      throw new Error(
        `TableTransform ${nameType} "${tableName}" duplicates an explicit ${nameType}`
      );
    }
  }
}

function assertMatchingBufferLayouts(
  tableBufferLayout: BufferLayout[],
  explicitBufferLayout: BufferLayout[],
  candidateBufferLayout: BufferLayout[],
  errorMessage: string
): void {
  const expectedBufferLayout = [...explicitBufferLayout, ...tableBufferLayout];
  if (JSON.stringify(expectedBufferLayout) !== JSON.stringify(candidateBufferLayout)) {
    throw new Error(errorMessage);
  }
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {type BufferLayout, Device, type RenderPassProps, type ShaderLayout} from '@luma.gl/core';
6	import {BufferTransform, type BufferTransformProps} from '@luma.gl/engine';
7	import * as arrow from 'apache-arrow';
8	import type {ArrowVertexFormatOptions} from './arrow-shader-layout';
9	import type {GPUVectorProps} from './arrow-gpu-vector';
10	import {GPURecordBatch} from './arrow-gpu-record-batch';
11	import {GPUTable, type GPUTableProps} from './plain-gpu-table';
12
13	type TableTransformBufferMap = NonNullable<Parameters<BufferTransform['run']>[0]>['outputBuffers'];
14
15	/** Options supplied for one {@link TableTransform.runBatches} dispatch. */
16	export type TableTransformBatchOptions = RenderPassProps & {
17	/** Output transform-feedback buffers for the current batch. */
18	outputBuffers?:
19	\| TableTransformBufferMap
20	\| ((batch: GPURecordBatch, batchIndex: number) => TableTransformBufferMap);
21	};
22
23	/** Props for creating a WebGL transform backed by a GPU-resident Arrow table. */
24	export type TableTransformProps = BufferTransformProps &
25	ArrowVertexFormatOptions & {
26	/** Existing GPU table used as the source for transform input buffers. */
27	table?: GPUTable;
28	/** Arrow table convenience input converted into a {@link GPUTable}. */
29	arrowTable?: arrow.Table;
30	/** Maps shader attribute names to Arrow column paths. Defaults to using attribute names. */
31	arrowPaths?: Record<string, string>;
32	/** Buffer props applied when `arrowTable` is materialized into GPU vectors. */
33	arrowBufferProps?: GPUVectorProps;
34	/** Controls whether GPU table row count is assigned to vertexCount. */
35	tableCount?: 'vertex' \| 'none';
36	};
37
38	type TableTransformState = {
39	table: GPUTable;
40	ownsTable: boolean;
41	transformProps: BufferTransformProps;
42	explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
43	explicitBufferLayout: BufferLayout[];
44	inferVertexCount: boolean;
45	};
46
47	/**
48	* A WebGL transform-feedback program whose input buffers come from a {@link GPUTable}.
49	*
50	* `TableTransform` keeps the regular {@link BufferTransform} execution model,
51	* while adding table construction and batch-by-batch execution helpers.
52	*/
53	export class TableTransform extends BufferTransform {
54	/** GPU table backing the transform input attributes. */
55	readonly table: GPUTable;
56	private readonly ownsTable: boolean;
57	private readonly explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
58	private readonly explicitBufferLayout: BufferLayout[];
59	private readonly inferVertexCount: boolean;
NEW 60	private tableTransformDestroyed = false;	×
61
62	constructor(device: Device, props: TableTransformProps) {
63	const {
64	table,
65	ownsTable,
66	transformProps,
67	explicitAttributes,
68	explicitBufferLayout,
69	inferVertexCount
NEW 70	} = getTableTransformState(device, props);	×
71
NEW 72	try {	×
NEW 73	super(device, transformProps);	×
74	} catch (error) {
NEW 75	if (ownsTable) {	×
NEW 76	table.destroy();	×
77	}
NEW 78	throw error;	×
79	}
80
NEW 81	this.table = table;	×
NEW 82	this.ownsTable = ownsTable;	×
NEW 83	this.explicitAttributes = explicitAttributes;	×
NEW 84	this.explicitBufferLayout = explicitBufferLayout;	×
NEW 85	this.inferVertexCount = inferVertexCount;	×
86	}
87
88	/** Runs the transform once per preserved GPU record batch. */
89	runBatches(options: TableTransformBatchOptions = {}): void {	×
NEW 90	assertMatchingBufferLayouts(	×
91	this.table.bufferLayout,
92	this.explicitBufferLayout,
93	this.model.bufferLayout,
94	'TableTransform.runBatches() model buffer layout does not match its GPU table'
95	);
96
NEW 97	try {	×
NEW 98	this.table.batches.forEach((batch, batchIndex) => {	×
NEW 99	assertMatchingBufferLayouts(	×
100	this.table.bufferLayout,
101	[],
102	batch.bufferLayout,
103	'TableTransform.runBatches() requires every GPU batch to use the table buffer layout'
104	);
NEW 105	this.model.setAttributes({	×
106	...this.explicitAttributes,
107	...batch.attributes
108	});
NEW 109	if (this.inferVertexCount) {	×
NEW 110	this.model.setVertexCount(batch.numRows);	×
111	}
112
113	const outputBuffers =
NEW 114	typeof options.outputBuffers === 'function'	×
115	? options.outputBuffers(batch, batchIndex)
116	: options.outputBuffers;
NEW 117	const {outputBuffers: _ignoredOutputBuffers, ...renderPassProps} = options;	×
NEW 118	super.run({...renderPassProps, outputBuffers});	×
119	});
120	} finally {
NEW 121	this.model.setAttributes({	×
122	...this.explicitAttributes,
123	...this.table.attributes
124	});
NEW 125	if (this.inferVertexCount) {	×
NEW 126	this.model.setVertexCount(this.table.numRows);	×
127	}
128	}
129	}
130
131	override destroy(): void {
NEW 132	if (this.tableTransformDestroyed) {	×
NEW 133	return;	×
134	}
NEW 135	super.destroy();	×
NEW 136	if (this.ownsTable) {	×
NEW 137	this.table.destroy();	×
138	}
NEW 139	this.tableTransformDestroyed = true;	×
140	}
141	}
142
143	function getTableTransformState(device: Device, props: TableTransformProps): TableTransformState {
144	const {
145	table: explicitTable,
146	arrowTable,
147	arrowPaths,
148	arrowBufferProps,
149	tableCount = 'vertex',	×
150	allowWebGLOnlyFormats,
151	...transformProps
NEW 152	} = props;	×
153
NEW 154	validateTableTransformSources({table: explicitTable, arrowTable});	×
NEW 155	if (!transformProps.shaderLayout) {	×
NEW 156	throw new Error('TableTransform requires shaderLayout');	×
157	}
158
NEW 159	const {table, ownsTable} = getInitialTable({	×
160	device,
161	table: explicitTable,
162	arrowTable,
163	shaderLayout: transformProps.shaderLayout,
164	arrowPaths,
165	arrowBufferProps,
166	allowWebGLOnlyFormats
167	});
168
NEW 169	const explicitAttributes = transformProps.attributes \|\| {};	×
NEW 170	const explicitBufferLayout = transformProps.bufferLayout \|\| [];	×
NEW 171	const inferVertexCount = tableCount === 'vertex' && transformProps.vertexCount === undefined;	×
172
NEW 173	try {	×
NEW 174	assertNoDuplicateNames(	×
175	Object.keys(explicitAttributes),
176	Object.keys(table.attributes),
177	'attribute'
178	);
NEW 179	assertNoDuplicateNames(	×
180	getBufferLayoutNames(explicitBufferLayout),
181	getBufferLayoutNames(table.bufferLayout),
182	'buffer layout'
183	);
184	} catch (error) {
NEW 185	if (ownsTable) {	×
NEW 186	table.destroy();	×
187	}
NEW 188	throw error;	×
189	}
190
NEW 191	return {	×
192	table,
193	ownsTable,
194	explicitAttributes,
195	explicitBufferLayout,
196	inferVertexCount,
197	transformProps: {
198	...transformProps,
199	attributes: {...explicitAttributes, ...table.attributes},
200	bufferLayout: [...explicitBufferLayout, ...table.bufferLayout],
201	...(inferVertexCount ? {vertexCount: table.numRows} : {})	×
202	}
203	};
204	}
205
206	function getInitialTable(props: {
207	device: Device;
208	table?: GPUTable;
209	arrowTable?: arrow.Table;
210	shaderLayout: ShaderLayout;
211	arrowPaths?: Record<string, string>;
212	arrowBufferProps?: GPUVectorProps;
213	allowWebGLOnlyFormats?: boolean;
214	}): {table: GPUTable; ownsTable: boolean} {
NEW 215	if (props.table) {	×
NEW 216	return {table: props.table, ownsTable: false};	×
217	}
218
NEW 219	return {	×
220	table: new GPUTable(props.device, props.arrowTable!, {
221	shaderLayout: props.shaderLayout,
222	arrowPaths: props.arrowPaths,
223	bufferProps: props.arrowBufferProps,
224	allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
225	} satisfies GPUTableProps),
226	ownsTable: true
227	};
228	}
229
230	function validateTableTransformSources(props: {table?: GPUTable; arrowTable?: arrow.Table}): void {
NEW 231	const sourceCount = Number(Boolean(props.table)) + Number(Boolean(props.arrowTable));	×
NEW 232	if (sourceCount > 1) {	×
NEW 233	throw new Error('TableTransform requires only one of table or arrowTable');	×
234	}
NEW 235	if (sourceCount === 0) {	×
NEW 236	throw new Error('TableTransform requires table or arrowTable');	×
237	}
238	}
239
240	function getBufferLayoutNames(bufferLayout: BufferLayout[]): string[] {
NEW 241	return bufferLayout.map(layout => layout.name);	×
242	}
243
244	function assertNoDuplicateNames(
245	explicitNames: string[],
246	tableNames: string[],
247	nameType: string
248	): void {
NEW 249	const explicitNameSet = new Set(explicitNames);	×
NEW 250	for (const tableName of tableNames) {	×
NEW 251	if (explicitNameSet.has(tableName)) {	×
NEW 252	throw new Error(	×
253	`TableTransform ${nameType} "${tableName}" duplicates an explicit ${nameType}`
254	);
255	}
256	}
257	}
258
259	function assertMatchingBufferLayouts(
260	tableBufferLayout: BufferLayout[],
261	explicitBufferLayout: BufferLayout[],
262	candidateBufferLayout: BufferLayout[],
263	errorMessage: string
264	): void {
NEW 265	const expectedBufferLayout = [...explicitBufferLayout, ...tableBufferLayout];	×
NEW 266	if (JSON.stringify(expectedBufferLayout) !== JSON.stringify(candidateBufferLayout)) {	×
NEW 267	throw new Error(errorMessage);	×
268	}
269	}

visgl / luma.gl / 25990278810

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous