25990278810

Committed 17 May 2026 12:01PM UTC coverage: 75.092% (+0.2%) from 74.881%

Build # 25990278810

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Columnar GPU-data stack (#2616)

Coverage Stats

6711 of 10084 branches covered (66.55%)

Branch coverage included in aggregate %.

625 of 865 new or added lines in 22 files covered. (72.25%)

1 existing line in 1 file now uncovered.

14631 of 18337 relevant lines covered (79.79%)

792.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/modules/arrow/src/arrow/table-computation.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {type Binding, type ComputePass, Device} from '@luma.gl/core';
import {Computation, type ComputationProps} from '@luma.gl/engine';
import {DynamicBuffer} from '@luma.gl/engine';
import type {GPUData} from './arrow-gpu-data';
import type {GPUVector} from './arrow-gpu-vector';

/** Metadata supplied to one table-computation batch dispatch. */
export type TableComputationBatch = {
  /** Zero-based batch index. */
  batchIndex: number;
  /** Shared logical row count for the current batched vector slice. */
  numRows: number;
};

/** Props for creating a WebGPU computation backed by GPU table vectors. */
export type TableComputationProps = Omit<ComputationProps, 'bindings'> & {
  /** Ordinary non-table bindings forwarded to {@link Computation}. */
  bindings?: Record<string, Binding>;
  /** GPU vectors converted to storage-buffer bindings by name. */
  vectorBindings?: Record<string, GPUVector>;
};

/**
 * A WebGPU computation that accepts {@link GPUVector} storage bindings.
 *
 * Direct vectors bind once during construction. Aggregate multi-buffer vectors
 * can be dispatched batch-by-batch with {@link dispatchBatches}.
 */
export class TableComputation extends Computation {
  /** GPU vectors supplied when the computation was created. */
  readonly vectorBindings: Record<string, GPUVector>;
  private readonly baseBindings: Record<string, Binding>;
  private readonly batchState: TableComputationBatchState;

  constructor(device: Device, props: TableComputationProps) {
    const {vectorBindings = {}, bindings = {}, ...computationProps} = props;
    assertNoDuplicateBindingNames(vectorBindings, bindings);

    const batchState = getTableComputationBatchState(vectorBindings);
    const baseBindings = {
      ...getDirectVectorBindings(vectorBindings),
      ...bindings
    };

    super(device, {
      ...computationProps,
      bindings: baseBindings
    });

    this.vectorBindings = {...vectorBindings};
    this.baseBindings = baseBindings;
    this.batchState = batchState;
  }

  /** Dispatches once per vector batch, rebinding storage ranges before each dispatch. */
  dispatchBatches(
    computePass: ComputePass,
    getWorkgroupCount: number | ((batch: TableComputationBatch) => number),
    y?: number,
    z?: number
  ): void {
    for (let batchIndex = 0; batchIndex < this.batchState.batchCount; batchIndex++) {
      const batch = {
        batchIndex,
        numRows: this.batchState.batchRowCounts[batchIndex]
      } satisfies TableComputationBatch;
      this.setBindings({
        ...this.baseBindings,
        ...getBatchVectorBindings(this.vectorBindings, batchIndex)
      });

      const workgroupCount =
        typeof getWorkgroupCount === 'function' ? getWorkgroupCount(batch) : getWorkgroupCount;
      super.dispatch(computePass, workgroupCount, y, z);
    }

    this.setBindings(this.baseBindings);
  }
}

type TableComputationBatchState = {
  batchCount: number;
  batchRowCounts: number[];
};

function getTableComputationBatchState(
  vectorBindings: Record<string, GPUVector>
): TableComputationBatchState {
  const batchedVectorEntries = Object.entries(vectorBindings).filter(([, vector]) =>
    requiresBatchBinding(vector)
  );
  if (batchedVectorEntries.length === 0) {
    const firstVector = Object.values(vectorBindings)[0];
    return {
      batchCount: 1,
      batchRowCounts: [firstVector?.length ?? 0]
    };
  }

  const [referenceName, referenceVector] = batchedVectorEntries[0];
  const batchCount = referenceVector.data.length;
  const batchRowCounts = referenceVector.data.map(data => data.length);

  for (const [name, vector] of batchedVectorEntries.slice(1)) {
    if (vector.data.length !== batchCount) {
      throw new Error(
        `TableComputation vector "${name}" batch count does not match "${referenceName}"`
      );
    }
    vector.data.forEach((data, batchIndex) => {
      if (data.length !== batchRowCounts[batchIndex]) {
        throw new Error(
          `TableComputation vector "${name}" batch ${batchIndex} rows do not match "${referenceName}"`
        );
      }
    });
  }

  return {batchCount, batchRowCounts};
}

function requiresBatchBinding(vector: GPUVector): boolean {
  try {
    vector.buffer;
    return false;
  } catch {
    return true;
  }
}

function getDirectVectorBindings(
  vectorBindings: Record<string, GPUVector>
): Record<string, Binding> {
  const bindings: Record<string, Binding> = {};
  for (const [name, vector] of Object.entries(vectorBindings)) {
    if (!requiresBatchBinding(vector)) {
      bindings[name] = getDirectVectorBinding(vector);
    }
  }
  return bindings;
}

function getDirectVectorBinding(vector: GPUVector): Binding {
  const buffer = vector.buffer;
  return buffer instanceof DynamicBuffer ? buffer.buffer : buffer;
}

function getBatchVectorBindings(
  vectorBindings: Record<string, GPUVector>,
  batchIndex: number
): Record<string, Binding> {
  const bindings: Record<string, Binding> = {};
  for (const [name, vector] of Object.entries(vectorBindings)) {
    if (!requiresBatchBinding(vector)) {
      continue;
    }
    const data = vector.data[batchIndex];
    if (!data) {
      throw new Error(`TableComputation vector "${name}" is missing batch ${batchIndex}`);
    }
    bindings[name] = getGPUDataBinding(data);
  }
  return bindings;
}

function getGPUDataBinding(data: GPUData): Binding {
  return {
    buffer: data.buffer.buffer,
    offset: data.byteOffset,
    size: data.length * data.byteStride
  };
}

function assertNoDuplicateBindingNames(
  vectorBindings: Record<string, GPUVector>,
  bindings: Record<string, Binding>
): void {
  for (const name of Object.keys(vectorBindings)) {
    if (name in bindings) {
      throw new Error(`TableComputation binding "${name}" duplicates an explicit binding`);
    }
  }
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {type Binding, type ComputePass, Device} from '@luma.gl/core';
6	import {Computation, type ComputationProps} from '@luma.gl/engine';
7	import {DynamicBuffer} from '@luma.gl/engine';
8	import type {GPUData} from './arrow-gpu-data';
9	import type {GPUVector} from './arrow-gpu-vector';
10
11	/** Metadata supplied to one table-computation batch dispatch. */
12	export type TableComputationBatch = {
13	/** Zero-based batch index. */
14	batchIndex: number;
15	/** Shared logical row count for the current batched vector slice. */
16	numRows: number;
17	};
18
19	/** Props for creating a WebGPU computation backed by GPU table vectors. */
20	export type TableComputationProps = Omit<ComputationProps, 'bindings'> & {
21	/** Ordinary non-table bindings forwarded to {@link Computation}. */
22	bindings?: Record<string, Binding>;
23	/** GPU vectors converted to storage-buffer bindings by name. */
24	vectorBindings?: Record<string, GPUVector>;
25	};
26
27	/**
28	* A WebGPU computation that accepts {@link GPUVector} storage bindings.
29	*
30	* Direct vectors bind once during construction. Aggregate multi-buffer vectors
31	* can be dispatched batch-by-batch with {@link dispatchBatches}.
32	*/
33	export class TableComputation extends Computation {
34	/** GPU vectors supplied when the computation was created. */
35	readonly vectorBindings: Record<string, GPUVector>;
36	private readonly baseBindings: Record<string, Binding>;
37	private readonly batchState: TableComputationBatchState;
38
39	constructor(device: Device, props: TableComputationProps) {
NEW 40	const {vectorBindings = {}, bindings = {}, ...computationProps} = props;	×
NEW 41	assertNoDuplicateBindingNames(vectorBindings, bindings);	×
42
NEW 43	const batchState = getTableComputationBatchState(vectorBindings);	×
NEW 44	const baseBindings = {	×
45	...getDirectVectorBindings(vectorBindings),
46	...bindings
47	};
48
NEW 49	super(device, {	×
50	...computationProps,
51	bindings: baseBindings
52	});
53
NEW 54	this.vectorBindings = {...vectorBindings};	×
NEW 55	this.baseBindings = baseBindings;	×
NEW 56	this.batchState = batchState;	×
57	}
58
59	/** Dispatches once per vector batch, rebinding storage ranges before each dispatch. */
60	dispatchBatches(
61	computePass: ComputePass,
62	getWorkgroupCount: number \| ((batch: TableComputationBatch) => number),
63	y?: number,
64	z?: number
65	): void {
NEW 66	for (let batchIndex = 0; batchIndex < this.batchState.batchCount; batchIndex++) {	×
NEW 67	const batch = {	×
68	batchIndex,
69	numRows: this.batchState.batchRowCounts[batchIndex]
70	} satisfies TableComputationBatch;
NEW 71	this.setBindings({	×
72	...this.baseBindings,
73	...getBatchVectorBindings(this.vectorBindings, batchIndex)
74	});
75
76	const workgroupCount =
NEW 77	typeof getWorkgroupCount === 'function' ? getWorkgroupCount(batch) : getWorkgroupCount;	×
NEW 78	super.dispatch(computePass, workgroupCount, y, z);	×
79	}
80
NEW 81	this.setBindings(this.baseBindings);	×
82	}
83	}
84
85	type TableComputationBatchState = {
86	batchCount: number;
87	batchRowCounts: number[];
88	};
89
90	function getTableComputationBatchState(
91	vectorBindings: Record<string, GPUVector>
92	): TableComputationBatchState {
NEW 93	const batchedVectorEntries = Object.entries(vectorBindings).filter(([, vector]) =>	×
NEW 94	requiresBatchBinding(vector)	×
95	);
NEW 96	if (batchedVectorEntries.length === 0) {	×
NEW 97	const firstVector = Object.values(vectorBindings)[0];	×
NEW 98	return {	×
99	batchCount: 1,
100	batchRowCounts: [firstVector?.length ?? 0]	×
101	};
102	}
103
NEW 104	const [referenceName, referenceVector] = batchedVectorEntries[0];	×
NEW 105	const batchCount = referenceVector.data.length;	×
NEW 106	const batchRowCounts = referenceVector.data.map(data => data.length);	×
107
NEW 108	for (const [name, vector] of batchedVectorEntries.slice(1)) {	×
NEW 109	if (vector.data.length !== batchCount) {	×
NEW 110	throw new Error(	×
111	`TableComputation vector "${name}" batch count does not match "${referenceName}"`
112	);
113	}
NEW 114	vector.data.forEach((data, batchIndex) => {	×
NEW 115	if (data.length !== batchRowCounts[batchIndex]) {	×
NEW 116	throw new Error(	×
117	`TableComputation vector "${name}" batch ${batchIndex} rows do not match "${referenceName}"`
118	);
119	}
120	});
121	}
122
NEW 123	return {batchCount, batchRowCounts};	×
124	}
125
126	function requiresBatchBinding(vector: GPUVector): boolean {
NEW 127	try {	×
NEW 128	vector.buffer;	×
NEW 129	return false;	×
130	} catch {
NEW 131	return true;	×
132	}
133	}
134
135	function getDirectVectorBindings(
136	vectorBindings: Record<string, GPUVector>
137	): Record<string, Binding> {
NEW 138	const bindings: Record<string, Binding> = {};	×
NEW 139	for (const [name, vector] of Object.entries(vectorBindings)) {	×
NEW 140	if (!requiresBatchBinding(vector)) {	×
NEW 141	bindings[name] = getDirectVectorBinding(vector);	×
142	}
143	}
NEW 144	return bindings;	×
145	}
146
147	function getDirectVectorBinding(vector: GPUVector): Binding {
NEW 148	const buffer = vector.buffer;	×
NEW 149	return buffer instanceof DynamicBuffer ? buffer.buffer : buffer;	×
150	}
151
152	function getBatchVectorBindings(
153	vectorBindings: Record<string, GPUVector>,
154	batchIndex: number
155	): Record<string, Binding> {
NEW 156	const bindings: Record<string, Binding> = {};	×
NEW 157	for (const [name, vector] of Object.entries(vectorBindings)) {	×
NEW 158	if (!requiresBatchBinding(vector)) {	×
NEW 159	continue;	×
160	}
NEW 161	const data = vector.data[batchIndex];	×
NEW 162	if (!data) {	×
NEW 163	throw new Error(`TableComputation vector "${name}" is missing batch ${batchIndex}`);	×
164	}
NEW 165	bindings[name] = getGPUDataBinding(data);	×
166	}
NEW 167	return bindings;	×
168	}
169
170	function getGPUDataBinding(data: GPUData): Binding {
NEW 171	return {	×
172	buffer: data.buffer.buffer,
173	offset: data.byteOffset,
174	size: data.length * data.byteStride
175	};
176	}
177
178	function assertNoDuplicateBindingNames(
179	vectorBindings: Record<string, GPUVector>,
180	bindings: Record<string, Binding>
181	): void {
NEW 182	for (const name of Object.keys(vectorBindings)) {	×
NEW 183	if (name in bindings) {	×
NEW 184	throw new Error(`TableComputation binding "${name}" duplicates an explicit binding`);	×
185	}
186	}
187	}

visgl / luma.gl / 25990278810

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous