25992522693

Committed 17 May 2026 01:42PM UTC coverage: 74.98% (+0.04%) from 74.942%

Build # 25992522693

Build Type

push

github

Committed by

web-flow

Commit Message

feat(arrow) WebGL transform handles Vectors and writeback (#2618)

Coverage Stats

6814 of 10253 branches covered (66.46%)

Branch coverage included in aggregate %.

81 of 159 new or added lines in 3 files covered. (50.94%)

2 existing lines in 2 files now uncovered.

14811 of 18588 relevant lines covered (79.68%)

845.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.4

/modules/arrow/src/arrow/table-transform.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {
  Buffer,
  type BufferLayout,
  Device,
  type RenderPassProps,
  type ShaderLayout
} from '@luma.gl/core';
import {BufferTransform, DynamicBuffer, type BufferTransformProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import type {ArrowVertexFormatOptions} from './arrow-shader-layout';
import {GPUVector, type GPUVectorProps} from './arrow-gpu-vector';
import {GPURecordBatch} from './arrow-gpu-record-batch';
import {GPUTable, type GPUTableProps} from './plain-gpu-table';
import type {AttributeArrowType} from './arrow-types';
import {getArrowTypeByteStride} from './arrow-gpu-data';

type TableTransformBufferMap = NonNullable<Parameters<BufferTransform['run']>[0]>['outputBuffers'];
type TableTransformRunOptions = Parameters<BufferTransform['run']>[0];
type TableTransformInputVectors = Record<string, GPUVector> | GPUVector[];

/** Maps transform-feedback output varying names back to the input GPU vector they should update. */
export type TableTransformOutputCopyMap = Record<string, string>;

/** Options supplied for one {@link TableTransform.runBatches} dispatch. */
export type TableTransformBatchOptions = RenderPassProps & {
  /** Output transform-feedback buffers for the current batch. */
  outputBuffers?:
    | TableTransformBufferMap
    | ((batch: GPURecordBatch, batchIndex: number) => TableTransformBufferMap);
};

/** Props for creating a WebGL transform backed by a GPU-resident Arrow table. */
export type TableTransformProps = BufferTransformProps &
  ArrowVertexFormatOptions & {
    /** Existing GPU table used as the source for transform input buffers. */
    table?: GPUTable;
    /** Arrow table convenience input converted into a {@link GPUTable}. */
    arrowTable?: arrow.Table;
    /** Existing GPU vectors converted into the transform input table. */
    inputVectors?: TableTransformInputVectors;
    /** Maps shader attribute names to Arrow column paths. Defaults to using attribute names. */
    arrowPaths?: Record<string, string>;
    /** Buffer props applied when `arrowTable` is materialized into GPU vectors. */
    arrowBufferProps?: GPUVectorProps;
    /**
     * Allocates dense transform-feedback output vectors and copies them back into named input
     * vectors after each run.
     *
     * Output names are inferred as transform-feedback outputs when `outputs` is omitted.
     */
    copyOutputToInputVectors?: TableTransformOutputCopyMap;
    /** Controls whether GPU table row count is assigned to vertexCount. */
    tableCount?: 'vertex' | 'none';
  };

type TableTransformState = {
  table: GPUTable;
  ownsTable: boolean;
  transformProps: BufferTransformProps;
  explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
  explicitBufferLayout: BufferLayout[];
  inferVertexCount: boolean;
  copyOutputToInputVectors: TableTransformOutputCopyMap;
  outputVectors: Record<string, GPUVector>;
};

/**
 * A WebGL transform-feedback program whose input buffers come from a {@link GPUTable}.
 *
 * `TableTransform` keeps the regular {@link BufferTransform} execution model,
 * while adding table construction and batch-by-batch execution helpers.
 */
export class TableTransform extends BufferTransform {
  /** GPU table backing the transform input attributes. */
  readonly table: GPUTable;
  /** GPU vectors backing the transform input attributes. */
  readonly inputVectors: Record<string, GPUVector>;
  /** Dense transform-feedback output vectors allocated for automatic input writeback. */
  readonly outputVectors: Record<string, GPUVector>;
  private readonly ownsTable: boolean;
  private readonly explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
  private readonly explicitBufferLayout: BufferLayout[];
  private readonly inferVertexCount: boolean;
  private readonly copyOutputToInputVectors: TableTransformOutputCopyMap;
  private tableTransformDestroyed = false;

  constructor(device: Device, props: TableTransformProps) {
    const {
      table,
      ownsTable,
      transformProps,
      explicitAttributes,
      explicitBufferLayout,
      inferVertexCount,
      copyOutputToInputVectors,
      outputVectors
    } = getTableTransformState(device, props);

    try {
      super(device, transformProps);
    } catch (error) {
      destroyGPUVectors(outputVectors);
      if (ownsTable) {
        table.destroy();
      }
      throw error;
    }

    this.table = table;
    this.inputVectors = table.gpuVectors;
    this.outputVectors = outputVectors;
    this.ownsTable = ownsTable;
    this.explicitAttributes = explicitAttributes;
    this.explicitBufferLayout = explicitBufferLayout;
    this.inferVertexCount = inferVertexCount;
    this.copyOutputToInputVectors = copyOutputToInputVectors;
  }

  /** Runs the transform once and optionally copies transform outputs back into input vectors. */
  override run(options: TableTransformRunOptions = {}): void {
    if (!this.hasAutomaticOutputWriteback()) {
      super.run(options);
      return;
    }
    assertNoExplicitOutputBuffers(
      options?.outputBuffers,
      'TableTransform.run() cannot combine outputBuffers with copyOutputToInputVectors'
    );
    super.run({...options, outputBuffers: this.getAutomaticOutputBuffers()});
    this.copyOutputsToInputVectors();
  }

  /** Runs the transform once per preserved GPU record batch. */
  runBatches(options: TableTransformBatchOptions = {}): void {
    if (this.hasAutomaticOutputWriteback() && options.outputBuffers) {
      throw new Error(
        'TableTransform.runBatches() cannot combine outputBuffers with copyOutputToInputVectors'
      );
    }
    assertMatchingBufferLayouts(
      this.table.bufferLayout,
      this.explicitBufferLayout,
      this.model.bufferLayout,
      'TableTransform.runBatches() model buffer layout does not match its GPU table'
    );

    try {
      this.table.batches.forEach((batch, batchIndex) => {
        assertMatchingBufferLayouts(
          this.table.bufferLayout,
          [],
          batch.bufferLayout,
          'TableTransform.runBatches() requires every GPU batch to use the table buffer layout'
        );
        this.model.setAttributes({
          ...this.explicitAttributes,
          ...batch.attributes
        });
        if (this.inferVertexCount) {
          this.model.setVertexCount(batch.numRows);
        }

        const outputBuffers =
          typeof options.outputBuffers === 'function'
            ? options.outputBuffers(batch, batchIndex)
            : options.outputBuffers;
        const {outputBuffers: _ignoredOutputBuffers, ...renderPassProps} = options;
        if (this.hasAutomaticOutputWriteback()) {
          super.run({...renderPassProps, outputBuffers: this.getAutomaticOutputBuffers()});
        } else {
          super.run({...renderPassProps, outputBuffers});
        }
      });
      if (this.hasAutomaticOutputWriteback()) {
        this.copyOutputsToInputVectors();
      }
    } finally {
      this.model.setAttributes({
        ...this.explicitAttributes,
        ...this.table.attributes
      });
      if (this.inferVertexCount) {
        this.model.setVertexCount(this.table.numRows);
      }
    }
  }

  override destroy(): void {
    if (this.tableTransformDestroyed) {
      return;
    }
    super.destroy();
    destroyGPUVectors(this.outputVectors);
    if (this.ownsTable) {
      this.table.destroy();
    }
    this.tableTransformDestroyed = true;
  }

  private hasAutomaticOutputWriteback(): boolean {
    return Object.keys(this.copyOutputToInputVectors).length > 0;
  }

  private getAutomaticOutputBuffers(): TableTransformBufferMap {
    return Object.fromEntries(
      Object.entries(this.outputVectors).map(([outputName, vector]) => [
        outputName,
        getConcreteBuffer(vector.buffer)
      ])
    );
  }

  private copyOutputsToInputVectors(): void {
    const commandEncoder = this.device.createCommandEncoder();
    let copyCount = 0;

    for (const [outputName, inputName] of Object.entries(this.copyOutputToInputVectors)) {
      const outputVector = this.outputVectors[outputName];
      const inputVector = this.inputVectors[inputName];
      if (!outputVector || !inputVector) {
        throw new Error(`TableTransform writeback mapping "${outputName}" is incomplete`);
      }
      const size = inputVector.length * inputVector.byteStride;
      if (size === 0) {
        continue;
      }
      commandEncoder.copyBufferToBuffer({
        sourceBuffer: getConcreteBuffer(outputVector.buffer),
        destinationBuffer: getConcreteBuffer(inputVector.buffer),
        size
      });
      copyCount++;
    }

    if (copyCount > 0) {
      this.device.submit(commandEncoder.finish());
      return;
    }
    commandEncoder.destroy();
  }
}

function getTableTransformState(device: Device, props: TableTransformProps): TableTransformState {
  const {
    table: explicitTable,
    arrowTable,
    inputVectors,
    arrowPaths,
    arrowBufferProps,
    copyOutputToInputVectors = {},
    tableCount = 'vertex',
    allowWebGLOnlyFormats,
    ...transformProps
  } = props;

  validateTableTransformSources({table: explicitTable, arrowTable, inputVectors});
  if (!transformProps.shaderLayout) {
    throw new Error('TableTransform requires shaderLayout');
  }

  const {table, ownsTable} = getInitialTable({
    device,
    table: explicitTable,
    arrowTable,
    inputVectors,
    shaderLayout: transformProps.shaderLayout,
    arrowPaths,
    arrowBufferProps,
    allowWebGLOnlyFormats
  });

  const explicitAttributes = transformProps.attributes || {};
  const explicitBufferLayout = transformProps.bufferLayout || [];
  const inferVertexCount = tableCount === 'vertex' && transformProps.vertexCount === undefined;
  let outputVectors: Record<string, GPUVector> = {};
  let transformOutputs: string[] | undefined;

  try {
    assertNoDuplicateNames(
      Object.keys(explicitAttributes),
      Object.keys(table.attributes),
      'attribute'
    );
    assertNoDuplicateNames(
      getBufferLayoutNames(explicitBufferLayout),
      getBufferLayoutNames(table.bufferLayout),
      'buffer layout'
    );
    outputVectors = createAutomaticOutputVectors(device, table, copyOutputToInputVectors);
    transformOutputs = getTableTransformOutputs(transformProps, copyOutputToInputVectors);
  } catch (error) {
    destroyGPUVectors(outputVectors);
    if (ownsTable) {
      table.destroy();
    }
    throw error;
  }

  return {
    table,
    ownsTable,
    explicitAttributes,
    explicitBufferLayout,
    inferVertexCount,
    copyOutputToInputVectors,
    outputVectors,
    transformProps: {
      ...transformProps,
      ...(transformOutputs ? {outputs: transformOutputs} : {}),
      attributes: {...explicitAttributes, ...table.attributes},
      bufferLayout: [...explicitBufferLayout, ...table.bufferLayout],
      ...(inferVertexCount ? {vertexCount: table.numRows} : {})
    }
  };
}

function getInitialTable(props: {
  device: Device;
  table?: GPUTable;
  arrowTable?: arrow.Table;
  inputVectors?: TableTransformInputVectors;
  shaderLayout: ShaderLayout;
  arrowPaths?: Record<string, string>;
  arrowBufferProps?: GPUVectorProps;
  allowWebGLOnlyFormats?: boolean;
}): {table: GPUTable; ownsTable: boolean} {
  if (props.table) {
    return {table: props.table, ownsTable: false};
  }
  if (props.inputVectors) {
    return {table: new GPUTable({vectors: props.inputVectors}), ownsTable: true};
  }

  return {
    table: new GPUTable(props.device, props.arrowTable!, {
      shaderLayout: props.shaderLayout,
      arrowPaths: props.arrowPaths,
      bufferProps: props.arrowBufferProps,
      allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
    } satisfies GPUTableProps),
    ownsTable: true
  };
}

function validateTableTransformSources(props: {
  table?: GPUTable;
  arrowTable?: arrow.Table;
  inputVectors?: TableTransformInputVectors;
}): void {
  const sourceCount =
    Number(Boolean(props.table)) +
    Number(Boolean(props.arrowTable)) +
    Number(Boolean(props.inputVectors));
  if (sourceCount > 1) {
    throw new Error('TableTransform requires only one of table, arrowTable, or inputVectors');
  }
  if (sourceCount === 0) {
    throw new Error('TableTransform requires table, arrowTable, or inputVectors');
  }
}

function createAutomaticOutputVectors(
  device: Device,
  table: GPUTable,
  copyOutputToInputVectors: TableTransformOutputCopyMap
): Record<string, GPUVector> {
  const outputEntries = Object.entries(copyOutputToInputVectors);
  if (outputEntries.length === 0) {
    return {};
  }
  if (table.batches.length !== 1) {
    throw new Error(
      'TableTransform copyOutputToInputVectors currently requires one directly bindable GPU batch'
    );
  }

  const copiedInputNames = new Set<string>();
  const outputVectors: Record<string, GPUVector> = {};

  try {
    for (const [outputName, inputName] of outputEntries) {
      if (copiedInputNames.has(inputName)) {
        throw new Error(
          `TableTransform copyOutputToInputVectors maps more than one output to "${inputName}"`
        );
      }
      copiedInputNames.add(inputName);

      const inputVector = table.gpuVectors[inputName];
      if (!inputVector) {
        throw new Error(
          `TableTransform copyOutputToInputVectors references missing input vector "${inputName}"`
        );
      }
      validateAutomaticWritebackVector(inputName, inputVector);

      const byteLength = inputVector.length * inputVector.byteStride;
      const outputBuffer = device.createBuffer({
        usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_SRC | Buffer.COPY_DST,
        byteLength: Math.max(1, byteLength)
      });
      outputVectors[outputName] = new GPUVector({
        type: 'buffer',
        name: outputName,
        buffer: outputBuffer,
        arrowType: inputVector.type as AttributeArrowType,
        length: inputVector.length,
        byteStride: inputVector.byteStride,
        ownsBuffer: true
      } as any);
    }
  } catch (error) {
    destroyGPUVectors(outputVectors);
    throw error;
  }

  return outputVectors;
}

function validateAutomaticWritebackVector(name: string, vector: GPUVector): void {
  if (vector.bufferLayout) {
    throw new Error(
      `TableTransform copyOutputToInputVectors does not support interleaved input vector "${name}"`
    );
  }
  if (vector.byteOffset !== 0) {
    throw new Error(
      `TableTransform copyOutputToInputVectors requires zero byteOffset for input vector "${name}"`
    );
  }
  const packedByteStride = getArrowTypeByteStride(vector.type as AttributeArrowType);
  if (vector.byteStride !== packedByteStride) {
    throw new Error(
      `TableTransform copyOutputToInputVectors requires tightly packed input vector "${name}"`
    );
  }
  getConcreteBuffer(vector.buffer);
}

function getTableTransformOutputs(
  transformProps: BufferTransformProps,
  copyOutputToInputVectors: TableTransformOutputCopyMap
): string[] | undefined {
  const inferredOutputs = Object.keys(copyOutputToInputVectors);
  if (inferredOutputs.length === 0) {
    return transformProps.outputs;
  }

  const declaredOutputs = transformProps.outputs || transformProps.varyings;
  if (!declaredOutputs) {
    return inferredOutputs;
  }
  assertSameOutputNames(declaredOutputs, inferredOutputs);
  return transformProps.outputs;
}

function assertSameOutputNames(declaredOutputs: string[], inferredOutputs: string[]): void {
  if (
    declaredOutputs.length !== inferredOutputs.length ||
    declaredOutputs.some(outputName => !inferredOutputs.includes(outputName))
  ) {
    throw new Error(
      'TableTransform outputs must match copyOutputToInputVectors output names when both are supplied'
    );
  }
}

function assertNoExplicitOutputBuffers(
  outputBuffers: TableTransformBufferMap | undefined,
  errorMessage: string
): void {
  if (outputBuffers) {
    throw new Error(errorMessage);
  }
}

function getConcreteBuffer(buffer: Buffer | DynamicBuffer): Buffer {
  return buffer instanceof DynamicBuffer ? buffer.buffer : buffer;
}

function destroyGPUVectors(vectors: Record<string, GPUVector>): void {
  for (const vector of Object.values(vectors)) {
    vector.destroy();
  }
}

function getBufferLayoutNames(bufferLayout: BufferLayout[]): string[] {
  return bufferLayout.map(layout => layout.name);
}

function assertNoDuplicateNames(
  explicitNames: string[],
  tableNames: string[],
  nameType: string
): void {
  const explicitNameSet = new Set(explicitNames);
  for (const tableName of tableNames) {
    if (explicitNameSet.has(tableName)) {
      throw new Error(
        `TableTransform ${nameType} "${tableName}" duplicates an explicit ${nameType}`
      );
    }
  }
}

function assertMatchingBufferLayouts(
  tableBufferLayout: BufferLayout[],
  explicitBufferLayout: BufferLayout[],
  candidateBufferLayout: BufferLayout[],
  errorMessage: string
): void {
  const expectedBufferLayout = [...explicitBufferLayout, ...tableBufferLayout];
  if (JSON.stringify(expectedBufferLayout) !== JSON.stringify(candidateBufferLayout)) {
    throw new Error(errorMessage);
  }
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {
6	Buffer,
7	type BufferLayout,
8	Device,
9	type RenderPassProps,
10	type ShaderLayout
11	} from '@luma.gl/core';
12	import {BufferTransform, DynamicBuffer, type BufferTransformProps} from '@luma.gl/engine';
13	import * as arrow from 'apache-arrow';
14	import type {ArrowVertexFormatOptions} from './arrow-shader-layout';
15	import {GPUVector, type GPUVectorProps} from './arrow-gpu-vector';
16	import {GPURecordBatch} from './arrow-gpu-record-batch';
17	import {GPUTable, type GPUTableProps} from './plain-gpu-table';
18	import type {AttributeArrowType} from './arrow-types';
19	import {getArrowTypeByteStride} from './arrow-gpu-data';
20
21	type TableTransformBufferMap = NonNullable<Parameters<BufferTransform['run']>[0]>['outputBuffers'];
22	type TableTransformRunOptions = Parameters<BufferTransform['run']>[0];
23	type TableTransformInputVectors = Record<string, GPUVector> \| GPUVector[];
24
25	/** Maps transform-feedback output varying names back to the input GPU vector they should update. */
26	export type TableTransformOutputCopyMap = Record<string, string>;
27
28	/** Options supplied for one {@link TableTransform.runBatches} dispatch. */
29	export type TableTransformBatchOptions = RenderPassProps & {
30	/** Output transform-feedback buffers for the current batch. */
31	outputBuffers?:
32	\| TableTransformBufferMap
33	\| ((batch: GPURecordBatch, batchIndex: number) => TableTransformBufferMap);
34	};
35
36	/** Props for creating a WebGL transform backed by a GPU-resident Arrow table. */
37	export type TableTransformProps = BufferTransformProps &
38	ArrowVertexFormatOptions & {
39	/** Existing GPU table used as the source for transform input buffers. */
40	table?: GPUTable;
41	/** Arrow table convenience input converted into a {@link GPUTable}. */
42	arrowTable?: arrow.Table;
43	/** Existing GPU vectors converted into the transform input table. */
44	inputVectors?: TableTransformInputVectors;
45	/** Maps shader attribute names to Arrow column paths. Defaults to using attribute names. */
46	arrowPaths?: Record<string, string>;
47	/** Buffer props applied when `arrowTable` is materialized into GPU vectors. */
48	arrowBufferProps?: GPUVectorProps;
49	/**
50	* Allocates dense transform-feedback output vectors and copies them back into named input
51	* vectors after each run.
52	*
53	* Output names are inferred as transform-feedback outputs when `outputs` is omitted.
54	*/
55	copyOutputToInputVectors?: TableTransformOutputCopyMap;
56	/** Controls whether GPU table row count is assigned to vertexCount. */
57	tableCount?: 'vertex' \| 'none';
58	};
59
60	type TableTransformState = {
61	table: GPUTable;
62	ownsTable: boolean;
63	transformProps: BufferTransformProps;
64	explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
65	explicitBufferLayout: BufferLayout[];
66	inferVertexCount: boolean;
67	copyOutputToInputVectors: TableTransformOutputCopyMap;
68	outputVectors: Record<string, GPUVector>;
69	};
70
71	/**
72	* A WebGL transform-feedback program whose input buffers come from a {@link GPUTable}.
73	*
74	* `TableTransform` keeps the regular {@link BufferTransform} execution model,
75	* while adding table construction and batch-by-batch execution helpers.
76	*/
77	export class TableTransform extends BufferTransform {
78	/** GPU table backing the transform input attributes. */
79	readonly table: GPUTable;
80	/** GPU vectors backing the transform input attributes. */
81	readonly inputVectors: Record<string, GPUVector>;
82	/** Dense transform-feedback output vectors allocated for automatic input writeback. */
83	readonly outputVectors: Record<string, GPUVector>;
84	private readonly ownsTable: boolean;
85	private readonly explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
86	private readonly explicitBufferLayout: BufferLayout[];
87	private readonly inferVertexCount: boolean;
88	private readonly copyOutputToInputVectors: TableTransformOutputCopyMap;
89	private tableTransformDestroyed = false;	1✔
90
91	constructor(device: Device, props: TableTransformProps) {
92	const {
93	table,
94	ownsTable,
95	transformProps,
96	explicitAttributes,
97	explicitBufferLayout,
98	inferVertexCount,
99	copyOutputToInputVectors,
100	outputVectors
101	} = getTableTransformState(device, props);	2✔
102
103	try {	2✔
104	super(device, transformProps);	2✔
105	} catch (error) {
NEW 106	destroyGPUVectors(outputVectors);	×
107	if (ownsTable) {	×
108	table.destroy();	×
109	}
110	throw error;	×
111	}
112
113	this.table = table;	1✔
114	this.inputVectors = table.gpuVectors;	1✔
115	this.outputVectors = outputVectors;	1✔
116	this.ownsTable = ownsTable;	1✔
117	this.explicitAttributes = explicitAttributes;	1✔
118	this.explicitBufferLayout = explicitBufferLayout;	1✔
119	this.inferVertexCount = inferVertexCount;	1✔
120	this.copyOutputToInputVectors = copyOutputToInputVectors;	1✔
121	}
122
123	/** Runs the transform once and optionally copies transform outputs back into input vectors. */
124	override run(options: TableTransformRunOptions = {}): void {	1✔
125	if (!this.hasAutomaticOutputWriteback()) {	1!
NEW 126	super.run(options);	×
NEW 127	return;	×
128	}
129	assertNoExplicitOutputBuffers(	1✔
130	options?.outputBuffers,
131	'TableTransform.run() cannot combine outputBuffers with copyOutputToInputVectors'
132	);
133	super.run({...options, outputBuffers: this.getAutomaticOutputBuffers()});	1✔
134	this.copyOutputsToInputVectors();	1✔
135	}
136
137	/** Runs the transform once per preserved GPU record batch. */
138	runBatches(options: TableTransformBatchOptions = {}): void {	×
NEW 139	if (this.hasAutomaticOutputWriteback() && options.outputBuffers) {	×
NEW 140	throw new Error(	×
141	'TableTransform.runBatches() cannot combine outputBuffers with copyOutputToInputVectors'
142	);
143	}
UNCOV 144	assertMatchingBufferLayouts(	×
145	this.table.bufferLayout,
146	this.explicitBufferLayout,
147	this.model.bufferLayout,
148	'TableTransform.runBatches() model buffer layout does not match its GPU table'
149	);
150
151	try {	×
152	this.table.batches.forEach((batch, batchIndex) => {	×
153	assertMatchingBufferLayouts(	×
154	this.table.bufferLayout,
155	[],
156	batch.bufferLayout,
157	'TableTransform.runBatches() requires every GPU batch to use the table buffer layout'
158	);
159	this.model.setAttributes({	×
160	...this.explicitAttributes,
161	...batch.attributes
162	});
163	if (this.inferVertexCount) {	×
164	this.model.setVertexCount(batch.numRows);	×
165	}
166
167	const outputBuffers =
168	typeof options.outputBuffers === 'function'	×
169	? options.outputBuffers(batch, batchIndex)
170	: options.outputBuffers;
171	const {outputBuffers: _ignoredOutputBuffers, ...renderPassProps} = options;	×
NEW 172	if (this.hasAutomaticOutputWriteback()) {	×
NEW 173	super.run({...renderPassProps, outputBuffers: this.getAutomaticOutputBuffers()});	×
174	} else {
NEW 175	super.run({...renderPassProps, outputBuffers});	×
176	}
177	});
NEW 178	if (this.hasAutomaticOutputWriteback()) {	×
NEW 179	this.copyOutputsToInputVectors();	×
180	}
181	} finally {
182	this.model.setAttributes({	×
183	...this.explicitAttributes,
184	...this.table.attributes
185	});
186	if (this.inferVertexCount) {	×
187	this.model.setVertexCount(this.table.numRows);	×
188	}
189	}
190	}
191
192	override destroy(): void {
193	if (this.tableTransformDestroyed) {	1!
194	return;	×
195	}
196	super.destroy();	1✔
197	destroyGPUVectors(this.outputVectors);	1✔
198	if (this.ownsTable) {	1!
199	this.table.destroy();	1✔
200	}
201	this.tableTransformDestroyed = true;	1✔
202	}
203
204	private hasAutomaticOutputWriteback(): boolean {
205	return Object.keys(this.copyOutputToInputVectors).length > 0;	1✔
206	}
207
208	private getAutomaticOutputBuffers(): TableTransformBufferMap {
209	return Object.fromEntries(	1✔
210	Object.entries(this.outputVectors).map(([outputName, vector]) => [	1✔
211	outputName,
212	getConcreteBuffer(vector.buffer)
213	])
214	);
215	}
216
217	private copyOutputsToInputVectors(): void {
218	const commandEncoder = this.device.createCommandEncoder();	1✔
219	let copyCount = 0;	1✔
220
221	for (const [outputName, inputName] of Object.entries(this.copyOutputToInputVectors)) {	1✔
222	const outputVector = this.outputVectors[outputName];	1✔
223	const inputVector = this.inputVectors[inputName];	1✔
224	if (!outputVector \|\| !inputVector) {	1!
NEW 225	throw new Error(`TableTransform writeback mapping "${outputName}" is incomplete`);	×
226	}
227	const size = inputVector.length * inputVector.byteStride;	1✔
228	if (size === 0) {	1!
NEW 229	continue;	×
230	}
231	commandEncoder.copyBufferToBuffer({	1✔
232	sourceBuffer: getConcreteBuffer(outputVector.buffer),
233	destinationBuffer: getConcreteBuffer(inputVector.buffer),
234	size
235	});
236	copyCount++;	1✔
237	}
238
239	if (copyCount > 0) {	1!
240	this.device.submit(commandEncoder.finish());	1✔
241	return;	1✔
242	}
NEW 243	commandEncoder.destroy();	×
244	}
245	}
246
247	function getTableTransformState(device: Device, props: TableTransformProps): TableTransformState {
248	const {
249	table: explicitTable,
250	arrowTable,
251	inputVectors,
252	arrowPaths,
253	arrowBufferProps,
254	copyOutputToInputVectors = {},	2✔
255	tableCount = 'vertex',	2✔
256	allowWebGLOnlyFormats,
257	...transformProps
258	} = props;	2✔
259
260	validateTableTransformSources({table: explicitTable, arrowTable, inputVectors});	2✔
261	if (!transformProps.shaderLayout) {	2!
262	throw new Error('TableTransform requires shaderLayout');	×
263	}
264
265	const {table, ownsTable} = getInitialTable({	2✔
266	device,
267	table: explicitTable,
268	arrowTable,
269	inputVectors,
270	shaderLayout: transformProps.shaderLayout,
271	arrowPaths,
272	arrowBufferProps,
273	allowWebGLOnlyFormats
274	});
275
276	const explicitAttributes = transformProps.attributes \|\| {};	2✔
277	const explicitBufferLayout = transformProps.bufferLayout \|\| [];	2✔
278	const inferVertexCount = tableCount === 'vertex' && transformProps.vertexCount === undefined;	2✔
279	let outputVectors: Record<string, GPUVector> = {};	2✔
280	let transformOutputs: string[] \| undefined;
281
282	try {	2✔
283	assertNoDuplicateNames(	2✔
284	Object.keys(explicitAttributes),
285	Object.keys(table.attributes),
286	'attribute'
287	);
288	assertNoDuplicateNames(	2✔
289	getBufferLayoutNames(explicitBufferLayout),
290	getBufferLayoutNames(table.bufferLayout),
291	'buffer layout'
292	);
293	outputVectors = createAutomaticOutputVectors(device, table, copyOutputToInputVectors);	2✔
294	transformOutputs = getTableTransformOutputs(transformProps, copyOutputToInputVectors);	2✔
295	} catch (error) {
296	destroyGPUVectors(outputVectors);	1✔
297	if (ownsTable) {	1!
298	table.destroy();	1✔
299	}
300	throw error;	1✔
301	}
302
303	return {	1✔
304	table,
305	ownsTable,
306	explicitAttributes,
307	explicitBufferLayout,
308	inferVertexCount,
309	copyOutputToInputVectors,
310	outputVectors,
311	transformProps: {
312	...transformProps,
313	...(transformOutputs ? {outputs: transformOutputs} : {}),	1!
314	attributes: {...explicitAttributes, ...table.attributes},
315	bufferLayout: [...explicitBufferLayout, ...table.bufferLayout],
316	...(inferVertexCount ? {vertexCount: table.numRows} : {})	1!
317	}
318	};
319	}
320
321	function getInitialTable(props: {
322	device: Device;
323	table?: GPUTable;
324	arrowTable?: arrow.Table;
325	inputVectors?: TableTransformInputVectors;
326	shaderLayout: ShaderLayout;
327	arrowPaths?: Record<string, string>;
328	arrowBufferProps?: GPUVectorProps;
329	allowWebGLOnlyFormats?: boolean;
330	}): {table: GPUTable; ownsTable: boolean} {
331	if (props.table) {	2!
332	return {table: props.table, ownsTable: false};	×
333	}
334	if (props.inputVectors) {	2!
335	return {table: new GPUTable({vectors: props.inputVectors}), ownsTable: true};	2✔
336	}
337
338	return {	×
339	table: new GPUTable(props.device, props.arrowTable!, {
340	shaderLayout: props.shaderLayout,
341	arrowPaths: props.arrowPaths,
342	bufferProps: props.arrowBufferProps,
343	allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
344	} satisfies GPUTableProps),
345	ownsTable: true
346	};
347	}
348
349	function validateTableTransformSources(props: {
350	table?: GPUTable;
351	arrowTable?: arrow.Table;
352	inputVectors?: TableTransformInputVectors;
353	}): void {
354	const sourceCount =
355	Number(Boolean(props.table)) +	2✔
356	Number(Boolean(props.arrowTable)) +
357	Number(Boolean(props.inputVectors));
358	if (sourceCount > 1) {	2!
NEW 359	throw new Error('TableTransform requires only one of table, arrowTable, or inputVectors');	×
360	}
361	if (sourceCount === 0) {	2!
NEW 362	throw new Error('TableTransform requires table, arrowTable, or inputVectors');	×
363	}
364	}
365
366	function createAutomaticOutputVectors(
367	device: Device,
368	table: GPUTable,
369	copyOutputToInputVectors: TableTransformOutputCopyMap
370	): Record<string, GPUVector> {
371	const outputEntries = Object.entries(copyOutputToInputVectors);	2✔
372	if (outputEntries.length === 0) {	2!
NEW 373	return {};	×
374	}
375	if (table.batches.length !== 1) {	2!
NEW 376	throw new Error(	×
377	'TableTransform copyOutputToInputVectors currently requires one directly bindable GPU batch'
378	);
379	}
380
381	const copiedInputNames = new Set<string>();	2✔
382	const outputVectors: Record<string, GPUVector> = {};	2✔
383
384	try {	2✔
385	for (const [outputName, inputName] of outputEntries) {	2✔
386	if (copiedInputNames.has(inputName)) {	2!
NEW 387	throw new Error(	×
388	`TableTransform copyOutputToInputVectors maps more than one output to "${inputName}"`
389	);
390	}
391	copiedInputNames.add(inputName);	2✔
392
393	const inputVector = table.gpuVectors[inputName];	2✔
394	if (!inputVector) {	2!
NEW 395	throw new Error(	×
396	`TableTransform copyOutputToInputVectors references missing input vector "${inputName}"`
397	);
398	}
399	validateAutomaticWritebackVector(inputName, inputVector);	2✔
400
401	const byteLength = inputVector.length * inputVector.byteStride;	2✔
402	const outputBuffer = device.createBuffer({	2✔
403	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_SRC \| Buffer.COPY_DST,
404	byteLength: Math.max(1, byteLength)
405	});
406	outputVectors[outputName] = new GPUVector({	2✔
407	type: 'buffer',
408	name: outputName,
409	buffer: outputBuffer,
410	arrowType: inputVector.type as AttributeArrowType,
411	length: inputVector.length,
412	byteStride: inputVector.byteStride,
413	ownsBuffer: true
414	} as any);
415	}
416	} catch (error) {
417	destroyGPUVectors(outputVectors);	1✔
418	throw error;	1✔
419	}
420
421	return outputVectors;	1✔
422	}
423
424	function validateAutomaticWritebackVector(name: string, vector: GPUVector): void {
425	if (vector.bufferLayout) {	2!
NEW 426	throw new Error(	×
427	`TableTransform copyOutputToInputVectors does not support interleaved input vector "${name}"`
428	);
429	}
430	if (vector.byteOffset !== 0) {	2!
NEW 431	throw new Error(	×
432	`TableTransform copyOutputToInputVectors requires zero byteOffset for input vector "${name}"`
433	);
434	}
435	const packedByteStride = getArrowTypeByteStride(vector.type as AttributeArrowType);	2✔
436	if (vector.byteStride !== packedByteStride) {	2✔
437	throw new Error(	1✔
438	`TableTransform copyOutputToInputVectors requires tightly packed input vector "${name}"`
439	);
440	}
441	getConcreteBuffer(vector.buffer);	1✔
442	}
443
444	function getTableTransformOutputs(
445	transformProps: BufferTransformProps,
446	copyOutputToInputVectors: TableTransformOutputCopyMap
447	): string[] \| undefined {
448	const inferredOutputs = Object.keys(copyOutputToInputVectors);	1✔
449	if (inferredOutputs.length === 0) {	1!
NEW 450	return transformProps.outputs;	×
451	}
452
453	const declaredOutputs = transformProps.outputs \|\| transformProps.varyings;	1✔
454	if (!declaredOutputs) {	1!
455	return inferredOutputs;	1✔
456	}
NEW 457	assertSameOutputNames(declaredOutputs, inferredOutputs);	×
NEW 458	return transformProps.outputs;	×
459	}
460
461	function assertSameOutputNames(declaredOutputs: string[], inferredOutputs: string[]): void {
NEW 462	if (	×
463	declaredOutputs.length !== inferredOutputs.length \|\|	×
NEW 464	declaredOutputs.some(outputName => !inferredOutputs.includes(outputName))	×
465	) {
NEW 466	throw new Error(	×
467	'TableTransform outputs must match copyOutputToInputVectors output names when both are supplied'
468	);
469	}
470	}
471
472	function assertNoExplicitOutputBuffers(
473	outputBuffers: TableTransformBufferMap \| undefined,
474	errorMessage: string
475	): void {
476	if (outputBuffers) {	1!
NEW 477	throw new Error(errorMessage);	×
478	}
479	}
480
481	function getConcreteBuffer(buffer: Buffer \| DynamicBuffer): Buffer {
482	return buffer instanceof DynamicBuffer ? buffer.buffer : buffer;	4!
483	}
484
485	function destroyGPUVectors(vectors: Record<string, GPUVector>): void {
486	for (const vector of Object.values(vectors)) {	3✔
487	vector.destroy();	1✔
488	}
489	}
490
491	function getBufferLayoutNames(bufferLayout: BufferLayout[]): string[] {
492	return bufferLayout.map(layout => layout.name);	4✔
493	}
494
495	function assertNoDuplicateNames(
496	explicitNames: string[],
497	tableNames: string[],
498	nameType: string
499	): void {
500	const explicitNameSet = new Set(explicitNames);	4✔
501	for (const tableName of tableNames) {	4✔
502	if (explicitNameSet.has(tableName)) {	4!
503	throw new Error(	×
504	`TableTransform ${nameType} "${tableName}" duplicates an explicit ${nameType}`
505	);
506	}
507	}
508	}
509
510	function assertMatchingBufferLayouts(
511	tableBufferLayout: BufferLayout[],
512	explicitBufferLayout: BufferLayout[],
513	candidateBufferLayout: BufferLayout[],
514	errorMessage: string
515	): void {
516	const expectedBufferLayout = [...explicitBufferLayout, ...tableBufferLayout];	×
517	if (JSON.stringify(expectedBufferLayout) !== JSON.stringify(candidateBufferLayout)) {	×
518	throw new Error(errorMessage);	×
519	}
520	}

visgl / luma.gl / 25992522693

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous