• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

visgl / luma.gl / 25992522693

17 May 2026 01:42PM UTC coverage: 74.98% (+0.04%) from 74.942%
25992522693

push

github

web-flow
feat(arrow) WebGL transform handles Vectors and writeback (#2618)

6814 of 10253 branches covered (66.46%)

Branch coverage included in aggregate %.

81 of 159 new or added lines in 3 files covered. (50.94%)

2 existing lines in 2 files now uncovered.

14811 of 18588 relevant lines covered (79.68%)

845.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.4
/modules/arrow/src/arrow/table-transform.ts
1
// luma.gl
2
// SPDX-License-Identifier: MIT
3
// Copyright (c) vis.gl contributors
4

5
import {
6
  Buffer,
7
  type BufferLayout,
8
  Device,
9
  type RenderPassProps,
10
  type ShaderLayout
11
} from '@luma.gl/core';
12
import {BufferTransform, DynamicBuffer, type BufferTransformProps} from '@luma.gl/engine';
13
import * as arrow from 'apache-arrow';
14
import type {ArrowVertexFormatOptions} from './arrow-shader-layout';
15
import {GPUVector, type GPUVectorProps} from './arrow-gpu-vector';
16
import {GPURecordBatch} from './arrow-gpu-record-batch';
17
import {GPUTable, type GPUTableProps} from './plain-gpu-table';
18
import type {AttributeArrowType} from './arrow-types';
19
import {getArrowTypeByteStride} from './arrow-gpu-data';
20

21
type TableTransformBufferMap = NonNullable<Parameters<BufferTransform['run']>[0]>['outputBuffers'];
22
type TableTransformRunOptions = Parameters<BufferTransform['run']>[0];
23
type TableTransformInputVectors = Record<string, GPUVector> | GPUVector[];
24

25
/** Maps transform-feedback output varying names back to the input GPU vector they should update. */
26
export type TableTransformOutputCopyMap = Record<string, string>;
27

28
/** Options supplied for one {@link TableTransform.runBatches} dispatch. */
29
export type TableTransformBatchOptions = RenderPassProps & {
30
  /** Output transform-feedback buffers for the current batch. */
31
  outputBuffers?:
32
    | TableTransformBufferMap
33
    | ((batch: GPURecordBatch, batchIndex: number) => TableTransformBufferMap);
34
};
35

36
/** Props for creating a WebGL transform backed by a GPU-resident Arrow table. */
37
export type TableTransformProps = BufferTransformProps &
38
  ArrowVertexFormatOptions & {
39
    /** Existing GPU table used as the source for transform input buffers. */
40
    table?: GPUTable;
41
    /** Arrow table convenience input converted into a {@link GPUTable}. */
42
    arrowTable?: arrow.Table;
43
    /** Existing GPU vectors converted into the transform input table. */
44
    inputVectors?: TableTransformInputVectors;
45
    /** Maps shader attribute names to Arrow column paths. Defaults to using attribute names. */
46
    arrowPaths?: Record<string, string>;
47
    /** Buffer props applied when `arrowTable` is materialized into GPU vectors. */
48
    arrowBufferProps?: GPUVectorProps;
49
    /**
50
     * Allocates dense transform-feedback output vectors and copies them back into named input
51
     * vectors after each run.
52
     *
53
     * Output names are inferred as transform-feedback outputs when `outputs` is omitted.
54
     */
55
    copyOutputToInputVectors?: TableTransformOutputCopyMap;
56
    /** Controls whether GPU table row count is assigned to vertexCount. */
57
    tableCount?: 'vertex' | 'none';
58
  };
59

60
type TableTransformState = {
61
  table: GPUTable;
62
  ownsTable: boolean;
63
  transformProps: BufferTransformProps;
64
  explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
65
  explicitBufferLayout: BufferLayout[];
66
  inferVertexCount: boolean;
67
  copyOutputToInputVectors: TableTransformOutputCopyMap;
68
  outputVectors: Record<string, GPUVector>;
69
};
70

71
/**
72
 * A WebGL transform-feedback program whose input buffers come from a {@link GPUTable}.
73
 *
74
 * `TableTransform` keeps the regular {@link BufferTransform} execution model,
75
 * while adding table construction and batch-by-batch execution helpers.
76
 */
77
export class TableTransform extends BufferTransform {
78
  /** GPU table backing the transform input attributes. */
79
  readonly table: GPUTable;
80
  /** GPU vectors backing the transform input attributes. */
81
  readonly inputVectors: Record<string, GPUVector>;
82
  /** Dense transform-feedback output vectors allocated for automatic input writeback. */
83
  readonly outputVectors: Record<string, GPUVector>;
84
  private readonly ownsTable: boolean;
85
  private readonly explicitAttributes: NonNullable<BufferTransformProps['attributes']>;
86
  private readonly explicitBufferLayout: BufferLayout[];
87
  private readonly inferVertexCount: boolean;
88
  private readonly copyOutputToInputVectors: TableTransformOutputCopyMap;
89
  private tableTransformDestroyed = false;
1✔
90

91
  constructor(device: Device, props: TableTransformProps) {
92
    const {
93
      table,
94
      ownsTable,
95
      transformProps,
96
      explicitAttributes,
97
      explicitBufferLayout,
98
      inferVertexCount,
99
      copyOutputToInputVectors,
100
      outputVectors
101
    } = getTableTransformState(device, props);
2✔
102

103
    try {
2✔
104
      super(device, transformProps);
2✔
105
    } catch (error) {
NEW
106
      destroyGPUVectors(outputVectors);
×
107
      if (ownsTable) {
×
108
        table.destroy();
×
109
      }
110
      throw error;
×
111
    }
112

113
    this.table = table;
1✔
114
    this.inputVectors = table.gpuVectors;
1✔
115
    this.outputVectors = outputVectors;
1✔
116
    this.ownsTable = ownsTable;
1✔
117
    this.explicitAttributes = explicitAttributes;
1✔
118
    this.explicitBufferLayout = explicitBufferLayout;
1✔
119
    this.inferVertexCount = inferVertexCount;
1✔
120
    this.copyOutputToInputVectors = copyOutputToInputVectors;
1✔
121
  }
122

123
  /** Runs the transform once and optionally copies transform outputs back into input vectors. */
124
  override run(options: TableTransformRunOptions = {}): void {
1✔
125
    if (!this.hasAutomaticOutputWriteback()) {
1!
NEW
126
      super.run(options);
×
NEW
127
      return;
×
128
    }
129
    assertNoExplicitOutputBuffers(
1✔
130
      options?.outputBuffers,
131
      'TableTransform.run() cannot combine outputBuffers with copyOutputToInputVectors'
132
    );
133
    super.run({...options, outputBuffers: this.getAutomaticOutputBuffers()});
1✔
134
    this.copyOutputsToInputVectors();
1✔
135
  }
136

137
  /** Runs the transform once per preserved GPU record batch. */
138
  runBatches(options: TableTransformBatchOptions = {}): void {
×
NEW
139
    if (this.hasAutomaticOutputWriteback() && options.outputBuffers) {
×
NEW
140
      throw new Error(
×
141
        'TableTransform.runBatches() cannot combine outputBuffers with copyOutputToInputVectors'
142
      );
143
    }
UNCOV
144
    assertMatchingBufferLayouts(
×
145
      this.table.bufferLayout,
146
      this.explicitBufferLayout,
147
      this.model.bufferLayout,
148
      'TableTransform.runBatches() model buffer layout does not match its GPU table'
149
    );
150

151
    try {
×
152
      this.table.batches.forEach((batch, batchIndex) => {
×
153
        assertMatchingBufferLayouts(
×
154
          this.table.bufferLayout,
155
          [],
156
          batch.bufferLayout,
157
          'TableTransform.runBatches() requires every GPU batch to use the table buffer layout'
158
        );
159
        this.model.setAttributes({
×
160
          ...this.explicitAttributes,
161
          ...batch.attributes
162
        });
163
        if (this.inferVertexCount) {
×
164
          this.model.setVertexCount(batch.numRows);
×
165
        }
166

167
        const outputBuffers =
168
          typeof options.outputBuffers === 'function'
×
169
            ? options.outputBuffers(batch, batchIndex)
170
            : options.outputBuffers;
171
        const {outputBuffers: _ignoredOutputBuffers, ...renderPassProps} = options;
×
NEW
172
        if (this.hasAutomaticOutputWriteback()) {
×
NEW
173
          super.run({...renderPassProps, outputBuffers: this.getAutomaticOutputBuffers()});
×
174
        } else {
NEW
175
          super.run({...renderPassProps, outputBuffers});
×
176
        }
177
      });
NEW
178
      if (this.hasAutomaticOutputWriteback()) {
×
NEW
179
        this.copyOutputsToInputVectors();
×
180
      }
181
    } finally {
182
      this.model.setAttributes({
×
183
        ...this.explicitAttributes,
184
        ...this.table.attributes
185
      });
186
      if (this.inferVertexCount) {
×
187
        this.model.setVertexCount(this.table.numRows);
×
188
      }
189
    }
190
  }
191

192
  override destroy(): void {
193
    if (this.tableTransformDestroyed) {
1!
194
      return;
×
195
    }
196
    super.destroy();
1✔
197
    destroyGPUVectors(this.outputVectors);
1✔
198
    if (this.ownsTable) {
1!
199
      this.table.destroy();
1✔
200
    }
201
    this.tableTransformDestroyed = true;
1✔
202
  }
203

204
  private hasAutomaticOutputWriteback(): boolean {
205
    return Object.keys(this.copyOutputToInputVectors).length > 0;
1✔
206
  }
207

208
  private getAutomaticOutputBuffers(): TableTransformBufferMap {
209
    return Object.fromEntries(
1✔
210
      Object.entries(this.outputVectors).map(([outputName, vector]) => [
1✔
211
        outputName,
212
        getConcreteBuffer(vector.buffer)
213
      ])
214
    );
215
  }
216

217
  private copyOutputsToInputVectors(): void {
218
    const commandEncoder = this.device.createCommandEncoder();
1✔
219
    let copyCount = 0;
1✔
220

221
    for (const [outputName, inputName] of Object.entries(this.copyOutputToInputVectors)) {
1✔
222
      const outputVector = this.outputVectors[outputName];
1✔
223
      const inputVector = this.inputVectors[inputName];
1✔
224
      if (!outputVector || !inputVector) {
1!
NEW
225
        throw new Error(`TableTransform writeback mapping "${outputName}" is incomplete`);
×
226
      }
227
      const size = inputVector.length * inputVector.byteStride;
1✔
228
      if (size === 0) {
1!
NEW
229
        continue;
×
230
      }
231
      commandEncoder.copyBufferToBuffer({
1✔
232
        sourceBuffer: getConcreteBuffer(outputVector.buffer),
233
        destinationBuffer: getConcreteBuffer(inputVector.buffer),
234
        size
235
      });
236
      copyCount++;
1✔
237
    }
238

239
    if (copyCount > 0) {
1!
240
      this.device.submit(commandEncoder.finish());
1✔
241
      return;
1✔
242
    }
NEW
243
    commandEncoder.destroy();
×
244
  }
245
}
246

247
function getTableTransformState(device: Device, props: TableTransformProps): TableTransformState {
248
  const {
249
    table: explicitTable,
250
    arrowTable,
251
    inputVectors,
252
    arrowPaths,
253
    arrowBufferProps,
254
    copyOutputToInputVectors = {},
2✔
255
    tableCount = 'vertex',
2✔
256
    allowWebGLOnlyFormats,
257
    ...transformProps
258
  } = props;
2✔
259

260
  validateTableTransformSources({table: explicitTable, arrowTable, inputVectors});
2✔
261
  if (!transformProps.shaderLayout) {
2!
262
    throw new Error('TableTransform requires shaderLayout');
×
263
  }
264

265
  const {table, ownsTable} = getInitialTable({
2✔
266
    device,
267
    table: explicitTable,
268
    arrowTable,
269
    inputVectors,
270
    shaderLayout: transformProps.shaderLayout,
271
    arrowPaths,
272
    arrowBufferProps,
273
    allowWebGLOnlyFormats
274
  });
275

276
  const explicitAttributes = transformProps.attributes || {};
2✔
277
  const explicitBufferLayout = transformProps.bufferLayout || [];
2✔
278
  const inferVertexCount = tableCount === 'vertex' && transformProps.vertexCount === undefined;
2✔
279
  let outputVectors: Record<string, GPUVector> = {};
2✔
280
  let transformOutputs: string[] | undefined;
281

282
  try {
2✔
283
    assertNoDuplicateNames(
2✔
284
      Object.keys(explicitAttributes),
285
      Object.keys(table.attributes),
286
      'attribute'
287
    );
288
    assertNoDuplicateNames(
2✔
289
      getBufferLayoutNames(explicitBufferLayout),
290
      getBufferLayoutNames(table.bufferLayout),
291
      'buffer layout'
292
    );
293
    outputVectors = createAutomaticOutputVectors(device, table, copyOutputToInputVectors);
2✔
294
    transformOutputs = getTableTransformOutputs(transformProps, copyOutputToInputVectors);
2✔
295
  } catch (error) {
296
    destroyGPUVectors(outputVectors);
1✔
297
    if (ownsTable) {
1!
298
      table.destroy();
1✔
299
    }
300
    throw error;
1✔
301
  }
302

303
  return {
1✔
304
    table,
305
    ownsTable,
306
    explicitAttributes,
307
    explicitBufferLayout,
308
    inferVertexCount,
309
    copyOutputToInputVectors,
310
    outputVectors,
311
    transformProps: {
312
      ...transformProps,
313
      ...(transformOutputs ? {outputs: transformOutputs} : {}),
1!
314
      attributes: {...explicitAttributes, ...table.attributes},
315
      bufferLayout: [...explicitBufferLayout, ...table.bufferLayout],
316
      ...(inferVertexCount ? {vertexCount: table.numRows} : {})
1!
317
    }
318
  };
319
}
320

321
function getInitialTable(props: {
322
  device: Device;
323
  table?: GPUTable;
324
  arrowTable?: arrow.Table;
325
  inputVectors?: TableTransformInputVectors;
326
  shaderLayout: ShaderLayout;
327
  arrowPaths?: Record<string, string>;
328
  arrowBufferProps?: GPUVectorProps;
329
  allowWebGLOnlyFormats?: boolean;
330
}): {table: GPUTable; ownsTable: boolean} {
331
  if (props.table) {
2!
332
    return {table: props.table, ownsTable: false};
×
333
  }
334
  if (props.inputVectors) {
2!
335
    return {table: new GPUTable({vectors: props.inputVectors}), ownsTable: true};
2✔
336
  }
337

338
  return {
×
339
    table: new GPUTable(props.device, props.arrowTable!, {
340
      shaderLayout: props.shaderLayout,
341
      arrowPaths: props.arrowPaths,
342
      bufferProps: props.arrowBufferProps,
343
      allowWebGLOnlyFormats: props.allowWebGLOnlyFormats
344
    } satisfies GPUTableProps),
345
    ownsTable: true
346
  };
347
}
348

349
function validateTableTransformSources(props: {
350
  table?: GPUTable;
351
  arrowTable?: arrow.Table;
352
  inputVectors?: TableTransformInputVectors;
353
}): void {
354
  const sourceCount =
355
    Number(Boolean(props.table)) +
2✔
356
    Number(Boolean(props.arrowTable)) +
357
    Number(Boolean(props.inputVectors));
358
  if (sourceCount > 1) {
2!
NEW
359
    throw new Error('TableTransform requires only one of table, arrowTable, or inputVectors');
×
360
  }
361
  if (sourceCount === 0) {
2!
NEW
362
    throw new Error('TableTransform requires table, arrowTable, or inputVectors');
×
363
  }
364
}
365

366
function createAutomaticOutputVectors(
367
  device: Device,
368
  table: GPUTable,
369
  copyOutputToInputVectors: TableTransformOutputCopyMap
370
): Record<string, GPUVector> {
371
  const outputEntries = Object.entries(copyOutputToInputVectors);
2✔
372
  if (outputEntries.length === 0) {
2!
NEW
373
    return {};
×
374
  }
375
  if (table.batches.length !== 1) {
2!
NEW
376
    throw new Error(
×
377
      'TableTransform copyOutputToInputVectors currently requires one directly bindable GPU batch'
378
    );
379
  }
380

381
  const copiedInputNames = new Set<string>();
2✔
382
  const outputVectors: Record<string, GPUVector> = {};
2✔
383

384
  try {
2✔
385
    for (const [outputName, inputName] of outputEntries) {
2✔
386
      if (copiedInputNames.has(inputName)) {
2!
NEW
387
        throw new Error(
×
388
          `TableTransform copyOutputToInputVectors maps more than one output to "${inputName}"`
389
        );
390
      }
391
      copiedInputNames.add(inputName);
2✔
392

393
      const inputVector = table.gpuVectors[inputName];
2✔
394
      if (!inputVector) {
2!
NEW
395
        throw new Error(
×
396
          `TableTransform copyOutputToInputVectors references missing input vector "${inputName}"`
397
        );
398
      }
399
      validateAutomaticWritebackVector(inputName, inputVector);
2✔
400

401
      const byteLength = inputVector.length * inputVector.byteStride;
2✔
402
      const outputBuffer = device.createBuffer({
2✔
403
        usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_SRC | Buffer.COPY_DST,
404
        byteLength: Math.max(1, byteLength)
405
      });
406
      outputVectors[outputName] = new GPUVector({
2✔
407
        type: 'buffer',
408
        name: outputName,
409
        buffer: outputBuffer,
410
        arrowType: inputVector.type as AttributeArrowType,
411
        length: inputVector.length,
412
        byteStride: inputVector.byteStride,
413
        ownsBuffer: true
414
      } as any);
415
    }
416
  } catch (error) {
417
    destroyGPUVectors(outputVectors);
1✔
418
    throw error;
1✔
419
  }
420

421
  return outputVectors;
1✔
422
}
423

424
function validateAutomaticWritebackVector(name: string, vector: GPUVector): void {
425
  if (vector.bufferLayout) {
2!
NEW
426
    throw new Error(
×
427
      `TableTransform copyOutputToInputVectors does not support interleaved input vector "${name}"`
428
    );
429
  }
430
  if (vector.byteOffset !== 0) {
2!
NEW
431
    throw new Error(
×
432
      `TableTransform copyOutputToInputVectors requires zero byteOffset for input vector "${name}"`
433
    );
434
  }
435
  const packedByteStride = getArrowTypeByteStride(vector.type as AttributeArrowType);
2✔
436
  if (vector.byteStride !== packedByteStride) {
2✔
437
    throw new Error(
1✔
438
      `TableTransform copyOutputToInputVectors requires tightly packed input vector "${name}"`
439
    );
440
  }
441
  getConcreteBuffer(vector.buffer);
1✔
442
}
443

444
function getTableTransformOutputs(
445
  transformProps: BufferTransformProps,
446
  copyOutputToInputVectors: TableTransformOutputCopyMap
447
): string[] | undefined {
448
  const inferredOutputs = Object.keys(copyOutputToInputVectors);
1✔
449
  if (inferredOutputs.length === 0) {
1!
NEW
450
    return transformProps.outputs;
×
451
  }
452

453
  const declaredOutputs = transformProps.outputs || transformProps.varyings;
1✔
454
  if (!declaredOutputs) {
1!
455
    return inferredOutputs;
1✔
456
  }
NEW
457
  assertSameOutputNames(declaredOutputs, inferredOutputs);
×
NEW
458
  return transformProps.outputs;
×
459
}
460

461
function assertSameOutputNames(declaredOutputs: string[], inferredOutputs: string[]): void {
NEW
462
  if (
×
463
    declaredOutputs.length !== inferredOutputs.length ||
×
NEW
464
    declaredOutputs.some(outputName => !inferredOutputs.includes(outputName))
×
465
  ) {
NEW
466
    throw new Error(
×
467
      'TableTransform outputs must match copyOutputToInputVectors output names when both are supplied'
468
    );
469
  }
470
}
471

472
function assertNoExplicitOutputBuffers(
473
  outputBuffers: TableTransformBufferMap | undefined,
474
  errorMessage: string
475
): void {
476
  if (outputBuffers) {
1!
NEW
477
    throw new Error(errorMessage);
×
478
  }
479
}
480

481
function getConcreteBuffer(buffer: Buffer | DynamicBuffer): Buffer {
482
  return buffer instanceof DynamicBuffer ? buffer.buffer : buffer;
4!
483
}
484

485
function destroyGPUVectors(vectors: Record<string, GPUVector>): void {
486
  for (const vector of Object.values(vectors)) {
3✔
487
    vector.destroy();
1✔
488
  }
489
}
490

491
function getBufferLayoutNames(bufferLayout: BufferLayout[]): string[] {
492
  return bufferLayout.map(layout => layout.name);
4✔
493
}
494

495
function assertNoDuplicateNames(
496
  explicitNames: string[],
497
  tableNames: string[],
498
  nameType: string
499
): void {
500
  const explicitNameSet = new Set(explicitNames);
4✔
501
  for (const tableName of tableNames) {
4✔
502
    if (explicitNameSet.has(tableName)) {
4!
503
      throw new Error(
×
504
        `TableTransform ${nameType} "${tableName}" duplicates an explicit ${nameType}`
505
      );
506
    }
507
  }
508
}
509

510
function assertMatchingBufferLayouts(
511
  tableBufferLayout: BufferLayout[],
512
  explicitBufferLayout: BufferLayout[],
513
  candidateBufferLayout: BufferLayout[],
514
  errorMessage: string
515
): void {
516
  const expectedBufferLayout = [...explicitBufferLayout, ...tableBufferLayout];
×
517
  if (JSON.stringify(expectedBufferLayout) !== JSON.stringify(candidateBufferLayout)) {
×
518
    throw new Error(errorMessage);
×
519
  }
520
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc