• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

visgl / luma.gl / 25990278810

17 May 2026 12:01PM UTC coverage: 75.092% (+0.2%) from 74.881%
25990278810

push

github

web-flow
feat: Columnar GPU-data stack (#2616)

6711 of 10084 branches covered (66.55%)

Branch coverage included in aggregate %.

625 of 865 new or added lines in 22 files covered. (72.25%)

1 existing line in 1 file now uncovered.

14631 of 18337 relevant lines covered (79.79%)

792.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.43
/modules/arrow/src/arrow/arrow-gpu-record-batch.ts
1
// luma.gl
2
// SPDX-License-Identifier: MIT
3
// Copyright (c) vis.gl contributors
4

5
import {Buffer, Device, type BufferLayout, type ShaderLayout} from '@luma.gl/core';
6
import type {DynamicBuffer} from '@luma.gl/engine';
7
import * as arrow from 'apache-arrow';
8
import {getArrowFieldByPath, getArrowVectorByPath} from './arrow-paths';
9
import {getArrowBufferLayout, type ArrowVertexFormatOptions} from './arrow-shader-layout';
10
import type {AttributeArrowType} from './arrow-types';
11
import {
12
  getAppendableGPUColumnData,
13
  getAppendableGPUColumns,
14
  type AppendableGPUColumn
15
} from './arrow-gpu-appendable';
16
import {
17
  GPUVector,
18
  type GPUVectorBufferProps,
19
  type GPUVectorDynamicBufferProps,
20
  type GPUVectorProps
21
} from './arrow-gpu-vector';
22
import {createGPUVectorCollection} from './arrow-gpu-vector-collection';
23

24
/** Options for creating GPU buffers from shader-compatible Arrow record batch columns. */
25
export type GPURecordBatchProps = ArrowVertexFormatOptions & {
26
  /** Shader layout that selects which Arrow columns should be uploaded. */
27
  shaderLayout: ShaderLayout;
28
  /** Maps shader attribute names to Arrow column paths. Defaults to using the attribute name. */
29
  arrowPaths?: Record<string, string>;
30
  /** Buffer props applied to every Arrow-backed GPU vector. */
31
  bufferProps?: GPUVectorProps;
32
};
33

34
/** Options for constructing an GPURecordBatch from existing GPU vectors. */
35
export type GPURecordBatchFromVectorsProps = {
36
  /** GPU vectors keyed by name, or a list of named GPU vectors. */
37
  vectors: Record<string, GPUVector> | GPUVector[];
38
  /** Optional precomputed batch buffer layouts. */
39
  bufferLayout?: BufferLayout[];
40
  /** Optional selected schema fields. Defaults to fields synthesized from vector names and types. */
41
  fields?: arrow.Field[];
42
  /** Optional batch-level schema metadata. */
43
  metadata?: Map<string, string>;
44
  /** Number of null rows in the generated GPU record batch. */
45
  nullCount?: number;
46
};
47

48
/** Props for constructing an empty appendable GPU record batch from a selected schema. */
49
export type GPURecordBatchAppendableProps = ArrowVertexFormatOptions & {
50
  /** Discriminator for appendable record batch construction. */
51
  type: 'appendable';
52
  /** Device that creates appendable vector storage. */
53
  device: Device;
54
  /** Source schema used to select shader-compatible columns. */
55
  schema: arrow.Schema;
56
  /** Shader layout that selects which Arrow columns should be uploaded. */
57
  shaderLayout: ShaderLayout;
58
  /** Maps shader attribute names to Arrow column paths. Defaults to using the attribute name. */
59
  arrowPaths?: Record<string, string>;
60
  /** Initial row capacity for each appendable vector. */
61
  initialCapacityRows?: number;
62
  /** Appendable vector capacity growth multiplier. */
63
  capacityGrowthFactor?: number;
64
  /** Dynamic buffer props forwarded to appendable vectors. */
65
  bufferProps?: GPUVectorDynamicBufferProps;
66
};
67

68
/** GPU memory and Arrow schema metadata for one selected Arrow RecordBatch. */
69
export class GPURecordBatch {
70
  /** GPU-facing schema for the selected shader attribute columns. */
71
  schema: arrow.Schema;
72
  /** Number of rows in the source Arrow record batch. */
73
  numRows: number;
74
  /** Number of selected GPU columns in {@link schema}. */
75
  numCols: number;
76
  /** Number of null rows in the source Arrow record batch. */
77
  nullCount: number;
78
  /** Buffer layout derived from the selected Arrow columns and shader layout. */
79
  readonly bufferLayout: BufferLayout[] = [];
51✔
80
  /** GPU vectors keyed by shader attribute name. */
81
  readonly gpuVectors: Record<string, GPUVector> = {};
51✔
82
  /** Model-ready attribute buffers keyed by shader attribute name. */
83
  readonly attributes: Record<string, Buffer | DynamicBuffer> = {};
51✔
84
  /** Model-ready storage bindings keyed by shader binding name. */
85
  readonly bindings: Record<string, Buffer | DynamicBuffer> = {};
51✔
86
  private readonly appendableColumns?: AppendableGPUColumn[];
87

88
  /** Creates GPU buffers and GPU-facing schema from one Arrow record batch. */
89
  constructor(device: Device, recordBatch: arrow.RecordBatch, props: GPURecordBatchProps);
90
  /** Creates a GPU-facing record batch from existing named GPU vectors. */
91
  constructor(props: GPURecordBatchFromVectorsProps);
92
  /** Creates an empty appendable GPU record batch from a selected schema. */
93
  constructor(props: GPURecordBatchAppendableProps);
94
  constructor(
95
    deviceOrProps: Device | GPURecordBatchFromVectorsProps | GPURecordBatchAppendableProps,
96
    recordBatch?: arrow.RecordBatch,
97
    props?: GPURecordBatchProps
98
  ) {
99
    if (!(deviceOrProps instanceof Device)) {
51✔
100
      if ('type' in deviceOrProps && deviceOrProps.type === 'appendable') {
16✔
101
        const options = deviceOrProps;
11✔
102
        const appendableColumns = getAppendableGPUColumns({
11✔
103
          schema: options.schema,
104
          shaderLayout: options.shaderLayout,
105
          arrowPaths: options.arrowPaths,
106
          allowWebGLOnlyFormats: options.allowWebGLOnlyFormats
107
        });
108
        this.numRows = 0;
11✔
109
        this.nullCount = 0;
11✔
110
        this.bufferLayout.push(...appendableColumns.map(column => column.bufferLayout));
21✔
111
        this.appendableColumns = appendableColumns;
11✔
112

113
        const fields: arrow.Field[] = [];
11✔
114
        for (const column of appendableColumns) {
11✔
115
          const {bufferLayout, field: sourceField} = column;
21✔
116
          const field = new arrow.Field(
21✔
117
            bufferLayout.name,
118
            sourceField.type,
119
            sourceField.nullable,
120
            new Map(sourceField.metadata)
121
          );
122
          const gpuVector = new GPUVector({
21✔
123
            type: 'appendable',
124
            name: bufferLayout.name,
125
            device: options.device,
126
            arrowType: sourceField.type as AttributeArrowType,
127
            initialCapacityRows: options.initialCapacityRows,
128
            capacityGrowthFactor: options.capacityGrowthFactor,
129
            bufferProps: options.bufferProps
130
          } as any);
131

132
          fields.push(field);
21✔
133
          this.gpuVectors[bufferLayout.name] = gpuVector;
21✔
134
          this.attributes[bufferLayout.name] = gpuVector.buffer;
21✔
135
        }
136

137
        this.schema = new arrow.Schema(fields, new Map(options.schema.metadata));
11✔
138
        this.numCols = fields.length;
11✔
139
        return;
11✔
140
      }
141

142
      const {vectors, bufferLayout, fields, metadata, nullCount = 0} = deviceOrProps;
5✔
143
      const vectorCollection = createGPUVectorCollection({
5✔
144
        ownerName: 'GPURecordBatch',
145
        vectors,
146
        bufferLayout,
147
        fields
148
      });
149

150
      this.numRows = vectorCollection.numRows;
5✔
151
      this.nullCount = nullCount;
5✔
152
      Object.assign(this.gpuVectors, vectorCollection.gpuVectors);
5✔
153
      Object.assign(this.attributes, vectorCollection.attributes);
5✔
154
      this.bufferLayout.push(...vectorCollection.bufferLayout);
5✔
155

156
      this.schema = new arrow.Schema(vectorCollection.fields, metadata);
5✔
157
      this.numCols = vectorCollection.fields.length;
5✔
158
      return;
5✔
159
    }
160

161
    const device = deviceOrProps;
35✔
162
    const batch = recordBatch!;
35✔
163
    const options = props!;
35✔
164
    const table = new arrow.Table([batch]);
35✔
165

166
    this.numRows = batch.numRows;
35✔
167
    this.nullCount = batch.nullCount;
35✔
168
    this.bufferLayout = getArrowBufferLayout(options.shaderLayout, {
35✔
169
      arrowTable: table,
170
      arrowPaths: options.arrowPaths,
171
      allowWebGLOnlyFormats: options.allowWebGLOnlyFormats
172
    });
173

174
    const fields: arrow.Field[] = [];
35✔
175
    const selectedNames = new Set<string>();
35✔
176
    for (const bufferLayout of this.bufferLayout) {
35✔
177
      const arrowPath = options.arrowPaths?.[bufferLayout.name] || bufferLayout.name;
65✔
178
      const vector = getArrowVectorByPath(table, arrowPath);
65✔
179
      const sourceField = getArrowFieldByPath(table, arrowPath);
65✔
180
      const field = new arrow.Field(
65✔
181
        bufferLayout.name,
182
        vector.type,
183
        sourceField.nullable,
184
        new Map(sourceField.metadata)
185
      );
186
      const gpuVector = new GPUVector(
65✔
187
        device,
188
        vector as arrow.Vector<AttributeArrowType>,
189
        options.bufferProps as GPUVectorBufferProps
190
      );
191

192
      fields.push(field);
65✔
193
      selectedNames.add(bufferLayout.name);
65✔
194
      this.gpuVectors[bufferLayout.name] = gpuVector;
65✔
195
      if (bufferLayout.attributes) {
65!
NEW
196
        for (const attribute of bufferLayout.attributes) {
×
NEW
197
          this.attributes[attribute.attribute] = gpuVector.buffer;
×
198
        }
199
      } else {
200
        this.attributes[bufferLayout.name] = gpuVector.buffer;
65✔
201
      }
202
    }
203

204
    for (const storageBinding of getArrowStorageBindings(options.shaderLayout)) {
35✔
205
      if (selectedNames.has(storageBinding.name)) {
3!
NEW
206
        throw new Error(
×
207
          `GPURecordBatch shader input "${storageBinding.name}" cannot be both an attribute and a storage binding`
208
        );
209
      }
210
      const arrowPath = options.arrowPaths?.[storageBinding.name] || storageBinding.name;
3✔
211
      const vector = tryGetArrowVectorByPath(table, arrowPath);
3✔
212
      const sourceField = tryGetArrowFieldByPath(table, arrowPath);
3✔
213
      if (!vector || !sourceField) {
3!
NEW
214
        continue;
×
215
      }
216
      const gpuVector = new GPUVector(
3✔
217
        device,
218
        vector as arrow.Vector<AttributeArrowType>,
219
        options.bufferProps as GPUVectorBufferProps
220
      );
221
      const field = new arrow.Field(
3✔
222
        storageBinding.name,
223
        vector.type,
224
        sourceField.nullable,
225
        new Map(sourceField.metadata)
226
      );
227

228
      fields.push(field);
3✔
229
      selectedNames.add(storageBinding.name);
3✔
230
      this.gpuVectors[storageBinding.name] = gpuVector;
3✔
231
      this.bindings[storageBinding.name] = gpuVector.buffer;
3✔
232
    }
233

234
    this.schema = new arrow.Schema(fields, new Map(batch.schema.metadata));
35✔
235
    this.numCols = fields.length;
35✔
236
  }
237

238
  /** Appends one Arrow record batch into this appendable GPU record batch. */
239
  addToLastBatch(recordBatch: arrow.RecordBatch): this {
240
    if (!this.appendableColumns) {
18!
241
      throw new Error('GPURecordBatch.addToLastBatch() requires appendable batch storage');
×
242
    }
243
    const pendingData = getAppendableGPUColumnData(
18✔
244
      recordBatch,
245
      this.appendableColumns,
246
      'GPURecordBatch.addToLastBatch()'
247
    );
248
    for (const {column, data} of pendingData) {
18✔
249
      const vector = this.gpuVectors[column.attributeName];
34✔
250
      vector.addToLastData(data as any);
34✔
251
    }
252
    this.numRows += recordBatch.numRows;
17✔
253
    this.nullCount += recordBatch.nullCount;
17✔
254
    return this;
17✔
255
  }
256

257
  /** Clears appendable vector rows while retaining reusable DynamicBuffer allocations. */
258
  resetLastBatch(): this {
259
    if (!this.appendableColumns) {
2!
260
      throw new Error('GPURecordBatch.resetLastBatch() requires appendable batch storage');
×
261
    }
262
    for (const vector of Object.values(this.gpuVectors)) {
2✔
263
      vector.resetLastBatch();
4✔
264
    }
265
    this.numRows = 0;
2✔
266
    this.nullCount = 0;
2✔
267
    return this;
2✔
268
  }
269

270
  destroy(): void {
271
    for (const gpuVector of Object.values(this.gpuVectors)) {
51✔
272
      gpuVector.destroy();
95✔
273
    }
274
  }
275
}
276

277
function getArrowStorageBindings(shaderLayout: ShaderLayout): Array<{name: string}> {
278
  return shaderLayout.bindings.filter(
35✔
279
    binding =>
3✔
280
      (binding.type === 'storage' || binding.type === 'read-only-storage') && !('format' in binding)
281
  );
282
}
283

284
function tryGetArrowVectorByPath(table: arrow.Table, path: string): arrow.Vector | null {
285
  try {
3✔
286
    return getArrowVectorByPath(table, path);
3✔
287
  } catch {
NEW
288
    return null;
×
289
  }
290
}
291

292
function tryGetArrowFieldByPath(table: arrow.Table, path: string): arrow.Field | null {
293
  try {
3✔
294
    return getArrowFieldByPath(table, path);
3✔
295
  } catch {
NEW
296
    return null;
×
297
  }
298
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc