• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

visgl / luma.gl / 26054616669

18 May 2026 07:09PM UTC coverage: 74.862% (+0.09%) from 74.769%
26054616669

push

github

web-flow
feat(arrow): Support List<> attributes (#2621)

6988 of 10527 branches covered (66.38%)

Branch coverage included in aggregate %.

144 of 170 new or added lines in 4 files covered. (84.71%)

1 existing line in 1 file now uncovered.

15148 of 19042 relevant lines covered (79.55%)

909.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.2
/modules/arrow/src/arrow/arrow-gpu-data.ts
1
// luma.gl
2
// SPDX-License-Identifier: MIT
3
// Copyright (c) vis.gl contributors
4

5
import {Buffer, Device, type BigTypedArray} from '@luma.gl/core';
6
import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
7
import * as arrow from 'apache-arrow';
8
import {
9
  isVariableLengthAttributeArrowType,
10
  type AttributeArrowType,
11
  type NumericArrowType,
12
  type VariableLengthAttributeArrowType
13
} from './arrow-types';
14

15
type GPUDataBufferProps = Omit<DynamicBufferProps, 'byteLength' | 'data' | 'buffer' | 'ownsBuffer'>;
16

17
/** Compact CPU metadata required to reconstruct variable-width Arrow chunks after GPU readback. */
18
export type GPUDataReadbackMetadata =
19
  | {
20
      kind: 'utf8';
21
      valueOffsets: Int32Array;
22
      nullCount: number;
23
      nullBitmap?: Uint8Array;
24
      valueByteLength: number;
25
    }
26
  | {
27
      kind: 'variable-length-attribute';
28
      valueOffsets: Int32Array;
29
      nullCount: number;
30
      nullBitmap?: Uint8Array;
31
      valueByteLength: number;
32
    };
33

34
/** Constructor props that wrap one existing typed GPU data buffer. */
35
export type GPUDataFromBufferProps<T extends arrow.DataType = AttributeArrowType> = {
36
  /** Stable dynamic GPU buffer wrapper for this data range. */
37
  buffer: DynamicBuffer;
38
  /** Arrow type that describes the values in the data chunk. */
39
  arrowType: T;
40
  /** Number of logical rows in the data chunk. */
41
  length: number;
42
  /** Byte offset of the first logical row. */
43
  byteOffset?: number;
44
  /** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
45
  byteStride?: number;
46
  /** Whether this data view should destroy the buffer. */
47
  ownsBuffer?: boolean;
48
  /** Optional compact metadata for reconstructing variable-width Arrow chunks after GPU readback. */
49
  readbackMetadata?: GPUDataReadbackMetadata;
50
};
51

52
type GPUVectorReadableBuffer = Pick<Buffer, 'readAsync'>;
53

54
type GPUVectorReadProps<T extends AttributeArrowType> = {
55
  type: T;
56
  buffer: GPUVectorReadableBuffer;
57
  length: number;
58
  byteOffset: number;
59
  byteStride: number;
60
};
61

62
type NumericTypedArrayConstructor = {
63
  readonly BYTES_PER_ELEMENT: number;
64
  new (buffer: ArrayBufferLike, byteOffset?: number, length?: number): BigTypedArray;
65
};
66

67
const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {
16✔
68
  type: T;
69
  length: number;
70
  data: T['TArray'];
71
}) => arrow.Data<T>;
72

73
const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {
16✔
74
  type: arrow.FixedSizeList<T>;
75
  length: number;
76
  nullCount: number;
77
  nullBitmap: null;
78
  child: arrow.Data<T>;
79
}) => arrow.Data<arrow.FixedSizeList<T>>;
80

81
/**
82
 * GPU memory and Arrow type metadata for one Arrow Data chunk.
83
 *
84
 * GPUData can own a dedicated buffer when constructed from Arrow Data, or
85
 * describe a byte-range view into a shared static or dynamic GPU buffer.
86
 */
87
export class GPUData<T extends arrow.DataType = AttributeArrowType> {
88
  /** GPU buffer containing the Arrow data chunk's attribute-compatible value memory. */
89
  readonly buffer: DynamicBuffer;
90
  /** Arrow type that describes the uploaded data chunk. */
91
  readonly type: T;
92
  /** Number of logical Arrow rows in this chunk. */
93
  readonly length: number;
94
  /** Number of scalar values per logical row. */
95
  readonly stride: number;
96
  /** Byte offset of the first logical row in {@link buffer}. */
97
  readonly byteOffset: number;
98
  /** Bytes between adjacent logical rows in {@link buffer}. */
99
  readonly byteStride: number;
100
  /** Optional compact metadata for reconstructing variable-width Arrow chunks after GPU readback. */
101
  readonly readbackMetadata?: GPUDataReadbackMetadata;
102
  /** Whether this data view is responsible for destroying {@link buffer}. */
103
  private _ownsBuffer: boolean;
104

105
  /** Creates a GPU representation from one Arrow Data chunk. */
106
  constructor(device: Device, data: arrow.Data<T>, props?: GPUDataBufferProps);
107
  /** Creates a data view over an existing GPU buffer. */
108
  constructor(props: GPUDataFromBufferProps<T>);
109
  constructor(
110
    deviceOrProps: Device | GPUDataFromBufferProps<any>,
111
    data?: arrow.Data<T>,
112
    props: GPUDataBufferProps = {}
210✔
113
  ) {
114
    if (deviceOrProps instanceof Device) {
210✔
115
      const arrowData = data!;
33✔
116
      this.type = arrowData.type as T;
33✔
117
      this.length = arrowData.length;
33✔
118
      this.readbackMetadata = getArrowGPUDataReadbackMetadata(arrowData);
33✔
119
      if (arrow.DataType.isUtf8(arrowData.type)) {
33✔
120
        this.stride = 1;
25✔
121
        this.byteOffset = 0;
25✔
122
        this.byteStride = 1;
25✔
123
        this.buffer = new DynamicBuffer(deviceOrProps, {
25✔
124
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
125
          ...props,
126
          data: getArrowUtf8DataBufferSource(arrowData as arrow.Data<arrow.Utf8>)
127
        });
128
        this._ownsBuffer = true;
25✔
129
        return;
25✔
130
      }
131
      if (isVariableLengthAttributeArrowType(arrowData.type)) {
8!
132
        this.stride = getArrowVariableLengthAttributeElementStride(arrowData.type);
8✔
133
        this.byteOffset = 0;
8✔
134
        this.byteStride = getArrowVariableLengthAttributeElementByteStride(arrowData.type);
8✔
135
        this.buffer = new DynamicBuffer(deviceOrProps, {
8✔
136
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
137
          ...props,
138
          data: getArrowVariableLengthAttributeDataBufferSource(
139
            arrowData as unknown as arrow.Data<VariableLengthAttributeArrowType>
140
          )
141
        });
142
        this._ownsBuffer = true;
8✔
143
        return;
8✔
144
      }
145
      this.stride = getArrowTypeStride(arrowData.type);
×
146
      this.byteOffset = 0;
×
147
      this.byteStride = getArrowTypeByteStride(arrowData.type);
×
148
      this.buffer = new DynamicBuffer(deviceOrProps, {
×
149
        usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
150
        ...props,
151
        data: getArrowDataBufferSource(arrowData as any)
152
      });
153
      this._ownsBuffer = true;
×
154
      return;
×
155
    }
156

157
    const {
158
      buffer,
159
      arrowType,
160
      length,
161
      byteOffset = 0,
177✔
162
      byteStride = getArrowTypeByteStride(arrowType),
177✔
163
      ownsBuffer = false,
177✔
164
      readbackMetadata
165
    } = deviceOrProps;
177✔
166
    this.buffer = buffer;
177✔
167
    this.type = arrowType as T;
177✔
168
    this.length = length;
177✔
169
    this.stride = getArrowTypeStride(arrowType);
177✔
170
    this.byteOffset = byteOffset;
177✔
171
    this.byteStride = byteStride;
177✔
172
    this.readbackMetadata = readbackMetadata;
177✔
173
    this._ownsBuffer = ownsBuffer;
177✔
174
  }
175

176
  get ownsBuffer(): boolean {
177
    return this._ownsBuffer;
×
178
  }
179

180
  /** Reads this GPU chunk back into a single non-null Arrow Data chunk. */
181
  async readAsync(): Promise<arrow.Data<T>> {
182
    if (arrow.DataType.isUtf8(this.type)) {
15✔
183
      const metadata = this.readbackMetadata;
4✔
184
      if (metadata?.kind !== 'utf8') {
4!
NEW
185
        throw new Error('GPUData.readAsync() requires UTF-8 readback metadata');
×
186
      }
187
      const bytes =
188
        metadata.valueByteLength === 0
4!
189
          ? new Uint8Array(0)
190
          : await this.buffer.readAsync(this.byteOffset, metadata.valueByteLength);
191
      return arrow.makeData({
4✔
192
        type: new arrow.Utf8(),
193
        length: this.length,
194
        nullCount: metadata.nullCount,
195
        nullBitmap: metadata.nullBitmap,
196
        valueOffsets: metadata.valueOffsets,
197
        data: bytes
198
      }) as arrow.Data<T>;
199
    }
200
    if (isVariableLengthAttributeArrowType(this.type)) {
11✔
201
      const metadata = this.readbackMetadata;
10✔
202
      if (metadata?.kind !== 'variable-length-attribute') {
10!
NEW
203
        throw new Error('GPUData.readAsync() requires variable-length attribute readback metadata');
×
204
      }
205
      const bytes =
206
        metadata.valueByteLength === 0
10!
207
          ? new Uint8Array(0)
208
          : await this.buffer.readAsync(this.byteOffset, metadata.valueByteLength);
209
      return makeArrowVariableLengthAttributeDataFromPackedBytes(
10✔
210
        this.type,
211
        this.length,
212
        metadata,
213
        bytes
214
      ) as unknown as arrow.Data<T>;
215
    }
216
    const vector = await readArrowGPUVectorAsync({
1✔
217
      type: this.type as unknown as AttributeArrowType,
218
      buffer: this.buffer,
219
      length: this.length,
220
      byteOffset: this.byteOffset,
221
      byteStride: this.byteStride
222
    });
223
    return vector.data[0] as unknown as arrow.Data<T>;
1✔
224
  }
225

226
  destroy(): void {
227
    if (this._ownsBuffer) {
33!
228
      this.buffer.destroy();
33✔
229
      this._ownsBuffer = false;
33✔
230
    }
231
  }
232
}
233

234
export function getArrowDataBufferSource<T extends NumericArrowType>(
235
  data: arrow.Data<T>
236
): T['TArray'];
237
export function getArrowDataBufferSource<T extends NumericArrowType>(
238
  data: arrow.Data<arrow.FixedSizeList<T>>
239
): T['TArray'];
240
export function getArrowDataBufferSource<T extends AttributeArrowType>(
241
  data: arrow.Data<T>
242
): NumericArrowType['TArray'];
243
/** Return the uploadable typed-array view for one Arrow Data chunk. */
244
export function getArrowDataBufferSource(data: arrow.Data): NumericArrowType['TArray'] {
245
  const {values, startElement, elementCount} = getArrowDataValueRange(data);
222✔
246
  if (values.length < elementCount) {
222!
247
    throw new Error('Arrow data values are shorter than the logical upload length');
×
248
  }
249
  if (values.length === elementCount) {
222✔
250
    return values;
221✔
251
  }
252

253
  const endElement = startElement + elementCount;
1✔
254
  if (endElement > values.length) {
1!
255
    throw new Error('Arrow data values are shorter than the logical upload length');
×
256
  }
257
  return values.subarray(startElement, endElement) as NumericArrowType['TArray'];
1✔
258
}
259

260
/** Return the UTF-8 value bytes referenced by one Arrow Utf8 data chunk. */
261
export function getArrowUtf8DataBufferSource(data: arrow.Data<arrow.Utf8>): Uint8Array {
262
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
56✔
263
  const values = data.values as Uint8Array | undefined;
56✔
264
  if (!valueOffsets || !values) {
56!
265
    return new Uint8Array(0);
×
266
  }
267
  const firstValueOffset = valueOffsets[0] ?? 0;
56!
268
  const lastValueOffset = valueOffsets[data.length] ?? firstValueOffset;
56!
269
  return values.subarray(firstValueOffset, lastValueOffset);
56✔
270
}
271

272
/** Return flattened numeric values referenced by one variable-length nested attribute chunk. */
273
export function getArrowVariableLengthAttributeDataBufferSource(
274
  data: arrow.Data<VariableLengthAttributeArrowType>
275
): NumericArrowType['TArray'] {
276
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
20✔
277
  const elementData = data.children[0] as arrow.Data | undefined;
20✔
278
  if (!valueOffsets || !elementData) {
20!
NEW
279
    return new Float32Array(0);
×
280
  }
281

282
  const firstElementOffset = valueOffsets[0] ?? 0;
20!
283
  const lastElementOffset = valueOffsets[data.length] ?? firstElementOffset;
20!
284

285
  if (arrow.DataType.isFixedSizeList(elementData.type)) {
20✔
286
    const numericData = elementData.children[0] as arrow.Data<NumericArrowType> | undefined;
18✔
287
    const values = numericData?.values as NumericArrowType['TArray'] | undefined;
18✔
288
    if (!numericData || !values) {
18!
NEW
289
      return new Float32Array(0);
×
290
    }
291

292
    const elementStride = elementData.type.listSize;
18✔
293
    const numericValueOffset = numericData.offset ?? 0;
18!
294
    const firstValueOffset = numericValueOffset + firstElementOffset * elementStride;
18✔
295
    const lastValueOffset = numericValueOffset + lastElementOffset * elementStride;
18✔
296
    return values.subarray(firstValueOffset, lastValueOffset) as NumericArrowType['TArray'];
18✔
297
  }
298

299
  const values = elementData.values as NumericArrowType['TArray'] | undefined;
2✔
300
  if (!values) {
2!
NEW
301
    return new Float32Array(0);
×
302
  }
303
  const numericValueOffset = elementData.offset ?? 0;
2!
304
  return values.subarray(
20✔
305
    numericValueOffset + firstElementOffset,
306
    numericValueOffset + lastElementOffset
307
  ) as NumericArrowType['TArray'];
308
}
309

310
/** Copy compact variable-width reconstruction metadata without retaining Arrow value buffers. */
311
export function getArrowGPUDataReadbackMetadata(
312
  data: arrow.Data
313
): GPUDataReadbackMetadata | undefined {
314
  if (arrow.DataType.isUtf8(data.type)) {
49✔
315
    const values = getArrowUtf8DataBufferSource(data as arrow.Data<arrow.Utf8>);
28✔
316
    return {
28✔
317
      kind: 'utf8',
318
      valueOffsets: copyNormalizedArrowValueOffsets(data),
319
      nullCount: data.nullCount,
320
      nullBitmap: copyNormalizedArrowNullBitmap(data),
321
      valueByteLength: values.byteLength
322
    };
323
  }
324

325
  if (isVariableLengthAttributeArrowType(data.type)) {
21✔
326
    const values = getArrowVariableLengthAttributeDataBufferSource(
10✔
327
      data as arrow.Data<VariableLengthAttributeArrowType>
328
    );
329
    return {
10✔
330
      kind: 'variable-length-attribute',
331
      valueOffsets: copyNormalizedArrowValueOffsets(data),
332
      nullCount: data.nullCount,
333
      nullBitmap: copyNormalizedArrowNullBitmap(data),
334
      valueByteLength: values.byteLength
335
    };
336
  }
337

338
  return undefined;
11✔
339
}
340

341
export function getArrowVectorBufferSource<T extends NumericArrowType>(
342
  vector: arrow.Vector<T>
343
): T['TArray'];
344
export function getArrowVectorBufferSource<T extends NumericArrowType>(
345
  vector: arrow.Vector<arrow.FixedSizeList<T>>
346
): T['TArray'];
347
export function getArrowVectorBufferSource<T extends AttributeArrowType>(
348
  vector: arrow.Vector<T>
349
): NumericArrowType['TArray'];
350
/** Return a typed array that can be passed directly to `device.createBuffer()`. */
351
export function getArrowVectorBufferSource(vector: arrow.Vector): NumericArrowType['TArray'] {
352
  const dataSources = vector.data.map(data => getArrowDataBufferSource(data));
211✔
353
  if (dataSources.length === 0) {
210!
354
    throw new Error('Arrow vector has no data');
×
355
  }
356
  if (dataSources.length === 1) {
210✔
357
    return dataSources[0];
209✔
358
  }
359

360
  const totalLength = dataSources.reduce((length, dataSource) => length + dataSource.length, 0);
2✔
361
  const values = createTypedArrayLike(dataSources[0], totalLength);
1✔
362
  let targetOffset = 0;
1✔
363
  for (const dataSource of dataSources) {
1✔
364
    values.set(dataSource as never, targetOffset);
2✔
365
    targetOffset += dataSource.length;
2✔
366
  }
367
  return values;
1✔
368
}
369

370
/** Number of scalar values in one logical Arrow row. */
371
export function getArrowTypeStride(type: arrow.DataType): number {
372
  if (isVariableLengthAttributeArrowType(type)) {
366✔
373
    return getArrowVariableLengthAttributeElementStride(type);
11✔
374
  }
375
  return arrow.DataType.isFixedSizeList(type) ? type.listSize : 1;
355✔
376
}
377

378
/** Number of uploaded bytes in one logical Arrow row. */
379
export function getArrowTypeByteStride(type: arrow.DataType): number {
380
  if (isVariableLengthAttributeArrowType(type)) {
353✔
381
    return getArrowVariableLengthAttributeElementByteStride(type);
9✔
382
  }
383
  if (arrow.DataType.isFixedSizeList(type)) {
344✔
384
    return type.listSize * getArrowTypeByteStride(type.children[0].type);
143✔
385
  }
386
  if (arrow.DataType.isInt(type)) {
201✔
387
    return type.bitWidth / 8;
70✔
388
  }
389
  if (arrow.DataType.isFloat(type)) {
131!
390
    switch (type.precision) {
131!
391
      case arrow.Precision.HALF:
392
        return 2;
×
393
      case arrow.Precision.SINGLE:
394
        return 4;
131✔
395
      case arrow.Precision.DOUBLE:
396
        return 8;
×
397
    }
398
  }
399
  throw new Error(`Cannot determine byte stride for Arrow type ${type}`);
×
400
}
401

402
/** Reject nullable Arrow chunks that cannot be uploaded directly into GPU attribute buffers. */
403
export function validateArrowGPUDataDirectUpload(
404
  name: string,
405
  data: arrow.Data<AttributeArrowType | arrow.Utf8 | VariableLengthAttributeArrowType>
406
): void {
407
  if (data.nullCount > 0) {
35✔
408
    throw new Error(`GPUVector "${name}" does not support nullable data`);
1✔
409
  }
410
  if (arrow.DataType.isFixedSizeList(data.type) && (data.children[0]?.nullCount ?? 0) > 0) {
34!
411
    throw new Error(`GPUVector "${name}" does not support nullable child data`);
×
412
  }
413
  if (isVariableLengthAttributeArrowType(data.type)) {
34✔
414
    const elementData = data.children[0];
10✔
415
    const nestedNumericData = arrow.DataType.isFixedSizeList(elementData?.type)
10✔
416
      ? elementData.children[0]
417
      : undefined;
418
    if ((elementData?.nullCount ?? 0) > 0 || (nestedNumericData?.nullCount ?? 0) > 0) {
10!
NEW
419
      throw new Error(`GPUVector "${name}" does not support nullable nested list data`);
×
420
    }
421
  }
422
}
423

424
/** Read GPU bytes and reconstruct one non-null Arrow vector with the supplied Arrow type. */
425
export async function readArrowGPUVectorAsync<T extends AttributeArrowType>(
426
  props: GPUVectorReadProps<T>
427
): Promise<arrow.Vector<T>> {
428
  const {buffer, type, length, byteOffset, byteStride} = props;
6✔
429
  const rowByteWidth = getArrowTypeByteStride(type);
6✔
430
  if (byteStride < rowByteWidth) {
6!
431
    throw new Error(
×
432
      `GPUVector.readAsync() byteStride ${byteStride} is smaller than row byte width ${rowByteWidth}`
433
    );
434
  }
435

436
  const readByteLength = length === 0 ? 0 : (length - 1) * byteStride + rowByteWidth;
6!
437
  const bytes =
438
    readByteLength === 0 ? new Uint8Array(0) : await buffer.readAsync(byteOffset, readByteLength);
6!
439
  const packedBytes =
440
    byteStride === rowByteWidth
6✔
441
      ? bytes
442
      : compactStridedRows(bytes, length, byteStride, rowByteWidth);
443

444
  return makeArrowVectorFromPackedBytes(type, length, packedBytes);
6✔
445
}
446

447
function getArrowDataValueRange(data: arrow.Data): {
448
  values: NumericArrowType['TArray'];
449
  startElement: number;
450
  elementCount: number;
451
} {
452
  if (arrow.DataType.isFixedSizeList(data.type)) {
222✔
453
    const childData = data.children[0];
170✔
454
    const values = childData?.values;
170✔
455
    if (!values) {
170!
456
      throw new Error('Arrow FixedSizeList data has no child values');
×
457
    }
458
    const elementCount = data.length * data.type.listSize;
170✔
459
    const startElement = (childData.offset ?? 0) + data.offset * data.type.listSize;
170!
460
    return {values: values as NumericArrowType['TArray'], startElement, elementCount};
170✔
461
  }
462

463
  const values = data.values;
52✔
464
  if (!values) {
52!
465
    throw new Error('Arrow data has no values');
×
466
  }
467
  return {
52✔
468
    values: values as NumericArrowType['TArray'],
469
    startElement: data.offset,
470
    elementCount: data.length
471
  };
472
}
473

474
function createTypedArrayLike(
475
  source: NumericArrowType['TArray'],
476
  length: number
477
): NumericArrowType['TArray'] {
478
  const TypedArrayConstructor = source.constructor as {
1✔
479
    new (length: number): NumericArrowType['TArray'];
480
  };
481
  return new TypedArrayConstructor(length);
1✔
482
}
483

484
function compactStridedRows(
485
  bytes: Uint8Array,
486
  length: number,
487
  byteStride: number,
488
  rowByteWidth: number
489
): Uint8Array {
490
  const packedBytes = new Uint8Array(length * rowByteWidth);
1✔
491
  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
1✔
492
    const sourceOffset = rowIndex * byteStride;
2✔
493
    const targetOffset = rowIndex * rowByteWidth;
2✔
494
    packedBytes.set(bytes.subarray(sourceOffset, sourceOffset + rowByteWidth), targetOffset);
2✔
495
  }
496
  return packedBytes;
1✔
497
}
498

499
function makeArrowVectorFromPackedBytes<T extends AttributeArrowType>(
500
  type: T,
501
  length: number,
502
  bytes: Uint8Array
503
): arrow.Vector<T> {
504
  if (arrow.DataType.isFixedSizeList(type)) {
6✔
505
    const childType = type.children[0].type as NumericArrowType;
1✔
506
    const values = makeNumericTypedArray(childType, bytes, length * type.listSize);
1✔
507
    const childData = makeNumericData({
1✔
508
      type: childType,
509
      length: length * type.listSize,
510
      data: values as NumericArrowType['TArray']
511
    });
512
    const listData = makeFixedSizeListData({
1✔
513
      type: type as arrow.FixedSizeList<NumericArrowType>,
514
      length,
515
      nullCount: 0,
516
      nullBitmap: null,
517
      child: childData
518
    });
519
    return arrow.makeVector(listData) as arrow.Vector<T>;
1✔
520
  }
521

522
  const numericType = type as NumericArrowType;
5✔
523
  const values = makeNumericTypedArray(numericType, bytes, length);
5✔
524
  const data = makeNumericData({
5✔
525
    type: numericType,
526
    length,
527
    data: values as NumericArrowType['TArray']
528
  });
529
  return arrow.makeVector(data) as arrow.Vector<T>;
5✔
530
}
531

532
function makeArrowVariableLengthAttributeDataFromPackedBytes(
533
  type: VariableLengthAttributeArrowType,
534
  length: number,
535
  metadata: Extract<GPUDataReadbackMetadata, {kind: 'variable-length-attribute'}>,
536
  bytes: Uint8Array
537
): arrow.Data<VariableLengthAttributeArrowType> {
538
  const elementType = type.children[0].type;
10✔
539
  const elementData = arrow.DataType.isFixedSizeList(elementType)
10✔
540
    ? makeArrowFixedSizeListElementDataFromPackedBytes(elementType, bytes)
541
    : makeArrowNumericElementDataFromPackedBytes(elementType as NumericArrowType, bytes);
542

543
  return new arrow.Data<VariableLengthAttributeArrowType>(
10✔
544
    type,
545
    0,
546
    length,
547
    metadata.nullCount,
548
    {
549
      [arrow.BufferType.OFFSET]: metadata.valueOffsets,
550
      ...(metadata.nullBitmap ? {[arrow.BufferType.VALIDITY]: metadata.nullBitmap} : {})
10!
551
    },
552
    [elementData]
553
  );
554
}
555

556
function copyNormalizedArrowValueOffsets(data: arrow.Data): Int32Array {
557
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
38✔
558
  const copiedOffsets = new Int32Array(data.length + 1);
38✔
559
  if (!valueOffsets) {
38!
NEW
560
    return copiedOffsets;
×
561
  }
562

563
  const firstValueOffset = valueOffsets[0] ?? 0;
38!
564
  for (let offsetIndex = 0; offsetIndex <= data.length; offsetIndex++) {
38✔
565
    copiedOffsets[offsetIndex] = Math.max(
99✔
566
      0,
567
      (valueOffsets[offsetIndex] ?? firstValueOffset) - firstValueOffset
99!
568
    );
569
  }
570
  return copiedOffsets;
38✔
571
}
572

573
function copyNormalizedArrowNullBitmap(data: arrow.Data): Uint8Array | undefined {
574
  if (data.nullCount === 0 || !data.nullBitmap) {
38✔
575
    return undefined;
36✔
576
  }
577

578
  const copiedBitmap = new Uint8Array(Math.ceil(data.length / 8));
2✔
579
  const sourceBitmap = data.nullBitmap as Uint8Array;
2✔
580
  const sourceRowOffset = data.offset ?? 0;
2!
581
  for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
38✔
582
    const sourceBitIndex = sourceRowOffset + rowIndex;
4✔
583
    const sourceByte = sourceBitmap[sourceBitIndex >> 3] ?? 0;
4!
584
    if ((sourceByte & (1 << (sourceBitIndex & 7))) !== 0) {
4✔
585
      copiedBitmap[rowIndex >> 3] |= 1 << (rowIndex & 7);
2✔
586
    }
587
  }
588
  return copiedBitmap;
2✔
589
}
590

591
function makeArrowFixedSizeListElementDataFromPackedBytes(
592
  type: arrow.FixedSizeList<NumericArrowType>,
593
  bytes: Uint8Array
594
): arrow.Data<arrow.FixedSizeList<NumericArrowType>> {
595
  const numericType = type.children[0].type as NumericArrowType;
9✔
596
  const values = makeNumericTypedArray(
9✔
597
    numericType,
598
    bytes,
599
    bytes.byteLength / getArrowTypeByteStride(numericType)
600
  ) as NumericArrowType['TArray'];
601
  const numericData = makeNumericData({
9✔
602
    type: numericType,
603
    length: values.length,
604
    data: values
605
  });
606
  return makeFixedSizeListData({
9✔
607
    type,
608
    length: type.listSize === 0 ? 0 : values.length / type.listSize,
9!
609
    nullCount: 0,
610
    nullBitmap: null,
611
    child: numericData
612
  });
613
}
614

615
function makeArrowNumericElementDataFromPackedBytes(
616
  type: NumericArrowType,
617
  bytes: Uint8Array
618
): arrow.Data<NumericArrowType> {
619
  const values = makeNumericTypedArray(
1✔
620
    type,
621
    bytes,
622
    bytes.byteLength / getArrowTypeByteStride(type)
623
  ) as NumericArrowType['TArray'];
624
  return makeNumericData({
1✔
625
    type,
626
    length: values.length,
627
    data: values
628
  });
629
}
630

631
function getArrowVariableLengthAttributeElementStride(
632
  type: VariableLengthAttributeArrowType
633
): number {
634
  return getArrowTypeStride(type.children[0].type);
19✔
635
}
636

637
function getArrowVariableLengthAttributeElementByteStride(
638
  type: VariableLengthAttributeArrowType
639
): number {
640
  return getArrowTypeByteStride(type.children[0].type);
17✔
641
}
642

643
function makeNumericTypedArray(
644
  type: NumericArrowType,
645
  bytes: Uint8Array,
646
  length: number
647
): BigTypedArray {
648
  if (arrow.DataType.isInt(type)) {
16✔
649
    if (type.isSigned) {
2!
650
      switch (type.bitWidth) {
2!
651
        case 8:
652
          return makeTypedArrayView(Int8Array, bytes, length);
×
653
        case 16:
654
          return makeTypedArrayView(Int16Array, bytes, length);
1✔
655
        case 32:
656
          return makeTypedArrayView(Int32Array, bytes, length);
1✔
657
        case 64:
658
          return makeTypedArrayView(BigInt64Array, bytes, length);
×
659
      }
660
    }
661

662
    switch (type.bitWidth) {
×
663
      case 8:
664
        return makeTypedArrayView(Uint8Array, bytes, length);
×
665
      case 16:
666
        return makeTypedArrayView(Uint16Array, bytes, length);
×
667
      case 32:
668
        return makeTypedArrayView(Uint32Array, bytes, length);
×
669
      case 64:
670
        return makeTypedArrayView(BigUint64Array, bytes, length);
×
671
    }
672
  }
673

674
  if (arrow.DataType.isFloat(type)) {
14!
675
    switch (type.precision) {
14!
676
      case arrow.Precision.HALF:
677
        return makeTypedArrayView(Uint16Array, bytes, length);
×
678
      case arrow.Precision.SINGLE:
679
        return makeTypedArrayView(Float32Array, bytes, length);
14✔
680
      case arrow.Precision.DOUBLE:
681
        return makeTypedArrayView(Float64Array, bytes, length);
×
682
    }
683
  }
684

685
  throw new Error(`GPUVector.readAsync() does not support Arrow type ${type}`);
×
686
}
687

688
function makeTypedArrayView<T extends BigTypedArray>(
689
  TypedArrayConstructor: NumericTypedArrayConstructor,
690
  bytes: Uint8Array,
691
  length: number
692
): T {
693
  const byteLength = length * TypedArrayConstructor.BYTES_PER_ELEMENT;
16✔
694
  if (bytes.byteOffset % TypedArrayConstructor.BYTES_PER_ELEMENT === 0) {
16!
695
    return new TypedArrayConstructor(bytes.buffer, bytes.byteOffset, length) as T;
16✔
696
  }
697

698
  const alignedBytes = new Uint8Array(byteLength);
×
699
  alignedBytes.set(bytes.subarray(0, byteLength));
×
700
  return new TypedArrayConstructor(alignedBytes.buffer, 0, length) as T;
×
701
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc