26054616669

Committed 18 May 2026 07:09PM UTC coverage: 74.862% (+0.09%) from 74.769%

Build # 26054616669

Build Type

push

github

Committed by

web-flow

Commit Message

feat(arrow): Support List<> attributes (#2621)

Coverage Stats

6988 of 10527 branches covered (66.38%)

Branch coverage included in aggregate %.

144 of 170 new or added lines in 4 files covered. (84.71%)

1 existing line in 1 file now uncovered.

15148 of 19042 relevant lines covered (79.55%)

909.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.2

/modules/arrow/src/arrow/arrow-gpu-data.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {Buffer, Device, type BigTypedArray} from '@luma.gl/core';
import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import {
  isVariableLengthAttributeArrowType,
  type AttributeArrowType,
  type NumericArrowType,
  type VariableLengthAttributeArrowType
} from './arrow-types';

type GPUDataBufferProps = Omit<DynamicBufferProps, 'byteLength' | 'data' | 'buffer' | 'ownsBuffer'>;

/** Compact CPU metadata required to reconstruct variable-width Arrow chunks after GPU readback. */
export type GPUDataReadbackMetadata =
  | {
      kind: 'utf8';
      valueOffsets: Int32Array;
      nullCount: number;
      nullBitmap?: Uint8Array;
      valueByteLength: number;
    }
  | {
      kind: 'variable-length-attribute';
      valueOffsets: Int32Array;
      nullCount: number;
      nullBitmap?: Uint8Array;
      valueByteLength: number;
    };

/** Constructor props that wrap one existing typed GPU data buffer. */
export type GPUDataFromBufferProps<T extends arrow.DataType = AttributeArrowType> = {
  /** Stable dynamic GPU buffer wrapper for this data range. */
  buffer: DynamicBuffer;
  /** Arrow type that describes the values in the data chunk. */
  arrowType: T;
  /** Number of logical rows in the data chunk. */
  length: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
  byteStride?: number;
  /** Whether this data view should destroy the buffer. */
  ownsBuffer?: boolean;
  /** Optional compact metadata for reconstructing variable-width Arrow chunks after GPU readback. */
  readbackMetadata?: GPUDataReadbackMetadata;
};

type GPUVectorReadableBuffer = Pick<Buffer, 'readAsync'>;

type GPUVectorReadProps<T extends AttributeArrowType> = {
  type: T;
  buffer: GPUVectorReadableBuffer;
  length: number;
  byteOffset: number;
  byteStride: number;
};

type NumericTypedArrayConstructor = {
  readonly BYTES_PER_ELEMENT: number;
  new (buffer: ArrayBufferLike, byteOffset?: number, length?: number): BigTypedArray;
};

const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: T;
  length: number;
  data: T['TArray'];
}) => arrow.Data<T>;

const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: arrow.FixedSizeList<T>;
  length: number;
  nullCount: number;
  nullBitmap: null;
  child: arrow.Data<T>;
}) => arrow.Data<arrow.FixedSizeList<T>>;

/**
 * GPU memory and Arrow type metadata for one Arrow Data chunk.
 *
 * GPUData can own a dedicated buffer when constructed from Arrow Data, or
 * describe a byte-range view into a shared static or dynamic GPU buffer.
 */
export class GPUData<T extends arrow.DataType = AttributeArrowType> {
  /** GPU buffer containing the Arrow data chunk's attribute-compatible value memory. */
  readonly buffer: DynamicBuffer;
  /** Arrow type that describes the uploaded data chunk. */
  readonly type: T;
  /** Number of logical Arrow rows in this chunk. */
  readonly length: number;
  /** Number of scalar values per logical row. */
  readonly stride: number;
  /** Byte offset of the first logical row in {@link buffer}. */
  readonly byteOffset: number;
  /** Bytes between adjacent logical rows in {@link buffer}. */
  readonly byteStride: number;
  /** Optional compact metadata for reconstructing variable-width Arrow chunks after GPU readback. */
  readonly readbackMetadata?: GPUDataReadbackMetadata;
  /** Whether this data view is responsible for destroying {@link buffer}. */
  private _ownsBuffer: boolean;

  /** Creates a GPU representation from one Arrow Data chunk. */
  constructor(device: Device, data: arrow.Data<T>, props?: GPUDataBufferProps);
  /** Creates a data view over an existing GPU buffer. */
  constructor(props: GPUDataFromBufferProps<T>);
  constructor(
    deviceOrProps: Device | GPUDataFromBufferProps<any>,
    data?: arrow.Data<T>,
    props: GPUDataBufferProps = {}
  ) {
    if (deviceOrProps instanceof Device) {
      const arrowData = data!;
      this.type = arrowData.type as T;
      this.length = arrowData.length;
      this.readbackMetadata = getArrowGPUDataReadbackMetadata(arrowData);
      if (arrow.DataType.isUtf8(arrowData.type)) {
        this.stride = 1;
        this.byteOffset = 0;
        this.byteStride = 1;
        this.buffer = new DynamicBuffer(deviceOrProps, {
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...props,
          data: getArrowUtf8DataBufferSource(arrowData as arrow.Data<arrow.Utf8>)
        });
        this._ownsBuffer = true;
        return;
      }
      if (isVariableLengthAttributeArrowType(arrowData.type)) {
        this.stride = getArrowVariableLengthAttributeElementStride(arrowData.type);
        this.byteOffset = 0;
        this.byteStride = getArrowVariableLengthAttributeElementByteStride(arrowData.type);
        this.buffer = new DynamicBuffer(deviceOrProps, {
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...props,
          data: getArrowVariableLengthAttributeDataBufferSource(
            arrowData as unknown as arrow.Data<VariableLengthAttributeArrowType>
          )
        });
        this._ownsBuffer = true;
        return;
      }
      this.stride = getArrowTypeStride(arrowData.type);
      this.byteOffset = 0;
      this.byteStride = getArrowTypeByteStride(arrowData.type);
      this.buffer = new DynamicBuffer(deviceOrProps, {
        usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
        ...props,
        data: getArrowDataBufferSource(arrowData as any)
      });
      this._ownsBuffer = true;
      return;
    }

    const {
      buffer,
      arrowType,
      length,
      byteOffset = 0,
      byteStride = getArrowTypeByteStride(arrowType),
      ownsBuffer = false,
      readbackMetadata
    } = deviceOrProps;
    this.buffer = buffer;
    this.type = arrowType as T;
    this.length = length;
    this.stride = getArrowTypeStride(arrowType);
    this.byteOffset = byteOffset;
    this.byteStride = byteStride;
    this.readbackMetadata = readbackMetadata;
    this._ownsBuffer = ownsBuffer;
  }

  get ownsBuffer(): boolean {
    return this._ownsBuffer;
  }

  /** Reads this GPU chunk back into a single non-null Arrow Data chunk. */
  async readAsync(): Promise<arrow.Data<T>> {
    if (arrow.DataType.isUtf8(this.type)) {
      const metadata = this.readbackMetadata;
      if (metadata?.kind !== 'utf8') {
        throw new Error('GPUData.readAsync() requires UTF-8 readback metadata');
      }
      const bytes =
        metadata.valueByteLength === 0
          ? new Uint8Array(0)
          : await this.buffer.readAsync(this.byteOffset, metadata.valueByteLength);
      return arrow.makeData({
        type: new arrow.Utf8(),
        length: this.length,
        nullCount: metadata.nullCount,
        nullBitmap: metadata.nullBitmap,
        valueOffsets: metadata.valueOffsets,
        data: bytes
      }) as arrow.Data<T>;
    }
    if (isVariableLengthAttributeArrowType(this.type)) {
      const metadata = this.readbackMetadata;
      if (metadata?.kind !== 'variable-length-attribute') {
        throw new Error('GPUData.readAsync() requires variable-length attribute readback metadata');
      }
      const bytes =
        metadata.valueByteLength === 0
          ? new Uint8Array(0)
          : await this.buffer.readAsync(this.byteOffset, metadata.valueByteLength);
      return makeArrowVariableLengthAttributeDataFromPackedBytes(
        this.type,
        this.length,
        metadata,
        bytes
      ) as unknown as arrow.Data<T>;
    }
    const vector = await readArrowGPUVectorAsync({
      type: this.type as unknown as AttributeArrowType,
      buffer: this.buffer,
      length: this.length,
      byteOffset: this.byteOffset,
      byteStride: this.byteStride
    });
    return vector.data[0] as unknown as arrow.Data<T>;
  }

  destroy(): void {
    if (this._ownsBuffer) {
      this.buffer.destroy();
      this._ownsBuffer = false;
    }
  }
}

export function getArrowDataBufferSource<T extends NumericArrowType>(
  data: arrow.Data<T>
): T['TArray'];
export function getArrowDataBufferSource<T extends NumericArrowType>(
  data: arrow.Data<arrow.FixedSizeList<T>>
): T['TArray'];
export function getArrowDataBufferSource<T extends AttributeArrowType>(
  data: arrow.Data<T>
): NumericArrowType['TArray'];
/** Return the uploadable typed-array view for one Arrow Data chunk. */
export function getArrowDataBufferSource(data: arrow.Data): NumericArrowType['TArray'] {
  const {values, startElement, elementCount} = getArrowDataValueRange(data);
  if (values.length < elementCount) {
    throw new Error('Arrow data values are shorter than the logical upload length');
  }
  if (values.length === elementCount) {
    return values;
  }

  const endElement = startElement + elementCount;
  if (endElement > values.length) {
    throw new Error('Arrow data values are shorter than the logical upload length');
  }
  return values.subarray(startElement, endElement) as NumericArrowType['TArray'];
}

/** Return the UTF-8 value bytes referenced by one Arrow Utf8 data chunk. */
export function getArrowUtf8DataBufferSource(data: arrow.Data<arrow.Utf8>): Uint8Array {
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
  const values = data.values as Uint8Array | undefined;
  if (!valueOffsets || !values) {
    return new Uint8Array(0);
  }
  const firstValueOffset = valueOffsets[0] ?? 0;
  const lastValueOffset = valueOffsets[data.length] ?? firstValueOffset;
  return values.subarray(firstValueOffset, lastValueOffset);
}

/** Return flattened numeric values referenced by one variable-length nested attribute chunk. */
export function getArrowVariableLengthAttributeDataBufferSource(
  data: arrow.Data<VariableLengthAttributeArrowType>
): NumericArrowType['TArray'] {
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
  const elementData = data.children[0] as arrow.Data | undefined;
  if (!valueOffsets || !elementData) {
    return new Float32Array(0);
  }

  const firstElementOffset = valueOffsets[0] ?? 0;
  const lastElementOffset = valueOffsets[data.length] ?? firstElementOffset;

  if (arrow.DataType.isFixedSizeList(elementData.type)) {
    const numericData = elementData.children[0] as arrow.Data<NumericArrowType> | undefined;
    const values = numericData?.values as NumericArrowType['TArray'] | undefined;
    if (!numericData || !values) {
      return new Float32Array(0);
    }

    const elementStride = elementData.type.listSize;
    const numericValueOffset = numericData.offset ?? 0;
    const firstValueOffset = numericValueOffset + firstElementOffset * elementStride;
    const lastValueOffset = numericValueOffset + lastElementOffset * elementStride;
    return values.subarray(firstValueOffset, lastValueOffset) as NumericArrowType['TArray'];
  }

  const values = elementData.values as NumericArrowType['TArray'] | undefined;
  if (!values) {
    return new Float32Array(0);
  }
  const numericValueOffset = elementData.offset ?? 0;
  return values.subarray(
    numericValueOffset + firstElementOffset,
    numericValueOffset + lastElementOffset
  ) as NumericArrowType['TArray'];
}

/** Copy compact variable-width reconstruction metadata without retaining Arrow value buffers. */
export function getArrowGPUDataReadbackMetadata(
  data: arrow.Data
): GPUDataReadbackMetadata | undefined {
  if (arrow.DataType.isUtf8(data.type)) {
    const values = getArrowUtf8DataBufferSource(data as arrow.Data<arrow.Utf8>);
    return {
      kind: 'utf8',
      valueOffsets: copyNormalizedArrowValueOffsets(data),
      nullCount: data.nullCount,
      nullBitmap: copyNormalizedArrowNullBitmap(data),
      valueByteLength: values.byteLength
    };
  }

  if (isVariableLengthAttributeArrowType(data.type)) {
    const values = getArrowVariableLengthAttributeDataBufferSource(
      data as arrow.Data<VariableLengthAttributeArrowType>
    );
    return {
      kind: 'variable-length-attribute',
      valueOffsets: copyNormalizedArrowValueOffsets(data),
      nullCount: data.nullCount,
      nullBitmap: copyNormalizedArrowNullBitmap(data),
      valueByteLength: values.byteLength
    };
  }

  return undefined;
}

export function getArrowVectorBufferSource<T extends NumericArrowType>(
  vector: arrow.Vector<T>
): T['TArray'];
export function getArrowVectorBufferSource<T extends NumericArrowType>(
  vector: arrow.Vector<arrow.FixedSizeList<T>>
): T['TArray'];
export function getArrowVectorBufferSource<T extends AttributeArrowType>(
  vector: arrow.Vector<T>
): NumericArrowType['TArray'];
/** Return a typed array that can be passed directly to `device.createBuffer()`. */
export function getArrowVectorBufferSource(vector: arrow.Vector): NumericArrowType['TArray'] {
  const dataSources = vector.data.map(data => getArrowDataBufferSource(data));
  if (dataSources.length === 0) {
    throw new Error('Arrow vector has no data');
  }
  if (dataSources.length === 1) {
    return dataSources[0];
  }

  const totalLength = dataSources.reduce((length, dataSource) => length + dataSource.length, 0);
  const values = createTypedArrayLike(dataSources[0], totalLength);
  let targetOffset = 0;
  for (const dataSource of dataSources) {
    values.set(dataSource as never, targetOffset);
    targetOffset += dataSource.length;
  }
  return values;
}

/** Number of scalar values in one logical Arrow row. */
export function getArrowTypeStride(type: arrow.DataType): number {
  if (isVariableLengthAttributeArrowType(type)) {
    return getArrowVariableLengthAttributeElementStride(type);
  }
  return arrow.DataType.isFixedSizeList(type) ? type.listSize : 1;
}

/** Number of uploaded bytes in one logical Arrow row. */
export function getArrowTypeByteStride(type: arrow.DataType): number {
  if (isVariableLengthAttributeArrowType(type)) {
    return getArrowVariableLengthAttributeElementByteStride(type);
  }
  if (arrow.DataType.isFixedSizeList(type)) {
    return type.listSize * getArrowTypeByteStride(type.children[0].type);
  }
  if (arrow.DataType.isInt(type)) {
    return type.bitWidth / 8;
  }
  if (arrow.DataType.isFloat(type)) {
    switch (type.precision) {
      case arrow.Precision.HALF:
        return 2;
      case arrow.Precision.SINGLE:
        return 4;
      case arrow.Precision.DOUBLE:
        return 8;
    }
  }
  throw new Error(`Cannot determine byte stride for Arrow type ${type}`);
}

/** Reject nullable Arrow chunks that cannot be uploaded directly into GPU attribute buffers. */
export function validateArrowGPUDataDirectUpload(
  name: string,
  data: arrow.Data<AttributeArrowType | arrow.Utf8 | VariableLengthAttributeArrowType>
): void {
  if (data.nullCount > 0) {
    throw new Error(`GPUVector "${name}" does not support nullable data`);
  }
  if (arrow.DataType.isFixedSizeList(data.type) && (data.children[0]?.nullCount ?? 0) > 0) {
    throw new Error(`GPUVector "${name}" does not support nullable child data`);
  }
  if (isVariableLengthAttributeArrowType(data.type)) {
    const elementData = data.children[0];
    const nestedNumericData = arrow.DataType.isFixedSizeList(elementData?.type)
      ? elementData.children[0]
      : undefined;
    if ((elementData?.nullCount ?? 0) > 0 || (nestedNumericData?.nullCount ?? 0) > 0) {
      throw new Error(`GPUVector "${name}" does not support nullable nested list data`);
    }
  }
}

/** Read GPU bytes and reconstruct one non-null Arrow vector with the supplied Arrow type. */
export async function readArrowGPUVectorAsync<T extends AttributeArrowType>(
  props: GPUVectorReadProps<T>
): Promise<arrow.Vector<T>> {
  const {buffer, type, length, byteOffset, byteStride} = props;
  const rowByteWidth = getArrowTypeByteStride(type);
  if (byteStride < rowByteWidth) {
    throw new Error(
      `GPUVector.readAsync() byteStride ${byteStride} is smaller than row byte width ${rowByteWidth}`
    );
  }

  const readByteLength = length === 0 ? 0 : (length - 1) * byteStride + rowByteWidth;
  const bytes =
    readByteLength === 0 ? new Uint8Array(0) : await buffer.readAsync(byteOffset, readByteLength);
  const packedBytes =
    byteStride === rowByteWidth
      ? bytes
      : compactStridedRows(bytes, length, byteStride, rowByteWidth);

  return makeArrowVectorFromPackedBytes(type, length, packedBytes);
}

function getArrowDataValueRange(data: arrow.Data): {
  values: NumericArrowType['TArray'];
  startElement: number;
  elementCount: number;
} {
  if (arrow.DataType.isFixedSizeList(data.type)) {
    const childData = data.children[0];
    const values = childData?.values;
    if (!values) {
      throw new Error('Arrow FixedSizeList data has no child values');
    }
    const elementCount = data.length * data.type.listSize;
    const startElement = (childData.offset ?? 0) + data.offset * data.type.listSize;
    return {values: values as NumericArrowType['TArray'], startElement, elementCount};
  }

  const values = data.values;
  if (!values) {
    throw new Error('Arrow data has no values');
  }
  return {
    values: values as NumericArrowType['TArray'],
    startElement: data.offset,
    elementCount: data.length
  };
}

function createTypedArrayLike(
  source: NumericArrowType['TArray'],
  length: number
): NumericArrowType['TArray'] {
  const TypedArrayConstructor = source.constructor as {
    new (length: number): NumericArrowType['TArray'];
  };
  return new TypedArrayConstructor(length);
}

function compactStridedRows(
  bytes: Uint8Array,
  length: number,
  byteStride: number,
  rowByteWidth: number
): Uint8Array {
  const packedBytes = new Uint8Array(length * rowByteWidth);
  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
    const sourceOffset = rowIndex * byteStride;
    const targetOffset = rowIndex * rowByteWidth;
    packedBytes.set(bytes.subarray(sourceOffset, sourceOffset + rowByteWidth), targetOffset);
  }
  return packedBytes;
}

function makeArrowVectorFromPackedBytes<T extends AttributeArrowType>(
  type: T,
  length: number,
  bytes: Uint8Array
): arrow.Vector<T> {
  if (arrow.DataType.isFixedSizeList(type)) {
    const childType = type.children[0].type as NumericArrowType;
    const values = makeNumericTypedArray(childType, bytes, length * type.listSize);
    const childData = makeNumericData({
      type: childType,
      length: length * type.listSize,
      data: values as NumericArrowType['TArray']
    });
    const listData = makeFixedSizeListData({
      type: type as arrow.FixedSizeList<NumericArrowType>,
      length,
      nullCount: 0,
      nullBitmap: null,
      child: childData
    });
    return arrow.makeVector(listData) as arrow.Vector<T>;
  }

  const numericType = type as NumericArrowType;
  const values = makeNumericTypedArray(numericType, bytes, length);
  const data = makeNumericData({
    type: numericType,
    length,
    data: values as NumericArrowType['TArray']
  });
  return arrow.makeVector(data) as arrow.Vector<T>;
}

function makeArrowVariableLengthAttributeDataFromPackedBytes(
  type: VariableLengthAttributeArrowType,
  length: number,
  metadata: Extract<GPUDataReadbackMetadata, {kind: 'variable-length-attribute'}>,
  bytes: Uint8Array
): arrow.Data<VariableLengthAttributeArrowType> {
  const elementType = type.children[0].type;
  const elementData = arrow.DataType.isFixedSizeList(elementType)
    ? makeArrowFixedSizeListElementDataFromPackedBytes(elementType, bytes)
    : makeArrowNumericElementDataFromPackedBytes(elementType as NumericArrowType, bytes);

  return new arrow.Data<VariableLengthAttributeArrowType>(
    type,
    0,
    length,
    metadata.nullCount,
    {
      [arrow.BufferType.OFFSET]: metadata.valueOffsets,
      ...(metadata.nullBitmap ? {[arrow.BufferType.VALIDITY]: metadata.nullBitmap} : {})
    },
    [elementData]
  );
}

function copyNormalizedArrowValueOffsets(data: arrow.Data): Int32Array {
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
  const copiedOffsets = new Int32Array(data.length + 1);
  if (!valueOffsets) {
    return copiedOffsets;
  }

  const firstValueOffset = valueOffsets[0] ?? 0;
  for (let offsetIndex = 0; offsetIndex <= data.length; offsetIndex++) {
    copiedOffsets[offsetIndex] = Math.max(
      0,
      (valueOffsets[offsetIndex] ?? firstValueOffset) - firstValueOffset
    );
  }
  return copiedOffsets;
}

function copyNormalizedArrowNullBitmap(data: arrow.Data): Uint8Array | undefined {
  if (data.nullCount === 0 || !data.nullBitmap) {
    return undefined;
  }

  const copiedBitmap = new Uint8Array(Math.ceil(data.length / 8));
  const sourceBitmap = data.nullBitmap as Uint8Array;
  const sourceRowOffset = data.offset ?? 0;
  for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
    const sourceBitIndex = sourceRowOffset + rowIndex;
    const sourceByte = sourceBitmap[sourceBitIndex >> 3] ?? 0;
    if ((sourceByte & (1 << (sourceBitIndex & 7))) !== 0) {
      copiedBitmap[rowIndex >> 3] |= 1 << (rowIndex & 7);
    }
  }
  return copiedBitmap;
}

function makeArrowFixedSizeListElementDataFromPackedBytes(
  type: arrow.FixedSizeList<NumericArrowType>,
  bytes: Uint8Array
): arrow.Data<arrow.FixedSizeList<NumericArrowType>> {
  const numericType = type.children[0].type as NumericArrowType;
  const values = makeNumericTypedArray(
    numericType,
    bytes,
    bytes.byteLength / getArrowTypeByteStride(numericType)
  ) as NumericArrowType['TArray'];
  const numericData = makeNumericData({
    type: numericType,
    length: values.length,
    data: values
  });
  return makeFixedSizeListData({
    type,
    length: type.listSize === 0 ? 0 : values.length / type.listSize,
    nullCount: 0,
    nullBitmap: null,
    child: numericData
  });
}

function makeArrowNumericElementDataFromPackedBytes(
  type: NumericArrowType,
  bytes: Uint8Array
): arrow.Data<NumericArrowType> {
  const values = makeNumericTypedArray(
    type,
    bytes,
    bytes.byteLength / getArrowTypeByteStride(type)
  ) as NumericArrowType['TArray'];
  return makeNumericData({
    type,
    length: values.length,
    data: values
  });
}

function getArrowVariableLengthAttributeElementStride(
  type: VariableLengthAttributeArrowType
): number {
  return getArrowTypeStride(type.children[0].type);
}

function getArrowVariableLengthAttributeElementByteStride(
  type: VariableLengthAttributeArrowType
): number {
  return getArrowTypeByteStride(type.children[0].type);
}

function makeNumericTypedArray(
  type: NumericArrowType,
  bytes: Uint8Array,
  length: number
): BigTypedArray {
  if (arrow.DataType.isInt(type)) {
    if (type.isSigned) {
      switch (type.bitWidth) {
        case 8:
          return makeTypedArrayView(Int8Array, bytes, length);
        case 16:
          return makeTypedArrayView(Int16Array, bytes, length);
        case 32:
          return makeTypedArrayView(Int32Array, bytes, length);
        case 64:
          return makeTypedArrayView(BigInt64Array, bytes, length);
      }
    }

    switch (type.bitWidth) {
      case 8:
        return makeTypedArrayView(Uint8Array, bytes, length);
      case 16:
        return makeTypedArrayView(Uint16Array, bytes, length);
      case 32:
        return makeTypedArrayView(Uint32Array, bytes, length);
      case 64:
        return makeTypedArrayView(BigUint64Array, bytes, length);
    }
  }

  if (arrow.DataType.isFloat(type)) {
    switch (type.precision) {
      case arrow.Precision.HALF:
        return makeTypedArrayView(Uint16Array, bytes, length);
      case arrow.Precision.SINGLE:
        return makeTypedArrayView(Float32Array, bytes, length);
      case arrow.Precision.DOUBLE:
        return makeTypedArrayView(Float64Array, bytes, length);
    }
  }

  throw new Error(`GPUVector.readAsync() does not support Arrow type ${type}`);
}

function makeTypedArrayView<T extends BigTypedArray>(
  TypedArrayConstructor: NumericTypedArrayConstructor,
  bytes: Uint8Array,
  length: number
): T {
  const byteLength = length * TypedArrayConstructor.BYTES_PER_ELEMENT;
  if (bytes.byteOffset % TypedArrayConstructor.BYTES_PER_ELEMENT === 0) {
    return new TypedArrayConstructor(bytes.buffer, bytes.byteOffset, length) as T;
  }

  const alignedBytes = new Uint8Array(byteLength);
  alignedBytes.set(bytes.subarray(0, byteLength));
  return new TypedArrayConstructor(alignedBytes.buffer, 0, length) as T;
}

visgl / luma.gl / 26054616669

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous