25990278810

Committed 17 May 2026 12:01PM UTC coverage: 75.092% (+0.2%) from 74.881%

Build # 25990278810

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Columnar GPU-data stack (#2616)

Coverage Stats

6711 of 10084 branches covered (66.55%)

Branch coverage included in aggregate %.

625 of 865 new or added lines in 22 files covered. (72.25%)

1 existing line in 1 file now uncovered.

14631 of 18337 relevant lines covered (79.79%)

792.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.0

/modules/arrow/src/arrow/arrow-gpu-data.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {Buffer, Device, type BigTypedArray} from '@luma.gl/core';
import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import type {AttributeArrowType, NumericArrowType} from './arrow-types';

type GPUDataBufferProps = Omit<DynamicBufferProps, 'byteLength' | 'data' | 'buffer' | 'ownsBuffer'>;

/** Constructor props that wrap one existing typed GPU data buffer. */
export type GPUDataFromBufferProps<T extends arrow.DataType = AttributeArrowType> = {
  /** Stable dynamic GPU buffer wrapper for this data range. */
  buffer: DynamicBuffer;
  /** Arrow type that describes the values in the data chunk. */
  arrowType: T;
  /** Number of logical rows in the data chunk. */
  length: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
  byteStride?: number;
  /** Whether this data view should destroy the buffer. */
  ownsBuffer?: boolean;
  /** Optional source Arrow data retained for CPU-side consumers such as text expansion. */
  sourceData?: arrow.Data<T>;
};

type GPUVectorReadableBuffer = Pick<Buffer, 'readAsync'>;

type GPUVectorReadProps<T extends AttributeArrowType> = {
  type: T;
  buffer: GPUVectorReadableBuffer;
  length: number;
  byteOffset: number;
  byteStride: number;
};

type NumericTypedArrayConstructor = {
  readonly BYTES_PER_ELEMENT: number;
  new (buffer: ArrayBufferLike, byteOffset?: number, length?: number): BigTypedArray;
};

const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: T;
  length: number;
  data: T['TArray'];
}) => arrow.Data<T>;

const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {
  type: arrow.FixedSizeList<T>;
  length: number;
  nullCount: number;
  nullBitmap: null;
  child: arrow.Data<T>;
}) => arrow.Data<arrow.FixedSizeList<T>>;

/**
 * GPU memory and Arrow type metadata for one Arrow Data chunk.
 *
 * GPUData can own a dedicated buffer when constructed from Arrow Data, or
 * describe a byte-range view into a shared static or dynamic GPU buffer.
 */
export class GPUData<T extends arrow.DataType = AttributeArrowType> {
  /** GPU buffer containing the Arrow data chunk's attribute-compatible value memory. */
  readonly buffer: DynamicBuffer;
  /** Arrow type that describes the uploaded data chunk. */
  readonly type: T;
  /** Number of logical Arrow rows in this chunk. */
  readonly length: number;
  /** Number of scalar values per logical row. */
  readonly stride: number;
  /** Byte offset of the first logical row in {@link buffer}. */
  readonly byteOffset: number;
  /** Bytes between adjacent logical rows in {@link buffer}. */
  readonly byteStride: number;
  /** Optional source Arrow data retained for CPU-side consumers such as text expansion. */
  readonly sourceData?: arrow.Data<T>;
  /** Whether this data view is responsible for destroying {@link buffer}. */
  private _ownsBuffer: boolean;

  /** Creates a GPU representation from one Arrow Data chunk. */
  constructor(device: Device, data: arrow.Data<T>, props?: GPUDataBufferProps);
  /** Creates a data view over an existing GPU buffer. */
  constructor(props: GPUDataFromBufferProps<T>);
  constructor(
    deviceOrProps: Device | GPUDataFromBufferProps<any>,
    data?: arrow.Data<T>,
    props: GPUDataBufferProps = {}
  ) {
    if (deviceOrProps instanceof Device) {
      const arrowData = data!;
      this.type = arrowData.type as T;
      this.length = arrowData.length;
      this.sourceData = arrowData;
      if (arrow.DataType.isUtf8(arrowData.type)) {
        this.stride = 1;
        this.byteOffset = 0;
        this.byteStride = 1;
        this.buffer = new DynamicBuffer(deviceOrProps, {
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...props,
          data: getArrowUtf8DataBufferSource(arrowData as arrow.Data<arrow.Utf8>)
        });
        this._ownsBuffer = true;
        return;
      }
      this.stride = getArrowTypeStride(arrowData.type);
      this.byteOffset = 0;
      this.byteStride = getArrowTypeByteStride(arrowData.type);
      this.buffer = new DynamicBuffer(deviceOrProps, {
        usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
        ...props,
        data: getArrowDataBufferSource(arrowData as any)
      });
      this._ownsBuffer = true;
      return;
    }

    const {
      buffer,
      arrowType,
      length,
      byteOffset = 0,
      byteStride = getArrowTypeByteStride(arrowType),
      ownsBuffer = false,
      sourceData
    } = deviceOrProps;
    this.buffer = buffer;
    this.type = arrowType as T;
    this.length = length;
    this.stride = getArrowTypeStride(arrowType);
    this.byteOffset = byteOffset;
    this.byteStride = byteStride;
    this.sourceData = sourceData;
    this._ownsBuffer = ownsBuffer;
  }

  get ownsBuffer(): boolean {
    return this._ownsBuffer;
  }

  /** Reads this GPU chunk back into a single non-null Arrow Data chunk. */
  async readAsync(): Promise<arrow.Data<T>> {
    if (arrow.DataType.isUtf8(this.type)) {
      if (!this.sourceData) {
        throw new Error('GPUData.readAsync() requires retained UTF-8 source offsets');
      }
      const sourceData = this.sourceData as unknown as arrow.Data<arrow.Utf8>;
      const values = getArrowUtf8DataBufferSource(sourceData);
      const bytes =
        values.byteLength === 0
          ? new Uint8Array(0)
          : await this.buffer.readAsync(this.byteOffset, values.byteLength);
      return arrow.makeData({
        type: new arrow.Utf8(),
        length: sourceData.length,
        nullCount: sourceData.nullCount,
        nullBitmap: sourceData.nullBitmap as Uint8Array | undefined,
        valueOffsets: sourceData.valueOffsets as Int32Array,
        data: bytes
      }) as arrow.Data<T>;
    }
    const vector = await readArrowGPUVectorAsync({
      type: this.type as unknown as AttributeArrowType,
      buffer: this.buffer,
      length: this.length,
      byteOffset: this.byteOffset,
      byteStride: this.byteStride
    });
    return vector.data[0] as unknown as arrow.Data<T>;
  }

  destroy(): void {
    if (this._ownsBuffer) {
      this.buffer.destroy();
      this._ownsBuffer = false;
    }
  }
}

export function getArrowDataBufferSource<T extends NumericArrowType>(
  data: arrow.Data<T>
): T['TArray'];
export function getArrowDataBufferSource<T extends NumericArrowType>(
  data: arrow.Data<arrow.FixedSizeList<T>>
): T['TArray'];
export function getArrowDataBufferSource<T extends AttributeArrowType>(
  data: arrow.Data<T>
): NumericArrowType['TArray'];
/** Return the uploadable typed-array view for one Arrow Data chunk. */
export function getArrowDataBufferSource(data: arrow.Data): NumericArrowType['TArray'] {
  const {values, startElement, elementCount} = getArrowDataValueRange(data);
  if (values.length < elementCount) {
    throw new Error('Arrow data values are shorter than the logical upload length');
  }
  if (values.length === elementCount) {
    return values;
  }

  const endElement = startElement + elementCount;
  if (endElement > values.length) {
    throw new Error('Arrow data values are shorter than the logical upload length');
  }
  return values.subarray(startElement, endElement) as NumericArrowType['TArray'];
}

/** Return the UTF-8 value bytes referenced by one Arrow Utf8 data chunk. */
export function getArrowUtf8DataBufferSource(data: arrow.Data<arrow.Utf8>): Uint8Array {
  const valueOffsets = data.valueOffsets as Int32Array | undefined;
  const values = data.values as Uint8Array | undefined;
  if (!valueOffsets || !values) {
    return new Uint8Array(0);
  }
  const firstValueOffset = valueOffsets[0] ?? 0;
  const lastValueOffset = valueOffsets[data.length] ?? firstValueOffset;
  return values.subarray(firstValueOffset, lastValueOffset);
}

export function getArrowVectorBufferSource<T extends NumericArrowType>(
  vector: arrow.Vector<T>
): T['TArray'];
export function getArrowVectorBufferSource<T extends NumericArrowType>(
  vector: arrow.Vector<arrow.FixedSizeList<T>>
): T['TArray'];
export function getArrowVectorBufferSource<T extends AttributeArrowType>(
  vector: arrow.Vector<T>
): NumericArrowType['TArray'];
/** Return a typed array that can be passed directly to `device.createBuffer()`. */
export function getArrowVectorBufferSource(vector: arrow.Vector): NumericArrowType['TArray'] {
  const dataSources = vector.data.map(data => getArrowDataBufferSource(data));
  if (dataSources.length === 0) {
    throw new Error('Arrow vector has no data');
  }
  if (dataSources.length === 1) {
    return dataSources[0];
  }

  const totalLength = dataSources.reduce((length, dataSource) => length + dataSource.length, 0);
  const values = createTypedArrayLike(dataSources[0], totalLength);
  let targetOffset = 0;
  for (const dataSource of dataSources) {
    values.set(dataSource as never, targetOffset);
    targetOffset += dataSource.length;
  }
  return values;
}

/** Number of scalar values in one logical Arrow row. */
export function getArrowTypeStride(type: arrow.DataType): number {
  return arrow.DataType.isFixedSizeList(type) ? type.listSize : 1;
}

/** Number of uploaded bytes in one logical Arrow row. */
export function getArrowTypeByteStride(type: arrow.DataType): number {
  if (arrow.DataType.isFixedSizeList(type)) {
    return type.listSize * getArrowTypeByteStride(type.children[0].type);
  }
  if (arrow.DataType.isInt(type)) {
    return type.bitWidth / 8;
  }
  if (arrow.DataType.isFloat(type)) {
    switch (type.precision) {
      case arrow.Precision.HALF:
        return 2;
      case arrow.Precision.SINGLE:
        return 4;
      case arrow.Precision.DOUBLE:
        return 8;
    }
  }
  throw new Error(`Cannot determine byte stride for Arrow type ${type}`);
}

/** Reject nullable Arrow chunks that cannot be uploaded directly into GPU attribute buffers. */
export function validateArrowGPUDataDirectUpload(
  name: string,
  data: arrow.Data<AttributeArrowType>
): void {
  if (data.nullCount > 0) {
    throw new Error(`GPUVector "${name}" does not support nullable data`);
  }
  if (arrow.DataType.isFixedSizeList(data.type) && (data.children[0]?.nullCount ?? 0) > 0) {
    throw new Error(`GPUVector "${name}" does not support nullable child data`);
  }
}

/** Read GPU bytes and reconstruct one non-null Arrow vector with the supplied Arrow type. */
export async function readArrowGPUVectorAsync<T extends AttributeArrowType>(
  props: GPUVectorReadProps<T>
): Promise<arrow.Vector<T>> {
  const {buffer, type, length, byteOffset, byteStride} = props;
  const rowByteWidth = getArrowTypeByteStride(type);
  if (byteStride < rowByteWidth) {
    throw new Error(
      `GPUVector.readAsync() byteStride ${byteStride} is smaller than row byte width ${rowByteWidth}`
    );
  }

  const readByteLength = length === 0 ? 0 : (length - 1) * byteStride + rowByteWidth;
  const bytes =
    readByteLength === 0 ? new Uint8Array(0) : await buffer.readAsync(byteOffset, readByteLength);
  const packedBytes =
    byteStride === rowByteWidth
      ? bytes
      : compactStridedRows(bytes, length, byteStride, rowByteWidth);

  return makeArrowVectorFromPackedBytes(type, length, packedBytes);
}

function getArrowDataValueRange(data: arrow.Data): {
  values: NumericArrowType['TArray'];
  startElement: number;
  elementCount: number;
} {
  if (arrow.DataType.isFixedSizeList(data.type)) {
    const childData = data.children[0];
    const values = childData?.values;
    if (!values) {
      throw new Error('Arrow FixedSizeList data has no child values');
    }
    const elementCount = data.length * data.type.listSize;
    const startElement = (childData.offset ?? 0) + data.offset * data.type.listSize;
    return {values: values as NumericArrowType['TArray'], startElement, elementCount};
  }

  const values = data.values;
  if (!values) {
    throw new Error('Arrow data has no values');
  }
  return {
    values: values as NumericArrowType['TArray'],
    startElement: data.offset,
    elementCount: data.length
  };
}

function createTypedArrayLike(
  source: NumericArrowType['TArray'],
  length: number
): NumericArrowType['TArray'] {
  const TypedArrayConstructor = source.constructor as {
    new (length: number): NumericArrowType['TArray'];
  };
  return new TypedArrayConstructor(length);
}

function compactStridedRows(
  bytes: Uint8Array,
  length: number,
  byteStride: number,
  rowByteWidth: number
): Uint8Array {
  const packedBytes = new Uint8Array(length * rowByteWidth);
  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
    const sourceOffset = rowIndex * byteStride;
    const targetOffset = rowIndex * rowByteWidth;
    packedBytes.set(bytes.subarray(sourceOffset, sourceOffset + rowByteWidth), targetOffset);
  }
  return packedBytes;
}

function makeArrowVectorFromPackedBytes<T extends AttributeArrowType>(
  type: T,
  length: number,
  bytes: Uint8Array
): arrow.Vector<T> {
  if (arrow.DataType.isFixedSizeList(type)) {
    const childType = type.children[0].type as NumericArrowType;
    const values = makeNumericTypedArray(childType, bytes, length * type.listSize);
    const childData = makeNumericData({
      type: childType,
      length: length * type.listSize,
      data: values as NumericArrowType['TArray']
    });
    const listData = makeFixedSizeListData({
      type: type as arrow.FixedSizeList<NumericArrowType>,
      length,
      nullCount: 0,
      nullBitmap: null,
      child: childData
    });
    return arrow.makeVector(listData) as arrow.Vector<T>;
  }

  const numericType = type as NumericArrowType;
  const values = makeNumericTypedArray(numericType, bytes, length);
  const data = makeNumericData({
    type: numericType,
    length,
    data: values as NumericArrowType['TArray']
  });
  return arrow.makeVector(data) as arrow.Vector<T>;
}

function makeNumericTypedArray(
  type: NumericArrowType,
  bytes: Uint8Array,
  length: number
): BigTypedArray {
  if (arrow.DataType.isInt(type)) {
    if (type.isSigned) {
      switch (type.bitWidth) {
        case 8:
          return makeTypedArrayView(Int8Array, bytes, length);
        case 16:
          return makeTypedArrayView(Int16Array, bytes, length);
        case 32:
          return makeTypedArrayView(Int32Array, bytes, length);
        case 64:
          return makeTypedArrayView(BigInt64Array, bytes, length);
      }
    }

    switch (type.bitWidth) {
      case 8:
        return makeTypedArrayView(Uint8Array, bytes, length);
      case 16:
        return makeTypedArrayView(Uint16Array, bytes, length);
      case 32:
        return makeTypedArrayView(Uint32Array, bytes, length);
      case 64:
        return makeTypedArrayView(BigUint64Array, bytes, length);
    }
  }

  if (arrow.DataType.isFloat(type)) {
    switch (type.precision) {
      case arrow.Precision.HALF:
        return makeTypedArrayView(Uint16Array, bytes, length);
      case arrow.Precision.SINGLE:
        return makeTypedArrayView(Float32Array, bytes, length);
      case arrow.Precision.DOUBLE:
        return makeTypedArrayView(Float64Array, bytes, length);
    }
  }

  throw new Error(`GPUVector.readAsync() does not support Arrow type ${type}`);
}

function makeTypedArrayView<T extends BigTypedArray>(
  TypedArrayConstructor: NumericTypedArrayConstructor,
  bytes: Uint8Array,
  length: number
): T {
  const byteLength = length * TypedArrayConstructor.BYTES_PER_ELEMENT;
  if (bytes.byteOffset % TypedArrayConstructor.BYTES_PER_ELEMENT === 0) {
    return new TypedArrayConstructor(bytes.buffer, bytes.byteOffset, length) as T;
  }

  const alignedBytes = new Uint8Array(byteLength);
  alignedBytes.set(bytes.subarray(0, byteLength));
  return new TypedArrayConstructor(alignedBytes.buffer, 0, length) as T;
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {Buffer, Device, type BigTypedArray} from '@luma.gl/core';
6	import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
7	import * as arrow from 'apache-arrow';
8	import type {AttributeArrowType, NumericArrowType} from './arrow-types';
9
10	type GPUDataBufferProps = Omit<DynamicBufferProps, 'byteLength' \| 'data' \| 'buffer' \| 'ownsBuffer'>;
11
12	/** Constructor props that wrap one existing typed GPU data buffer. */
13	export type GPUDataFromBufferProps<T extends arrow.DataType = AttributeArrowType> = {
14	/** Stable dynamic GPU buffer wrapper for this data range. */
15	buffer: DynamicBuffer;
16	/** Arrow type that describes the values in the data chunk. */
17	arrowType: T;
18	/** Number of logical rows in the data chunk. */
19	length: number;
20	/** Byte offset of the first logical row. */
21	byteOffset?: number;
22	/** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
23	byteStride?: number;
24	/** Whether this data view should destroy the buffer. */
25	ownsBuffer?: boolean;
26	/** Optional source Arrow data retained for CPU-side consumers such as text expansion. */
27	sourceData?: arrow.Data<T>;
28	};
29
30	type GPUVectorReadableBuffer = Pick<Buffer, 'readAsync'>;
31
32	type GPUVectorReadProps<T extends AttributeArrowType> = {
33	type: T;
34	buffer: GPUVectorReadableBuffer;
35	length: number;
36	byteOffset: number;
37	byteStride: number;
38	};
39
40	type NumericTypedArrayConstructor = {
41	readonly BYTES_PER_ELEMENT: number;
42	new (buffer: ArrayBufferLike, byteOffset?: number, length?: number): BigTypedArray;
43	};
44
45	const makeNumericData = arrow.makeData as <T extends NumericArrowType>(props: {	13✔
46	type: T;
47	length: number;
48	data: T['TArray'];
49	}) => arrow.Data<T>;
50
51	const makeFixedSizeListData = arrow.makeData as <T extends NumericArrowType>(props: {	13✔
52	type: arrow.FixedSizeList<T>;
53	length: number;
54	nullCount: number;
55	nullBitmap: null;
56	child: arrow.Data<T>;
57	}) => arrow.Data<arrow.FixedSizeList<T>>;
58
59	/**
60	* GPU memory and Arrow type metadata for one Arrow Data chunk.
61	*
62	* GPUData can own a dedicated buffer when constructed from Arrow Data, or
63	* describe a byte-range view into a shared static or dynamic GPU buffer.
64	*/
65	export class GPUData<T extends arrow.DataType = AttributeArrowType> {
66	/** GPU buffer containing the Arrow data chunk's attribute-compatible value memory. */
67	readonly buffer: DynamicBuffer;
68	/** Arrow type that describes the uploaded data chunk. */
69	readonly type: T;
70	/** Number of logical Arrow rows in this chunk. */
71	readonly length: number;
72	/** Number of scalar values per logical row. */
73	readonly stride: number;
74	/** Byte offset of the first logical row in {@link buffer}. */
75	readonly byteOffset: number;
76	/** Bytes between adjacent logical rows in {@link buffer}. */
77	readonly byteStride: number;
78	/** Optional source Arrow data retained for CPU-side consumers such as text expansion. */
79	readonly sourceData?: arrow.Data<T>;
80	/** Whether this data view is responsible for destroying {@link buffer}. */
81	private _ownsBuffer: boolean;
82
83	/** Creates a GPU representation from one Arrow Data chunk. */
84	constructor(device: Device, data: arrow.Data<T>, props?: GPUDataBufferProps);
85	/** Creates a data view over an existing GPU buffer. */
86	constructor(props: GPUDataFromBufferProps<T>);
87	constructor(
88	deviceOrProps: Device \| GPUDataFromBufferProps<any>,
89	data?: arrow.Data<T>,
90	props: GPUDataBufferProps = {}	177✔
91	) {
92	if (deviceOrProps instanceof Device) {	177✔
93	const arrowData = data!;	11✔
94	this.type = arrowData.type as T;	11✔
95	this.length = arrowData.length;	11✔
96	this.sourceData = arrowData;	11✔
97	if (arrow.DataType.isUtf8(arrowData.type)) {	11!
98	this.stride = 1;	11✔
99	this.byteOffset = 0;	11✔
100	this.byteStride = 1;	11✔
101	this.buffer = new DynamicBuffer(deviceOrProps, {	11✔
102	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_DST \| Buffer.COPY_SRC,
103	...props,
104	data: getArrowUtf8DataBufferSource(arrowData as arrow.Data<arrow.Utf8>)
105	});
106	this._ownsBuffer = true;	11✔
107	return;	11✔
108	}
109	this.stride = getArrowTypeStride(arrowData.type);	×
110	this.byteOffset = 0;	×
111	this.byteStride = getArrowTypeByteStride(arrowData.type);	×
112	this.buffer = new DynamicBuffer(deviceOrProps, {	×
113	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_DST \| Buffer.COPY_SRC,
114	...props,
115	data: getArrowDataBufferSource(arrowData as any)
116	});
117	this._ownsBuffer = true;	×
118	return;	×
119	}
120
121	const {
122	buffer,
123	arrowType,
124	length,
125	byteOffset = 0,	166✔
126	byteStride = getArrowTypeByteStride(arrowType),	166✔
127	ownsBuffer = false,	166✔
128	sourceData
129	} = deviceOrProps;	166✔
130	this.buffer = buffer;	166✔
131	this.type = arrowType as T;	166✔
132	this.length = length;	166✔
133	this.stride = getArrowTypeStride(arrowType);	166✔
134	this.byteOffset = byteOffset;	166✔
135	this.byteStride = byteStride;	166✔
136	this.sourceData = sourceData;	166✔
137	this._ownsBuffer = ownsBuffer;	166✔
138	}
139
140	get ownsBuffer(): boolean {
141	return this._ownsBuffer;	×
142	}
143
144	/** Reads this GPU chunk back into a single non-null Arrow Data chunk. */
145	async readAsync(): Promise<arrow.Data<T>> {
146	if (arrow.DataType.isUtf8(this.type)) {	4✔
147	if (!this.sourceData) {	3!
NEW 148	throw new Error('GPUData.readAsync() requires retained UTF-8 source offsets');	×
149	}
150	const sourceData = this.sourceData as unknown as arrow.Data<arrow.Utf8>;	3✔
151	const values = getArrowUtf8DataBufferSource(sourceData);	3✔
152	const bytes =
153	values.byteLength === 0	3!
154	? new Uint8Array(0)
155	: await this.buffer.readAsync(this.byteOffset, values.byteLength);
156	return arrow.makeData({	3✔
157	type: new arrow.Utf8(),
158	length: sourceData.length,
159	nullCount: sourceData.nullCount,
160	nullBitmap: sourceData.nullBitmap as Uint8Array \| undefined,
161	valueOffsets: sourceData.valueOffsets as Int32Array,
162	data: bytes
163	}) as arrow.Data<T>;
164	}
165	const vector = await readArrowGPUVectorAsync({	1✔
166	type: this.type as unknown as AttributeArrowType,
167	buffer: this.buffer,
168	length: this.length,
169	byteOffset: this.byteOffset,
170	byteStride: this.byteStride
171	});
172	return vector.data[0] as unknown as arrow.Data<T>;	1✔
173	}
174
175	destroy(): void {
176	if (this._ownsBuffer) {	11!
177	this.buffer.destroy();	11✔
178	this._ownsBuffer = false;	11✔
179	}
180	}
181	}
182
183	export function getArrowDataBufferSource<T extends NumericArrowType>(
184	data: arrow.Data<T>
185	): T['TArray'];
186	export function getArrowDataBufferSource<T extends NumericArrowType>(
187	data: arrow.Data<arrow.FixedSizeList<T>>
188	): T['TArray'];
189	export function getArrowDataBufferSource<T extends AttributeArrowType>(
190	data: arrow.Data<T>
191	): NumericArrowType['TArray'];
192	/** Return the uploadable typed-array view for one Arrow Data chunk. */
193	export function getArrowDataBufferSource(data: arrow.Data): NumericArrowType['TArray'] {
194	const {values, startElement, elementCount} = getArrowDataValueRange(data);	191✔
195	if (values.length < elementCount) {	191!
196	throw new Error('Arrow data values are shorter than the logical upload length');	×
197	}
198	if (values.length === elementCount) {	191✔
199	return values;	190✔
200	}
201
202	const endElement = startElement + elementCount;	1✔
203	if (endElement > values.length) {	1!
204	throw new Error('Arrow data values are shorter than the logical upload length');	×
205	}
206	return values.subarray(startElement, endElement) as NumericArrowType['TArray'];	1✔
207	}
208
209	/** Return the UTF-8 value bytes referenced by one Arrow Utf8 data chunk. */
210	export function getArrowUtf8DataBufferSource(data: arrow.Data<arrow.Utf8>): Uint8Array {
211	const valueOffsets = data.valueOffsets as Int32Array \| undefined;	14✔
212	const values = data.values as Uint8Array \| undefined;	14✔
213	if (!valueOffsets \|\| !values) {	14!
NEW 214	return new Uint8Array(0);	×
215	}
216	const firstValueOffset = valueOffsets[0] ?? 0;	14!
217	const lastValueOffset = valueOffsets[data.length] ?? firstValueOffset;	14!
218	return values.subarray(firstValueOffset, lastValueOffset);	14✔
219	}
220
221	export function getArrowVectorBufferSource<T extends NumericArrowType>(
222	vector: arrow.Vector<T>
223	): T['TArray'];
224	export function getArrowVectorBufferSource<T extends NumericArrowType>(
225	vector: arrow.Vector<arrow.FixedSizeList<T>>
226	): T['TArray'];
227	export function getArrowVectorBufferSource<T extends AttributeArrowType>(
228	vector: arrow.Vector<T>
229	): NumericArrowType['TArray'];
230	/** Return a typed array that can be passed directly to `device.createBuffer()`. */
231	export function getArrowVectorBufferSource(vector: arrow.Vector): NumericArrowType['TArray'] {
232	const dataSources = vector.data.map(data => getArrowDataBufferSource(data));	143✔
233	if (dataSources.length === 0) {	142!
234	throw new Error('Arrow vector has no data');	×
235	}
236	if (dataSources.length === 1) {	142✔
237	return dataSources[0];	141✔
238	}
239
240	const totalLength = dataSources.reduce((length, dataSource) => length + dataSource.length, 0);	2✔
241	const values = createTypedArrayLike(dataSources[0], totalLength);	1✔
242	let targetOffset = 0;	1✔
243	for (const dataSource of dataSources) {	1✔
244	values.set(dataSource as never, targetOffset);	2✔
245	targetOffset += dataSource.length;	2✔
246	}
247	return values;	1✔
248	}
249
250	/** Number of scalar values in one logical Arrow row. */
251	export function getArrowTypeStride(type: arrow.DataType): number {
252	return arrow.DataType.isFixedSizeList(type) ? type.listSize : 1;	310✔
253	}
254
255	/** Number of uploaded bytes in one logical Arrow row. */
256	export function getArrowTypeByteStride(type: arrow.DataType): number {
257	if (arrow.DataType.isFixedSizeList(type)) {	279✔
258	return type.listSize * getArrowTypeByteStride(type.children[0].type);	121✔
259	}
260	if (arrow.DataType.isInt(type)) {	158✔
261	return type.bitWidth / 8;	67✔
262	}
263	if (arrow.DataType.isFloat(type)) {	91!
264	switch (type.precision) {	91!
265	case arrow.Precision.HALF:
266	return 2;	×
267	case arrow.Precision.SINGLE:
268	return 4;	91✔
269	case arrow.Precision.DOUBLE:
270	return 8;	×
271	}
272	}
273	throw new Error(`Cannot determine byte stride for Arrow type ${type}`);	×
274	}
275
276	/** Reject nullable Arrow chunks that cannot be uploaded directly into GPU attribute buffers. */
277	export function validateArrowGPUDataDirectUpload(
278	name: string,
279	data: arrow.Data<AttributeArrowType>
280	): void {
281	if (data.nullCount > 0) {	83✔
282	throw new Error(`GPUVector "${name}" does not support nullable data`);	1✔
283	}
284	if (arrow.DataType.isFixedSizeList(data.type) && (data.children[0]?.nullCount ?? 0) > 0) {	82!
285	throw new Error(`GPUVector "${name}" does not support nullable child data`);	×
286	}
287	}
288
289	/** Read GPU bytes and reconstruct one non-null Arrow vector with the supplied Arrow type. */
290	export async function readArrowGPUVectorAsync<T extends AttributeArrowType>(
291	props: GPUVectorReadProps<T>
292	): Promise<arrow.Vector<T>> {
293	const {buffer, type, length, byteOffset, byteStride} = props;	7✔
294	const rowByteWidth = getArrowTypeByteStride(type);	7✔
295	if (byteStride < rowByteWidth) {	7!
296	throw new Error(	×
297	`GPUVector.readAsync() byteStride ${byteStride} is smaller than row byte width ${rowByteWidth}`
298	);
299	}
300
301	const readByteLength = length === 0 ? 0 : (length - 1) * byteStride + rowByteWidth;	7✔
302	const bytes =
303	readByteLength === 0 ? new Uint8Array(0) : await buffer.readAsync(byteOffset, readByteLength);	7✔
304	const packedBytes =
305	byteStride === rowByteWidth	6✔
306	? bytes
307	: compactStridedRows(bytes, length, byteStride, rowByteWidth);
308
309	return makeArrowVectorFromPackedBytes(type, length, packedBytes);	7✔
310	}
311
312	function getArrowDataValueRange(data: arrow.Data): {
313	values: NumericArrowType['TArray'];
314	startElement: number;
315	elementCount: number;
316	} {
317	if (arrow.DataType.isFixedSizeList(data.type)) {	191✔
318	const childData = data.children[0];	151✔
319	const values = childData?.values;	151✔
320	if (!values) {	151!
321	throw new Error('Arrow FixedSizeList data has no child values');	×
322	}
323	const elementCount = data.length * data.type.listSize;	151✔
324	const startElement = (childData.offset ?? 0) + data.offset * data.type.listSize;	151!
325	return {values: values as NumericArrowType['TArray'], startElement, elementCount};	151✔
326	}
327
328	const values = data.values;	40✔
329	if (!values) {	40!
330	throw new Error('Arrow data has no values');	×
331	}
332	return {	40✔
333	values: values as NumericArrowType['TArray'],
334	startElement: data.offset,
335	elementCount: data.length
336	};
337	}
338
339	function createTypedArrayLike(
340	source: NumericArrowType['TArray'],
341	length: number
342	): NumericArrowType['TArray'] {
343	const TypedArrayConstructor = source.constructor as {	1✔
344	new (length: number): NumericArrowType['TArray'];
345	};
346	return new TypedArrayConstructor(length);	1✔
347	}
348
349	function compactStridedRows(
350	bytes: Uint8Array,
351	length: number,
352	byteStride: number,
353	rowByteWidth: number
354	): Uint8Array {
355	const packedBytes = new Uint8Array(length * rowByteWidth);	1✔
356	for (let rowIndex = 0; rowIndex < length; rowIndex++) {	1✔
357	const sourceOffset = rowIndex * byteStride;	2✔
358	const targetOffset = rowIndex * rowByteWidth;	2✔
359	packedBytes.set(bytes.subarray(sourceOffset, sourceOffset + rowByteWidth), targetOffset);	2✔
360	}
361	return packedBytes;	1✔
362	}
363
364	function makeArrowVectorFromPackedBytes<T extends AttributeArrowType>(
365	type: T,
366	length: number,
367	bytes: Uint8Array
368	): arrow.Vector<T> {
369	if (arrow.DataType.isFixedSizeList(type)) {	7✔
370	const childType = type.children[0].type as NumericArrowType;	1✔
371	const values = makeNumericTypedArray(childType, bytes, length * type.listSize);	1✔
372	const childData = makeNumericData({	1✔
373	type: childType,
374	length: length * type.listSize,
375	data: values as NumericArrowType['TArray']
376	});
377	const listData = makeFixedSizeListData({	1✔
378	type: type as arrow.FixedSizeList<NumericArrowType>,
379	length,
380	nullCount: 0,
381	nullBitmap: null,
382	child: childData
383	});
384	return arrow.makeVector(listData) as arrow.Vector<T>;	1✔
385	}
386
387	const numericType = type as NumericArrowType;	6✔
388	const values = makeNumericTypedArray(numericType, bytes, length);	6✔
389	const data = makeNumericData({	6✔
390	type: numericType,
391	length,
392	data: values as NumericArrowType['TArray']
393	});
394	return arrow.makeVector(data) as arrow.Vector<T>;	6✔
395	}
396
397	function makeNumericTypedArray(
398	type: NumericArrowType,
399	bytes: Uint8Array,
400	length: number
401	): BigTypedArray {
402	if (arrow.DataType.isInt(type)) {	7✔
403	if (type.isSigned) {	1!
404	switch (type.bitWidth) {	1!
405	case 8:
406	return makeTypedArrayView(Int8Array, bytes, length);	×
407	case 16:
408	return makeTypedArrayView(Int16Array, bytes, length);	×
409	case 32:
410	return makeTypedArrayView(Int32Array, bytes, length);	1✔
411	case 64:
412	return makeTypedArrayView(BigInt64Array, bytes, length);	×
413	}
414	}
415
416	switch (type.bitWidth) {	×
417	case 8:
418	return makeTypedArrayView(Uint8Array, bytes, length);	×
419	case 16:
420	return makeTypedArrayView(Uint16Array, bytes, length);	×
421	case 32:
422	return makeTypedArrayView(Uint32Array, bytes, length);	×
423	case 64:
424	return makeTypedArrayView(BigUint64Array, bytes, length);	×
425	}
426	}
427
428	if (arrow.DataType.isFloat(type)) {	6!
429	switch (type.precision) {	6!
430	case arrow.Precision.HALF:
431	return makeTypedArrayView(Uint16Array, bytes, length);	×
432	case arrow.Precision.SINGLE:
433	return makeTypedArrayView(Float32Array, bytes, length);	6✔
434	case arrow.Precision.DOUBLE:
435	return makeTypedArrayView(Float64Array, bytes, length);	×
436	}
437	}
438
439	throw new Error(`GPUVector.readAsync() does not support Arrow type ${type}`);	×
440	}
441
442	function makeTypedArrayView<T extends BigTypedArray>(
443	TypedArrayConstructor: NumericTypedArrayConstructor,
444	bytes: Uint8Array,
445	length: number
446	): T {
447	const byteLength = length * TypedArrayConstructor.BYTES_PER_ELEMENT;	7✔
448	if (bytes.byteOffset % TypedArrayConstructor.BYTES_PER_ELEMENT === 0) {	7!
449	return new TypedArrayConstructor(bytes.buffer, bytes.byteOffset, length) as T;	7✔
450	}
451
452	const alignedBytes = new Uint8Array(byteLength);	×
453	alignedBytes.set(bytes.subarray(0, byteLength));	×
454	return new TypedArrayConstructor(alignedBytes.buffer, 0, length) as T;	×
455	}

visgl / luma.gl / 25990278810

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous