25879749619

Committed 14 May 2026 07:05PM UTC coverage: 74.881% (-0.2%) from 75.089%

Build # 25879749619

Build Type

push

github

Committed by

web-flow

Commit Message

feat(text) TextArrowModel (#2615)

Coverage Stats

6380 of 9600 branches covered (66.46%)

Branch coverage included in aggregate %.

462 of 672 new or added lines in 6 files covered. (68.75%)

123 existing lines in 9 files now uncovered.

13975 of 17583 relevant lines covered (79.48%)

782.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.24

/modules/arrow/src/arrow/arrow-gpu-vector.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {
  Buffer,
  Device,
  type BufferAttributeLayout,
  type BufferLayout,
  type BufferProps
} from '@luma.gl/core';
import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import {isInstanceArrowType, type AttributeArrowType} from './arrow-types';
import {
  GPUData,
  getArrowDataBufferSource,
  getArrowTypeByteStride,
  getArrowTypeStride,
  getArrowVectorBufferSource,
  readArrowGPUVectorAsync,
  validateArrowGPUDataDirectUpload
} from './arrow-gpu-data';

export {
  GPUData,
  readArrowGPUVectorAsync,
  type GPUDataFromBufferProps
} from './arrow-gpu-data';

/** Buffer creation props forwarded when uploading Arrow vector memory to the GPU. */
export type GPUVectorBufferProps = Omit<BufferProps, 'byteLength' | 'data'>;
/** Dynamic buffer props forwarded when creating appendable Arrow GPU vectors. */
export type GPUVectorDynamicBufferProps = Omit<DynamicBufferProps, 'byteLength' | 'data'>;

/** @deprecated Use {@link GPUVectorBufferProps}. */
export type GPUVectorProps = GPUVectorBufferProps;

/** Constructor props that upload an Arrow vector into a new GPU buffer. */
export type GPUVectorFromArrowProps<T extends AttributeArrowType = AttributeArrowType> = {
  /** Discriminator for Arrow-vector upload construction. */
  type: 'arrow';
  /** Name used when this vector is added to an {@link GPUTable}. */
  name: string;
  /** Device that creates the GPU buffer. */
  device: Device;
  /** Arrow vector whose value memory is uploaded. */
  vector: arrow.Vector<T>;
  /** Buffer creation props forwarded to the GPU buffer. */
  bufferProps?: GPUVectorBufferProps;
};

/** Constructor props that wrap an existing typed GPU buffer. */
export type GPUVectorFromBufferProps<T extends AttributeArrowType = AttributeArrowType> = {
  /** Discriminator for existing-buffer construction. */
  type: 'buffer';
  /** Name used when this vector is added to an {@link GPUTable}. */
  name: string;
  /** Existing GPU buffer. */
  buffer: Buffer;
  /** Arrow type that describes the values in the buffer. */
  arrowType: T;
  /** Number of logical rows in the buffer. */
  length: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
  byteStride?: number;
  /**
   * Whether this vector should destroy the buffer.
   *
   * Defaults to `false` for wrapped buffers because ownership remains with the caller unless
   * explicitly transferred or opted in.
   */
  ownsBuffer?: boolean;
};

/** Constructor props that wrap one interleaved GPU buffer as opaque Arrow binary rows. */
export type GPUVectorFromInterleavedProps = {
  /** Discriminator for interleaved-buffer construction. */
  type: 'interleaved';
  /** Name used when this vector is added to an {@link GPUTable}. */
  name: string;
  /** Existing interleaved GPU buffer. */
  buffer: Buffer;
  /** Number of logical rows in the buffer. */
  length: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. */
  byteStride: number;
  /** Attribute views stored in each interleaved row. */
  attributes: BufferAttributeLayout[];
  /**
   * Whether this vector should destroy the buffer.
   *
   * Defaults to `false` for wrapped buffers because ownership remains with the caller unless
   * explicitly transferred or opted in.
   */
  ownsBuffer?: boolean;
};

/** Constructor props that expose pre-existing Arrow GPU data chunks as one logical vector. */
export type GPUVectorFromDataProps<T extends arrow.DataType = AttributeArrowType> = {
  /** Discriminator for chunk-backed construction. */
  type: 'data';
  /** Name used when this vector is added to an {@link GPUTable}. */
  name: string;
  /** Arrow type that describes every chunk in `data`. */
  arrowType: T;
  /** Existing GPU data chunks to expose through this vector. */
  data: GPUData<T>[];
  /** Bytes between adjacent logical rows. Defaults to the first chunk stride when available. */
  byteStride?: number;
  /** Optional buffer layout retained for interleaved chunk collections. */
  bufferLayout?: BufferLayout;
};

/** Constructor props for an appendable DynamicBuffer-backed Arrow vector. */
export type GPUVectorFromAppendableProps<T extends AttributeArrowType = AttributeArrowType> = {
  /** Discriminator for appendable DynamicBuffer-backed construction. */
  type: 'appendable';
  /** Name used when this vector is added to an {@link GPUTable}. */
  name: string;
  /** Device that creates the DynamicBuffer. */
  device: Device;
  /** Arrow type that describes appended Arrow data. */
  arrowType: T;
  /** Initial row capacity. Defaults to `0`. */
  initialCapacityRows?: number;
  /** Capacity growth multiplier. Defaults to `1.5`. */
  capacityGrowthFactor?: number;
  /** DynamicBuffer construction props. */
  bufferProps?: GPUVectorDynamicBufferProps;
};

/** Discriminated constructor props for {@link GPUVector}. */
export type GPUVectorCreateProps<T extends arrow.DataType = AttributeArrowType> =
  | (T extends AttributeArrowType
      ? GPUVectorFromArrowProps<T> | GPUVectorFromBufferProps<T>
      : never)
  | GPUVectorFromInterleavedProps
  | GPUVectorFromDataProps<T>
  | (T extends AttributeArrowType ? GPUVectorFromAppendableProps<T> : never);

/**
 * GPU memory and Arrow type metadata derived from one Arrow vector.
 *
 * The Arrow vector is a construction input only. GPUVector does not retain
 * the source vector; it keeps a GPU buffer plus the type, length, and stride that
 * describe the uploaded memory.
 *
 * Ownership is tracked separately from the buffer reference. Vectors constructed
 * from Arrow data allocate and own their buffers. Vectors wrapping an existing
 * buffer default to non-owning unless `ownsBuffer` is supplied. In-place
 * operations can use {@link transferBufferOwnership} to consume one logical
 * vector and return another view that becomes responsible for destroying the
 * shared buffer.
 */
export class GPUVector<T extends arrow.DataType = AttributeArrowType> {
  /** Name used when this vector is added to an {@link GPUTable}. */
  readonly name: string;
  /** Arrow type that describes the uploaded vector memory. */
  readonly type: T;
  /** Number of logical Arrow vector rows uploaded into the GPU buffer. */
  length: number;
  /** Number of scalar values per logical vector row. */
  readonly stride: number;
  /** Byte offset of the first logical row in {@link buffer}. */
  readonly byteOffset: number;
  /** Bytes between adjacent logical rows in {@link buffer}. */
  readonly byteStride: number;
  /** Optional GPU buffer layout described by this vector. */
  readonly bufferLayout?: BufferLayout;
  /** GPU data chunk views preserving the source Arrow vector's chunk boundaries. */
  readonly data: GPUData<T>[] = [];
  /** Single concrete GPU buffer when this vector is directly bindable as one buffer. */
  private _buffer?: Buffer | DynamicBuffer;
  /** Whether this vector is responsible for destroying {@link buffer}. */
  private _ownsBuffer: boolean;
  /** Detached batch-local vectors whose buffers are now owned by this aggregate view. */
  private readonly _ownedVectors: GPUVector[] = [];
  /** Capacity growth multiplier for appendable DynamicBuffer-backed vectors. */
  private readonly capacityGrowthFactor?: number;

  /** Creates a GPU representation from an Arrow vector without retaining the source vector. */
  constructor(
    device: Device,
    vector: arrow.Vector<T & AttributeArrowType>,
    props?: GPUVectorBufferProps
  );
  /** Creates a GPU representation using discriminated construction props. */
  constructor(props: GPUVectorCreateProps<T>);
  constructor(
    deviceOrProps: Device | GPUVectorCreateProps<any>,
    vector?: arrow.Vector<T & AttributeArrowType>,
    props: GPUVectorBufferProps = {}
  ) {
    const constructionProps =
      deviceOrProps instanceof Device
        ? ({
            type: 'arrow',
            name: 'vector',
            device: deviceOrProps,
            vector: vector!,
            bufferProps: props
          } satisfies GPUVectorFromArrowProps)
        : deviceOrProps;

    switch (constructionProps.type) {
      case 'arrow': {
        const {name, device, vector: arrowVector, bufferProps = {}} = constructionProps;
        this.name = name;
        this.type = arrowVector.type;
        this.length = arrowVector.length;
        this.stride = getArrowVectorStride(arrowVector);
        this.byteOffset = 0;
        this.byteStride = getArrowTypeByteStride(arrowVector.type);
        this._buffer = device.createBuffer({
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...bufferProps,
          data: getArrowVectorBufferSource(arrowVector as any)
        });
        const dataBuffer = createArrowGPUDataBuffer(this._buffer);
        let byteOffset = 0;
        for (const arrowData of arrowVector.data) {
          const gpuData = new GPUData({
            buffer: dataBuffer,
            arrowType: arrowData.type as AttributeArrowType,
            length: arrowData.length,
            byteOffset,
            byteStride: this.byteStride,
            ownsBuffer: false
          }) as unknown as GPUData<T>;
          this.data.push(gpuData);
          byteOffset += arrowData.length * this.byteStride;
        }
        this._ownsBuffer = true;
        return;
      }

      case 'buffer': {
        const {
          name,
          buffer,
          arrowType,
          length,
          byteOffset = 0,
          byteStride = getArrowTypeByteStride(arrowType),
          ownsBuffer = false
        } = constructionProps;
        this.name = name;
        this._buffer = buffer;
        this.type = arrowType;
        this.length = length;
        this.stride = getArrowTypeStride(arrowType);
        this.byteOffset = byteOffset;
        this.byteStride = byteStride;
        const dataBuffer = createArrowGPUDataBuffer(buffer);
        this.data.push(
          new GPUData({
            buffer: dataBuffer,
            arrowType,
            length,
            byteOffset,
            byteStride,
            ownsBuffer: false
          }) as GPUData<T>
        );
        this._ownsBuffer = ownsBuffer;
        return;
      }

      case 'interleaved': {
        const {
          name,
          buffer,
          length,
          byteOffset = 0,
          byteStride,
          attributes,
          ownsBuffer = false
        } = constructionProps;
        this.name = name;
        this._buffer = buffer;
        this.type = new arrow.Binary() as T;
        this.length = length;
        this.stride = byteStride;
        this.byteOffset = byteOffset;
        this.byteStride = byteStride;
        this.bufferLayout = {name, byteStride, attributes};
        const dataBuffer = createArrowGPUDataBuffer(buffer);
        this.data.push(
          new GPUData({
            buffer: dataBuffer,
            arrowType: this.type,
            length,
            byteOffset,
            byteStride,
            ownsBuffer: false
          })
        );
        this._ownsBuffer = ownsBuffer;
        return;
      }

      case 'data': {
        const {name, arrowType, data, byteStride, bufferLayout} = constructionProps;
        if (data.some(chunk => !arrow.util.compareTypes(chunk.type, arrowType))) {
          throw new Error('GPUVector data chunks must share the declared Arrow type');
        }

        this.name = name;
        this.type = arrowType;
        this.length = data.reduce((length, chunk) => length + chunk.length, 0);
        this.stride = data[0]?.stride ?? getArrowTypeStride(arrowType);
        this.byteOffset = data.length === 1 ? data[0].byteOffset : 0;
        this.byteStride = byteStride ?? data[0]?.byteStride ?? getArrowTypeByteStride(arrowType);
        this.bufferLayout = bufferLayout;
        this.data.push(...data);
        this._buffer = data.length === 1 ? data[0].buffer : undefined;
        this._ownsBuffer = false;
        return;
      }

      case 'appendable': {
        const {
          name,
          device,
          arrowType,
          initialCapacityRows = 0,
          capacityGrowthFactor = 1.5,
          bufferProps
        } = constructionProps;
        if (!isInstanceArrowType(arrowType)) {
          throw new Error(`GPUVector does not support Arrow type ${arrowType}`);
        }
        this.name = name;
        this.type = arrowType as unknown as T;
        this.length = 0;
        this.stride = getArrowTypeStride(arrowType);
        this.byteOffset = 0;
        this.byteStride = getArrowTypeByteStride(arrowType);
        this._buffer = new DynamicBuffer(device, {
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...bufferProps,
          id: bufferProps?.id ?? `${name}-appendable-arrow-vector`,
          byteLength: Math.max(1, initialCapacityRows * this.byteStride)
        });
        this.capacityGrowthFactor = capacityGrowthFactor;
        this._ownsBuffer = true;
        return;
      }
    }
  }

  /**
   * Directly bindable GPU buffer when this vector has one concrete backing buffer.
   *
   * Aggregate table vectors may span multiple batch-owned buffers. Those vectors
   * intentionally require callers to use {@link data} or batch-local vectors.
   */
  get buffer(): Buffer | DynamicBuffer {
    if (!this._buffer) {
      throw new Error('GPUVector.buffer is unavailable for multi-buffer vectors; use data[]');
    }
    return this._buffer;
  }

  /**
   * Whether this vector is responsible for destroying {@link buffer}.
   *
   * `destroy()` only releases the buffer when this is `true`. This value can
   * change when ownership is transferred to another same-buffer view.
   */
  get ownsBuffer(): boolean {
    return this._ownsBuffer || this._ownedVectors.some(vector => vector.ownsBuffer);
  }

  /**
   * Adds one already-materialized GPU data chunk to this logical vector.
   *
   * This preserves ownership on the supplied {@link GPUData}; the vector
   * only aggregates metadata and never adopts or destroys that buffer through
   * this method.
   */
  addData(data: GPUData<T>): this {
    if (!arrow.util.compareTypes(data.type, this.type)) {
      throw new Error('GPUVector.addData() requires matching Arrow data types');
    }
    if (data.byteStride !== this.byteStride) {
      throw new Error('GPUVector.addData() requires matching byteStride');
    }

    this.data.push(data);
    this.length += data.length;
    if (this.data.length > 1) {
      this._buffer = undefined;
    }
    return this;
  }

  /** Number of rows the appendable backing DynamicBuffer can hold without reallocating. */
  get capacityRows(): number | undefined {
    return this._buffer instanceof DynamicBuffer
      ? Math.floor(this._buffer.byteLength / this.byteStride)
      : undefined;
  }

  /** Appends one Arrow Data chunk into this vector's trailing DynamicBuffer-backed data storage. */
  addToLastData(data: arrow.Data<T & AttributeArrowType>): this {
    if (!(this._buffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector.addToLastData() requires appendable vector storage');
    }
    if (!arrow.util.compareTypes(data.type, this.type)) {
      throw new Error('GPUVector.addToLastData() requires matching Arrow data types');
    }
    validateArrowGPUDataDirectUpload(this.name, data);

    const requiredRows = this.length + data.length;
    this.ensureAppendableCapacity(requiredRows);

    const byteOffset = this.length * this.byteStride;
    this._buffer.write(getArrowDataBufferSource(data), byteOffset);
    this.data.push(
      new GPUData({
        buffer: this._buffer,
        arrowType: data.type,
        length: data.length,
        byteOffset,
        byteStride: this.byteStride,
        ownsBuffer: false
      }) as GPUData<T>
    );
    this.length = requiredRows;
    return this;
  }

  /** @deprecated Use {@link addToLastData}. */
  addToLastBatch(data: arrow.Data<T & AttributeArrowType>): this {
    return this.addToLastData(data);
  }

  /** Appends every Arrow Data chunk from one Arrow vector into appendable batch storage. */
  addVectorToLastBatch(vector: arrow.Vector<T & AttributeArrowType>): this {
    if (!arrow.util.compareTypes(vector.type, this.type)) {
      throw new Error('GPUVector.addVectorToLastBatch() requires matching Arrow data types');
    }
    for (const data of vector.data) {
      this.addToLastData(data as arrow.Data<T & AttributeArrowType>);
    }
    return this;
  }

  /** Clears appendable logical rows while retaining the DynamicBuffer allocation. */
  resetLastBatch(): this {
    if (!(this._buffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector.resetLastBatch() requires appendable vector storage');
    }
    this.length = 0;
    this.data.length = 0;
    return this;
  }

  /** @internal Retains detached batch-local vector ownership under this aggregate vector. */
  retainOwnedVectors(vectors: GPUVector[]): this {
    this._ownedVectors.push(...vectors);
    return this;
  }

  /**
   * Transfers buffer ownership to another vector that views the same GPU buffer.
   *
   * This is intended for in-place operations that consume one logical vector and
   * return a new logical interpretation of the same bytes. After transfer,
   * destroying this vector will not destroy the buffer; destroying `target` will
   * destroy it if this vector previously owned it.
   */
  transferBufferOwnership(target: GPUVector): void {
    if (!this._buffer || !target._buffer || target._buffer !== this._buffer) {
      throw new Error('GPUVector ownership can only be transferred to the same buffer');
    }
    target._ownsBuffer = this._ownsBuffer;
    this._ownsBuffer = false;
  }

  /** Reads the GPU buffer contents back into a single non-null Arrow vector. */
  async readAsync(): Promise<arrow.Vector<T>> {
    if (this.bufferLayout) {
      throw new Error('GPUVector.readAsync() does not support interleaved vectors');
    }

    if (!this._buffer) {
      const data = await Promise.all(this.data.map(chunk => chunk.readAsync()));
      return new arrow.Vector(data) as arrow.Vector<T>;
    }

    return readArrowGPUVectorAsync({
      type: this.type as unknown as AttributeArrowType,
      buffer: this._buffer,
      length: this.length,
      byteOffset: this.byteOffset,
      byteStride: this.byteStride
    }) as unknown as Promise<arrow.Vector<T>>;
  }

  destroy(): void {
    if (this._ownsBuffer && this._buffer) {
      this._buffer.destroy();
      this._ownsBuffer = false;
    }
    for (const vector of this._ownedVectors.splice(0)) {
      vector.destroy();
    }
  }

  private ensureAppendableCapacity(requiredRows: number): void {
    if (!(this._buffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector append capacity requires DynamicBuffer storage');
    }
    const capacityRows = this.capacityRows ?? 0;
    if (requiredRows <= capacityRows) {
      return;
    }
    const grownRows = Math.ceil(Math.max(capacityRows, 1) * (this.capacityGrowthFactor ?? 1.5));
    const nextCapacityRows = Math.max(requiredRows, grownRows);
    this._buffer.ensureSize(nextCapacityRows * this.byteStride, {preserveData: true});
  }
}

function getArrowVectorStride(vector: arrow.Vector<AttributeArrowType>): number {
  return getArrowTypeStride(vector.type);
}

function createArrowGPUDataBuffer(buffer: Buffer | DynamicBuffer): DynamicBuffer {
  return buffer instanceof DynamicBuffer
    ? buffer
    : new DynamicBuffer(buffer.device, {
        buffer,
        ownsBuffer: false
      });
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {
6	Buffer,
7	Device,
8	type BufferAttributeLayout,
9	type BufferLayout,
10	type BufferProps
11	} from '@luma.gl/core';
12	import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
13	import * as arrow from 'apache-arrow';
14	import {isInstanceArrowType, type AttributeArrowType} from './arrow-types';
15	import {
16	GPUData,
17	getArrowDataBufferSource,
18	getArrowTypeByteStride,
19	getArrowTypeStride,
20	getArrowVectorBufferSource,
21	readArrowGPUVectorAsync,
22	validateArrowGPUDataDirectUpload
23	} from './arrow-gpu-data';
24
25	export {
26	GPUData,
27	readArrowGPUVectorAsync,
28	type GPUDataFromBufferProps
29	} from './arrow-gpu-data';
30
31	/** Buffer creation props forwarded when uploading Arrow vector memory to the GPU. */
32	export type GPUVectorBufferProps = Omit<BufferProps, 'byteLength' \| 'data'>;
33	/** Dynamic buffer props forwarded when creating appendable Arrow GPU vectors. */
34	export type GPUVectorDynamicBufferProps = Omit<DynamicBufferProps, 'byteLength' \| 'data'>;
35
36	/** @deprecated Use {@link GPUVectorBufferProps}. */
37	export type GPUVectorProps = GPUVectorBufferProps;
38
39	/** Constructor props that upload an Arrow vector into a new GPU buffer. */
40	export type GPUVectorFromArrowProps<T extends AttributeArrowType = AttributeArrowType> = {
41	/** Discriminator for Arrow-vector upload construction. */
42	type: 'arrow';
43	/** Name used when this vector is added to an {@link GPUTable}. */
44	name: string;
45	/** Device that creates the GPU buffer. */
46	device: Device;
47	/** Arrow vector whose value memory is uploaded. */
48	vector: arrow.Vector<T>;
49	/** Buffer creation props forwarded to the GPU buffer. */
50	bufferProps?: GPUVectorBufferProps;
51	};
52
53	/** Constructor props that wrap an existing typed GPU buffer. */
54	export type GPUVectorFromBufferProps<T extends AttributeArrowType = AttributeArrowType> = {
55	/** Discriminator for existing-buffer construction. */
56	type: 'buffer';
57	/** Name used when this vector is added to an {@link GPUTable}. */
58	name: string;
59	/** Existing GPU buffer. */
60	buffer: Buffer;
61	/** Arrow type that describes the values in the buffer. */
62	arrowType: T;
63	/** Number of logical rows in the buffer. */
64	length: number;
65	/** Byte offset of the first logical row. */
66	byteOffset?: number;
67	/** Bytes between adjacent logical rows. Defaults to the byte width of `arrowType`. */
68	byteStride?: number;
69	/**
70	* Whether this vector should destroy the buffer.
71	*
72	* Defaults to `false` for wrapped buffers because ownership remains with the caller unless
73	* explicitly transferred or opted in.
74	*/
75	ownsBuffer?: boolean;
76	};
77
78	/** Constructor props that wrap one interleaved GPU buffer as opaque Arrow binary rows. */
79	export type GPUVectorFromInterleavedProps = {
80	/** Discriminator for interleaved-buffer construction. */
81	type: 'interleaved';
82	/** Name used when this vector is added to an {@link GPUTable}. */
83	name: string;
84	/** Existing interleaved GPU buffer. */
85	buffer: Buffer;
86	/** Number of logical rows in the buffer. */
87	length: number;
88	/** Byte offset of the first logical row. */
89	byteOffset?: number;
90	/** Bytes between adjacent logical rows. */
91	byteStride: number;
92	/** Attribute views stored in each interleaved row. */
93	attributes: BufferAttributeLayout[];
94	/**
95	* Whether this vector should destroy the buffer.
96	*
97	* Defaults to `false` for wrapped buffers because ownership remains with the caller unless
98	* explicitly transferred or opted in.
99	*/
100	ownsBuffer?: boolean;
101	};
102
103	/** Constructor props that expose pre-existing Arrow GPU data chunks as one logical vector. */
104	export type GPUVectorFromDataProps<T extends arrow.DataType = AttributeArrowType> = {
105	/** Discriminator for chunk-backed construction. */
106	type: 'data';
107	/** Name used when this vector is added to an {@link GPUTable}. */
108	name: string;
109	/** Arrow type that describes every chunk in `data`. */
110	arrowType: T;
111	/** Existing GPU data chunks to expose through this vector. */
112	data: GPUData<T>[];
113	/** Bytes between adjacent logical rows. Defaults to the first chunk stride when available. */
114	byteStride?: number;
115	/** Optional buffer layout retained for interleaved chunk collections. */
116	bufferLayout?: BufferLayout;
117	};
118
119	/** Constructor props for an appendable DynamicBuffer-backed Arrow vector. */
120	export type GPUVectorFromAppendableProps<T extends AttributeArrowType = AttributeArrowType> = {
121	/** Discriminator for appendable DynamicBuffer-backed construction. */
122	type: 'appendable';
123	/** Name used when this vector is added to an {@link GPUTable}. */
124	name: string;
125	/** Device that creates the DynamicBuffer. */
126	device: Device;
127	/** Arrow type that describes appended Arrow data. */
128	arrowType: T;
129	/** Initial row capacity. Defaults to `0`. */
130	initialCapacityRows?: number;
131	/** Capacity growth multiplier. Defaults to `1.5`. */
132	capacityGrowthFactor?: number;
133	/** DynamicBuffer construction props. */
134	bufferProps?: GPUVectorDynamicBufferProps;
135	};
136
137	/** Discriminated constructor props for {@link GPUVector}. */
138	export type GPUVectorCreateProps<T extends arrow.DataType = AttributeArrowType> =
139	\| (T extends AttributeArrowType
140	? GPUVectorFromArrowProps<T> \| GPUVectorFromBufferProps<T>
141	: never)
142	\| GPUVectorFromInterleavedProps
143	\| GPUVectorFromDataProps<T>
144	\| (T extends AttributeArrowType ? GPUVectorFromAppendableProps<T> : never);
145
146	/**
147	* GPU memory and Arrow type metadata derived from one Arrow vector.
148	*
149	* The Arrow vector is a construction input only. GPUVector does not retain
150	* the source vector; it keeps a GPU buffer plus the type, length, and stride that
151	* describe the uploaded memory.
152	*
153	* Ownership is tracked separately from the buffer reference. Vectors constructed
154	* from Arrow data allocate and own their buffers. Vectors wrapping an existing
155	* buffer default to non-owning unless `ownsBuffer` is supplied. In-place
156	* operations can use {@link transferBufferOwnership} to consume one logical
157	* vector and return another view that becomes responsible for destroying the
158	* shared buffer.
159	*/
160	export class GPUVector<T extends arrow.DataType = AttributeArrowType> {
161	/** Name used when this vector is added to an {@link GPUTable}. */
162	readonly name: string;
163	/** Arrow type that describes the uploaded vector memory. */
164	readonly type: T;
165	/** Number of logical Arrow vector rows uploaded into the GPU buffer. */
166	length: number;
167	/** Number of scalar values per logical vector row. */
168	readonly stride: number;
169	/** Byte offset of the first logical row in {@link buffer}. */
170	readonly byteOffset: number;
171	/** Bytes between adjacent logical rows in {@link buffer}. */
172	readonly byteStride: number;
173	/** Optional GPU buffer layout described by this vector. */
174	readonly bufferLayout?: BufferLayout;
175	/** GPU data chunk views preserving the source Arrow vector's chunk boundaries. */
176	readonly data: GPUData<T>[] = [];	141✔
177	/** Single concrete GPU buffer when this vector is directly bindable as one buffer. */
178	private _buffer?: Buffer \| DynamicBuffer;
179	/** Whether this vector is responsible for destroying {@link buffer}. */
180	private _ownsBuffer: boolean;
181	/** Detached batch-local vectors whose buffers are now owned by this aggregate view. */
182	private readonly _ownedVectors: GPUVector[] = [];	141✔
183	/** Capacity growth multiplier for appendable DynamicBuffer-backed vectors. */
184	private readonly capacityGrowthFactor?: number;
185
186	/** Creates a GPU representation from an Arrow vector without retaining the source vector. */
187	constructor(
188	device: Device,
189	vector: arrow.Vector<T & AttributeArrowType>,
190	props?: GPUVectorBufferProps
191	);
192	/** Creates a GPU representation using discriminated construction props. */
193	constructor(props: GPUVectorCreateProps<T>);
194	constructor(
195	deviceOrProps: Device \| GPUVectorCreateProps<any>,
196	vector?: arrow.Vector<T & AttributeArrowType>,
197	props: GPUVectorBufferProps = {}	141✔
198	) {
199	const constructionProps =
200	deviceOrProps instanceof Device	141✔
201	? ({
202	type: 'arrow',
203	name: 'vector',
204	device: deviceOrProps,
205	vector: vector!,
206	bufferProps: props
207	} satisfies GPUVectorFromArrowProps)
208	: deviceOrProps;
209
210	switch (constructionProps.type) {	141✔
211	case 'arrow': {
212	const {name, device, vector: arrowVector, bufferProps = {}} = constructionProps;	72✔
213	this.name = name;	72✔
214	this.type = arrowVector.type;	72✔
215	this.length = arrowVector.length;	72✔
216	this.stride = getArrowVectorStride(arrowVector);	72✔
217	this.byteOffset = 0;	72✔
218	this.byteStride = getArrowTypeByteStride(arrowVector.type);	72✔
219	this._buffer = device.createBuffer({	72✔
220	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_DST \| Buffer.COPY_SRC,
221	...bufferProps,
222	data: getArrowVectorBufferSource(arrowVector as any)
223	});
224	const dataBuffer = createArrowGPUDataBuffer(this._buffer);	72✔
225	let byteOffset = 0;	72✔
226	for (const arrowData of arrowVector.data) {	72✔
227	const gpuData = new GPUData({	73✔
228	buffer: dataBuffer,
229	arrowType: arrowData.type as AttributeArrowType,
230	length: arrowData.length,
231	byteOffset,
232	byteStride: this.byteStride,
233	ownsBuffer: false
234	}) as unknown as GPUData<T>;
235	this.data.push(gpuData);	73✔
236	byteOffset += arrowData.length * this.byteStride;	73✔
237	}
238	this._ownsBuffer = true;	72✔
239	return;	72✔
240	}
241
242	case 'buffer': {
243	const {
244	name,
245	buffer,
246	arrowType,
247	length,
248	byteOffset = 0,	12✔
249	byteStride = getArrowTypeByteStride(arrowType),	12✔
250	ownsBuffer = false	12✔
251	} = constructionProps;	12✔
252	this.name = name;	12✔
253	this._buffer = buffer;	12✔
254	this.type = arrowType;	12✔
255	this.length = length;	12✔
256	this.stride = getArrowTypeStride(arrowType);	12✔
257	this.byteOffset = byteOffset;	12✔
258	this.byteStride = byteStride;	12✔
259	const dataBuffer = createArrowGPUDataBuffer(buffer);	12✔
260	this.data.push(	12✔
261	new GPUData({
262	buffer: dataBuffer,
263	arrowType,
264	length,
265	byteOffset,
266	byteStride,
267	ownsBuffer: false
268	}) as GPUData<T>
269	);
270	this._ownsBuffer = ownsBuffer;	12✔
271	return;	12✔
272	}
273
274	case 'interleaved': {
275	const {
276	name,
277	buffer,
278	length,
279	byteOffset = 0,	3✔
280	byteStride,
281	attributes,
282	ownsBuffer = false	3✔
283	} = constructionProps;	3✔
284	this.name = name;	3✔
285	this._buffer = buffer;	3✔
286	this.type = new arrow.Binary() as T;	3✔
287	this.length = length;	3✔
288	this.stride = byteStride;	3✔
289	this.byteOffset = byteOffset;	3✔
290	this.byteStride = byteStride;	3✔
291	this.bufferLayout = {name, byteStride, attributes};	3✔
292	const dataBuffer = createArrowGPUDataBuffer(buffer);	3✔
293	this.data.push(	3✔
294	new GPUData({
295	buffer: dataBuffer,
296	arrowType: this.type,
297	length,
298	byteOffset,
299	byteStride,
300	ownsBuffer: false
301	})
302	);
303	this._ownsBuffer = ownsBuffer;	3✔
304	return;	3✔
305	}
306
307	case 'data': {
308	const {name, arrowType, data, byteStride, bufferLayout} = constructionProps;	21✔
309	if (data.some(chunk => !arrow.util.compareTypes(chunk.type, arrowType))) {	21!
UNCOV 310	throw new Error('GPUVector data chunks must share the declared Arrow type');	×
311	}
312
313	this.name = name;	21✔
314	this.type = arrowType;	21✔
315	this.length = data.reduce((length, chunk) => length + chunk.length, 0);	21✔
316	this.stride = data[0]?.stride ?? getArrowTypeStride(arrowType);	21!
317	this.byteOffset = data.length === 1 ? data[0].byteOffset : 0;	21!
318	this.byteStride = byteStride ?? data[0]?.byteStride ?? getArrowTypeByteStride(arrowType);	21!
319	this.bufferLayout = bufferLayout;	21✔
320	this.data.push(...data);	21✔
321	this._buffer = data.length === 1 ? data[0].buffer : undefined;	21!
322	this._ownsBuffer = false;	21✔
323	return;	21✔
324	}
325
326	case 'appendable': {
327	const {
328	name,
329	device,
330	arrowType,
331	initialCapacityRows = 0,	33✔
332	capacityGrowthFactor = 1.5,	33✔
333	bufferProps
334	} = constructionProps;	33✔
335	if (!isInstanceArrowType(arrowType)) {	33✔
336	throw new Error(`GPUVector does not support Arrow type ${arrowType}`);	1✔
337	}
338	this.name = name;	32✔
339	this.type = arrowType as unknown as T;	32✔
340	this.length = 0;	32✔
341	this.stride = getArrowTypeStride(arrowType);	32✔
342	this.byteOffset = 0;	32✔
343	this.byteStride = getArrowTypeByteStride(arrowType);	32✔
344	this._buffer = new DynamicBuffer(device, {	32✔
345	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_DST \| Buffer.COPY_SRC,
346	...bufferProps,
347	id: bufferProps?.id ?? `${name}-appendable-arrow-vector`,	64✔
348	byteLength: Math.max(1, initialCapacityRows * this.byteStride)
349	});
350	this.capacityGrowthFactor = capacityGrowthFactor;	33✔
351	this._ownsBuffer = true;	33✔
352	return;	33✔
353	}
354	}
355	}
356
357	/**
358	* Directly bindable GPU buffer when this vector has one concrete backing buffer.
359	*
360	* Aggregate table vectors may span multiple batch-owned buffers. Those vectors
361	* intentionally require callers to use {@link data} or batch-local vectors.
362	*/
363	get buffer(): Buffer \| DynamicBuffer {
364	if (!this._buffer) {	170✔
365	throw new Error('GPUVector.buffer is unavailable for multi-buffer vectors; use data[]');	2✔
366	}
367	return this._buffer;	168✔
368	}
369
370	/**
371	* Whether this vector is responsible for destroying {@link buffer}.
372	*
373	* `destroy()` only releases the buffer when this is `true`. This value can
374	* change when ownership is transferred to another same-buffer view.
375	*/
376	get ownsBuffer(): boolean {
377	return this._ownsBuffer \|\| this._ownedVectors.some(vector => vector.ownsBuffer);	5✔
378	}
379
380	/**
381	* Adds one already-materialized GPU data chunk to this logical vector.
382	*
383	* This preserves ownership on the supplied {@link GPUData}; the vector
384	* only aggregates metadata and never adopts or destroys that buffer through
385	* this method.
386	*/
387	addData(data: GPUData<T>): this {
388	if (!arrow.util.compareTypes(data.type, this.type)) {	25!
UNCOV 389	throw new Error('GPUVector.addData() requires matching Arrow data types');	×
390	}
391	if (data.byteStride !== this.byteStride) {	25!
UNCOV 392	throw new Error('GPUVector.addData() requires matching byteStride');	×
393	}
394
395	this.data.push(data);	25✔
396	this.length += data.length;	25✔
397	if (this.data.length > 1) {	25!
398	this._buffer = undefined;	25✔
399	}
400	return this;	25✔
401	}
402
403	/** Number of rows the appendable backing DynamicBuffer can hold without reallocating. */
404	get capacityRows(): number \| undefined {
405	return this._buffer instanceof DynamicBuffer	51!
406	? Math.floor(this._buffer.byteLength / this.byteStride)
407	: undefined;
408	}
409
410	/** Appends one Arrow Data chunk into this vector's trailing DynamicBuffer-backed data storage. */
411	addToLastData(data: arrow.Data<T & AttributeArrowType>): this {
412	if (!(this._buffer instanceof DynamicBuffer)) {	49!
413	throw new Error('GPUVector.addToLastData() requires appendable vector storage');	×
414	}
415	if (!arrow.util.compareTypes(data.type, this.type)) {	49!
UNCOV 416	throw new Error('GPUVector.addToLastData() requires matching Arrow data types');	×
417	}
418	validateArrowGPUDataDirectUpload(this.name, data);	49✔
419
420	const requiredRows = this.length + data.length;	49✔
421	this.ensureAppendableCapacity(requiredRows);	49✔
422
423	const byteOffset = this.length * this.byteStride;	49✔
424	this._buffer.write(getArrowDataBufferSource(data), byteOffset);	49✔
425	this.data.push(	49✔
426	new GPUData({
427	buffer: this._buffer,
428	arrowType: data.type,
429	length: data.length,
430	byteOffset,
431	byteStride: this.byteStride,
432	ownsBuffer: false
433	}) as GPUData<T>
434	);
435	this.length = requiredRows;	49✔
436	return this;	49✔
437	}
438
439	/** @deprecated Use {@link addToLastData}. */
440	addToLastBatch(data: arrow.Data<T & AttributeArrowType>): this {
UNCOV 441	return this.addToLastData(data);	×
442	}
443
444	/** Appends every Arrow Data chunk from one Arrow vector into appendable batch storage. */
445	addVectorToLastBatch(vector: arrow.Vector<T & AttributeArrowType>): this {
446	if (!arrow.util.compareTypes(vector.type, this.type)) {	3!
447	throw new Error('GPUVector.addVectorToLastBatch() requires matching Arrow data types');	×
448	}
449	for (const data of vector.data) {	3✔
450	this.addToLastData(data as arrow.Data<T & AttributeArrowType>);	4✔
451	}
452	return this;	3✔
453	}
454
455	/** Clears appendable logical rows while retaining the DynamicBuffer allocation. */
456	resetLastBatch(): this {
457	if (!(this._buffer instanceof DynamicBuffer)) {	7!
UNCOV 458	throw new Error('GPUVector.resetLastBatch() requires appendable vector storage');	×
459	}
460	this.length = 0;	7✔
461	this.data.length = 0;	7✔
462	return this;	7✔
463	}
464
465	/** @internal Retains detached batch-local vector ownership under this aggregate vector. */
466	retainOwnedVectors(vectors: GPUVector[]): this {
467	this._ownedVectors.push(...vectors);	1✔
468	return this;	1✔
469	}
470
471	/**
472	* Transfers buffer ownership to another vector that views the same GPU buffer.
473	*
474	* This is intended for in-place operations that consume one logical vector and
475	* return a new logical interpretation of the same bytes. After transfer,
476	* destroying this vector will not destroy the buffer; destroying `target` will
477	* destroy it if this vector previously owned it.
478	*/
479	transferBufferOwnership(target: GPUVector): void {
480	if (!this._buffer \|\| !target._buffer \|\| target._buffer !== this._buffer) {	1!
UNCOV 481	throw new Error('GPUVector ownership can only be transferred to the same buffer');	×
482	}
483	target._ownsBuffer = this._ownsBuffer;	1✔
484	this._ownsBuffer = false;	1✔
485	}
486
487	/** Reads the GPU buffer contents back into a single non-null Arrow vector. */
488	async readAsync(): Promise<arrow.Vector<T>> {
489	if (this.bufferLayout) {	7✔
490	throw new Error('GPUVector.readAsync() does not support interleaved vectors');	1✔
491	}
492
493	if (!this._buffer) {	6!
UNCOV 494	const data = await Promise.all(this.data.map(chunk => chunk.readAsync()));	×
UNCOV 495	return new arrow.Vector(data) as arrow.Vector<T>;	×
496	}
497
498	return readArrowGPUVectorAsync({	6✔
499	type: this.type as unknown as AttributeArrowType,
500	buffer: this._buffer,
501	length: this.length,
502	byteOffset: this.byteOffset,
503	byteStride: this.byteStride
504	}) as unknown as Promise<arrow.Vector<T>>;
505	}
506
507	destroy(): void {
508	if (this._ownsBuffer && this._buffer) {	121✔
509	this._buffer.destroy();	116✔
510	this._ownsBuffer = false;	116✔
511	}
512	for (const vector of this._ownedVectors.splice(0)) {	121✔
513	vector.destroy();	2✔
514	}
515	}
516
517	private ensureAppendableCapacity(requiredRows: number): void {
518	if (!(this._buffer instanceof DynamicBuffer)) {	48!
UNCOV 519	throw new Error('GPUVector append capacity requires DynamicBuffer storage');	×
520	}
521	const capacityRows = this.capacityRows ?? 0;	48!
522	if (requiredRows <= capacityRows) {	48✔
523	return;	8✔
524	}
525	const grownRows = Math.ceil(Math.max(capacityRows, 1) * (this.capacityGrowthFactor ?? 1.5));	40!
526	const nextCapacityRows = Math.max(requiredRows, grownRows);	48✔
527	this._buffer.ensureSize(nextCapacityRows * this.byteStride, {preserveData: true});	48✔
528	}
529	}
530
531	function getArrowVectorStride(vector: arrow.Vector<AttributeArrowType>): number {
532	return getArrowTypeStride(vector.type);	72✔
533	}
534
535	function createArrowGPUDataBuffer(buffer: Buffer \| DynamicBuffer): DynamicBuffer {
536	return buffer instanceof DynamicBuffer	87!
537	? buffer
538	: new DynamicBuffer(buffer.device, {
539	buffer,
540	ownsBuffer: false
541	});
542	}

visgl / luma.gl / 25879749619

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous