26120831014

Committed 19 May 2026 07:40PM UTC coverage: 74.867% (-0.1%) from 74.965%

Build # 26120831014

Build Type

push

github

Committed by

ibgreen

Commit Message

feat(tables) Split out tables module from arrow

Coverage Stats

7324 of 11059 branches covered (66.23%)

Branch coverage included in aggregate %.

879 of 1079 new or added lines in 16 files covered. (81.46%)

22 existing lines in 2 files now uncovered.

15967 of 20051 relevant lines covered (79.63%)

1015.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.08

/modules/tables/src/table/gpu-vector.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {
  Buffer,
  Device,
  type BufferAttributeLayout,
  type BufferLayout,
  type BufferProps
} from '@luma.gl/core';
import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
import * as arrow from 'apache-arrow';
import {GPUData} from './gpu-data';

/** Buffer creation props used by format-specific producers before wrapping storage in a GPUVector. */
export type GPUVectorBufferProps = Omit<BufferProps, 'byteLength' | 'data'>;
/** Dynamic buffer props used when creating appendable GPU vectors. */
export type GPUVectorDynamicBufferProps = Omit<DynamicBufferProps, 'byteLength' | 'data'>;
/** @deprecated Use {@link GPUVectorBufferProps}. */
export type GPUVectorProps = GPUVectorBufferProps;

/** Arrow upload props moved to `@luma.gl/arrow`; retained only as a typed migration sentinel. */
export type GPUVectorFromArrowProps<_T extends arrow.DataType = arrow.DataType> = never;

/** Constructor props that wrap an existing typed GPU buffer. */
export type GPUVectorFromBufferProps<T extends arrow.DataType = arrow.DataType> = {
  /** Discriminator for existing-buffer construction. */
  type: 'buffer';
  /** Stable vector name. */
  name: string;
  /** Existing GPU buffer. */
  buffer: Buffer | DynamicBuffer;
  /** Logical schema/type descriptor for values stored in the buffer. */
  dataType: T;
  /** Number of logical rows in the buffer. */
  length: number;
  /** Number of scalar values represented by one logical row. */
  stride?: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. */
  byteStride: number;
  /** Number of bytes occupied by one logical row payload. */
  rowByteLength?: number;
  /** Whether this vector should destroy the wrapped buffer. */
  ownsBuffer?: boolean;
};

/** Constructor props that wrap one interleaved GPU buffer. */
export type GPUVectorFromInterleavedProps<T extends arrow.DataType = arrow.DataType> = {
  /** Discriminator for interleaved-buffer construction. */
  type: 'interleaved';
  /** Stable vector name. */
  name: string;
  /** Existing interleaved GPU buffer. */
  buffer: Buffer | DynamicBuffer;
  /** Logical schema/type descriptor for the interleaved rows. */
  dataType: T;
  /** Number of logical rows in the buffer. */
  length: number;
  /** Byte offset of the first logical row. */
  byteOffset?: number;
  /** Bytes between adjacent logical rows. */
  byteStride: number;
  /** Attribute views stored in each interleaved row. */
  attributes: BufferAttributeLayout[];
  /** Whether this vector should destroy the wrapped buffer. */
  ownsBuffer?: boolean;
};

/** Constructor props that expose existing GPU data chunks as one logical vector. */
export type GPUVectorFromDataProps<T extends arrow.DataType = arrow.DataType> = {
  /** Discriminator for chunk-backed construction. */
  type: 'data';
  /** Stable vector name. */
  name: string;
  /** Logical schema/type descriptor shared by every chunk. */
  dataType: T;
  /** Existing GPU data chunks to expose through this vector. */
  data: GPUData<T>[];
  /** Optional concrete aggregate buffer shared by the supplied data chunks. */
  buffer?: Buffer | DynamicBuffer;
  /** Number of scalar values represented by one logical row. */
  stride?: number;
  /** Bytes between adjacent logical rows. Defaults to the first chunk stride when available. */
  byteStride?: number;
  /** Number of bytes occupied by one logical row payload. */
  rowByteLength?: number;
  /** Optional buffer layout retained for interleaved chunk collections. */
  bufferLayout?: BufferLayout;
  /** Whether this vector should destroy the supplied GPU data chunks. */
  ownsData?: boolean;
  /** Whether this vector should destroy the optional aggregate buffer. */
  ownsBuffer?: boolean;
};

/** Constructor props for an appendable DynamicBuffer-backed vector. */
export type GPUVectorFromAppendableProps<T extends arrow.DataType = arrow.DataType> = {
  /** Discriminator for appendable DynamicBuffer-backed construction. */
  type: 'appendable';
  /** Stable vector name. */
  name: string;
  /** Device that creates the DynamicBuffer. */
  device: Device;
  /** Logical schema/type descriptor for appended data. */
  dataType: T;
  /** Number of scalar values represented by one logical row. */
  stride: number;
  /** Bytes between adjacent logical rows. */
  byteStride: number;
  /** Number of bytes occupied by one logical row payload. */
  rowByteLength?: number;
  /** Initial row capacity. Defaults to `0`. */
  initialCapacityRows?: number;
  /** Capacity growth multiplier. Defaults to `1.5`. */
  capacityGrowthFactor?: number;
  /** DynamicBuffer construction props. */
  bufferProps?: GPUVectorDynamicBufferProps;
};

/** Discriminated constructor props for {@link GPUVector}. */
export type GPUVectorCreateProps<T extends arrow.DataType = arrow.DataType> =
  | GPUVectorFromBufferProps<T>
  | GPUVectorFromInterleavedProps<T>
  | GPUVectorFromDataProps<T>
  | GPUVectorFromAppendableProps<T>;

/**
 * GPU memory and logical type metadata for one vector-valued table column.
 *
 * Format-specific modules upload bytes and use these vectors to expose shared
 * lifecycle, chunking, batching, and ownership semantics.
 */
export class GPUVector<T extends arrow.DataType = arrow.DataType> {
  /** Stable vector name. */
  readonly name: string;
  /** Logical schema/type descriptor for the uploaded bytes. */
  readonly type: T;
  /** Number of logical rows represented by the vector. */
  length: number;
  /** Number of scalar values represented by one logical row. */
  readonly stride: number;
  /** Byte offset of the first logical row in {@link buffer}. */
  readonly byteOffset: number;
  /** Bytes between adjacent logical rows in {@link buffer}. */
  readonly byteStride: number;
  /** Bytes occupied by one logical row payload. */
  readonly rowByteLength: number;
  /** Optional GPU buffer layout described by this vector. */
  readonly bufferLayout?: BufferLayout;
  /** GPU data chunk views preserved across table/batch aggregation. */
  readonly data: GPUData<T>[] = [];
  private concreteBuffer?: Buffer | DynamicBuffer;
  private ownsConcreteBuffer: boolean;
  private ownsDataChunks = false;
  private readonly ownedVectors: GPUVector[] = [];
  private readonly capacityGrowthFactor?: number;
  private appendableByteLength = 0;

  constructor(props: GPUVectorCreateProps<T>) {
    switch (props.type) {
      case 'buffer': {
        const {
          name,
          buffer,
          dataType,
          length,
          stride = 1,
          byteOffset = 0,
          byteStride,
          rowByteLength = byteStride,
          ownsBuffer = false
        } = props;
        this.name = name;
        this.type = dataType;
        this.length = length;
        this.stride = stride;
        this.byteOffset = byteOffset;
        this.byteStride = byteStride;
        this.rowByteLength = rowByteLength;
        this.concreteBuffer = buffer;
        this.ownsConcreteBuffer = ownsBuffer;
        this.data.push(
          new GPUData({
            buffer: createGPUDataBuffer(buffer),
            dataType,
            length,
            stride,
            byteOffset,
            byteStride,
            rowByteLength,
            ownsBuffer: false
          }) as GPUData<T>
        );
        return;
      }

      case 'interleaved': {
        const {
          name,
          buffer,
          dataType,
          length,
          byteOffset = 0,
          byteStride,
          attributes,
          ownsBuffer = false
        } = props;
        this.name = name;
        this.type = dataType;
        this.length = length;
        this.stride = byteStride;
        this.byteOffset = byteOffset;
        this.byteStride = byteStride;
        this.rowByteLength = byteStride;
        this.bufferLayout = {name, byteStride, attributes};
        this.concreteBuffer = buffer;
        this.ownsConcreteBuffer = ownsBuffer;
        this.data.push(
          new GPUData({
            buffer: createGPUDataBuffer(buffer),
            dataType,
            length,
            stride: byteStride,
            byteOffset,
            byteStride,
            rowByteLength: byteStride,
            ownsBuffer: false
          }) as GPUData<T>
        );
        return;
      }

      case 'data': {
        const {
          name,
          dataType,
          data,
          buffer,
          stride = data[0]?.stride ?? 1,
          byteStride = data[0]?.byteStride ?? 0,
          rowByteLength = data[0]?.rowByteLength ?? byteStride,
          bufferLayout,
          ownsData = false,
          ownsBuffer = false
        } = props;
        if (data.some(chunk => !arrow.util.compareTypes(chunk.type, dataType))) {
          throw new Error('GPUVector data chunks must share the declared logical type');
        }
        this.name = name;
        this.type = dataType;
        this.length = data.reduce((totalLength, chunk) => totalLength + chunk.length, 0);
        this.stride = stride;
        this.byteOffset = data.length === 1 ? data[0].byteOffset : 0;
        this.byteStride = byteStride;
        this.rowByteLength = rowByteLength;
        this.bufferLayout = bufferLayout;
        this.data.push(...data);
        this.concreteBuffer = buffer ?? (data.length === 1 ? data[0].buffer : undefined);
        this.ownsConcreteBuffer = ownsBuffer;
        this.ownsDataChunks = ownsData;
        return;
      }

      case 'appendable': {
        const {
          name,
          device,
          dataType,
          stride,
          byteStride,
          rowByteLength = byteStride,
          initialCapacityRows = 0,
          capacityGrowthFactor = 1.5,
          bufferProps
        } = props;
        this.name = name;
        this.type = dataType;
        this.length = 0;
        this.stride = stride;
        this.byteOffset = 0;
        this.byteStride = byteStride;
        this.rowByteLength = rowByteLength;
        this.concreteBuffer = new DynamicBuffer(device, {
          usage: Buffer.VERTEX | Buffer.STORAGE | Buffer.COPY_DST | Buffer.COPY_SRC,
          ...bufferProps,
          id: bufferProps?.id ?? `${name}-appendable-gpu-vector`,
          byteLength: Math.max(1, initialCapacityRows * byteStride)
        });
        this.capacityGrowthFactor = capacityGrowthFactor;
        this.ownsConcreteBuffer = true;
        return;
      }
    }
  }

  /** Directly bindable GPU buffer when this vector has one concrete backing buffer. */
  get buffer(): Buffer | DynamicBuffer {
    if (!this.concreteBuffer) {
      throw new Error('GPUVector.buffer is unavailable for multi-buffer vectors; use data[]');
    }
    return this.concreteBuffer;
  }

  /** Whether destroying this vector releases any retained GPU storage. */
  get ownsBuffer(): boolean {
    return this.ownsConcreteBuffer || this.ownedVectors.some(vector => vector.ownsBuffer);
  }

  /** Number of rows the appendable backing DynamicBuffer can hold without reallocating. */
  get capacityRows(): number | undefined {
    return this.concreteBuffer instanceof DynamicBuffer
      ? Math.floor(this.concreteBuffer.byteLength / this.byteStride)
      : undefined;
  }

  /** Bytes occupied by already-appended payloads in appendable storage. */
  get appendedByteLength(): number {
    return this.appendableByteLength;
  }

  /** Adds one already-materialized GPU data chunk to this logical vector. */
  addData(data: GPUData<T>): this {
    if (!arrow.util.compareTypes(data.type, this.type)) {
      throw new Error('GPUVector.addData() requires matching logical types');
    }
    if (data.byteStride !== this.byteStride) {
      throw new Error('GPUVector.addData() requires matching byteStride');
    }
    if (data.rowByteLength !== this.rowByteLength) {
      throw new Error('GPUVector.addData() requires matching rowByteLength');
    }

    this.data.push(data);
    this.length += data.length;
    if (this.data.length > 1) {
      this.concreteBuffer = undefined;
    }
    return this;
  }

  /**
   * Reserves appendable storage and writes raw bytes supplied by a format-specific adapter.
   */
  writeAppendableBytes(
    data: ArrayBufferView,
    byteOffset: number,
    requiredByteLength: number
  ): void {
    if (!(this.concreteBuffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector append writes require appendable DynamicBuffer storage');
    }
    this.ensureAppendableByteCapacity(requiredByteLength);
    if (data.byteLength > 0) {
      this.concreteBuffer.write(data, byteOffset);
    }
  }

  /**
   * Adds one appendable data view while preserving the concrete backing buffer.
   */
  appendDataChunk(data: GPUData<T>, appendedByteLength: number): this {
    if (!(this.concreteBuffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector append views require appendable DynamicBuffer storage');
    }
    if (!arrow.util.compareTypes(data.type, this.type)) {
      throw new Error('GPUVector.appendDataChunk() requires matching logical types');
    }
    if (data.byteStride !== this.byteStride || data.rowByteLength !== this.rowByteLength) {
      throw new Error('GPUVector.appendDataChunk() requires matching byte layout metadata');
    }
    this.data.push(data);
    this.length += data.length;
    this.appendableByteLength = appendedByteLength;
    return this;
  }

  /** Clears appendable logical rows while retaining the DynamicBuffer allocation. */
  resetLastBatch(): this {
    if (!(this.concreteBuffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector.resetLastBatch() requires appendable vector storage');
    }
    this.length = 0;
    this.appendableByteLength = 0;
    this.data.length = 0;
    return this;
  }

  /** @internal Retains detached batch-local vector ownership under this aggregate vector. */
  retainOwnedVectors(vectors: GPUVector[]): this {
    this.ownedVectors.push(...vectors);
    return this;
  }

  /** Transfers same-buffer ownership to another vector view. */
  transferBufferOwnership(target: GPUVector): void {
    if (
      !this.concreteBuffer ||
      !target.concreteBuffer ||
      target.concreteBuffer !== this.concreteBuffer
    ) {
      throw new Error('GPUVector ownership can only be transferred to the same buffer');
    }
    target.ownsConcreteBuffer = this.ownsConcreteBuffer;
    this.ownsConcreteBuffer = false;
  }

  /** Releases owned GPU data, detached ownership handles, and appendable storage. */
  destroy(): void {
    if (this.ownsConcreteBuffer && this.concreteBuffer) {
      this.concreteBuffer.destroy();
      this.ownsConcreteBuffer = false;
    }
    for (const vector of this.ownedVectors.splice(0)) {
      vector.destroy();
    }
    if (this.ownsDataChunks) {
      for (const data of this.data) {
        data.destroy();
      }
      this.ownsDataChunks = false;
    }
  }

  private ensureAppendableByteCapacity(requiredByteLength: number): void {
    if (!(this.concreteBuffer instanceof DynamicBuffer)) {
      throw new Error('GPUVector append capacity requires DynamicBuffer storage');
    }
    const capacityByteLength = this.concreteBuffer.byteLength;
    if (requiredByteLength <= capacityByteLength) {
      return;
    }
    const grownByteLength = Math.ceil(
      Math.max(capacityByteLength, 1) * (this.capacityGrowthFactor ?? 1.5)
    );
    const nextCapacityByteLength = Math.max(requiredByteLength, grownByteLength);
    this.concreteBuffer.resize({
      byteLength: nextCapacityByteLength,
      preserveData: this.appendableByteLength > 0,
      copyByteLength: this.appendableByteLength
    });
  }
}

function createGPUDataBuffer(buffer: Buffer | DynamicBuffer): DynamicBuffer {
  return buffer instanceof DynamicBuffer
    ? buffer
    : new DynamicBuffer(buffer.device, {
        buffer,
        ownsBuffer: false
      });
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {
6	Buffer,
7	Device,
8	type BufferAttributeLayout,
9	type BufferLayout,
10	type BufferProps
11	} from '@luma.gl/core';
12	import {DynamicBuffer, type DynamicBufferProps} from '@luma.gl/engine';
13	import * as arrow from 'apache-arrow';
14	import {GPUData} from './gpu-data';
15
16	/** Buffer creation props used by format-specific producers before wrapping storage in a GPUVector. */
17	export type GPUVectorBufferProps = Omit<BufferProps, 'byteLength' \| 'data'>;
18	/** Dynamic buffer props used when creating appendable GPU vectors. */
19	export type GPUVectorDynamicBufferProps = Omit<DynamicBufferProps, 'byteLength' \| 'data'>;
20	/** @deprecated Use {@link GPUVectorBufferProps}. */
21	export type GPUVectorProps = GPUVectorBufferProps;
22
23	/** Arrow upload props moved to `@luma.gl/arrow`; retained only as a typed migration sentinel. */
24	export type GPUVectorFromArrowProps<_T extends arrow.DataType = arrow.DataType> = never;
25
26	/** Constructor props that wrap an existing typed GPU buffer. */
27	export type GPUVectorFromBufferProps<T extends arrow.DataType = arrow.DataType> = {
28	/** Discriminator for existing-buffer construction. */
29	type: 'buffer';
30	/** Stable vector name. */
31	name: string;
32	/** Existing GPU buffer. */
33	buffer: Buffer \| DynamicBuffer;
34	/** Logical schema/type descriptor for values stored in the buffer. */
35	dataType: T;
36	/** Number of logical rows in the buffer. */
37	length: number;
38	/** Number of scalar values represented by one logical row. */
39	stride?: number;
40	/** Byte offset of the first logical row. */
41	byteOffset?: number;
42	/** Bytes between adjacent logical rows. */
43	byteStride: number;
44	/** Number of bytes occupied by one logical row payload. */
45	rowByteLength?: number;
46	/** Whether this vector should destroy the wrapped buffer. */
47	ownsBuffer?: boolean;
48	};
49
50	/** Constructor props that wrap one interleaved GPU buffer. */
51	export type GPUVectorFromInterleavedProps<T extends arrow.DataType = arrow.DataType> = {
52	/** Discriminator for interleaved-buffer construction. */
53	type: 'interleaved';
54	/** Stable vector name. */
55	name: string;
56	/** Existing interleaved GPU buffer. */
57	buffer: Buffer \| DynamicBuffer;
58	/** Logical schema/type descriptor for the interleaved rows. */
59	dataType: T;
60	/** Number of logical rows in the buffer. */
61	length: number;
62	/** Byte offset of the first logical row. */
63	byteOffset?: number;
64	/** Bytes between adjacent logical rows. */
65	byteStride: number;
66	/** Attribute views stored in each interleaved row. */
67	attributes: BufferAttributeLayout[];
68	/** Whether this vector should destroy the wrapped buffer. */
69	ownsBuffer?: boolean;
70	};
71
72	/** Constructor props that expose existing GPU data chunks as one logical vector. */
73	export type GPUVectorFromDataProps<T extends arrow.DataType = arrow.DataType> = {
74	/** Discriminator for chunk-backed construction. */
75	type: 'data';
76	/** Stable vector name. */
77	name: string;
78	/** Logical schema/type descriptor shared by every chunk. */
79	dataType: T;
80	/** Existing GPU data chunks to expose through this vector. */
81	data: GPUData<T>[];
82	/** Optional concrete aggregate buffer shared by the supplied data chunks. */
83	buffer?: Buffer \| DynamicBuffer;
84	/** Number of scalar values represented by one logical row. */
85	stride?: number;
86	/** Bytes between adjacent logical rows. Defaults to the first chunk stride when available. */
87	byteStride?: number;
88	/** Number of bytes occupied by one logical row payload. */
89	rowByteLength?: number;
90	/** Optional buffer layout retained for interleaved chunk collections. */
91	bufferLayout?: BufferLayout;
92	/** Whether this vector should destroy the supplied GPU data chunks. */
93	ownsData?: boolean;
94	/** Whether this vector should destroy the optional aggregate buffer. */
95	ownsBuffer?: boolean;
96	};
97
98	/** Constructor props for an appendable DynamicBuffer-backed vector. */
99	export type GPUVectorFromAppendableProps<T extends arrow.DataType = arrow.DataType> = {
100	/** Discriminator for appendable DynamicBuffer-backed construction. */
101	type: 'appendable';
102	/** Stable vector name. */
103	name: string;
104	/** Device that creates the DynamicBuffer. */
105	device: Device;
106	/** Logical schema/type descriptor for appended data. */
107	dataType: T;
108	/** Number of scalar values represented by one logical row. */
109	stride: number;
110	/** Bytes between adjacent logical rows. */
111	byteStride: number;
112	/** Number of bytes occupied by one logical row payload. */
113	rowByteLength?: number;
114	/** Initial row capacity. Defaults to `0`. */
115	initialCapacityRows?: number;
116	/** Capacity growth multiplier. Defaults to `1.5`. */
117	capacityGrowthFactor?: number;
118	/** DynamicBuffer construction props. */
119	bufferProps?: GPUVectorDynamicBufferProps;
120	};
121
122	/** Discriminated constructor props for {@link GPUVector}. */
123	export type GPUVectorCreateProps<T extends arrow.DataType = arrow.DataType> =
124	\| GPUVectorFromBufferProps<T>
125	\| GPUVectorFromInterleavedProps<T>
126	\| GPUVectorFromDataProps<T>
127	\| GPUVectorFromAppendableProps<T>;
128
129	/**
130	* GPU memory and logical type metadata for one vector-valued table column.
131	*
132	* Format-specific modules upload bytes and use these vectors to expose shared
133	* lifecycle, chunking, batching, and ownership semantics.
134	*/
135	export class GPUVector<T extends arrow.DataType = arrow.DataType> {
136	/** Stable vector name. */
137	readonly name: string;
138	/** Logical schema/type descriptor for the uploaded bytes. */
139	readonly type: T;
140	/** Number of logical rows represented by the vector. */
141	length: number;
142	/** Number of scalar values represented by one logical row. */
143	readonly stride: number;
144	/** Byte offset of the first logical row in {@link buffer}. */
145	readonly byteOffset: number;
146	/** Bytes between adjacent logical rows in {@link buffer}. */
147	readonly byteStride: number;
148	/** Bytes occupied by one logical row payload. */
149	readonly rowByteLength: number;
150	/** Optional GPU buffer layout described by this vector. */
151	readonly bufferLayout?: BufferLayout;
152	/** GPU data chunk views preserved across table/batch aggregation. */
153	readonly data: GPUData<T>[] = [];	269✔
154	private concreteBuffer?: Buffer \| DynamicBuffer;
155	private ownsConcreteBuffer: boolean;
156	private ownsDataChunks = false;	269✔
157	private readonly ownedVectors: GPUVector[] = [];	269✔
158	private readonly capacityGrowthFactor?: number;
159	private appendableByteLength = 0;	269✔
160
161	constructor(props: GPUVectorCreateProps<T>) {
162	switch (props.type) {	269✔
163	case 'buffer': {
164	const {
165	name,
166	buffer,
167	dataType,
168	length,
169	stride = 1,	18✔
170	byteOffset = 0,	18✔
171	byteStride,
172	rowByteLength = byteStride,	18✔
173	ownsBuffer = false	18✔
174	} = props;	18✔
175	this.name = name;	18✔
176	this.type = dataType;	18✔
177	this.length = length;	18✔
178	this.stride = stride;	18✔
179	this.byteOffset = byteOffset;	18✔
180	this.byteStride = byteStride;	18✔
181	this.rowByteLength = rowByteLength;	18✔
182	this.concreteBuffer = buffer;	18✔
183	this.ownsConcreteBuffer = ownsBuffer;	18✔
184	this.data.push(	18✔
185	new GPUData({
186	buffer: createGPUDataBuffer(buffer),
187	dataType,
188	length,
189	stride,
190	byteOffset,
191	byteStride,
192	rowByteLength,
193	ownsBuffer: false
194	}) as GPUData<T>
195	);
196	return;	18✔
197	}
198
199	case 'interleaved': {
200	const {
201	name,
202	buffer,
203	dataType,
204	length,
205	byteOffset = 0,	9✔
206	byteStride,
207	attributes,
208	ownsBuffer = false	9✔
209	} = props;	9✔
210	this.name = name;	9✔
211	this.type = dataType;	9✔
212	this.length = length;	9✔
213	this.stride = byteStride;	9✔
214	this.byteOffset = byteOffset;	9✔
215	this.byteStride = byteStride;	9✔
216	this.rowByteLength = byteStride;	9✔
217	this.bufferLayout = {name, byteStride, attributes};	9✔
218	this.concreteBuffer = buffer;	9✔
219	this.ownsConcreteBuffer = ownsBuffer;	9✔
220	this.data.push(	9✔
221	new GPUData({
222	buffer: createGPUDataBuffer(buffer),
223	dataType,
224	length,
225	stride: byteStride,
226	byteOffset,
227	byteStride,
228	rowByteLength: byteStride,
229	ownsBuffer: false
230	}) as GPUData<T>
231	);
232	return;	9✔
233	}
234
235	case 'data': {
236	const {
237	name,
238	dataType,
239	data,
240	buffer,
241	stride = data[0]?.stride ?? 1,	466!
242	byteStride = data[0]?.byteStride ?? 0,	466!
243	rowByteLength = data[0]?.rowByteLength ?? byteStride,	466!
244	bufferLayout,
245	ownsData = false,	233✔
246	ownsBuffer = false	233✔
247	} = props;	233✔
248	if (data.some(chunk => !arrow.util.compareTypes(chunk.type, dataType))) {	237!
NEW 249	throw new Error('GPUVector data chunks must share the declared logical type');	×
250	}
251	this.name = name;	233✔
252	this.type = dataType;	233✔
253	this.length = data.reduce((totalLength, chunk) => totalLength + chunk.length, 0);	237✔
254	this.stride = stride;	233✔
255	this.byteOffset = data.length === 1 ? data[0].byteOffset : 0;	233✔
256	this.byteStride = byteStride;	233✔
257	this.rowByteLength = rowByteLength;	233✔
258	this.bufferLayout = bufferLayout;	233✔
259	this.data.push(...data);	233✔
260	this.concreteBuffer = buffer ?? (data.length === 1 ? data[0].buffer : undefined);	233✔
261	this.ownsConcreteBuffer = ownsBuffer;	233✔
262	this.ownsDataChunks = ownsData;	233✔
263	return;	233✔
264	}
265
266	case 'appendable': {
267	const {
268	name,
269	device,
270	dataType,
271	stride,
272	byteStride,
273	rowByteLength = byteStride,	9✔
274	initialCapacityRows = 0,	9✔
275	capacityGrowthFactor = 1.5,	9✔
276	bufferProps
277	} = props;	9✔
278	this.name = name;	9✔
279	this.type = dataType;	9✔
280	this.length = 0;	9✔
281	this.stride = stride;	9✔
282	this.byteOffset = 0;	9✔
283	this.byteStride = byteStride;	9✔
284	this.rowByteLength = rowByteLength;	9✔
285	this.concreteBuffer = new DynamicBuffer(device, {	9✔
286	usage: Buffer.VERTEX \| Buffer.STORAGE \| Buffer.COPY_DST \| Buffer.COPY_SRC,
287	...bufferProps,
288	id: bufferProps?.id ?? `${name}-appendable-gpu-vector`,	18✔
289	byteLength: Math.max(1, initialCapacityRows * byteStride)
290	});
291	this.capacityGrowthFactor = capacityGrowthFactor;	9✔
292	this.ownsConcreteBuffer = true;	9✔
293	return;	9✔
294	}
295	}
296	}
297
298	/** Directly bindable GPU buffer when this vector has one concrete backing buffer. */
299	get buffer(): Buffer \| DynamicBuffer {
300	if (!this.concreteBuffer) {	266✔
301	throw new Error('GPUVector.buffer is unavailable for multi-buffer vectors; use data[]');	2✔
302	}
303	return this.concreteBuffer;	264✔
304	}
305
306	/** Whether destroying this vector releases any retained GPU storage. */
307	get ownsBuffer(): boolean {
308	return this.ownsConcreteBuffer \|\| this.ownedVectors.some(vector => vector.ownsBuffer);	5✔
309	}
310
311	/** Number of rows the appendable backing DynamicBuffer can hold without reallocating. */
312	get capacityRows(): number \| undefined {
313	return this.concreteBuffer instanceof DynamicBuffer	1!
314	? Math.floor(this.concreteBuffer.byteLength / this.byteStride)
315	: undefined;
316	}
317
318	/** Bytes occupied by already-appended payloads in appendable storage. */
319	get appendedByteLength(): number {
320	return this.appendableByteLength;	16✔
321	}
322
323	/** Adds one already-materialized GPU data chunk to this logical vector. */
324	addData(data: GPUData<T>): this {
325	if (!arrow.util.compareTypes(data.type, this.type)) {	31!
NEW 326	throw new Error('GPUVector.addData() requires matching logical types');	×
327	}
328	if (data.byteStride !== this.byteStride) {	31!
NEW 329	throw new Error('GPUVector.addData() requires matching byteStride');	×
330	}
331	if (data.rowByteLength !== this.rowByteLength) {	31!
NEW 332	throw new Error('GPUVector.addData() requires matching rowByteLength');	×
333	}
334
335	this.data.push(data);	31✔
336	this.length += data.length;	31✔
337	if (this.data.length > 1) {	31!
338	this.concreteBuffer = undefined;	31✔
339	}
340	return this;	31✔
341	}
342
343	/**
344	* Reserves appendable storage and writes raw bytes supplied by a format-specific adapter.
345	*/
346	writeAppendableBytes(
347	data: ArrayBufferView,
348	byteOffset: number,
349	requiredByteLength: number
350	): void {
351	if (!(this.concreteBuffer instanceof DynamicBuffer)) {	16!
NEW 352	throw new Error('GPUVector append writes require appendable DynamicBuffer storage');	×
353	}
354	this.ensureAppendableByteCapacity(requiredByteLength);	16✔
355	if (data.byteLength > 0) {	16!
356	this.concreteBuffer.write(data, byteOffset);	16✔
357	}
358	}
359
360	/**
361	* Adds one appendable data view while preserving the concrete backing buffer.
362	*/
363	appendDataChunk(data: GPUData<T>, appendedByteLength: number): this {
364	if (!(this.concreteBuffer instanceof DynamicBuffer)) {	16!
NEW 365	throw new Error('GPUVector append views require appendable DynamicBuffer storage');	×
366	}
367	if (!arrow.util.compareTypes(data.type, this.type)) {	16!
NEW 368	throw new Error('GPUVector.appendDataChunk() requires matching logical types');	×
369	}
370	if (data.byteStride !== this.byteStride \|\| data.rowByteLength !== this.rowByteLength) {	16!
NEW 371	throw new Error('GPUVector.appendDataChunk() requires matching byte layout metadata');	×
372	}
373	this.data.push(data);	16✔
374	this.length += data.length;	16✔
375	this.appendableByteLength = appendedByteLength;	16✔
376	return this;	16✔
377	}
378
379	/** Clears appendable logical rows while retaining the DynamicBuffer allocation. */
380	resetLastBatch(): this {
381	if (!(this.concreteBuffer instanceof DynamicBuffer)) {	3!
NEW 382	throw new Error('GPUVector.resetLastBatch() requires appendable vector storage');	×
383	}
384	this.length = 0;	3✔
385	this.appendableByteLength = 0;	3✔
386	this.data.length = 0;	3✔
387	return this;	3✔
388	}
389
390	/** @internal Retains detached batch-local vector ownership under this aggregate vector. */
391	retainOwnedVectors(vectors: GPUVector[]): this {
392	this.ownedVectors.push(...vectors);	1✔
393	return this;	1✔
394	}
395
396	/** Transfers same-buffer ownership to another vector view. */
397	transferBufferOwnership(target: GPUVector): void {
398	if (	1!
399	!this.concreteBuffer \|\|	3✔
400	!target.concreteBuffer \|\|
401	target.concreteBuffer !== this.concreteBuffer
402	) {
NEW 403	throw new Error('GPUVector ownership can only be transferred to the same buffer');	×
404	}
405	target.ownsConcreteBuffer = this.ownsConcreteBuffer;	1✔
406	this.ownsConcreteBuffer = false;	1✔
407	}
408
409	/** Releases owned GPU data, detached ownership handles, and appendable storage. */
410	destroy(): void {
411	if (this.ownsConcreteBuffer && this.concreteBuffer) {	246✔
412	this.concreteBuffer.destroy();	200✔
413	this.ownsConcreteBuffer = false;	200✔
414	}
415	for (const vector of this.ownedVectors.splice(0)) {	246✔
416	vector.destroy();	2✔
417	}
418	if (this.ownsDataChunks) {	246✔
419	for (const data of this.data) {	39✔
420	data.destroy();	42✔
421	}
422	this.ownsDataChunks = false;	39✔
423	}
424	}
425
426	private ensureAppendableByteCapacity(requiredByteLength: number): void {
427	if (!(this.concreteBuffer instanceof DynamicBuffer)) {	16!
NEW 428	throw new Error('GPUVector append capacity requires DynamicBuffer storage');	×
429	}
430	const capacityByteLength = this.concreteBuffer.byteLength;	16✔
431	if (requiredByteLength <= capacityByteLength) {	16!
NEW 432	return;	×
433	}
434	const grownByteLength = Math.ceil(	16✔
435	Math.max(capacityByteLength, 1) * (this.capacityGrowthFactor ?? 1.5)	16!
436	);
437	const nextCapacityByteLength = Math.max(requiredByteLength, grownByteLength);	16✔
438	this.concreteBuffer.resize({	16✔
439	byteLength: nextCapacityByteLength,
440	preserveData: this.appendableByteLength > 0,
441	copyByteLength: this.appendableByteLength
442	});
443	}
444	}
445
446	function createGPUDataBuffer(buffer: Buffer \| DynamicBuffer): DynamicBuffer {
447	return buffer instanceof DynamicBuffer	27!
448	? buffer
449	: new DynamicBuffer(buffer.device, {
450	buffer,
451	ownsBuffer: false
452	});
453	}

visgl / luma.gl / 26120831014

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous