25756190722

Committed 12 May 2026 07:06PM UTC coverage: 75.119% (+0.2%) from 74.932%

Build # 25756190722

Build Type

push

github

Committed by

web-flow

Commit Message

feat(arrow) Support RecordBatch stream to ArrowGPUTable (#2611)

Coverage Stats

5973 of 8932 branches covered (66.87%)

Branch coverage included in aggregate %.

271 of 333 new or added lines in 9 files covered. (81.38%)

3 existing lines in 2 files now uncovered.

13032 of 16368 relevant lines covered (79.62%)

831.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.5

/modules/engine/src/material/material.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Binding, BindingsByGroup, CommandEncoder, Device} from '@luma.gl/core';
import {Buffer, Texture, TextureView, UniformStore} from '@luma.gl/core';
import type {ShaderModule} from '@luma.gl/shadertools';
import {
  DynamicBuffer,
  type DynamicBufferRange,
  getDynamicBufferFromBinding,
  isBufferRangeBinding,
  resolveBufferRangeBinding
} from '../dynamic-buffer/dynamic-buffer';
import {DynamicTexture} from '../dynamic-texture/dynamic-texture';
import {ShaderInputs} from '../shader-inputs';
import {shaderModuleHasUniforms} from '../utils/shader-module-utils';
import {uid} from '../utils/uid';
import {
  getModuleNameFromUniformBinding,
  MATERIAL_BIND_GROUP,
  MaterialFactory
} from './material-factory';

type MaterialModuleProps = Partial<Record<string, Record<string, unknown>>>;
type MaterialBinding = Binding | DynamicTexture | DynamicBuffer | DynamicBufferRange;
type MaterialBindings = Record<string, MaterialBinding>;
type MaterialPropsUpdate<TModuleProps extends MaterialModuleProps> = Partial<{
  [P in keyof TModuleProps]?: Partial<TModuleProps[P]>;
}>;

/** Construction props for one typed {@link Material}. */
export type MaterialProps<
  TModuleProps extends MaterialModuleProps = MaterialModuleProps,
  TBindings extends MaterialBindings = MaterialBindings
> = {
  /** Optional application-provided identifier. */
  id?: string;
  /** Factory that owns the material schema. */
  factory?: MaterialFactory<TModuleProps, TBindings>;
  /** Optional pre-created shader inputs for the material modules. */
  shaderInputs?: ShaderInputs<TModuleProps>;
  /** Shader modules used when a factory is not supplied. */
  modules?: ShaderModule[];
  /** Initial material-owned resource bindings. */
  bindings?: Partial<TBindings>;
};

/** Structural overrides applied when cloning a {@link Material}. */
export type MaterialCloneProps<
  TModuleProps extends MaterialModuleProps = MaterialModuleProps,
  TBindings extends MaterialBindings = MaterialBindings
> = {
  /** Optional identifier for the cloned material. */
  id?: string;
  /** Replacement material-owned resource bindings. */
  bindings?: Partial<TBindings>;
  /** Additional uniform/module props applied to the clone. */
  moduleProps?: MaterialPropsUpdate<TModuleProps>;
  /** Optional full replacement shader-input store. */
  shaderInputs?: ShaderInputs<TModuleProps>;
};

/**
 * Material owns bind group `3` resources and uniforms for one material instance.
 *
 * `setProps()` mutates uniform values in place. Structural resource changes are
 * expressed through `clone({...})`, which creates a new material identity.
 */
export class Material<
  TModuleProps extends MaterialModuleProps = MaterialModuleProps,
  TBindings extends MaterialBindings = MaterialBindings
> {
  /** Application-provided identifier. */
  readonly id: string;
  /** Device that owns the material resources. */
  readonly device: Device;
  /** Factory that defines the material schema. */
  readonly factory: MaterialFactory<TModuleProps, TBindings>;
  /** Shader inputs for the material-owned modules. */
  readonly shaderInputs: ShaderInputs<TModuleProps>;
  /** Internal binding store including uniform buffers and resource bindings. */
  readonly bindings: Record<string, MaterialBinding> = {};

  private _uniformStore: UniformStore;
  private _bindGroupCacheToken: object = {};
  private _dynamicResourceGenerations: Record<string, number> = {};

  constructor(device: Device, props: MaterialProps<TModuleProps, TBindings> = {}) {
    this.id = props.id || uid('material');
    this.device = device;

    this.factory =
      props.factory ||
      new MaterialFactory<TModuleProps, TBindings>(device, {
        modules: props.modules || props.shaderInputs?.getModules() || []
      });

    const moduleMap = Object.fromEntries(
      (props.shaderInputs?.getModules() || this.factory.modules).map(module => [
        module.name,
        module
      ])
    ) as {[P in keyof TModuleProps]?: ShaderModule[] extends never ? never : any};
    this.shaderInputs = props.shaderInputs || new ShaderInputs<TModuleProps>(moduleMap);
    this._uniformStore = new UniformStore(this.device, this.shaderInputs.modules);

    for (const [moduleName, module] of Object.entries(this.shaderInputs.modules)) {
      if (this.ownsModule(moduleName) && shaderModuleHasUniforms(module)) {
        const uniformBuffer = this._uniformStore.getManagedUniformBuffer(moduleName);
        this.bindings[`${moduleName}Uniforms`] = uniformBuffer;
      }
    }

    this.updateShaderInputs();
    if (props.bindings) {
      this._replaceOwnedBindings(props.bindings);
    }
  }

  /** Destroys managed uniform-buffer resources owned by this material. */
  destroy(): void {
    this._uniformStore.destroy();
  }

  /** Creates a new material variant with optional structural and uniform overrides. */
  clone(
    props: MaterialCloneProps<TModuleProps, TBindings> = {}
  ): Material<TModuleProps, TBindings> {
    const material = this.factory.createMaterial({
      id: props.id,
      shaderInputs: props.shaderInputs,
      bindings: {
        ...this.getResourceBindings(),
        ...props.bindings
      }
    });

    if (!props.shaderInputs) {
      material.setProps(this.shaderInputs.getUniformValues() as MaterialPropsUpdate<TModuleProps>);
    }
    if (props.moduleProps) {
      material.setProps(props.moduleProps);
    }
    material.updateShaderInputs();
    return material;
  }

  /** Returns `true` if this material owns the supplied binding name. */
  ownsBinding(bindingName: string): boolean {
    return this.factory.ownsBinding(bindingName);
  }

  /** Returns `true` if this material owns the supplied shader module. */
  ownsModule(moduleName: string): boolean {
    return this.factory.ownsModule(moduleName);
  }

  /** Updates material uniform/module props in place without changing material identity. */
  setProps(props: MaterialPropsUpdate<TModuleProps>): void {
    this.shaderInputs.setProps(props);
  }

  /**
   * Updates managed uniform buffers and shader-input-owned bindings.
   *
   * @param commandEncoder - Optional encoder used to order material uniform
   * uploads with subsequent draw commands.
   */
  updateShaderInputs(commandEncoder?: CommandEncoder): void {
    this._uniformStore.setUniforms(this.shaderInputs.getUniformValues(), commandEncoder);
    const didChange = this._setOwnedBindings(this.shaderInputs.getBindingValues());
    if (didChange) {
      this._bindGroupCacheToken = {};
    }
  }

  /** Returns the material-owned resource bindings without internal uniform buffers. */
  getResourceBindings(): Partial<TBindings> {
    const resourceBindings = {} as Partial<TBindings>;

    for (const [name, binding] of Object.entries(this.bindings)) {
      if (!getModuleNameFromUniformBinding(name)) {
        (resourceBindings as Record<string, MaterialBinding>)[name] = binding;
      }
    }

    return resourceBindings;
  }

  /** Returns the resolved bindings, including internal uniform buffers and ready textures. */
  getBindings(): Partial<{[K in keyof TBindings]: Binding}> & Record<string, Binding> {
    this._syncDynamicResourceCacheToken();

    const validBindings = {} as Partial<{[K in keyof TBindings]: Binding}> &
      Record<string, Binding>;
    const validBindingsMap = validBindings as Record<string, Binding>;

    for (const [name, binding] of Object.entries(this.bindings)) {
      if (binding instanceof DynamicTexture) {
        if (binding.isReady) {
          validBindingsMap[name] = binding.texture;
        }
      } else if (binding instanceof DynamicBuffer) {
        validBindingsMap[name] = binding.buffer;
      } else if (isBufferRangeBinding(binding)) {
        validBindingsMap[name] = resolveBufferRangeBinding(binding);
      } else {
        validBindingsMap[name] = binding;
      }
    }

    return validBindings;
  }

  /** Packages resolved material bindings into logical bind group `3`. */
  getBindingsByGroup(): BindingsByGroup {
    return this.factory.getBindingsByGroup(this.getBindings());
  }

  /** Returns the stable bind-group cache token for the requested bind group. */
  getBindGroupCacheKey(group: number): object | null {
    this._syncDynamicResourceCacheToken();
    return group === MATERIAL_BIND_GROUP ? this._bindGroupCacheToken : null;
  }

  /** Returns the latest update timestamp across material-owned resources. */
  getBindingsUpdateTimestamp(): number {
    let timestamp = 0;
    for (const binding of Object.values(this.bindings)) {
      if (binding instanceof TextureView) {
        timestamp = Math.max(timestamp, binding.texture.updateTimestamp);
      } else if (binding instanceof Buffer || binding instanceof Texture) {
        timestamp = Math.max(timestamp, binding.updateTimestamp);
      } else if (binding instanceof DynamicBuffer) {
        timestamp = Math.max(timestamp, binding.updateTimestamp);
      } else if (binding instanceof DynamicTexture) {
        timestamp = binding.isReady ? Math.max(timestamp, binding.updateTimestamp) : Infinity;
      } else if (isBufferRangeBinding(binding)) {
        timestamp = Math.max(
          timestamp,
          binding.buffer instanceof DynamicBuffer
            ? binding.buffer.updateTimestamp
            : binding.buffer.updateTimestamp
        );
      }
    }
    return timestamp;
  }

  /** Replaces owned resource bindings and invalidates the material cache identity when needed. */
  private _replaceOwnedBindings(bindings: Partial<TBindings>): void {
    const didChange = this._setOwnedBindings(bindings);
    if (didChange) {
      this._bindGroupCacheToken = {};
    }
  }

  private _setOwnedBindings(
    bindings: Partial<TBindings> | Record<string, MaterialBinding>
  ): boolean {
    let didChange = false;

    for (const [name, binding] of Object.entries(bindings)) {
      if (binding === undefined) {
        continue;
      }
      if (!this.ownsBinding(name)) {
        continue;
      }

      if (this.bindings[name] !== binding) {
        this.bindings[name] = binding;
        didChange = true;
      }
    }

    return didChange;
  }

  private _syncDynamicResourceCacheToken(): void {
    const nextGenerations: Record<string, number> = {};
    let didChange = false;

    for (const [name, binding] of Object.entries(this.bindings)) {
      const dynamicResourceGeneration = getDynamicResourceGeneration(binding);
      if (dynamicResourceGeneration !== null) {
        nextGenerations[name] = dynamicResourceGeneration;
        if (this._dynamicResourceGenerations[name] !== dynamicResourceGeneration) {
          didChange = true;
        }
      }
    }

    if (
      Object.keys(nextGenerations).length !== Object.keys(this._dynamicResourceGenerations).length
    ) {
      didChange = true;
    }

    this._dynamicResourceGenerations = nextGenerations;
    if (didChange) {
      this._bindGroupCacheToken = {};
    }
  }
}

function getDynamicResourceGeneration(binding: MaterialBinding): number | null {
  if (binding instanceof DynamicTexture) {
    return binding.generation;
  }

  return getDynamicBufferFromBinding(binding)?.generation ?? null;
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {Binding, BindingsByGroup, CommandEncoder, Device} from '@luma.gl/core';
6	import {Buffer, Texture, TextureView, UniformStore} from '@luma.gl/core';
7	import type {ShaderModule} from '@luma.gl/shadertools';
8	import {
9	DynamicBuffer,
10	type DynamicBufferRange,
11	getDynamicBufferFromBinding,
12	isBufferRangeBinding,
13	resolveBufferRangeBinding
14	} from '../dynamic-buffer/dynamic-buffer';
15	import {DynamicTexture} from '../dynamic-texture/dynamic-texture';
16	import {ShaderInputs} from '../shader-inputs';
17	import {shaderModuleHasUniforms} from '../utils/shader-module-utils';
18	import {uid} from '../utils/uid';
19	import {
20	getModuleNameFromUniformBinding,
21	MATERIAL_BIND_GROUP,
22	MaterialFactory
23	} from './material-factory';
24
25	type MaterialModuleProps = Partial<Record<string, Record<string, unknown>>>;
26	type MaterialBinding = Binding \| DynamicTexture \| DynamicBuffer \| DynamicBufferRange;
27	type MaterialBindings = Record<string, MaterialBinding>;
28	type MaterialPropsUpdate<TModuleProps extends MaterialModuleProps> = Partial<{
29	[P in keyof TModuleProps]?: Partial<TModuleProps[P]>;
30	}>;
31
32	/** Construction props for one typed {@link Material}. */
33	export type MaterialProps<
34	TModuleProps extends MaterialModuleProps = MaterialModuleProps,
35	TBindings extends MaterialBindings = MaterialBindings
36	> = {
37	/** Optional application-provided identifier. */
38	id?: string;
39	/** Factory that owns the material schema. */
40	factory?: MaterialFactory<TModuleProps, TBindings>;
41	/** Optional pre-created shader inputs for the material modules. */
42	shaderInputs?: ShaderInputs<TModuleProps>;
43	/** Shader modules used when a factory is not supplied. */
44	modules?: ShaderModule[];
45	/** Initial material-owned resource bindings. */
46	bindings?: Partial<TBindings>;
47	};
48
49	/** Structural overrides applied when cloning a {@link Material}. */
50	export type MaterialCloneProps<
51	TModuleProps extends MaterialModuleProps = MaterialModuleProps,
52	TBindings extends MaterialBindings = MaterialBindings
53	> = {
54	/** Optional identifier for the cloned material. */
55	id?: string;
56	/** Replacement material-owned resource bindings. */
57	bindings?: Partial<TBindings>;
58	/** Additional uniform/module props applied to the clone. */
59	moduleProps?: MaterialPropsUpdate<TModuleProps>;
60	/** Optional full replacement shader-input store. */
61	shaderInputs?: ShaderInputs<TModuleProps>;
62	};
63
64	/**
65	* Material owns bind group `3` resources and uniforms for one material instance.
66	*
67	* `setProps()` mutates uniform values in place. Structural resource changes are
68	* expressed through `clone({...})`, which creates a new material identity.
69	*/
70	export class Material<
71	TModuleProps extends MaterialModuleProps = MaterialModuleProps,
72	TBindings extends MaterialBindings = MaterialBindings
73	> {
74	/** Application-provided identifier. */
75	readonly id: string;
76	/** Device that owns the material resources. */
77	readonly device: Device;
78	/** Factory that defines the material schema. */
79	readonly factory: MaterialFactory<TModuleProps, TBindings>;
80	/** Shader inputs for the material-owned modules. */
81	readonly shaderInputs: ShaderInputs<TModuleProps>;
82	/** Internal binding store including uniform buffers and resource bindings. */
83	readonly bindings: Record<string, MaterialBinding> = {};	16✔
84
85	private _uniformStore: UniformStore;
86	private _bindGroupCacheToken: object = {};	16✔
87	private _dynamicResourceGenerations: Record<string, number> = {};	16✔
88
89	constructor(device: Device, props: MaterialProps<TModuleProps, TBindings> = {}) {	16✔
90	this.id = props.id \|\| uid('material');	16✔
91	this.device = device;	16✔
92
93	this.factory =	16✔
94	props.factory \|\|	16!
95	new MaterialFactory<TModuleProps, TBindings>(device, {
96	modules: props.modules \|\| props.shaderInputs?.getModules() \|\| []	×
97	});
98
99	const moduleMap = Object.fromEntries(	16✔
100	(props.shaderInputs?.getModules() \|\| this.factory.modules).map(module => [	16✔
101	module.name,
102	module
103	])
104	) as {[P in keyof TModuleProps]?: ShaderModule[] extends never ? never : any};
105	this.shaderInputs = props.shaderInputs \|\| new ShaderInputs<TModuleProps>(moduleMap);	16✔
106	this._uniformStore = new UniformStore(this.device, this.shaderInputs.modules);	16✔
107
108	for (const [moduleName, module] of Object.entries(this.shaderInputs.modules)) {	16✔
109	if (this.ownsModule(moduleName) && shaderModuleHasUniforms(module)) {	53✔
110	const uniformBuffer = this._uniformStore.getManagedUniformBuffer(moduleName);	14✔
111	this.bindings[`${moduleName}Uniforms`] = uniformBuffer;	14✔
112	}
113	}
114
115	this.updateShaderInputs();	16✔
116	if (props.bindings) {	16✔
117	this._replaceOwnedBindings(props.bindings);	14✔
118	}
119	}
120
121	/** Destroys managed uniform-buffer resources owned by this material. */
122	destroy(): void {
123	this._uniformStore.destroy();	6✔
124	}
125
126	/** Creates a new material variant with optional structural and uniform overrides. */
127	clone(
128	props: MaterialCloneProps<TModuleProps, TBindings> = {}	×
129	): Material<TModuleProps, TBindings> {
130	const material = this.factory.createMaterial({	×
131	id: props.id,
132	shaderInputs: props.shaderInputs,
133	bindings: {
134	...this.getResourceBindings(),
135	...props.bindings
136	}
137	});
138
139	if (!props.shaderInputs) {	×
140	material.setProps(this.shaderInputs.getUniformValues() as MaterialPropsUpdate<TModuleProps>);	×
141	}
142	if (props.moduleProps) {	×
143	material.setProps(props.moduleProps);	×
144	}
145	material.updateShaderInputs();	×
146	return material;	×
147	}
148
149	/** Returns `true` if this material owns the supplied binding name. */
150	ownsBinding(bindingName: string): boolean {
151	return this.factory.ownsBinding(bindingName);	3✔
152	}
153
154	/** Returns `true` if this material owns the supplied shader module. */
155	ownsModule(moduleName: string): boolean {
156	return this.factory.ownsModule(moduleName);	149✔
157	}
158
159	/** Updates material uniform/module props in place without changing material identity. */
160	setProps(props: MaterialPropsUpdate<TModuleProps>): void {
161	this.shaderInputs.setProps(props);	16✔
162	}
163
164	/**
165	* Updates managed uniform buffers and shader-input-owned bindings.
166	*
167	* @param commandEncoder - Optional encoder used to order material uniform
168	* uploads with subsequent draw commands.
169	*/
170	updateShaderInputs(commandEncoder?: CommandEncoder): void {
171	this._uniformStore.setUniforms(this.shaderInputs.getUniformValues(), commandEncoder);	16✔
172	const didChange = this._setOwnedBindings(this.shaderInputs.getBindingValues());	16✔
173	if (didChange) {	16!
174	this._bindGroupCacheToken = {};	×
175	}
176	}
177
178	/** Returns the material-owned resource bindings without internal uniform buffers. */
179	getResourceBindings(): Partial<TBindings> {
180	const resourceBindings = {} as Partial<TBindings>;	×
181
182	for (const [name, binding] of Object.entries(this.bindings)) {	×
183	if (!getModuleNameFromUniformBinding(name)) {	×
184	(resourceBindings as Record<string, MaterialBinding>)[name] = binding;	×
185	}
186	}
187
188	return resourceBindings;	×
189	}
190
191	/** Returns the resolved bindings, including internal uniform buffers and ready textures. */
192	getBindings(): Partial<{[K in keyof TBindings]: Binding}> & Record<string, Binding> {
193	this._syncDynamicResourceCacheToken();	19✔
194
195	const validBindings = {} as Partial<{[K in keyof TBindings]: Binding}> &	19✔
196	Record<string, Binding>;
197	const validBindingsMap = validBindings as Record<string, Binding>;	19✔
198
199	for (const [name, binding] of Object.entries(this.bindings)) {	19✔
200	if (binding instanceof DynamicTexture) {	21✔
201	if (binding.isReady) {	3!
202	validBindingsMap[name] = binding.texture;	3✔
203	}
204	} else if (binding instanceof DynamicBuffer) {	18✔
205	validBindingsMap[name] = binding.buffer;	3✔
206	} else if (isBufferRangeBinding(binding)) {	15!
207	validBindingsMap[name] = resolveBufferRangeBinding(binding);	×
208	} else {
209	validBindingsMap[name] = binding;	15✔
210	}
211	}
212
213	return validBindings;	19✔
214	}
215
216	/** Packages resolved material bindings into logical bind group `3`. */
217	getBindingsByGroup(): BindingsByGroup {
218	return this.factory.getBindingsByGroup(this.getBindings());	10✔
219	}
220
221	/** Returns the stable bind-group cache token for the requested bind group. */
222	getBindGroupCacheKey(group: number): object \| null {
223	this._syncDynamicResourceCacheToken();	4✔
224	return group === MATERIAL_BIND_GROUP ? this._bindGroupCacheToken : null;	4!
225	}
226
227	/** Returns the latest update timestamp across material-owned resources. */
228	getBindingsUpdateTimestamp(): number {
229	let timestamp = 0;	×
230	for (const binding of Object.values(this.bindings)) {	×
231	if (binding instanceof TextureView) {	×
232	timestamp = Math.max(timestamp, binding.texture.updateTimestamp);	×
233	} else if (binding instanceof Buffer \|\| binding instanceof Texture) {	×
234	timestamp = Math.max(timestamp, binding.updateTimestamp);	×
235	} else if (binding instanceof DynamicBuffer) {	×
236	timestamp = Math.max(timestamp, binding.updateTimestamp);	×
237	} else if (binding instanceof DynamicTexture) {	×
NEW 238	timestamp = binding.isReady ? Math.max(timestamp, binding.updateTimestamp) : Infinity;	×
UNCOV 239	} else if (isBufferRangeBinding(binding)) {	×
240	timestamp = Math.max(	×
241	timestamp,
242	binding.buffer instanceof DynamicBuffer	×
243	? binding.buffer.updateTimestamp
244	: binding.buffer.updateTimestamp
245	);
246	}
247	}
248	return timestamp;	×
249	}
250
251	/** Replaces owned resource bindings and invalidates the material cache identity when needed. */
252	private _replaceOwnedBindings(bindings: Partial<TBindings>): void {
253	const didChange = this._setOwnedBindings(bindings);	14✔
254	if (didChange) {	14✔
255	this._bindGroupCacheToken = {};	3✔
256	}
257	}
258
259	private _setOwnedBindings(
260	bindings: Partial<TBindings> \| Record<string, MaterialBinding>
261	): boolean {
262	let didChange = false;	30✔
263
264	for (const [name, binding] of Object.entries(bindings)) {	30✔
265	if (binding === undefined) {	3!
266	continue;	×
267	}
268	if (!this.ownsBinding(name)) {	3!
269	continue;	×
270	}
271
272	if (this.bindings[name] !== binding) {	3!
273	this.bindings[name] = binding;	3✔
274	didChange = true;	3✔
275	}
276	}
277
278	return didChange;	30✔
279	}
280
281	private _syncDynamicResourceCacheToken(): void {
282	const nextGenerations: Record<string, number> = {};	23✔
283	let didChange = false;	23✔
284
285	for (const [name, binding] of Object.entries(this.bindings)) {	23✔
286	const dynamicResourceGeneration = getDynamicResourceGeneration(binding);	25✔
287	if (dynamicResourceGeneration !== null) {	25✔
288	nextGenerations[name] = dynamicResourceGeneration;	10✔
289	if (this._dynamicResourceGenerations[name] !== dynamicResourceGeneration) {	10✔
290	didChange = true;	4✔
291	}
292	}
293	}
294
295	if (	23✔
296	Object.keys(nextGenerations).length !== Object.keys(this._dynamicResourceGenerations).length
297	) {
298	didChange = true;	2✔
299	}
300
301	this._dynamicResourceGenerations = nextGenerations;	23✔
302	if (didChange) {	23✔
303	this._bindGroupCacheToken = {};	4✔
304	}
305	}
306	}
307
308	function getDynamicResourceGeneration(binding: MaterialBinding): number \| null {
309	if (binding instanceof DynamicTexture) {	25✔
310	return binding.generation;	5✔
311	}
312
313	return getDynamicBufferFromBinding(binding)?.generation ?? null;	20✔
314	}

visgl / luma.gl / 25756190722

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous