25256585712

Committed 02 May 2026 04:35PM UTC coverage: 59.717% (-0.06%) from 59.776%

Build # 25256585712

Build Type

push

github

Committed by

web-flow

Commit Message

chore(loader-utils): Consolidate `parseWithWorker` with `processOnWorker` (#1564)

Coverage Stats

12514 of 23182 branches covered (53.98%)

Branch coverage included in aggregate %.

497 of 804 new or added lines in 22 files covered. (61.82%)

25 existing lines in 4 files now uncovered.

25948 of 41225 relevant lines covered (62.94%)

14803.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/modules/core/src/lib/loader-utils/get-data.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  DataType,
  SyncDataType,
  BatchableDataType,
  Loader,
  LoaderOptions
} from '@loaders.gl/loader-utils';
import {
  concatenateArrayBuffersAsync,
  isPromise,
  isResponse,
  isReadableStream,
  isAsyncIterable,
  isIterable,
  isIterator,
  isBlob,
  isBuffer,
  isArrayBufferLike,
  toArrayBuffer,
  toArrayBufferView
} from '@loaders.gl/loader-utils';
import {makeIterator} from '../../iterators/make-iterator/make-iterator';
import {checkResponse, makeResponse} from '../utils/response-utils';

const ERR_DATA = 'Cannot convert supplied data type';

/**
 * Returns an {@link ArrayBuffer} or string from the provided data synchronously.
 * Supports `ArrayBuffer`, `ArrayBufferView`, and `ArrayBufferLike` (e.g. `SharedArrayBuffer`)
 * while preserving typed array view offsets.
 */
// eslint-disable-next-line complexity
export function getArrayBufferOrStringFromDataSync(
  data: SyncDataType,
  loader: Loader,
  options: LoaderOptions
): ArrayBuffer | string {
  if (loader.text && typeof data === 'string') {
    return data;
  }

  if (isBuffer(data)) {
    data = data.buffer;
  }

  if (isArrayBufferLike(data)) {
    const bufferSource = toArrayBufferView(data);
    if (loader.text && !loader.binary) {
      const textDecoder = new TextDecoder('utf8');
      return textDecoder.decode(bufferSource);
    }
    return toArrayBuffer(bufferSource);
  }

  throw new Error(ERR_DATA);
}

/**
 * Resolves the provided data into an {@link ArrayBuffer} or string asynchronously.
 * Accepts the full {@link DataType} surface including responses and async iterables.
 */
export async function getArrayBufferOrStringFromData(
  data: DataType,
  loader: Loader,
  options: LoaderOptions
): Promise<ArrayBuffer | string> {
  if (typeof data === 'string' || isArrayBufferLike(data)) {
    return getArrayBufferOrStringFromDataSync(data as SyncDataType, loader, options);
  }

  // Blobs and files are FileReader compatible
  if (isBlob(data)) {
    data = await makeResponse(data);
  }

  if (isResponse(data)) {
    await checkResponse(data);
    return loader.binary ? await data.arrayBuffer() : await data.text();
  }

  if (isReadableStream(data)) {
    // @ts-expect-error TS2559 options type
    data = makeIterator(data as ReadableStream, options);
  }

  if (isIterable(data) || isAsyncIterable(data)) {
    // Assume arrayBuffer iterator - attempt to concatenate
    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
  }

  throw new Error(ERR_DATA);
}

/**
 * Resolves the provided data into an {@link ArrayBuffer}, preserving bytes for worker transfer.
 */
export async function getArrayBufferFromData(
  data: DataType,
  options: LoaderOptions
): Promise<ArrayBuffer> {
  if (typeof data === 'string') {
    return new TextEncoder().encode(data).buffer;
  }

  if (isArrayBufferLike(data)) {
    return toArrayBuffer(toArrayBufferView(data));
  }

  if (isBlob(data)) {
    data = await makeResponse(data);
  }

  if (isResponse(data)) {
    await checkResponse(data);
    return await data.arrayBuffer();
  }

  if (isReadableStream(data)) {
    // @ts-expect-error TS2559 options type
    data = makeIterator(data as ReadableStream, options);
  }

  if (isIterable(data) || isAsyncIterable(data)) {
    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
  }

  throw new Error(ERR_DATA);
}

/**
 * Normalizes batchable inputs into async iterables for batch parsing flows.
 * Supports synchronous iterables, async iterables, fetch responses, readable streams, and
 * single binary chunks (including typed array views and `ArrayBufferLike` values).
 */
export async function getAsyncIterableFromData(
  data: BatchableDataType,
  options: LoaderOptions
): Promise<
  AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
> {
  if (isPromise(data)) {
    data = await data;
  }

  if (isIterator(data)) {
    return data as AsyncIterable<ArrayBuffer>;
  }

  if (isResponse(data)) {
    // Note Since this function is not async, we currently can't load error message, just status
    await checkResponse(data);
    // TODO - bug in polyfill, body can be a Promise under Node.js
    // eslint-disable-next-line @typescript-eslint/await-thenable
    const body = await data.body;
    if (!body) {
      throw new Error(ERR_DATA);
    }
    return makeIterator(body, options as any);
  }

  if (isBlob(data) || isReadableStream(data)) {
    return makeIterator(data as Blob | ReadableStream, options as any);
  }

  if (isAsyncIterable(data)) {
    return data as AsyncIterable<ArrayBufferLike | ArrayBufferView>;
  }

  if (isIterable(data)) {
    return data as Iterable<ArrayBufferLike | ArrayBufferView>;
  }

  // @ts-expect-error TODO - fix type mess
  return getIterableFromData(data);
}

/**
 * Returns a readable stream for streaming loader inputs when available.
 */
export async function getReadableStream(data: BatchableDataType): Promise<ReadableStream> {
  if (isReadableStream(data)) {
    return data as ReadableStream;
  }
  if (isResponse(data)) {
    // @ts-ignore
    if (!data.body) {
      throw new Error(ERR_DATA);
    }
    return data.body;
  }
  const response = await makeResponse(data);
  // @ts-ignore
  if (!response.body) {
    throw new Error(ERR_DATA);
  }
  return response.body;
}

// HELPERS

function getIterableFromData(data: string | ArrayBuffer | SharedArrayBuffer | ArrayBufferView) {
  // generate an iterator that emits a single chunk
  if (ArrayBuffer.isView(data)) {
    return (function* oneChunk() {
      yield toArrayBuffer(data);
    })();
  }

  if (isArrayBufferLike(data)) {
    return (function* oneChunk() {
      yield toArrayBuffer(data);
    })();
  }

  if (isIterator(data)) {
    return data;
  }

  if (isIterable(data)) {
    return data[Symbol.iterator]();
  }

  throw new Error(ERR_DATA);
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	DataType,
7	SyncDataType,
8	BatchableDataType,
9	Loader,
10	LoaderOptions
11	} from '@loaders.gl/loader-utils';
12	import {
13	concatenateArrayBuffersAsync,
14	isPromise,
15	isResponse,
16	isReadableStream,
17	isAsyncIterable,
18	isIterable,
19	isIterator,
20	isBlob,
21	isBuffer,
22	isArrayBufferLike,
23	toArrayBuffer,
24	toArrayBufferView
25	} from '@loaders.gl/loader-utils';
26	import {makeIterator} from '../../iterators/make-iterator/make-iterator';
27	import {checkResponse, makeResponse} from '../utils/response-utils';
28
29	const ERR_DATA = 'Cannot convert supplied data type';	368✔
30
31	/**
32	* Returns an {@link ArrayBuffer} or string from the provided data synchronously.
33	* Supports `ArrayBuffer`, `ArrayBufferView`, and `ArrayBufferLike` (e.g. `SharedArrayBuffer`)
34	* while preserving typed array view offsets.
35	*/
36	// eslint-disable-next-line complexity
37	export function getArrayBufferOrStringFromDataSync(
38	data: SyncDataType,
39	loader: Loader,
40	options: LoaderOptions
41	): ArrayBuffer \| string {
42	if (loader.text && typeof data === 'string') {	161,247✔
43	return data;	160,382✔
44	}
45
46	if (isBuffer(data)) {	865!
47	data = data.buffer;	×
48	}
49
50	if (isArrayBufferLike(data)) {	865✔
51	const bufferSource = toArrayBufferView(data);	863✔
52	if (loader.text && !loader.binary) {	863✔
53	const textDecoder = new TextDecoder('utf8');	41✔
54	return textDecoder.decode(bufferSource);	41✔
55	}
56	return toArrayBuffer(bufferSource);	822✔
57	}
58
59	throw new Error(ERR_DATA);	2✔
60	}
61
62	/**
63	* Resolves the provided data into an {@link ArrayBuffer} or string asynchronously.
64	* Accepts the full {@link DataType} surface including responses and async iterables.
65	*/
66	export async function getArrayBufferOrStringFromData(
67	data: DataType,
68	loader: Loader,
69	options: LoaderOptions
70	): Promise<ArrayBuffer \| string> {
71	if (typeof data === 'string' \|\| isArrayBufferLike(data)) {	1,993✔
72	return getArrayBufferOrStringFromDataSync(data as SyncDataType, loader, options);	799✔
73	}
74
75	// Blobs and files are FileReader compatible
76	if (isBlob(data)) {	1,194!
77	data = await makeResponse(data);	×
78	}
79
80	if (isResponse(data)) {	1,194✔
81	await checkResponse(data);	1,190✔
82	return loader.binary ? await data.arrayBuffer() : await data.text();	1,190✔
83	}
84
85	if (isReadableStream(data)) {	4✔
86	// @ts-expect-error TS2559 options type
87	data = makeIterator(data as ReadableStream, options);	3✔
88	}
89
90	if (isIterable(data) \|\| isAsyncIterable(data)) {	4!
91	// Assume arrayBuffer iterator - attempt to concatenate
92	return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);	4✔
93	}
94
95	throw new Error(ERR_DATA);	×
96	}
97
98	/**
99	* Resolves the provided data into an {@link ArrayBuffer}, preserving bytes for worker transfer.
100	*/
101	export async function getArrayBufferFromData(
102	data: DataType,
103	options: LoaderOptions
104	): Promise<ArrayBuffer> {
105	if (typeof data === 'string') {	109!
NEW 106	return new TextEncoder().encode(data).buffer;	×
107	}
108
109	if (isArrayBufferLike(data)) {	109✔
110	return toArrayBuffer(toArrayBufferView(data));	64✔
111	}
112
113	if (isBlob(data)) {	45!
NEW 114	data = await makeResponse(data);	×
115	}
116
117	if (isResponse(data)) {	45!
118	await checkResponse(data);	45✔
119	return await data.arrayBuffer();	45✔
120	}
121
NEW 122	if (isReadableStream(data)) {	×
123	// @ts-expect-error TS2559 options type
NEW 124	data = makeIterator(data as ReadableStream, options);	×
125	}
126
NEW 127	if (isIterable(data) \|\| isAsyncIterable(data)) {	×
NEW 128	return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);	×
129	}
130
NEW 131	throw new Error(ERR_DATA);	×
132	}
133
134	/**
135	* Normalizes batchable inputs into async iterables for batch parsing flows.
136	* Supports synchronous iterables, async iterables, fetch responses, readable streams, and
137	* single binary chunks (including typed array views and `ArrayBufferLike` values).
138	*/
139	export async function getAsyncIterableFromData(
140	data: BatchableDataType,
141	options: LoaderOptions
142	): Promise<
143	AsyncIterable<ArrayBufferLike \| ArrayBufferView> \| Iterable<ArrayBufferLike \| ArrayBufferView>
144	> {
145	if (isPromise(data)) {	448!
146	data = await data;	×
147	}
148
149	if (isIterator(data)) {	448✔
150	return data as AsyncIterable<ArrayBuffer>;	123✔
151	}
152
153	if (isResponse(data)) {	325✔
154	// Note Since this function is not async, we currently can't load error message, just status
155	await checkResponse(data);	267✔
156	// TODO - bug in polyfill, body can be a Promise under Node.js
157	// eslint-disable-next-line @typescript-eslint/await-thenable
158	const body = await data.body;	267✔
159	if (!body) {	267!
160	throw new Error(ERR_DATA);	×
161	}
162	return makeIterator(body, options as any);	267✔
163	}
164
165	if (isBlob(data) \|\| isReadableStream(data)) {	58✔
166	return makeIterator(data as Blob \| ReadableStream, options as any);	40✔
167	}
168
169	if (isAsyncIterable(data)) {	18!
170	return data as AsyncIterable<ArrayBufferLike \| ArrayBufferView>;	×
171	}
172
173	if (isIterable(data)) {	18✔
174	return data as Iterable<ArrayBufferLike \| ArrayBufferView>;	10✔
175	}
176
177	// @ts-expect-error TODO - fix type mess
178	return getIterableFromData(data);	8✔
179	}
180
181	/**
182	* Returns a readable stream for streaming loader inputs when available.
183	*/
184	export async function getReadableStream(data: BatchableDataType): Promise<ReadableStream> {
185	if (isReadableStream(data)) {	×
186	return data as ReadableStream;	×
187	}
188	if (isResponse(data)) {	×
189	// @ts-ignore
190	if (!data.body) {	×
191	throw new Error(ERR_DATA);	×
192	}
193	return data.body;	×
194	}
195	const response = await makeResponse(data);	×
196	// @ts-ignore
197	if (!response.body) {	×
198	throw new Error(ERR_DATA);	×
199	}
200	return response.body;	×
201	}
202
203	// HELPERS
204
205	function getIterableFromData(data: string \| ArrayBuffer \| SharedArrayBuffer \| ArrayBufferView) {
206	// generate an iterator that emits a single chunk
207	if (ArrayBuffer.isView(data)) {	8✔
208	return (function* oneChunk() {	2✔
209	yield toArrayBuffer(data);	2✔
210	})();
211	}
212
213	if (isArrayBufferLike(data)) {	6✔
214	return (function* oneChunk() {	4✔
215	yield toArrayBuffer(data);	4✔
216	})();
217	}
218
219	if (isIterator(data)) {	2!
220	return data;	×
221	}
222
223	if (isIterable(data)) {	2!
224	return data[Symbol.iterator]();	×
225	}
226
227	throw new Error(ERR_DATA);	2✔
228	}

visgl / loaders.gl / 25256585712

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous