25256585712

Committed 02 May 2026 04:35PM UTC coverage: 59.717% (-0.06%) from 59.776%

Build # 25256585712

Build Type

push

github

Committed by

web-flow

Commit Message

chore(loader-utils): Consolidate `parseWithWorker` with `processOnWorker` (#1564)

Coverage Stats

12514 of 23182 branches covered (53.98%)

Branch coverage included in aggregate %.

497 of 804 new or added lines in 22 files covered. (61.82%)

25 existing lines in 4 files now uncovered.

25948 of 41225 relevant lines covered (62.94%)

14803.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.94

/modules/core/src/lib/api/parse.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  Loader,
  LoaderContext,
  LoaderOptions,
  LoaderOptionsWithShape,
  DataType,
  LoaderWithParser,
  LoaderOptionsType,
  LoaderShapeType,
  LoaderReturnType,
  LoaderArrayOptionsType,
  LoaderArrayReturnType,
  StrictLoaderOptions
} from '@loaders.gl/loader-utils';
import {
  parseWithWorker,
  canParseWithWorker,
  mergeOptions,
  isResponse,
  isSourceLoader
} from '@loaders.gl/loader-utils';
import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
import {isLoaderObject} from '../loader-utils/normalize-loader';
import {normalizeOptions} from '../loader-utils/option-utils';
import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
import {getArrayBufferFromData} from '../loader-utils/get-data';
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
import {getResourceUrl} from '../utils/resource-utils';
import {getLoaderImplementation} from './load-loader';
import {selectLoader} from './select-loader';

// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T

/**
 * Parses `data` asynchronously using the supplied loader
 */
export async function parse<
  LoaderT extends Loader,
  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
    LoaderOptionsType<LoaderT>,
    LoaderShapeType<LoaderT>
  >
>(
  data: DataType | Promise<DataType>,
  loader: LoaderT,
  options?: OptionsT,
  context?: LoaderContext
): Promise<LoaderReturnType<LoaderT>>;

/**
 * Parses `data` asynchronously by matching one of the supplied loader
 */
export async function parse<
  LoaderArrayT extends Loader[],
  OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
>(
  data: DataType | Promise<DataType>,
  loaders: LoaderArrayT,
  options?: OptionsT,
  context?: LoaderContext
): Promise<LoaderArrayReturnType<LoaderArrayT>>;

/**
 * Parses data asynchronously by matching a pre-registered loader
 * @deprecated Loader registration is deprecated, use parse(data, loaders, options) instead
 */
export async function parse(
  data: DataType | Promise<DataType>,
  options?: LoaderOptions
): Promise<unknown>;

/**
 * Parses `data` using a specified loader
 * @param data
 * @param loaders
 * @param options
 * @param context
 */
// implementation signature
export async function parse(
  data: DataType | Promise<DataType>,
  loaders?: Loader | Loader[] | LoaderOptions,
  options?: LoaderOptions,
  context?: LoaderContext
): Promise<unknown> {
  // Signature: parse(data, options, context | url)
  // Uses registered loaders
  if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
    context = undefined; // context not supported in short signature
    options = loaders as LoaderOptions;
    loaders = undefined;
  }

  data = await data; // Resolve any promise
  options = options || ({} as LoaderOptions); // Could be invalid...

  // Extract a url for auto detection
  const url = getResourceUrl(data);

  // Chooses a loader (and normalizes it)
  // Also use any loaders in the context, new loaders take priority
  const typedLoaders = loaders as Loader | Loader[] | undefined;
  const candidateLoaders = getLoadersFromContext(typedLoaders, context);
  // todo hacky type cast
  const loader = await selectLoader(data as ArrayBuffer, candidateLoaders, options);
  // Note: if no loader was found, if so just return null
  if (!loader) {
    return null;
  }

  if (isSourceLoader(loader)) {
    throw new Error(
      `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
    );
  }

  // Normalize options
  // @ts-expect-error candidateLoaders
  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...

  // Get a context (if already present, will be unchanged)
  context = getLoaderContext(
    // @ts-expect-error
    {url, _parse: parse, loaders: candidateLoaders},
    strictOptions,
    context || null
  );

  return await parseWithLoader(loader, data, strictOptions, context);
}

// TODO: support progress and abort
// TODO - should accept loader.parseAsyncIterator and concatenate.
async function parseWithLoader(
  loader: Loader,
  data,
  options: StrictLoaderOptions,
  context: LoaderContext
): Promise<unknown> {
  validateWorkerVersion(loader);

  options = mergeOptions(loader.options, options);

  if (isResponse(data)) {
    // Serialize to support passing the response to web worker
    const {ok, redirected, status, statusText, type, url} = data;
    const headers = Object.fromEntries(data.headers.entries());
    // @ts-expect-error TODO - fix this
    context.response = {headers, ok, redirected, status, statusText, type, url};
  }

  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);

  if (canParseWithWorker(loaderWithParser, options)) {
    data = await getArrayBufferFromData(data, options);
    return await parseWithWorker(loaderWithParser, data, options, context, parse);
  }

  data = await getArrayBufferOrStringFromData(data, loader, options);

  return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
}

async function parseWithLoaderImplementation(
  loader: LoaderWithParser,
  data: string | ArrayBuffer,
  options: StrictLoaderOptions,
  context: LoaderContext
): Promise<unknown> {
  if (loader.parseText && typeof data === 'string') {
    return await loader.parseText(data, options, context);
  }

  // Fall back to synchronous text parser, wrap results in promises
  if (loader.parseTextSync && typeof data === 'string') {
    return loader.parseTextSync(data, options, context);
  }

  if (loader.parse) {
    return await loader.parse(data as ArrayBuffer, options, context);
  }

  // This should not happen, all sync loaders should also offer `parse` function
  assert(!loader.parseSync);

  // TBD - If asynchronous parser not available, return null
  throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	Loader,
7	LoaderContext,
8	LoaderOptions,
9	LoaderOptionsWithShape,
10	DataType,
11	LoaderWithParser,
12	LoaderOptionsType,
13	LoaderShapeType,
14	LoaderReturnType,
15	LoaderArrayOptionsType,
16	LoaderArrayReturnType,
17	StrictLoaderOptions
18	} from '@loaders.gl/loader-utils';
19	import {
20	parseWithWorker,
21	canParseWithWorker,
22	mergeOptions,
23	isResponse,
24	isSourceLoader
25	} from '@loaders.gl/loader-utils';
26	import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
27	import {isLoaderObject} from '../loader-utils/normalize-loader';
28	import {normalizeOptions} from '../loader-utils/option-utils';
29	import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
30	import {getArrayBufferFromData} from '../loader-utils/get-data';
31	import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
32	import {getResourceUrl} from '../utils/resource-utils';
33	import {getLoaderImplementation} from './load-loader';
34	import {selectLoader} from './select-loader';
35
36	// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
37
38	/**
39	* Parses `data` asynchronously using the supplied loader
40	*/
41	export async function parse<
42	LoaderT extends Loader,
43	OptionsT extends LoaderOptions = LoaderOptionsWithShape<
44	LoaderOptionsType<LoaderT>,
45	LoaderShapeType<LoaderT>
46	>
47	>(
48	data: DataType \| Promise<DataType>,
49	loader: LoaderT,
50	options?: OptionsT,
51	context?: LoaderContext
52	): Promise<LoaderReturnType<LoaderT>>;
53
54	/**
55	* Parses `data` asynchronously by matching one of the supplied loader
56	*/
57	export async function parse<
58	LoaderArrayT extends Loader[],
59	OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
60	>(
61	data: DataType \| Promise<DataType>,
62	loaders: LoaderArrayT,
63	options?: OptionsT,
64	context?: LoaderContext
65	): Promise<LoaderArrayReturnType<LoaderArrayT>>;
66
67	/**
68	* Parses data asynchronously by matching a pre-registered loader
69	* @deprecated Loader registration is deprecated, use parse(data, loaders, options) instead
70	*/
71	export async function parse(
72	data: DataType \| Promise<DataType>,
73	options?: LoaderOptions
74	): Promise<unknown>;
75
76	/**
77	* Parses `data` using a specified loader
78	* @param data
79	* @param loaders
80	* @param options
81	* @param context
82	*/
83	// implementation signature
84	export async function parse(
85	data: DataType \| Promise<DataType>,
86	loaders?: Loader \| Loader[] \| LoaderOptions,
87	options?: LoaderOptions,
88	context?: LoaderContext
89	): Promise<unknown> {
90	// Signature: parse(data, options, context \| url)
91	// Uses registered loaders
92	if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {	2,105!
UNCOV 93	context = undefined; // context not supported in short signature	×
UNCOV 94	options = loaders as LoaderOptions;	×
UNCOV 95	loaders = undefined;	×
96	}
97
98	data = await data; // Resolve any promise	2,105✔
99	options = options \|\| ({} as LoaderOptions); // Could be invalid...	2,105✔
100
101	// Extract a url for auto detection
102	const url = getResourceUrl(data);	2,105✔
103
104	// Chooses a loader (and normalizes it)
105	// Also use any loaders in the context, new loaders take priority
106	const typedLoaders = loaders as Loader \| Loader[] \| undefined;	2,105✔
107	const candidateLoaders = getLoadersFromContext(typedLoaders, context);	2,105✔
108	// todo hacky type cast
109	const loader = await selectLoader(data as ArrayBuffer, candidateLoaders, options);	2,105✔
110	// Note: if no loader was found, if so just return null
111	if (!loader) {	2,103!
112	return null;	×
113	}
114
115	if (isSourceLoader(loader)) {	2,103✔
116	throw new Error(	2✔
117	`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
118	);
119	}
120
121	// Normalize options
122	// @ts-expect-error candidateLoaders
123	const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...	2,101✔
124
125	// Get a context (if already present, will be unchanged)
126	context = getLoaderContext(	2,101✔
127	// @ts-expect-error
128	{url, _parse: parse, loaders: candidateLoaders},
129	strictOptions,
130	context \|\| null	3,503✔
131	);
132
133	return await parseWithLoader(loader, data, strictOptions, context);	2,105✔
134	}
135
136	// TODO: support progress and abort
137	// TODO - should accept loader.parseAsyncIterator and concatenate.
138	async function parseWithLoader(
139	loader: Loader,
140	data,
141	options: StrictLoaderOptions,
142	context: LoaderContext
143	): Promise<unknown> {
144	validateWorkerVersion(loader);	2,101✔
145
146	options = mergeOptions(loader.options, options);	2,101✔
147
148	if (isResponse(data)) {	2,101✔
149	// Serialize to support passing the response to web worker
150	const {ok, redirected, status, statusText, type, url} = data;	1,235✔
151	const headers = Object.fromEntries(data.headers.entries());	1,235✔
152	// @ts-expect-error TODO - fix this
153	context.response = {headers, ok, redirected, status, statusText, type, url};	1,235✔
154	}
155
156	const loaderWithParser = await getLoaderImplementation(loader, options, context.url);	2,101✔
157
158	if (canParseWithWorker(loaderWithParser, options)) {	2,101✔
159	data = await getArrayBufferFromData(data, options);	109✔
160	return await parseWithWorker(loaderWithParser, data, options, context, parse);	109✔
161	}
162
163	data = await getArrayBufferOrStringFromData(data, loader, options);	1,992✔
164
165	return await parseWithLoaderImplementation(loaderWithParser, data, options, context);	1,992✔
166	}
167
168	async function parseWithLoaderImplementation(
169	loader: LoaderWithParser,
170	data: string \| ArrayBuffer,
171	options: StrictLoaderOptions,
172	context: LoaderContext
173	): Promise<unknown> {
174	if (loader.parseText && typeof data === 'string') {	1,992✔
175	return await loader.parseText(data, options, context);	174✔
176	}
177
178	// Fall back to synchronous text parser, wrap results in promises
179	if (loader.parseTextSync && typeof data === 'string') {	1,818✔
180	return loader.parseTextSync(data, options, context);	161✔
181	}
182
183	if (loader.parse) {	1,657!
184	return await loader.parse(data as ArrayBuffer, options, context);	1,657✔
185	}
186
187	// This should not happen, all sync loaders should also offer `parse` function
188	assert(!loader.parseSync);	×
189
190	// TBD - If asynchronous parser not available, return null
191	throw new Error(`${loader.id} loader - no parser found and worker is disabled`);	×
192	}

visgl / loaders.gl / 25256585712

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous