24907303489

Committed 24 Apr 2026 07:12PM UTC coverage: 59.423% (+0.09%) from 59.334%

Build # 24907303489

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Dynamic import loaders (#3405)

Coverage Stats

11252 of 20783 branches covered (54.14%)

Branch coverage included in aggregate %.

1164 of 1518 new or added lines in 244 files covered. (76.68%)

41 existing lines in 18 files now uncovered.

23432 of 37585 relevant lines covered (62.34%)

16317.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.06

/modules/core/src/lib/api/parse.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  Loader,
  LoaderContext,
  LoaderOptions,
  LoaderOptionsWithShape,
  DataType,
  LoaderWithParser,
  LoaderOptionsType,
  LoaderShapeType,
  LoaderReturnType,
  LoaderArrayOptionsType,
  LoaderArrayReturnType,
  StrictLoaderOptions
} from '@loaders.gl/loader-utils';
import {
  parseWithWorker,
  canParseWithWorker,
  mergeOptions,
  isResponse,
  isSourceLoader
} from '@loaders.gl/loader-utils';
import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
import {isLoaderObject} from '../loader-utils/normalize-loader';
import {normalizeOptions} from '../loader-utils/option-utils';
import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
import {getResourceUrl} from '../utils/resource-utils';
import {getLoaderImplementation} from './load-loader';
import {selectLoader} from './select-loader';

// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T

/**
 * Parses `data` asynchronously using the supplied loader
 */
export async function parse<
  LoaderT extends Loader,
  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
    LoaderOptionsType<LoaderT>,
    LoaderShapeType<LoaderT>
  >
>(
  data: DataType | Promise<DataType>,
  loader: LoaderT,
  options?: OptionsT,
  context?: LoaderContext
): Promise<LoaderReturnType<LoaderT>>;

/**
 * Parses `data` asynchronously by matching one of the supplied loader
 */
export async function parse<
  LoaderArrayT extends Loader[],
  OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
>(
  data: DataType | Promise<DataType>,
  loaders: LoaderArrayT,
  options?: OptionsT,
  context?: LoaderContext
): Promise<LoaderArrayReturnType<LoaderArrayT>>;

/**
 * Parses data asynchronously by matching a pre-registered loader
 * @deprecated Loader registration is deprecated, use parse(data, loaders, options) instead
 */
export async function parse(
  data: DataType | Promise<DataType>,
  options?: LoaderOptions
): Promise<unknown>;

/**
 * Parses `data` using a specified loader
 * @param data
 * @param loaders
 * @param options
 * @param context
 */
// implementation signature
export async function parse(
  data: DataType | Promise<DataType>,
  loaders?: Loader | Loader[] | LoaderOptions,
  options?: LoaderOptions,
  context?: LoaderContext
): Promise<unknown> {
  // Signature: parse(data, options, context | url)
  // Uses registered loaders
  if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
    context = undefined; // context not supported in short signature
    options = loaders as LoaderOptions;
    loaders = undefined;
  }

  data = await data; // Resolve any promise
  options = options || ({} as LoaderOptions); // Could be invalid...

  // Extract a url for auto detection
  const url = getResourceUrl(data);

  // Chooses a loader (and normalizes it)
  // Also use any loaders in the context, new loaders take priority
  const typedLoaders = loaders as Loader | Loader[] | undefined;
  const candidateLoaders = getLoadersFromContext(typedLoaders, context);
  // todo hacky type cast
  const loader = await selectLoader(data as ArrayBuffer, candidateLoaders, options);
  // Note: if no loader was found, if so just return null
  if (!loader) {
    return null;
  }

  if (isSourceLoader(loader)) {
    throw new Error(
      `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
    );
  }

  // Normalize options
  // @ts-expect-error candidateLoaders
  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...

  // Get a context (if already present, will be unchanged)
  context = getLoaderContext(
    // @ts-expect-error
    {url, _parse: parse, loaders: candidateLoaders},
    strictOptions,
    context || null
  );

  return await parseWithLoader(loader, data, strictOptions, context);
}

// TODO: support progress and abort
// TODO - should accept loader.parseAsyncIterator and concatenate.
async function parseWithLoader(
  loader: Loader,
  data,
  options: StrictLoaderOptions,
  context: LoaderContext
): Promise<unknown> {
  validateWorkerVersion(loader);

  options = mergeOptions(loader.options, options);

  if (isResponse(data)) {
    // Serialize to support passing the response to web worker
    const {ok, redirected, status, statusText, type, url} = data;
    const headers = Object.fromEntries(data.headers.entries());
    // @ts-expect-error TODO - fix this
    context.response = {headers, ok, redirected, status, statusText, type, url};
  }

  data = await getArrayBufferOrStringFromData(data, loader, options);

  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);

  return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
}

async function parseWithLoaderImplementation(
  loader: LoaderWithParser,
  data: string | ArrayBuffer,
  options: StrictLoaderOptions,
  context: LoaderContext
): Promise<unknown> {
  if (loader.parseText && typeof data === 'string') {
    return await loader.parseText(data, options, context);
  }

  // Fall back to synchronous text parser, wrap results in promises
  if (loader.parseTextSync && typeof data === 'string') {
    return loader.parseTextSync(data, options, context);
  }

  // If we have a workerUrl and the loader can parse the given options efficiently in a worker
  if (canParseWithWorker(loader, options)) {
    return await parseWithWorker(loader, data, options, context, parse);
  }

  if (loader.parse) {
    return await loader.parse(data as ArrayBuffer, options, context);
  }

  // This should not happen, all sync loaders should also offer `parse` function
  assert(!loader.parseSync);

  // TBD - If asynchronous parser not available, return null
  throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	Loader,
7	LoaderContext,
8	LoaderOptions,
9	LoaderOptionsWithShape,
10	DataType,
11	LoaderWithParser,
12	LoaderOptionsType,
13	LoaderShapeType,
14	LoaderReturnType,
15	LoaderArrayOptionsType,
16	LoaderArrayReturnType,
17	StrictLoaderOptions
18	} from '@loaders.gl/loader-utils';
19	import {
20	parseWithWorker,
21	canParseWithWorker,
22	mergeOptions,
23	isResponse,
24	isSourceLoader
25	} from '@loaders.gl/loader-utils';
26	import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
27	import {isLoaderObject} from '../loader-utils/normalize-loader';
28	import {normalizeOptions} from '../loader-utils/option-utils';
29	import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
30	import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
31	import {getResourceUrl} from '../utils/resource-utils';
32	import {getLoaderImplementation} from './load-loader';
33	import {selectLoader} from './select-loader';
34
35	// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
36
37	/**
38	* Parses `data` asynchronously using the supplied loader
39	*/
40	export async function parse<
41	LoaderT extends Loader,
42	OptionsT extends LoaderOptions = LoaderOptionsWithShape<
43	LoaderOptionsType<LoaderT>,
44	LoaderShapeType<LoaderT>
45	>
46	>(
47	data: DataType \| Promise<DataType>,
48	loader: LoaderT,
49	options?: OptionsT,
50	context?: LoaderContext
51	): Promise<LoaderReturnType<LoaderT>>;
52
53	/**
54	* Parses `data` asynchronously by matching one of the supplied loader
55	*/
56	export async function parse<
57	LoaderArrayT extends Loader[],
58	OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
59	>(
60	data: DataType \| Promise<DataType>,
61	loaders: LoaderArrayT,
62	options?: OptionsT,
63	context?: LoaderContext
64	): Promise<LoaderArrayReturnType<LoaderArrayT>>;
65
66	/**
67	* Parses data asynchronously by matching a pre-registered loader
68	* @deprecated Loader registration is deprecated, use parse(data, loaders, options) instead
69	*/
70	export async function parse(
71	data: DataType \| Promise<DataType>,
72	options?: LoaderOptions
73	): Promise<unknown>;
74
75	/**
76	* Parses `data` using a specified loader
77	* @param data
78	* @param loaders
79	* @param options
80	* @param context
81	*/
82	// implementation signature
83	export async function parse(
84	data: DataType \| Promise<DataType>,
85	loaders?: Loader \| Loader[] \| LoaderOptions,
86	options?: LoaderOptions,
87	context?: LoaderContext
88	): Promise<unknown> {
89	// Signature: parse(data, options, context \| url)
90	// Uses registered loaders
91	if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {	2,092✔
92	context = undefined; // context not supported in short signature	7✔
93	options = loaders as LoaderOptions;	7✔
94	loaders = undefined;	7✔
95	}
96
97	data = await data; // Resolve any promise	2,092✔
98	options = options \|\| ({} as LoaderOptions); // Could be invalid...	2,092✔
99
100	// Extract a url for auto detection
101	const url = getResourceUrl(data);	2,092✔
102
103	// Chooses a loader (and normalizes it)
104	// Also use any loaders in the context, new loaders take priority
105	const typedLoaders = loaders as Loader \| Loader[] \| undefined;	2,092✔
106	const candidateLoaders = getLoadersFromContext(typedLoaders, context);	2,092✔
107	// todo hacky type cast
108	const loader = await selectLoader(data as ArrayBuffer, candidateLoaders, options);	2,092✔
109	// Note: if no loader was found, if so just return null
110	if (!loader) {	2,090!
UNCOV 111	return null;	×
112	}
113
114	if (isSourceLoader(loader)) {	2,090✔
115	throw new Error(	2✔
116	`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
117	);
118	}
119
120	// Normalize options
121	// @ts-expect-error candidateLoaders
122	const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...	2,088✔
123
124	// Get a context (if already present, will be unchanged)
125	context = getLoaderContext(	2,088✔
126	// @ts-expect-error
127	{url, _parse: parse, loaders: candidateLoaders},
128	strictOptions,
129	context \|\| null	3,474✔
130	);
131
132	return await parseWithLoader(loader, data, strictOptions, context);	2,092✔
133	}
134
135	// TODO: support progress and abort
136	// TODO - should accept loader.parseAsyncIterator and concatenate.
137	async function parseWithLoader(
138	loader: Loader,
139	data,
140	options: StrictLoaderOptions,
141	context: LoaderContext
142	): Promise<unknown> {
143	validateWorkerVersion(loader);	2,088✔
144
145	options = mergeOptions(loader.options, options);	2,088✔
146
147	if (isResponse(data)) {	2,088✔
148	// Serialize to support passing the response to web worker
149	const {ok, redirected, status, statusText, type, url} = data;	1,242✔
150	const headers = Object.fromEntries(data.headers.entries());	1,242✔
151	// @ts-expect-error TODO - fix this
152	context.response = {headers, ok, redirected, status, statusText, type, url};	1,242✔
153	}
154
155	data = await getArrayBufferOrStringFromData(data, loader, options);	2,088✔
156
157	const loaderWithParser = await getLoaderImplementation(loader, options, context.url);	2,088✔
158
159	return await parseWithLoaderImplementation(loaderWithParser, data, options, context);	2,088✔
160	}
161
162	async function parseWithLoaderImplementation(
163	loader: LoaderWithParser,
164	data: string \| ArrayBuffer,
165	options: StrictLoaderOptions,
166	context: LoaderContext
167	): Promise<unknown> {
168	if (loader.parseText && typeof data === 'string') {	2,088✔
169	return await loader.parseText(data, options, context);	148✔
170	}
171
172	// Fall back to synchronous text parser, wrap results in promises
173	if (loader.parseTextSync && typeof data === 'string') {	1,940✔
174	return loader.parseTextSync(data, options, context);	176✔
175	}
176
177	// If we have a workerUrl and the loader can parse the given options efficiently in a worker
178	if (canParseWithWorker(loader, options)) {	1,764✔
179	return await parseWithWorker(loader, data, options, context, parse);	99✔
180	}
181
182	if (loader.parse) {	1,665!
183	return await loader.parse(data as ArrayBuffer, options, context);	1,665✔
184	}
185
186	// This should not happen, all sync loaders should also offer `parse` function
UNCOV 187	assert(!loader.parseSync);	×
188
189	// TBD - If asynchronous parser not available, return null
UNCOV 190	throw new Error(`${loader.id} loader - no parser found and worker is disabled`);	×
191	}

visgl / loaders.gl / 24907303489

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous