25256585712

Committed 02 May 2026 04:35PM UTC coverage: 59.717% (-0.06%) from 59.776%

Build # 25256585712

Build Type

push

github

Committed by

web-flow

Commit Message

chore(loader-utils): Consolidate `parseWithWorker` with `processOnWorker` (#1564)

Coverage Stats

12514 of 23182 branches covered (53.98%)

Branch coverage included in aggregate %.

497 of 804 new or added lines in 22 files covered. (61.82%)

25 existing lines in 4 files now uncovered.

25948 of 41225 relevant lines covered (62.94%)

14803.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/modules/loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts

/* eslint-disable no-restricted-globals */
import type {CoreAPI} from '../sources/data-source';
import type {LoaderWithParser, LoaderOptions, LoaderContext, Loader} from '../../loader-types';
import {createWorker} from '@loaders.gl/worker-utils';
// import {validateLoaderVersion} from './validate-loader-version';

/**
 * Set up a WebWorkerGlobalScope to talk with the main thread
 * @param loader
 */
export async function createLoaderWorker(loader: LoaderWithParser) {
  await createWorker(
    async (input: any, options: {[key: string]: any} = {}, workerContext, loaderContext = {}) => {
      // validateLoaderVersion(loader, data.source.split('@')[1]);

      const result = await parseData({
        loader,
        arrayBuffer: input,
        options,
        context: {
          ...loaderContext,
          coreApi: createWorkerCoreApi(),
          _parse: createParseOnMainThread(workerContext?.process)
        } as LoaderContext
      });

      return loader.serializeWorkerResult
        ? loader.serializeWorkerResult(result, options, loaderContext as LoaderContext)
        : result;
    }
  );
}

/**
 * Create a minimal core API implementation available inside worker loaders.
 */
function createWorkerCoreApi(): CoreAPI {
  const unavailable = (methodName: keyof CoreAPI) => () => {
    throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
  };

  return {
    fetchFile: async (urlOrData, fetchOptions) =>
      await fetch(urlOrData as RequestInfo | URL, fetchOptions),
    parseSync: unavailable('parseSync'),
    parse: unavailable('parse'),
    parseInBatches: unavailable('parseInBatches'),
    load: unavailable('load'),
    loadInBatches: unavailable('loadInBatches')
  };
}

/**
 * Create a loader context parse callback that redirects subloader parsing to the main thread.
 * @param processOnMainThread
 */
function createParseOnMainThread(
  processOnMainThread?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any
) {
  return (
    arrayBuffer: ArrayBuffer,
    loaders?: Loader | Loader[] | LoaderOptions,
    options?: LoaderOptions,
    context?: LoaderContext
  ) => {
    if (!processOnMainThread) {
      throw new Error('Worker not set up to parse on main thread');
    }

    const parseArguments = getMainThreadParseArguments(loaders, options, context);
    return processOnMainThread(arrayBuffer, parseArguments.options, parseArguments.context);
  };
}

/**
 * Extract parse options and context from the overloaded loader context parse signature.
 * @param loaders
 * @param options
 * @param context
 */
function getMainThreadParseArguments(
  loaders?: Loader | Loader[] | LoaderOptions,
  options?: LoaderOptions,
  context?: LoaderContext
): {options?: LoaderOptions; context?: Record<string, any>} {
  if (options) {
    return {options, context: getSerializableLoaderContext(context)};
  }
  if (Array.isArray(loaders) || (loaders && isLoaderObject(loaders))) {
    return {options: undefined, context: getSerializableLoaderContext(context)};
  }
  if (loaders && !Array.isArray(loaders)) {
    return {options: loaders};
  }
  return {options: undefined, context: getSerializableLoaderContext(context)};
}

/**
 * Checks whether a value is a loader object.
 * @param value
 */
function isLoaderObject(value: Loader | LoaderOptions): value is Loader {
  return 'id' in value && 'extensions' in value;
}

/**
 * Create a serializable loader context for a main-thread parse request.
 * @param context
 */
function getSerializableLoaderContext(context?: LoaderContext) {
  if (!context) {
    return undefined;
  }
  const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
    context;
  return JSON.parse(JSON.stringify(serializableContext));
}

// TODO - Support byteOffset and byteLength (enabling parsing of embedded binaries without copies)
// TODO - Why not support async loader.parse* funcs here?
// TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
// TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
async function parseData({
  loader,
  arrayBuffer,
  options,
  context
}: {
  loader: LoaderWithParser;
  arrayBuffer: ArrayBuffer;
  options: LoaderOptions;
  context: LoaderContext;
}) {
  let data;
  let parser;
  if (loader.parse || loader.parseSync) {
    data = arrayBuffer;
    parser = loader.parse || loader.parseSync;
  } else if (loader.parseTextSync) {
    const textDecoder = new TextDecoder();
    data = textDecoder.decode(arrayBuffer);
    parser = loader.parseTextSync;
  } else {
    throw new Error(`Could not load data with ${loader.name} loader`);
  }

  // TODO - proper merge in of loader options...
  options = {
    ...options,
    modules: (loader && loader.options && loader.options.modules) || {},
    core: {
      ...options.core,
      worker: false
    }
  };

  return await parser(data, {...options}, context, loader);
}

1	/* eslint-disable no-restricted-globals */
2	import type {CoreAPI} from '../sources/data-source';
3	import type {LoaderWithParser, LoaderOptions, LoaderContext, Loader} from '../../loader-types';
4	import {createWorker} from '@loaders.gl/worker-utils';
5	// import {validateLoaderVersion} from './validate-loader-version';
6
7	/**
8	* Set up a WebWorkerGlobalScope to talk with the main thread
9	* @param loader
10	*/
11	export async function createLoaderWorker(loader: LoaderWithParser) {
NEW 12	await createWorker(	×
13	async (input: any, options: {[key: string]: any} = {}, workerContext, loaderContext = {}) => {	×
14	// validateLoaderVersion(loader, data.source.split('@')[1]);
15
NEW 16	const result = await parseData({	×
17	loader,
18	arrayBuffer: input,
19	options,
20	context: {
21	...loaderContext,
22	coreApi: createWorkerCoreApi(),
23	_parse: createParseOnMainThread(workerContext?.process)
24	} as LoaderContext
25	});
26
NEW 27	return loader.serializeWorkerResult	×
28	? loader.serializeWorkerResult(result, options, loaderContext as LoaderContext)
29	: result;
30	}
31	);
32	}
33
34	/**
35	* Create a minimal core API implementation available inside worker loaders.
36	*/
37	function createWorkerCoreApi(): CoreAPI {
38	const unavailable = (methodName: keyof CoreAPI) => () => {	×
39	throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);	×
40	};
41
42	return {	×
43	fetchFile: async (urlOrData, fetchOptions) =>
NEW 44	await fetch(urlOrData as RequestInfo \| URL, fetchOptions),	×
45	parseSync: unavailable('parseSync'),
46	parse: unavailable('parse'),
47	parseInBatches: unavailable('parseInBatches'),
48	load: unavailable('load'),
49	loadInBatches: unavailable('loadInBatches')
50	};
51	}
52
53	/**
54	* Create a loader context parse callback that redirects subloader parsing to the main thread.
55	* @param processOnMainThread
56	*/
57	function createParseOnMainThread(
58	processOnMainThread?: (data: any, options?: LoaderOptions, context?: Record<string, any>) => any
59	) {
NEW 60	return (	×
61	arrayBuffer: ArrayBuffer,
62	loaders?: Loader \| Loader[] \| LoaderOptions,
63	options?: LoaderOptions,
64	context?: LoaderContext
65	) => {
NEW 66	if (!processOnMainThread) {	×
NEW 67	throw new Error('Worker not set up to parse on main thread');	×
68	}
69
NEW 70	const parseArguments = getMainThreadParseArguments(loaders, options, context);	×
NEW 71	return processOnMainThread(arrayBuffer, parseArguments.options, parseArguments.context);	×
72	};
73	}
74
75	/**
76	* Extract parse options and context from the overloaded loader context parse signature.
77	* @param loaders
78	* @param options
79	* @param context
80	*/
81	function getMainThreadParseArguments(
82	loaders?: Loader \| Loader[] \| LoaderOptions,
83	options?: LoaderOptions,
84	context?: LoaderContext
85	): {options?: LoaderOptions; context?: Record<string, any>} {
NEW 86	if (options) {	×
NEW 87	return {options, context: getSerializableLoaderContext(context)};	×
88	}
NEW 89	if (Array.isArray(loaders) \|\| (loaders && isLoaderObject(loaders))) {	×
NEW 90	return {options: undefined, context: getSerializableLoaderContext(context)};	×
91	}
NEW 92	if (loaders && !Array.isArray(loaders)) {	×
NEW 93	return {options: loaders};	×
94	}
NEW 95	return {options: undefined, context: getSerializableLoaderContext(context)};	×
96	}
97
98	/**
99	* Checks whether a value is a loader object.
100	* @param value
101	*/
102	function isLoaderObject(value: Loader \| LoaderOptions): value is Loader {
NEW 103	return 'id' in value && 'extensions' in value;	×
104	}
105
106	/**
107	* Create a serializable loader context for a main-thread parse request.
108	* @param context
109	*/
110	function getSerializableLoaderContext(context?: LoaderContext) {
NEW 111	if (!context) {	×
NEW 112	return undefined;	×
113	}
114	const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
NEW 115	context;	×
NEW 116	return JSON.parse(JSON.stringify(serializableContext));	×
117	}
118
119	// TODO - Support byteOffset and byteLength (enabling parsing of embedded binaries without copies)
120	// TODO - Why not support async loader.parse* funcs here?
121	// TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
122	// TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
123	async function parseData({
124	loader,
125	arrayBuffer,
126	options,
127	context
128	}: {
129	loader: LoaderWithParser;
130	arrayBuffer: ArrayBuffer;
131	options: LoaderOptions;
132	context: LoaderContext;
133	}) {
134	let data;
135	let parser;
NEW 136	if (loader.parse \|\| loader.parseSync) {	×
137	data = arrayBuffer;	×
NEW 138	parser = loader.parse \|\| loader.parseSync;	×
139	} else if (loader.parseTextSync) {	×
140	const textDecoder = new TextDecoder();	×
141	data = textDecoder.decode(arrayBuffer);	×
142	parser = loader.parseTextSync;	×
143	} else {
144	throw new Error(`Could not load data with ${loader.name} loader`);	×
145	}
146
147	// TODO - proper merge in of loader options...
148	options = {	×
149	...options,
150	modules: (loader && loader.options && loader.options.modules) \|\| {},	×
151	core: {
152	...options.core,
153	worker: false
154	}
155	};
156
157	return await parser(data, {...options}, context, loader);	×
158	}

visgl / loaders.gl / 25256585712

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous