25256585712

Committed 02 May 2026 04:35PM UTC coverage: 59.717% (-0.06%) from 59.776%

Build # 25256585712

Build Type

push

github

Committed by

web-flow

Commit Message

chore(loader-utils): Consolidate `parseWithWorker` with `processOnWorker` (#1564)

Coverage Stats

12514 of 23182 branches covered (53.98%)

Branch coverage included in aggregate %.

497 of 804 new or added lines in 22 files covered. (61.82%)

25 existing lines in 4 files now uncovered.

25948 of 41225 relevant lines covered (62.94%)

14803.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.71

/modules/loader-utils/src/lib/worker-loader-utils/parse-with-worker.ts

import {canProcessOnWorker, isBrowser, processOnWorker} from '@loaders.gl/worker-utils';
import type {Loader, StrictLoaderOptions, LoaderContext} from '../../loader-types';

type ParseOnMainThread = (
  arrayBuffer: ArrayBuffer,
  loaders?: Loader | Loader[] | StrictLoaderOptions,
  options?: StrictLoaderOptions,
  context?: LoaderContext
) => Promise<unknown>;

/**
 * Determines if a loader can parse with worker
 * @param loader
 * @param options
 */
export function canParseWithWorker(loader: Loader, options?: StrictLoaderOptions) {
  const workerOptions = getWorkerOptions(options);
  const nodeWorkers = workerOptions._nodeWorkers;
  if (!isBrowser && !nodeWorkers) {
    return false;
  }

  // Some Arrow table outputs need main-thread class instances; structured clone
  // preserves data but strips methods like `table.getChild()` from Arrow tables.
  if (
    (loader.id === 'excel' &&
      (options as {excel?: {shape?: string}} | undefined)?.excel?.shape === 'arrow-table') ||
    (loader.id === 'ply' &&
      (options as {ply?: {shape?: string}} | undefined)?.ply?.shape === 'arrow-table')
  ) {
    return false;
  }

  if (loader.id === 'csv' && !shouldParseCSVWithWorker(options)) {
    return false;
  }

  return Boolean(canProcessOnWorker(loader, workerOptions));
}

/**
 * this function expects that the worker function sends certain messages,
 * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
 */
export async function parseWithWorker(
  loader: Loader,
  data: any,
  options?: StrictLoaderOptions,
  context?: LoaderContext,
  parseOnMainThread?: ParseOnMainThread
) {
  const result = await processOnWorker(
    loader,
    data,
    getWorkerOptions(options),
    {
      process: async (input, processOptions, _workerContext, parseContext) => {
        if (!parseOnMainThread) {
          throw new Error('Worker not set up to parse on main thread');
        }
        const mainThreadContext = context
          ? ({...context, ...(parseContext || {})} as LoaderContext)
          : undefined;
        return await callParseOnMainThread(
          parseOnMainThread,
          input,
          processOptions,
          mainThreadContext
        );
      }
    },
    getSerializableLoaderContext(context)
  );
  return isLoaderWithWorkerResultDeserializer(loader)
    ? loader.deserializeWorkerResult(result, options, context)
    : result;
}

/**
 * Calls either the legacy two-argument parse callback or the loader-utils callback.
 * @param parseOnMainThread Main-thread parse callback.
 * @param input Data to parse on the main thread.
 * @param options Loader options from the worker.
 * @param context Loader context merged from the worker and caller.
 */
function callParseOnMainThread(
  parseOnMainThread: ParseOnMainThread,
  input: ArrayBuffer,
  options?: StrictLoaderOptions,
  context?: LoaderContext
): Promise<unknown> {
  if (parseOnMainThread.length <= 2) {
    return parseOnMainThread(input, options);
  }
  return parseOnMainThread(input, undefined, options, context);
}

/**
 * Create worker options with deprecated top-level worker fields available to worker-utils.
 * @param options
 */
function getWorkerOptions(options: StrictLoaderOptions = {}) {
  const serializedOptions = JSON.parse(JSON.stringify(options));
  return {
    ...serializedOptions.core,
    ...serializedOptions
  };
}

/**
 * Create a serializable loader context for worker jobs.
 * @param context
 */
function getSerializableLoaderContext(context?: LoaderContext) {
  if (!context) {
    return {};
  }
  const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
    context;
  return JSON.parse(JSON.stringify(serializableContext));
}

/**
 * Checks whether CSV options request Arrow output that can be transported from a worker.
 * @param options Loader options.
 * @returns True when CSV should parse on a worker.
 */
function shouldParseCSVWithWorker(options?: StrictLoaderOptions): boolean {
  const csvOptions = options as {csv?: {shape?: string}; core?: {shape?: string}} | undefined;
  return (csvOptions?.csv?.shape ?? csvOptions?.core?.shape) === 'arrow-table';
}

/**
 * Tests whether a loader can deserialize worker results.
 * @param loader Loader object.
 * @returns True when the loader exposes a worker result deserializer.
 */
function isLoaderWithWorkerResultDeserializer(
  loader: Loader
): loader is Loader & Required<Pick<Loader, 'deserializeWorkerResult'>> {
  return (
    typeof (loader as Loader & {deserializeWorkerResult?: unknown}).deserializeWorkerResult ===
    'function'
  );
}

1	import {canProcessOnWorker, isBrowser, processOnWorker} from '@loaders.gl/worker-utils';
2	import type {Loader, StrictLoaderOptions, LoaderContext} from '../../loader-types';
3
4	type ParseOnMainThread = (
5	arrayBuffer: ArrayBuffer,
6	loaders?: Loader \| Loader[] \| StrictLoaderOptions,
7	options?: StrictLoaderOptions,
8	context?: LoaderContext
9	) => Promise<unknown>;
10
11	/**
12	* Determines if a loader can parse with worker
13	* @param loader
14	* @param options
15	*/
16	export function canParseWithWorker(loader: Loader, options?: StrictLoaderOptions) {
17	const workerOptions = getWorkerOptions(options);	2,101✔
18	const nodeWorkers = workerOptions._nodeWorkers;	2,101✔
19	if (!isBrowser && !nodeWorkers) {	2,101✔
20	return false;	996✔
21	}
22
23	// Some Arrow table outputs need main-thread class instances; structured clone
24	// preserves data but strips methods like `table.getChild()` from Arrow tables.
25	if (	1,105✔
26	(loader.id === 'excel' &&	2,223✔
27	(options as {excel?: {shape?: string}} \| undefined)?.excel?.shape === 'arrow-table') \|\|
28	(loader.id === 'ply' &&
29	(options as {ply?: {shape?: string}} \| undefined)?.ply?.shape === 'arrow-table')
30	) {
31	return false;	4✔
32	}
33
34	if (loader.id === 'csv' && !shouldParseCSVWithWorker(options)) {	1,101✔
35	return false;	27✔
36	}
37
38	return Boolean(canProcessOnWorker(loader, workerOptions));	1,074✔
39	}
40
41	/**
42	* this function expects that the worker function sends certain messages,
43	* this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
44	*/
45	export async function parseWithWorker(
46	loader: Loader,
47	data: any,
48	options?: StrictLoaderOptions,
49	context?: LoaderContext,
50	parseOnMainThread?: ParseOnMainThread
51	) {
52	const result = await processOnWorker(	113✔
53	loader,
54	data,
55	getWorkerOptions(options),
56	{
57	process: async (input, processOptions, _workerContext, parseContext) => {
58	if (!parseOnMainThread) {	10!
NEW 59	throw new Error('Worker not set up to parse on main thread');	×
60	}
61	const mainThreadContext = context	10!
62	? ({...context, ...(parseContext \|\| {})} as LoaderContext)	10!
63	: undefined;
64	return await callParseOnMainThread(	10✔
65	parseOnMainThread,
66	input,
67	processOptions,
68	mainThreadContext
69	);
70	}
71	},
72	getSerializableLoaderContext(context)
73	);
74	return isLoaderWithWorkerResultDeserializer(loader)	113!
75	? loader.deserializeWorkerResult(result, options, context)
76	: result;
77	}
78
79	/**
80	* Calls either the legacy two-argument parse callback or the loader-utils callback.
81	* @param parseOnMainThread Main-thread parse callback.
82	* @param input Data to parse on the main thread.
83	* @param options Loader options from the worker.
84	* @param context Loader context merged from the worker and caller.
85	*/
86	function callParseOnMainThread(
87	parseOnMainThread: ParseOnMainThread,
88	input: ArrayBuffer,
89	options?: StrictLoaderOptions,
90	context?: LoaderContext
91	): Promise<unknown> {
92	if (parseOnMainThread.length <= 2) {	10✔
93	return parseOnMainThread(input, options);	1✔
94	}
95	return parseOnMainThread(input, undefined, options, context);	9✔
96	}
97
98	/**
99	* Create worker options with deprecated top-level worker fields available to worker-utils.
100	* @param options
101	*/
102	function getWorkerOptions(options: StrictLoaderOptions = {}) {	2,214✔
103	const serializedOptions = JSON.parse(JSON.stringify(options));	2,214✔
104	return {	2,214✔
105	...serializedOptions.core,
106	...serializedOptions
107	};
108	}
109
110	/**
111	* Create a serializable loader context for worker jobs.
112	* @param context
113	*/
114	function getSerializableLoaderContext(context?: LoaderContext) {
115	if (!context) {	113!
NEW 116	return {};	×
117	}
118	const {fetch, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext} =
119	context;	113✔
120	return JSON.parse(JSON.stringify(serializableContext));	113✔
121	}
122
123	/**
124	* Checks whether CSV options request Arrow output that can be transported from a worker.
125	* @param options Loader options.
126	* @returns True when CSV should parse on a worker.
127	*/
128	function shouldParseCSVWithWorker(options?: StrictLoaderOptions): boolean {
129	const csvOptions = options as {csv?: {shape?: string}; core?: {shape?: string}} \| undefined;	59✔
130	return (csvOptions?.csv?.shape ?? csvOptions?.core?.shape) === 'arrow-table';	59!
131	}
132
133	/**
134	* Tests whether a loader can deserialize worker results.
135	* @param loader Loader object.
136	* @returns True when the loader exposes a worker result deserializer.
137	*/
138	function isLoaderWithWorkerResultDeserializer(
139	loader: Loader
140	): loader is Loader & Required<Pick<Loader, 'deserializeWorkerResult'>> {
141	return (	113✔
142	typeof (loader as Loader & {deserializeWorkerResult?: unknown}).deserializeWorkerResult ===
143	'function'
144	);
145	}

visgl / loaders.gl / 25256585712

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous