20352515932

Committed 18 Dec 2025 09:56PM UTC coverage: 35.115% (-28.4%) from 63.485%

Build # 20352515932

Build Type

push

github

Committed by

web-flow

Commit Message

feat(loader-utils): Export is-type helpers (#3258)

Coverage Stats

1188 of 1998 branches covered (59.46%)

Branch coverage included in aggregate %.

147 of 211 new or added lines in 13 files covered. (69.67%)

30011 existing lines in 424 files now uncovered.

37457 of 108056 relevant lines covered (34.66%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.66

/modules/core/src/lib/api/encode.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  WriterOptions,
  WriterWithEncoder,
  WriterOptionsType,
  WriterDataType,
  WriterBatchType
} from '@loaders.gl/loader-utils';
import {canEncodeWithWorker, NodeFile, resolvePath, isBrowser} from '@loaders.gl/loader-utils';
import {processOnWorker} from '@loaders.gl/worker-utils';
import {fetchFile} from '../fetch/fetch-file';
import {getLoaderOptions} from './loader-options';

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export async function encode<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options_?: WriterOptionsType<WriterT>
): Promise<ArrayBuffer> {
  const globalOptions = getLoaderOptions() as WriterOptions;
  // const globalOptions: WriterOptions = {}; // getWriterOptions();
  const options = {...globalOptions, ...options_};

  // Handle the special case where we are invoking external command-line tools
  if (writer.encodeURLtoURL) {
    return encodeWithCommandLineTool(writer, data, options);
  }

  // Worker support
  if (canEncodeWithWorker(writer, options)) {
    return await processOnWorker(writer, data, options);
  }

  // TODO Merge default writer options with options argument like it is done in load module.
  return await writer.encode(data, options);
}

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export function encodeSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): ArrayBuffer {
  if (writer.encodeSync) {
    return writer.encodeSync(data, options);
  }
  if (writer.encodeTextSync) {
    return new TextEncoder().encode(writer.encodeTextSync(data, options));
  }
  throw new Error(`Writer ${writer.name} could not synchronously encode data`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export async function encodeText<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): Promise<string> {
  if (writer.encodeText) {
    return await writer.encodeText(data, options);
  }

  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text) {
    const arrayBuffer = await writer.encode(data, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export function encodeTextSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): string {
  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text && writer.encodeSync) {
    const arrayBuffer = encodeSync(data, writer, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeInBatches<WriterT extends WriterWithEncoder>(
  data: WriterBatchType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode in batches`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeTextInBatches(
  data: unknown,
  writer: WriterWithEncoder,
  options?: WriterOptions
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeTextInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeTextInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode text in batches`);
}

/**
 * Encode data stored in a file (on disk) to another file.
 * @note Node.js only. This function enables using command-line converters as "writers".
 */
export async function encodeURLtoURL(
  inputUrl: string,
  outputUrl: string,
  writer: Omit<WriterWithEncoder, 'encode'>,
  options?: WriterOptions
): Promise<string> {
  inputUrl = resolvePath(inputUrl);
  outputUrl = resolvePath(outputUrl);
  if (isBrowser || !writer.encodeURLtoURL) {
    throw new Error();
  }
  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
  return outputFilename;
}

/** Helper function to encode via external tool (typically command line execution in Node.js) */
async function encodeWithCommandLineTool(
  writer: WriterWithEncoder,
  data: unknown,
  options: WriterOptions
): Promise<ArrayBuffer> {
  if (isBrowser) {
    throw new Error(`Writer ${writer.name} not supported in browser`);
  }
  // TODO - how to generate filenames with correct extensions?
  const tmpInputFilename = getTemporaryFilename('input');
  const file = new NodeFile(tmpInputFilename, 'w');
  await file.write(data as ArrayBuffer);

  const tmpOutputFilename = getTemporaryFilename('output');

  const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);

  const response = await fetchFile(outputFilename);
  return response.arrayBuffer();
}

/**
 * @todo TODO - this is an unacceptable hack!!!
 */
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
  const dataIterator = [{...data, start: 0, end: data.length}];
  return dataIterator;
}

/**
 * @todo Move to utils
 */
function getTemporaryFilename(filename: string): string {
  return `/tmp/${filename}`;
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {	1✔
6	WriterOptions,	1✔
7	WriterWithEncoder,	1✔
8	WriterOptionsType,	1✔
9	WriterDataType,	1✔
10	WriterBatchType	1✔
11	} from '@loaders.gl/loader-utils';	1✔
12	import {canEncodeWithWorker, NodeFile, resolvePath, isBrowser} from '@loaders.gl/loader-utils';	1✔
13	import {processOnWorker} from '@loaders.gl/worker-utils';	1✔
14	import {fetchFile} from '../fetch/fetch-file';	1✔
15	import {getLoaderOptions} from './loader-options';	1✔
16		1✔
17	/**	1✔
18	* Encode loaded data into a binary ArrayBuffer using the specified Writer.	1✔
19	*/	1✔
UNCOV 20	export async function encode<WriterT extends WriterWithEncoder>(	×
UNCOV 21	data: WriterDataType<WriterT>,	×
UNCOV 22	writer: WriterT,	×
UNCOV 23	options_?: WriterOptionsType<WriterT>	×
UNCOV 24	): Promise<ArrayBuffer> {	×
UNCOV 25	const globalOptions = getLoaderOptions() as WriterOptions;	×
UNCOV 26	// const globalOptions: WriterOptions = {}; // getWriterOptions();	×
UNCOV 27	const options = {...globalOptions, ...options_};	×
UNCOV 28		×
UNCOV 29	// Handle the special case where we are invoking external command-line tools	×
UNCOV 30	if (writer.encodeURLtoURL) {	×
31	return encodeWithCommandLineTool(writer, data, options);	×
32	}	×
UNCOV 33		×
UNCOV 34	// Worker support	×
UNCOV 35	if (canEncodeWithWorker(writer, options)) {	×
36	return await processOnWorker(writer, data, options);	×
37	}	×
UNCOV 38		×
UNCOV 39	// TODO Merge default writer options with options argument like it is done in load module.	×
UNCOV 40	return await writer.encode(data, options);	×
UNCOV 41	}	×
42		1✔
43	/**	1✔
44	* Encode loaded data into a binary ArrayBuffer using the specified Writer.	1✔
45	*/	1✔
46	export function encodeSync<WriterT extends WriterWithEncoder>(	1✔
UNCOV 47	data: WriterDataType<WriterT>,	×
UNCOV 48	writer: WriterT,	×
UNCOV 49	options?: WriterOptionsType<WriterT>	×
UNCOV 50	): ArrayBuffer {	×
UNCOV 51	if (writer.encodeSync) {	×
UNCOV 52	return writer.encodeSync(data, options);	×
UNCOV 53	}	×
54	if (writer.encodeTextSync) {	×
55	return new TextEncoder().encode(writer.encodeTextSync(data, options));	×
56	}	×
57	throw new Error(`Writer ${writer.name} could not synchronously encode data`);	×
58	}	×
59		1✔
60	/**	1✔
61	* Encode loaded data to text using the specified Writer	1✔
62	* @note This is a convenience function not intended for production use on large input data.	1✔
63	* It is not optimized for performance. Data maybe converted from text to binary and back.	1✔
64	* @throws if the writer does not generate text output	1✔
65	*/	1✔
66	export async function encodeText<WriterT extends WriterWithEncoder>(	×
67	data: WriterDataType<WriterT>,	×
68	writer: WriterT,	×
69	options?: WriterOptionsType<WriterT>	×
70	): Promise<string> {	×
71	if (writer.encodeText) {	×
72	return await writer.encodeText(data, options);	×
73	}	×
74		×
75	if (writer.encodeTextSync) {	×
76	return writer.encodeTextSync(data, options);	×
77	}	×
78		×
79	if (writer.text) {	×
80	const arrayBuffer = await writer.encode(data, options);	×
81	return new TextDecoder().decode(arrayBuffer);	×
82	}	×
83		×
84	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
85	}	×
86		1✔
87	/**	1✔
88	* Encode loaded data to text using the specified Writer	1✔
89	* @note This is a convenience function not intended for production use on large input data.	1✔
90	* It is not optimized for performance. Data maybe converted from text to binary and back.	1✔
91	* @throws if the writer does not generate text output	1✔
92	*/	1✔
93	export function encodeTextSync<WriterT extends WriterWithEncoder>(	1✔
UNCOV 94	data: WriterDataType<WriterT>,	×
UNCOV 95	writer: WriterT,	×
UNCOV 96	options?: WriterOptionsType<WriterT>	×
UNCOV 97	): string {	×
UNCOV 98	if (writer.encodeTextSync) {	×
UNCOV 99	return writer.encodeTextSync(data, options);	×
UNCOV 100	}	×
101		×
UNCOV 102	if (writer.text && writer.encodeSync) {	×
103	const arrayBuffer = encodeSync(data, writer, options);	×
104	return new TextDecoder().decode(arrayBuffer);	×
105	}	×
106		×
107	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
108	}	×
109		1✔
110	/**	1✔
111	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.	1✔
112	*/	1✔
113	export function encodeInBatches<WriterT extends WriterWithEncoder>(	1✔
114	data: WriterBatchType<WriterT>,	×
115	writer: WriterT,	×
116	options?: WriterOptionsType<WriterT>	×
117	): AsyncIterable<ArrayBuffer> {	×
118	if (writer.encodeInBatches) {	×
119	const dataIterator = getIterator(data);	×
120	// @ts-expect-error	×
121	return writer.encodeInBatches(dataIterator, options);	×
122	}	×
123	// TODO -fall back to atomic encode?	×
124	throw new Error(`Writer ${writer.name} could not encode in batches`);	×
125	}	×
126		1✔
127	/**	1✔
128	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.	1✔
129	*/	1✔
130	export function encodeTextInBatches(	1✔
131	data: unknown,	×
132	writer: WriterWithEncoder,	×
133	options?: WriterOptions	×
134	): AsyncIterable<ArrayBuffer> {	×
135	if (writer.encodeTextInBatches) {	×
136	const dataIterator = getIterator(data);	×
137	// @ts-expect-error	×
138	return writer.encodeTextInBatches(dataIterator, options);	×
139	}	×
140	// TODO -fall back to atomic encode?	×
141	throw new Error(`Writer ${writer.name} could not encode text in batches`);	×
142	}	×
143		1✔
144	/**	1✔
145	* Encode data stored in a file (on disk) to another file.	1✔
146	* @note Node.js only. This function enables using command-line converters as "writers".	1✔
147	*/	1✔
148	export async function encodeURLtoURL(	×
149	inputUrl: string,	×
150	outputUrl: string,	×
151	writer: Omit<WriterWithEncoder, 'encode'>,	×
152	options?: WriterOptions	×
153	): Promise<string> {	×
154	inputUrl = resolvePath(inputUrl);	×
155	outputUrl = resolvePath(outputUrl);	×
156	if (isBrowser \|\| !writer.encodeURLtoURL) {	×
157	throw new Error();	×
158	}	×
159	const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);	×
160	return outputFilename;	×
161	}	×
162		1✔
163	/** Helper function to encode via external tool (typically command line execution in Node.js) */	1✔
164	async function encodeWithCommandLineTool(	×
165	writer: WriterWithEncoder,	×
166	data: unknown,	×
167	options: WriterOptions	×
168	): Promise<ArrayBuffer> {	×
169	if (isBrowser) {	×
170	throw new Error(`Writer ${writer.name} not supported in browser`);	×
171	}	×
172	// TODO - how to generate filenames with correct extensions?	×
173	const tmpInputFilename = getTemporaryFilename('input');	×
174	const file = new NodeFile(tmpInputFilename, 'w');	×
175	await file.write(data as ArrayBuffer);	×
176		×
177	const tmpOutputFilename = getTemporaryFilename('output');	×
178		×
179	const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);	×
180		×
181	const response = await fetchFile(outputFilename);	×
182	return response.arrayBuffer();	×
183	}	×
184		1✔
185	/**	1✔
186	* @todo TODO - this is an unacceptable hack!!!	1✔
187	*/	1✔
188	function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {	×
189	const dataIterator = [{...data, start: 0, end: data.length}];	×
190	return dataIterator;	×
191	}	×
192		1✔
193	/**	1✔
194	* @todo Move to utils	1✔
195	*/	1✔
196	function getTemporaryFilename(filename: string): string {	×
197	return `/tmp/${filename}`;	×
198	}	×

visgl / loaders.gl / 20352515932

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous