20382848403

Committed 19 Dec 2025 09:20PM UTC coverage: 35.219% (+0.1%) from 35.095%

Build # 20382848403

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Upgrade to handle ArrayBufferLike (#3271)

Run Details

1190 of 2002 branches covered (59.44%)

Branch coverage included in aggregate %.

157 of 269 new or added lines in 41 files covered. (58.36%)

3 existing lines in 3 files now uncovered.

37536 of 107957 relevant lines covered (34.77%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.63

/modules/core/src/lib/api/encode.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  WriterOptions,
  WriterWithEncoder,
  WriterOptionsType,
  WriterDataType,
  WriterBatchType
} from '@loaders.gl/loader-utils';
import {
  canEncodeWithWorker,
  NodeFile,
  resolvePath,
  isBrowser,
  ensureArrayBuffer
} from '@loaders.gl/loader-utils';
import {processOnWorker} from '@loaders.gl/worker-utils';
import {fetchFile} from '../fetch/fetch-file';
import {getLoaderOptions} from './loader-options';

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export async function encode<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options_?: WriterOptionsType<WriterT>
): Promise<ArrayBuffer> {
  const globalOptions = getLoaderOptions() as WriterOptions;
  // const globalOptions: WriterOptions = {}; // getWriterOptions();
  const options = {...globalOptions, ...options_};

  // Handle the special case where we are invoking external command-line tools
  if (writer.encodeURLtoURL) {
    return encodeWithCommandLineTool(writer, data, options);
  }

  // Worker support
  if (canEncodeWithWorker(writer, options)) {
    return await processOnWorker(writer, data, options);
  }

  // TODO Merge default writer options with options argument like it is done in load module.
  return await writer.encode(data, options);
}

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export function encodeSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): ArrayBuffer {
  if (writer.encodeSync) {
    return writer.encodeSync(data, options);
  }
  if (writer.encodeTextSync) {
    return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));
  }
  throw new Error(`Writer ${writer.name} could not synchronously encode data`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export async function encodeText<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): Promise<string> {
  if (writer.encodeText) {
    return await writer.encodeText(data, options);
  }

  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text) {
    const arrayBuffer = await writer.encode(data, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export function encodeTextSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): string {
  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text && writer.encodeSync) {
    const arrayBuffer = encodeSync(data, writer, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeInBatches<WriterT extends WriterWithEncoder>(
  data: WriterBatchType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode in batches`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeTextInBatches(
  data: unknown,
  writer: WriterWithEncoder,
  options?: WriterOptions
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeTextInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeTextInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode text in batches`);
}

/**
 * Encode data stored in a file (on disk) to another file.
 * @note Node.js only. This function enables using command-line converters as "writers".
 */
export async function encodeURLtoURL(
  inputUrl: string,
  outputUrl: string,
  writer: Omit<WriterWithEncoder, 'encode'>,
  options?: WriterOptions
): Promise<string> {
  inputUrl = resolvePath(inputUrl);
  outputUrl = resolvePath(outputUrl);
  if (isBrowser || !writer.encodeURLtoURL) {
    throw new Error();
  }
  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
  return outputFilename;
}

/** Helper function to encode via external tool (typically command line execution in Node.js) */
async function encodeWithCommandLineTool(
  writer: WriterWithEncoder,
  data: unknown,
  options: WriterOptions
): Promise<ArrayBuffer> {
  if (isBrowser) {
    throw new Error(`Writer ${writer.name} not supported in browser`);
  }
  // TODO - how to generate filenames with correct extensions?
  const tmpInputFilename = getTemporaryFilename('input');
  const file = new NodeFile(tmpInputFilename, 'w');
  await file.write(data as ArrayBuffer);

  const tmpOutputFilename = getTemporaryFilename('output');

  const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);

  const response = await fetchFile(outputFilename);
  return response.arrayBuffer();
}

/**
 * @todo TODO - this is an unacceptable hack!!!
 */
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
  const dataIterator = [{...data, start: 0, end: data.length}];
  return dataIterator;
}

/**
 * @todo Move to utils
 */
function getTemporaryFilename(filename: string): string {
  return `/tmp/${filename}`;
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {	1✔
6	WriterOptions,	1✔
7	WriterWithEncoder,	1✔
8	WriterOptionsType,	1✔
9	WriterDataType,	1✔
10	WriterBatchType	1✔
11	} from '@loaders.gl/loader-utils';	1✔
12	import {	1✔
13	canEncodeWithWorker,	1✔
14	NodeFile,	1✔
15	resolvePath,	1✔
16	isBrowser,	1✔
17	ensureArrayBuffer	1✔
18	} from '@loaders.gl/loader-utils';	1✔
19	import {processOnWorker} from '@loaders.gl/worker-utils';	1✔
20	import {fetchFile} from '../fetch/fetch-file';	1✔
21	import {getLoaderOptions} from './loader-options';	1✔
22		1✔
23	/**	1✔
24	* Encode loaded data into a binary ArrayBuffer using the specified Writer.	1✔
25	*/	1✔
26	export async function encode<WriterT extends WriterWithEncoder>(	×
27	data: WriterDataType<WriterT>,	×
28	writer: WriterT,	×
29	options_?: WriterOptionsType<WriterT>	×
30	): Promise<ArrayBuffer> {	×
31	const globalOptions = getLoaderOptions() as WriterOptions;	×
32	// const globalOptions: WriterOptions = {}; // getWriterOptions();	×
33	const options = {...globalOptions, ...options_};	×
34		×
35	// Handle the special case where we are invoking external command-line tools	×
36	if (writer.encodeURLtoURL) {	×
37	return encodeWithCommandLineTool(writer, data, options);	×
38	}	×
39		×
40	// Worker support	×
41	if (canEncodeWithWorker(writer, options)) {	×
42	return await processOnWorker(writer, data, options);	×
43	}	×
44		×
45	// TODO Merge default writer options with options argument like it is done in load module.	×
46	return await writer.encode(data, options);	×
47	}	×
48		1✔
49	/**	1✔
50	* Encode loaded data into a binary ArrayBuffer using the specified Writer.	1✔
51	*/	1✔
52	export function encodeSync<WriterT extends WriterWithEncoder>(	1✔
53	data: WriterDataType<WriterT>,	×
54	writer: WriterT,	×
55	options?: WriterOptionsType<WriterT>	×
56	): ArrayBuffer {	×
57	if (writer.encodeSync) {	×
58	return writer.encodeSync(data, options);	×
59	}	×
60	if (writer.encodeTextSync) {	×
NEW 61	return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));	×
62	}	×
63	throw new Error(`Writer ${writer.name} could not synchronously encode data`);	×
64	}	×
65		1✔
66	/**	1✔
67	* Encode loaded data to text using the specified Writer	1✔
68	* @note This is a convenience function not intended for production use on large input data.	1✔
69	* It is not optimized for performance. Data maybe converted from text to binary and back.	1✔
70	* @throws if the writer does not generate text output	1✔
71	*/	1✔
72	export async function encodeText<WriterT extends WriterWithEncoder>(	×
73	data: WriterDataType<WriterT>,	×
74	writer: WriterT,	×
75	options?: WriterOptionsType<WriterT>	×
76	): Promise<string> {	×
77	if (writer.encodeText) {	×
78	return await writer.encodeText(data, options);	×
79	}	×
80		×
81	if (writer.encodeTextSync) {	×
82	return writer.encodeTextSync(data, options);	×
83	}	×
84		×
85	if (writer.text) {	×
86	const arrayBuffer = await writer.encode(data, options);	×
87	return new TextDecoder().decode(arrayBuffer);	×
88	}	×
89		×
90	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
91	}	×
92		1✔
93	/**	1✔
94	* Encode loaded data to text using the specified Writer	1✔
95	* @note This is a convenience function not intended for production use on large input data.	1✔
96	* It is not optimized for performance. Data maybe converted from text to binary and back.	1✔
97	* @throws if the writer does not generate text output	1✔
98	*/	1✔
99	export function encodeTextSync<WriterT extends WriterWithEncoder>(	1✔
100	data: WriterDataType<WriterT>,	×
101	writer: WriterT,	×
102	options?: WriterOptionsType<WriterT>	×
103	): string {	×
104	if (writer.encodeTextSync) {	×
105	return writer.encodeTextSync(data, options);	×
106	}	×
107		×
108	if (writer.text && writer.encodeSync) {	×
109	const arrayBuffer = encodeSync(data, writer, options);	×
110	return new TextDecoder().decode(arrayBuffer);	×
111	}	×
112		×
113	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
114	}	×
115		1✔
116	/**	1✔
117	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.	1✔
118	*/	1✔
119	export function encodeInBatches<WriterT extends WriterWithEncoder>(	1✔
120	data: WriterBatchType<WriterT>,	×
121	writer: WriterT,	×
122	options?: WriterOptionsType<WriterT>	×
123	): AsyncIterable<ArrayBuffer> {	×
124	if (writer.encodeInBatches) {	×
125	const dataIterator = getIterator(data);	×
126	// @ts-expect-error	×
127	return writer.encodeInBatches(dataIterator, options);	×
128	}	×
129	// TODO -fall back to atomic encode?	×
130	throw new Error(`Writer ${writer.name} could not encode in batches`);	×
131	}	×
132		1✔
133	/**	1✔
134	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.	1✔
135	*/	1✔
136	export function encodeTextInBatches(	1✔
137	data: unknown,	×
138	writer: WriterWithEncoder,	×
139	options?: WriterOptions	×
140	): AsyncIterable<ArrayBuffer> {	×
141	if (writer.encodeTextInBatches) {	×
142	const dataIterator = getIterator(data);	×
143	// @ts-expect-error	×
144	return writer.encodeTextInBatches(dataIterator, options);	×
145	}	×
146	// TODO -fall back to atomic encode?	×
147	throw new Error(`Writer ${writer.name} could not encode text in batches`);	×
148	}	×
149		1✔
150	/**	1✔
151	* Encode data stored in a file (on disk) to another file.	1✔
152	* @note Node.js only. This function enables using command-line converters as "writers".	1✔
153	*/	1✔
154	export async function encodeURLtoURL(	×
155	inputUrl: string,	×
156	outputUrl: string,	×
157	writer: Omit<WriterWithEncoder, 'encode'>,	×
158	options?: WriterOptions	×
159	): Promise<string> {	×
160	inputUrl = resolvePath(inputUrl);	×
161	outputUrl = resolvePath(outputUrl);	×
162	if (isBrowser \|\| !writer.encodeURLtoURL) {	×
163	throw new Error();	×
164	}	×
165	const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);	×
166	return outputFilename;	×
167	}	×
168		1✔
169	/** Helper function to encode via external tool (typically command line execution in Node.js) */	1✔
170	async function encodeWithCommandLineTool(	×
171	writer: WriterWithEncoder,	×
172	data: unknown,	×
173	options: WriterOptions	×
174	): Promise<ArrayBuffer> {	×
175	if (isBrowser) {	×
176	throw new Error(`Writer ${writer.name} not supported in browser`);	×
177	}	×
178	// TODO - how to generate filenames with correct extensions?	×
179	const tmpInputFilename = getTemporaryFilename('input');	×
180	const file = new NodeFile(tmpInputFilename, 'w');	×
181	await file.write(data as ArrayBuffer);	×
182		×
183	const tmpOutputFilename = getTemporaryFilename('output');	×
184		×
185	const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);	×
186		×
187	const response = await fetchFile(outputFilename);	×
188	return response.arrayBuffer();	×
189	}	×
190		1✔
191	/**	1✔
192	* @todo TODO - this is an unacceptable hack!!!	1✔
193	*/	1✔
194	function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {	×
195	const dataIterator = [{...data, start: 0, end: data.length}];	×
196	return dataIterator;	×
197	}	×
198		1✔
199	/**	1✔
200	* @todo Move to utils	1✔
201	*/	1✔
202	function getTemporaryFilename(filename: string): string {	×
203	return `/tmp/${filename}`;	×
204	}	×

visgl / loaders.gl / 20382848403

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous