25798238260

Committed 13 May 2026 12:10PM UTC coverage: 60.607% (+0.3%) from 60.27%

Build # 25798238260

Build Type

push

github

Committed by

web-flow

Commit Message

feat(json) GeoJSON -> geoarrow, schema, logging  (#3399)

Coverage Stats

13466 of 24516 branches covered (54.93%)

Branch coverage included in aggregate %.

448 of 541 new or added lines in 12 files covered. (82.81%)

1264 existing lines in 117 files now uncovered.

27516 of 43103 relevant lines covered (63.84%)

15056.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

25.61

/modules/core/src/lib/api/encode.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {
  WriterOptions,
  WriterWithEncoder,
  WriterOptionsType,
  WriterDataType,
  WriterBatchType
} from '@loaders.gl/loader-utils';
import {
  canEncodeWithWorker,
  NodeFile,
  resolvePath,
  isBrowser,
  ensureArrayBuffer
} from '@loaders.gl/loader-utils';
import {processOnWorker} from '@loaders.gl/worker-utils';
import {fetchFile} from '../fetch/fetch-file';
import {getLoaderOptions} from './loader-options';

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export async function encode<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options_?: WriterOptionsType<WriterT>
): Promise<ArrayBuffer> {
  const globalOptions = getLoaderOptions() as WriterOptions;
  // const globalOptions: WriterOptions = {}; // getWriterOptions();
  const options = {...globalOptions, ...options_};

  // Handle the special case where we are invoking external command-line tools
  if (writer.encodeURLtoURL) {
    return encodeWithCommandLineTool(writer, data, options);
  }

  // Worker support
  if (canEncodeWithWorker(writer, options)) {
    return await processOnWorker(writer, data, options);
  }

  // TODO Merge default writer options with options argument like it is done in load module.
  return await writer.encode(data, options);
}

/**
 * Encode loaded data into a binary ArrayBuffer using the specified Writer.
 */
export function encodeSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): ArrayBuffer {
  if (writer.encodeSync) {
    return writer.encodeSync(data, options);
  }
  if (writer.encodeTextSync) {
    return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));
  }
  throw new Error(`Writer ${writer.name} could not synchronously encode data`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export async function encodeText<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): Promise<string> {
  if (writer.encodeText) {
    return await writer.encodeText(data, options);
  }

  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text) {
    const arrayBuffer = await writer.encode(data, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data to text using the specified Writer
 * @note This is a convenience function not intended for production use on large input data.
 * It is not optimized for performance. Data maybe converted from text to binary and back.
 * @throws if the writer does not generate text output
 */
export function encodeTextSync<WriterT extends WriterWithEncoder>(
  data: WriterDataType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): string {
  if (writer.encodeTextSync) {
    return writer.encodeTextSync(data, options);
  }

  if (writer.text && writer.encodeSync) {
    const arrayBuffer = encodeSync(data, writer, options);
    return new TextDecoder().decode(arrayBuffer);
  }

  throw new Error(`Writer ${writer.name} could not encode data as text`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeInBatches<WriterT extends WriterWithEncoder>(
  data: WriterBatchType<WriterT>,
  writer: WriterT,
  options?: WriterOptionsType<WriterT>
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode in batches`);
}

/**
 * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
 */
export function encodeTextInBatches(
  data: unknown,
  writer: WriterWithEncoder,
  options?: WriterOptions
): AsyncIterable<ArrayBuffer> {
  if (writer.encodeTextInBatches) {
    const dataIterator = getIterator(data);
    // @ts-expect-error
    return writer.encodeTextInBatches(dataIterator, options);
  }
  // TODO -fall back to atomic encode?
  throw new Error(`Writer ${writer.name} could not encode text in batches`);
}

/**
 * Encode data stored in a file (on disk) to another file.
 * @note Node.js only. This function enables using command-line converters as "writers".
 */
export async function encodeURLtoURL(
  inputUrl: string,
  outputUrl: string,
  writer: Omit<WriterWithEncoder, 'encode'>,
  options?: WriterOptions
): Promise<string> {
  inputUrl = resolvePath(inputUrl);
  outputUrl = resolvePath(outputUrl);
  if (isBrowser || !writer.encodeURLtoURL) {
    throw new Error();
  }
  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
  return outputFilename;
}

/** Helper function to encode via external tool (typically command line execution in Node.js) */
async function encodeWithCommandLineTool(
  writer: WriterWithEncoder,
  data: unknown,
  options: WriterOptions
): Promise<ArrayBuffer> {
  if (isBrowser) {
    throw new Error(`Writer ${writer.name} not supported in browser`);
  }
  // TODO - how to generate filenames with correct extensions?
  const tmpInputFilename = getTemporaryFilename('input');
  const file = new NodeFile(tmpInputFilename, 'w');
  await file.write(data as ArrayBuffer);

  const tmpOutputFilename = getTemporaryFilename('output');

  const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);

  const response = await fetchFile(outputFilename);
  return response.arrayBuffer();
}

/**
 * @todo TODO - this is an unacceptable hack!!!
 */
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
  const dataIterator = [{...data, start: 0, end: data.length}];
  return dataIterator;
}

/**
 * @todo Move to utils
 */
function getTemporaryFilename(filename: string): string {
  return `/tmp/${filename}`;
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {
6	WriterOptions,
7	WriterWithEncoder,
8	WriterOptionsType,
9	WriterDataType,
10	WriterBatchType
11	} from '@loaders.gl/loader-utils';
12	import {
13	canEncodeWithWorker,
14	NodeFile,
15	resolvePath,
16	isBrowser,
17	ensureArrayBuffer
18	} from '@loaders.gl/loader-utils';
19	import {processOnWorker} from '@loaders.gl/worker-utils';
20	import {fetchFile} from '../fetch/fetch-file';
21	import {getLoaderOptions} from './loader-options';
22
23	/**
24	* Encode loaded data into a binary ArrayBuffer using the specified Writer.
25	*/
26	export async function encode<WriterT extends WriterWithEncoder>(
27	data: WriterDataType<WriterT>,
28	writer: WriterT,
29	options_?: WriterOptionsType<WriterT>
30	): Promise<ArrayBuffer> {
31	const globalOptions = getLoaderOptions() as WriterOptions;	55✔
32	// const globalOptions: WriterOptions = {}; // getWriterOptions();
33	const options = {...globalOptions, ...options_};	55✔
34
35	// Handle the special case where we are invoking external command-line tools
36	if (writer.encodeURLtoURL) {	55!
37	return encodeWithCommandLineTool(writer, data, options);	×
38	}
39
40	// Worker support
41	if (canEncodeWithWorker(writer, options)) {	55!
42	return await processOnWorker(writer, data, options);	×
43	}
44
45	// TODO Merge default writer options with options argument like it is done in load module.
46	return await writer.encode(data, options);	55✔
47	}
48
49	/**
50	* Encode loaded data into a binary ArrayBuffer using the specified Writer.
51	*/
52	export function encodeSync<WriterT extends WriterWithEncoder>(
53	data: WriterDataType<WriterT>,
54	writer: WriterT,
55	options?: WriterOptionsType<WriterT>
56	): ArrayBuffer {
57	if (writer.encodeSync) {	120!
58	return writer.encodeSync(data, options);	120✔
59	}
60	if (writer.encodeTextSync) {	×
61	return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));	×
62	}
63	throw new Error(`Writer ${writer.name} could not synchronously encode data`);	×
64	}
65
66	/**
67	* Encode loaded data to text using the specified Writer
68	* @note This is a convenience function not intended for production use on large input data.
69	* It is not optimized for performance. Data maybe converted from text to binary and back.
70	* @throws if the writer does not generate text output
71	*/
72	export async function encodeText<WriterT extends WriterWithEncoder>(
73	data: WriterDataType<WriterT>,
74	writer: WriterT,
75	options?: WriterOptionsType<WriterT>
76	): Promise<string> {
77	if (writer.encodeText) {	×
78	return await writer.encodeText(data, options);	×
79	}
80
81	if (writer.encodeTextSync) {	×
82	return writer.encodeTextSync(data, options);	×
83	}
84
85	if (writer.text) {	×
86	const arrayBuffer = await writer.encode(data, options);	×
87	return new TextDecoder().decode(arrayBuffer);	×
88	}
89
90	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
91	}
92
93	/**
94	* Encode loaded data to text using the specified Writer
95	* @note This is a convenience function not intended for production use on large input data.
96	* It is not optimized for performance. Data maybe converted from text to binary and back.
97	* @throws if the writer does not generate text output
98	*/
99	export function encodeTextSync<WriterT extends WriterWithEncoder>(
100	data: WriterDataType<WriterT>,
101	writer: WriterT,
102	options?: WriterOptionsType<WriterT>
103	): string {
104	if (writer.encodeTextSync) {	12!
105	return writer.encodeTextSync(data, options);	12✔
106	}
107
108	if (writer.text && writer.encodeSync) {	×
109	const arrayBuffer = encodeSync(data, writer, options);	×
110	return new TextDecoder().decode(arrayBuffer);	×
111	}
112
113	throw new Error(`Writer ${writer.name} could not encode data as text`);	×
114	}
115
116	/**
117	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
118	*/
119	export function encodeInBatches<WriterT extends WriterWithEncoder>(
120	data: WriterBatchType<WriterT>,
121	writer: WriterT,
122	options?: WriterOptionsType<WriterT>
123	): AsyncIterable<ArrayBuffer> {
124	if (writer.encodeInBatches) {	×
125	const dataIterator = getIterator(data);	×
126	// @ts-expect-error
127	return writer.encodeInBatches(dataIterator, options);	×
128	}
129	// TODO -fall back to atomic encode?
130	throw new Error(`Writer ${writer.name} could not encode in batches`);	×
131	}
132
133	/**
134	* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
135	*/
136	export function encodeTextInBatches(
137	data: unknown,
138	writer: WriterWithEncoder,
139	options?: WriterOptions
140	): AsyncIterable<ArrayBuffer> {
141	if (writer.encodeTextInBatches) {	×
142	const dataIterator = getIterator(data);	×
143	// @ts-expect-error
144	return writer.encodeTextInBatches(dataIterator, options);	×
145	}
146	// TODO -fall back to atomic encode?
147	throw new Error(`Writer ${writer.name} could not encode text in batches`);	×
148	}
149
150	/**
151	* Encode data stored in a file (on disk) to another file.
152	* @note Node.js only. This function enables using command-line converters as "writers".
153	*/
154	export async function encodeURLtoURL(
155	inputUrl: string,
156	outputUrl: string,
157	writer: Omit<WriterWithEncoder, 'encode'>,
158	options?: WriterOptions
159	): Promise<string> {
UNCOV 160	inputUrl = resolvePath(inputUrl);	1✔
UNCOV 161	outputUrl = resolvePath(outputUrl);	1✔
UNCOV 162	if (isBrowser \|\| !writer.encodeURLtoURL) {	1!
163	throw new Error();	×
164	}
UNCOV 165	const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);	1✔
UNCOV 166	return outputFilename;	1✔
167	}
168
169	/** Helper function to encode via external tool (typically command line execution in Node.js) */
170	async function encodeWithCommandLineTool(
171	writer: WriterWithEncoder,
172	data: unknown,
173	options: WriterOptions
174	): Promise<ArrayBuffer> {
175	if (isBrowser) {	×
176	throw new Error(`Writer ${writer.name} not supported in browser`);	×
177	}
178	// TODO - how to generate filenames with correct extensions?
179	const tmpInputFilename = getTemporaryFilename('input');	×
180	const file = new NodeFile(tmpInputFilename, 'w');	×
181	await file.write(data as ArrayBuffer);	×
182
183	const tmpOutputFilename = getTemporaryFilename('output');	×
184
185	const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);	×
186
187	const response = await fetchFile(outputFilename);	×
188	return response.arrayBuffer();	×
189	}
190
191	/**
192	* @todo TODO - this is an unacceptable hack!!!
193	*/
194	function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
195	const dataIterator = [{...data, start: 0, end: data.length}];	×
196	return dataIterator;	×
197	}
198
199	/**
200	* @todo Move to utils
201	*/
202	function getTemporaryFilename(filename: string): string {
203	return `/tmp/${filename}`;	×
204	}

visgl / loaders.gl / 25798238260

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous