24369398061

Committed 13 Apr 2026 10:07PM UTC coverage: 55.756% (+0.004%) from 55.752%

Build # 24369398061

Build Type

push

github

Committed by

web-flow

Commit Message

chore: Warn if core is imported by loader module (#3379)

Coverage Stats

9478 of 18401 branches covered (51.51%)

Branch coverage included in aggregate %.

85 of 136 new or added lines in 33 files covered. (62.5%)

4 existing lines in 4 files now uncovered.

19703 of 33936 relevant lines covered (58.06%)

4979.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.15

/modules/loader-utils/src/lib/sources/data-source.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Loader, LoaderContext, LoaderOptions, StrictLoaderOptions} from '../../loader-types';
import type {BatchableDataType, DataType, SyncDataType} from '../../types';
import type {RequiredOptions} from '../option-utils/merge-options';
import {mergeOptions} from '../option-utils/merge-options';
import {resolvePath} from '../path-utils/file-aliases';
import {log} from '../log-utils/log';

/** Common properties for all data sources */
export type DataSourceOptions = Partial<{
  core: {
    /** Allows application to specify which source should be selected. Matches `Source.type`. Defaults to 'auto' */
    type?: string;
    /** Any dataset attributions (in case underlying metadata does not include attributions) */
    attributions?: string[];
    /** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */
    loadOptions?: StrictLoaderOptions;
    /** Make additional loaders available to the data source */
    loaders?: Loader[];
    /** Called when source-level initialization or metadata loading fails. */
    onError?: (error: Error, source: DataSource<any, any>) => void;
  };
  [key: string]: Record<string, unknown>;
}>;

/** Runtime hooks injected when a DataSource is created through an integration layer such as `@loaders.gl/core`. */
export type CoreAPI = Readonly<{
  fetchFile: (urlOrData: string | Blob, fetchOptions?: RequestInit) => Promise<Response>;
  parse: (
    data: DataType | Promise<DataType>,
    loaders?: Loader | Loader[] | LoaderOptions,
    options?: LoaderOptions,
    context?: LoaderContext
  ) => Promise<unknown>;
  parseSync: (
    data: SyncDataType,
    loaders?: Loader | Loader[] | LoaderOptions,
    options?: LoaderOptions,
    context?: LoaderContext
  ) => unknown;
  parseInBatches: (
    data: BatchableDataType,
    loaders?: Loader | Loader[] | LoaderOptions,
    options?: LoaderOptions,
    context?: LoaderContext
  ) => Promise<AsyncIterable<unknown> | Iterable<unknown>>;
  load: (
    url: string | DataType,
    loaders?: Loader[] | LoaderOptions | Loader,
    options?: LoaderOptions | LoaderContext,
    context?: LoaderContext
  ) => Promise<unknown>;
  loadInBatches: (
    files: string | File | Blob | Response | (string | File | Blob | Response)[] | FileList,
    loaders?: Loader[] | LoaderOptions | Loader,
    options?: LoaderOptions,
    context?: LoaderContext
  ) => Promise<AsyncIterable<unknown>> | Promise<AsyncIterable<unknown>>[];
}>;

const UNAVAILABLE_CORE_API: CoreAPI = {
  fetchFile: unavailableCoreApiMethod('fetchFile'),
  parse: unavailableCoreApiMethod('parse'),
  parseSync: unavailableCoreApiMethod('parseSync'),
  parseInBatches: unavailableCoreApiMethod('parseInBatches'),
  load: unavailableCoreApiMethod('load'),
  loadInBatches: unavailableCoreApiMethod('loadInBatches')
};

/** base class of all data sources */
export abstract class DataSource<DataT, OptionsT extends DataSourceOptions> {
  static defaultOptions: Required<DataSourceOptions> = {
    core: {
      type: 'auto',
      attributions: [],
      loadOptions: {},
      loaders: [],
      onError: undefined!
    }
  };

  optionsType?: OptionsT & DataSourceOptions;
  options: Required<OptionsT & DataSourceOptions>;
  readonly data: DataT;
  readonly url: string;

  /** The actual load options, if calling a loaders.gl loader */
  loadOptions: StrictLoaderOptions;
  /** A resolved fetch function extracted from loadOptions prop */
  fetch: (url: string, options?: RequestInit) => Promise<Response>;
  /** Shared source-level runtime hooks, when supplied by the source factory. */
  readonly coreApi: CoreAPI;
  /** Whether a real CoreAPI instance was injected by the integration layer. */
  readonly hasCoreApi: boolean;
  _needsRefresh: boolean = true;

  constructor(
    data: DataT,
    options: OptionsT,
    defaultOptions?: Omit<RequiredOptions<OptionsT>, 'core'>,
    coreApi?: CoreAPI
  ) {
    if (defaultOptions) {
      // @ts-expect-error Typescript gets confused
      this.options = mergeOptions({...defaultOptions, core: DataSource.defaultOptions}, options);
    } else {
      // @ts-expect-error
      this.options = {...options};
    }
    this.data = data;
    this.url = typeof data === 'string' ? resolvePath(data) : '';
    const loadOptions = normalizeDirectLoaderOptions(this.options.core?.loadOptions);
    this.loadOptions = loadOptions;
    const fetch = getFetchFunction(loadOptions);
    this.coreApi = coreApi || UNAVAILABLE_CORE_API;
    this.hasCoreApi = Boolean(coreApi);
    this.fetch = fetch;
  }

  setProps(options: OptionsT) {
    this.options = Object.assign(this.options, options);
    // TODO - add a shallow compare to avoid setting refresh if no change?
    this.setNeedsRefresh();
  }

  /** Mark this data source as needing a refresh (redraw) */
  setNeedsRefresh(): void {
    this._needsRefresh = true;
  }

  /**
   * Does this data source need refreshing?
   * @note The specifics of the refresh mechanism depends on type of data source
   */
  getNeedsRefresh(clear: boolean = true) {
    const needsRefresh = this._needsRefresh;
    if (clear) {
      this._needsRefresh = false;
    }
    return needsRefresh;
  }

  /** Reports a source-level failure through the configured callback or the shared logger. */
  protected reportError(error: unknown, message: string): Error {
    const normalizedError = normalizeError(error, message);
    const callback = this.options.core?.onError;
    if (callback) {
      callback(normalizedError, this);
    } else {
      log.warn(`${this.constructor.name}: ${normalizedError.message}`)();
    }
    return normalizedError;
  }
}

/**
 * Gets the current fetch function from options
 * @todo - move to loader-utils module
 * @todo - use in core module counterpart
 * @param options
 * @param context
 */
export function getFetchFunction(options?: StrictLoaderOptions) {
  const fetchFunction = options?.core?.fetch;

  // options.fetch can be a function
  if (fetchFunction && typeof fetchFunction === 'function') {
    return (url: string, fetchOptions?: RequestInit) => fetchFunction(url, fetchOptions);
  }

  // options.fetch can be an options object, use global fetch with those options
  const fetchOptions = options?.fetch;
  if (fetchOptions && typeof fetchOptions !== 'function') {
    return (url, requestOptions) => fetch(url, mergeFetchOptions(fetchOptions, requestOptions));
  }

  // else return the global fetch function
  return (url, requestOptions) => fetch(url, requestOptions);
}

function mergeFetchOptions(fetchOptions: RequestInit, requestOptions?: RequestInit): RequestInit {
  const mergedOptions: RequestInit = {...fetchOptions, ...requestOptions};
  if (fetchOptions.headers || requestOptions?.headers) {
    mergedOptions.headers = mergeHeaders(fetchOptions.headers, requestOptions?.headers);
  }
  return mergedOptions;
}

function mergeHeaders(defaultHeaders?: HeadersInit, requestHeaders?: HeadersInit): Headers {
  const headers = new Headers(defaultHeaders);
  if (requestHeaders) {
    new Headers(requestHeaders).forEach((value, key) => headers.set(key, value));
  }
  return headers;
}

function normalizeDirectLoaderOptions(options?: StrictLoaderOptions): StrictLoaderOptions {
  const loadOptions = {...options};
  if (options?.core) {
    loadOptions.core = {...options.core};
  }

  const topLevelBaseUri = typeof loadOptions.baseUri === 'string' ? loadOptions.baseUri : undefined;
  const topLevelBaseUrl = typeof loadOptions.baseUrl === 'string' ? loadOptions.baseUrl : undefined;

  if (topLevelBaseUri !== undefined || topLevelBaseUrl !== undefined) {
    loadOptions.core ||= {};
    if (loadOptions.core.baseUrl === undefined) {
      loadOptions.core.baseUrl = topLevelBaseUrl ?? topLevelBaseUri;
    }
    delete loadOptions.baseUri;
    delete loadOptions.baseUrl;
  }

  return loadOptions;
}

/** Normalizes arbitrary thrown values to `Error` instances for source-level reporting. */
function normalizeError(error: unknown, message: string): Error {
  if (error instanceof Error) {
    return error;
  }
  if (typeof error === 'string') {
    return new Error(error);
  }
  return new Error(message);
}

function unavailableCoreApiMethod(methodName: keyof CoreAPI) {
  return () => {
    throw new Error(
      `CoreAPI.${methodName} is unavailable. Use @loaders.gl/core.createDataSource().`
    );
  };
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {Loader, LoaderContext, LoaderOptions, StrictLoaderOptions} from '../../loader-types';
6	import type {BatchableDataType, DataType, SyncDataType} from '../../types';
7	import type {RequiredOptions} from '../option-utils/merge-options';
8	import {mergeOptions} from '../option-utils/merge-options';
9	import {resolvePath} from '../path-utils/file-aliases';
10	import {log} from '../log-utils/log';
11
12	/** Common properties for all data sources */
13	export type DataSourceOptions = Partial<{
14	core: {
15	/** Allows application to specify which source should be selected. Matches `Source.type`. Defaults to 'auto' */
16	type?: string;
17	/** Any dataset attributions (in case underlying metadata does not include attributions) */
18	attributions?: string[];
19	/** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */
20	loadOptions?: StrictLoaderOptions;
21	/** Make additional loaders available to the data source */
22	loaders?: Loader[];
23	/** Called when source-level initialization or metadata loading fails. */
24	onError?: (error: Error, source: DataSource<any, any>) => void;
25	};
26	[key: string]: Record<string, unknown>;
27	}>;
28
29	/** Runtime hooks injected when a DataSource is created through an integration layer such as `@loaders.gl/core`. */
30	export type CoreAPI = Readonly<{
31	fetchFile: (urlOrData: string \| Blob, fetchOptions?: RequestInit) => Promise<Response>;
32	parse: (
33	data: DataType \| Promise<DataType>,
34	loaders?: Loader \| Loader[] \| LoaderOptions,
35	options?: LoaderOptions,
36	context?: LoaderContext
37	) => Promise<unknown>;
38	parseSync: (
39	data: SyncDataType,
40	loaders?: Loader \| Loader[] \| LoaderOptions,
41	options?: LoaderOptions,
42	context?: LoaderContext
43	) => unknown;
44	parseInBatches: (
45	data: BatchableDataType,
46	loaders?: Loader \| Loader[] \| LoaderOptions,
47	options?: LoaderOptions,
48	context?: LoaderContext
49	) => Promise<AsyncIterable<unknown> \| Iterable<unknown>>;
50	load: (
51	url: string \| DataType,
52	loaders?: Loader[] \| LoaderOptions \| Loader,
53	options?: LoaderOptions \| LoaderContext,
54	context?: LoaderContext
55	) => Promise<unknown>;
56	loadInBatches: (
57	files: string \| File \| Blob \| Response \| (string \| File \| Blob \| Response)[] \| FileList,
58	loaders?: Loader[] \| LoaderOptions \| Loader,
59	options?: LoaderOptions,
60	context?: LoaderContext
61	) => Promise<AsyncIterable<unknown>> \| Promise<AsyncIterable<unknown>>[];
62	}>;
63
64	const UNAVAILABLE_CORE_API: CoreAPI = {	403✔
65	fetchFile: unavailableCoreApiMethod('fetchFile'),
66	parse: unavailableCoreApiMethod('parse'),
67	parseSync: unavailableCoreApiMethod('parseSync'),
68	parseInBatches: unavailableCoreApiMethod('parseInBatches'),
69	load: unavailableCoreApiMethod('load'),
70	loadInBatches: unavailableCoreApiMethod('loadInBatches')
71	};
72
73	/** base class of all data sources */
74	export abstract class DataSource<DataT, OptionsT extends DataSourceOptions> {
75	static defaultOptions: Required<DataSourceOptions> = {	403✔
76	core: {
77	type: 'auto',
78	attributions: [],
79	loadOptions: {},
80	loaders: [],
81	onError: undefined!
82	}
83	};
84
85	optionsType?: OptionsT & DataSourceOptions;
86	options: Required<OptionsT & DataSourceOptions>;
87	readonly data: DataT;
88	readonly url: string;
89
90	/** The actual load options, if calling a loaders.gl loader */
91	loadOptions: StrictLoaderOptions;
92	/** A resolved fetch function extracted from loadOptions prop */
93	fetch: (url: string, options?: RequestInit) => Promise<Response>;
94	/** Shared source-level runtime hooks, when supplied by the source factory. */
95	readonly coreApi: CoreAPI;
96	/** Whether a real CoreAPI instance was injected by the integration layer. */
97	readonly hasCoreApi: boolean;
98	_needsRefresh: boolean = true;	41✔
99
100	constructor(
101	data: DataT,
102	options: OptionsT,
103	defaultOptions?: Omit<RequiredOptions<OptionsT>, 'core'>,
104	coreApi?: CoreAPI
105	) {
106	if (defaultOptions) {	41✔
107	// @ts-expect-error Typescript gets confused
108	this.options = mergeOptions({...defaultOptions, core: DataSource.defaultOptions}, options);	36✔
109	} else {
110	// @ts-expect-error
111	this.options = {...options};	5✔
112	}
113	this.data = data;	41✔
114	this.url = typeof data === 'string' ? resolvePath(data) : '';	41✔
115	const loadOptions = normalizeDirectLoaderOptions(this.options.core?.loadOptions);	41✔
116	this.loadOptions = loadOptions;	41✔
117	const fetch = getFetchFunction(loadOptions);	41✔
118	this.coreApi = coreApi \|\| UNAVAILABLE_CORE_API;	41✔
119	this.hasCoreApi = Boolean(coreApi);	41✔
120	this.fetch = fetch;	41✔
121	}
122
123	setProps(options: OptionsT) {
124	this.options = Object.assign(this.options, options);	×
125	// TODO - add a shallow compare to avoid setting refresh if no change?
126	this.setNeedsRefresh();	×
127	}
128
129	/** Mark this data source as needing a refresh (redraw) */
130	setNeedsRefresh(): void {
131	this._needsRefresh = true;	×
132	}
133
134	/**
135	* Does this data source need refreshing?
136	* @note The specifics of the refresh mechanism depends on type of data source
137	*/
138	getNeedsRefresh(clear: boolean = true) {	×
139	const needsRefresh = this._needsRefresh;	×
140	if (clear) {	×
141	this._needsRefresh = false;	×
142	}
143	return needsRefresh;	×
144	}
145
146	/** Reports a source-level failure through the configured callback or the shared logger. */
147	protected reportError(error: unknown, message: string): Error {
148	const normalizedError = normalizeError(error, message);	×
149	const callback = this.options.core?.onError;	×
150	if (callback) {	×
151	callback(normalizedError, this);	×
152	} else {
153	log.warn(`${this.constructor.name}: ${normalizedError.message}`)();	×
154	}
155	return normalizedError;	×
156	}
157	}
158
159	/**
160	* Gets the current fetch function from options
161	* @todo - move to loader-utils module
162	* @todo - use in core module counterpart
163	* @param options
164	* @param context
165	*/
166	export function getFetchFunction(options?: StrictLoaderOptions) {
167	const fetchFunction = options?.core?.fetch;	41✔
168
169	// options.fetch can be a function
170	if (fetchFunction && typeof fetchFunction === 'function') {	41!
171	return (url: string, fetchOptions?: RequestInit) => fetchFunction(url, fetchOptions);	×
172	}
173
174	// options.fetch can be an options object, use global fetch with those options
175	const fetchOptions = options?.fetch;	41✔
176	if (fetchOptions && typeof fetchOptions !== 'function') {	41✔
177	return (url, requestOptions) => fetch(url, mergeFetchOptions(fetchOptions, requestOptions));	1✔
178	}
179
180	// else return the global fetch function
181	return (url, requestOptions) => fetch(url, requestOptions);	40✔
182	}
183
184	function mergeFetchOptions(fetchOptions: RequestInit, requestOptions?: RequestInit): RequestInit {
185	const mergedOptions: RequestInit = {...fetchOptions, ...requestOptions};	1✔
186	if (fetchOptions.headers \|\| requestOptions?.headers) {	1!
187	mergedOptions.headers = mergeHeaders(fetchOptions.headers, requestOptions?.headers);	1✔
188	}
189	return mergedOptions;	1✔
190	}
191
192	function mergeHeaders(defaultHeaders?: HeadersInit, requestHeaders?: HeadersInit): Headers {
193	const headers = new Headers(defaultHeaders);	1✔
194	if (requestHeaders) {	1!
195	new Headers(requestHeaders).forEach((value, key) => headers.set(key, value));	×
196	}
197	return headers;	1✔
198	}
199
200	function normalizeDirectLoaderOptions(options?: StrictLoaderOptions): StrictLoaderOptions {
201	const loadOptions = {...options};	41✔
202	if (options?.core) {	41!
203	loadOptions.core = {...options.core};	×
204	}
205
206	const topLevelBaseUri = typeof loadOptions.baseUri === 'string' ? loadOptions.baseUri : undefined;	41✔
207	const topLevelBaseUrl = typeof loadOptions.baseUrl === 'string' ? loadOptions.baseUrl : undefined;	41✔
208
209	if (topLevelBaseUri !== undefined \|\| topLevelBaseUrl !== undefined) {	41✔
210	loadOptions.core \|\|= {};	4✔
211	if (loadOptions.core.baseUrl === undefined) {	4!
212	loadOptions.core.baseUrl = topLevelBaseUrl ?? topLevelBaseUri;	4✔
213	}
214	delete loadOptions.baseUri;	4✔
215	delete loadOptions.baseUrl;	4✔
216	}
217
218	return loadOptions;	41✔
219	}
220
221	/** Normalizes arbitrary thrown values to `Error` instances for source-level reporting. */
222	function normalizeError(error: unknown, message: string): Error {
223	if (error instanceof Error) {	×
224	return error;	×
225	}
226	if (typeof error === 'string') {	×
227	return new Error(error);	×
228	}
229	return new Error(message);	×
230	}
231
232	function unavailableCoreApiMethod(methodName: keyof CoreAPI) {
233	return () => {	2,418✔
NEW 234	throw new Error(	×
235	`CoreAPI.${methodName} is unavailable. Use @loaders.gl/core.createDataSource().`
236	);
237	};
238	}

visgl / loaders.gl / 24369398061

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous