• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

visgl / loaders.gl / 20382848403

19 Dec 2025 09:20PM UTC coverage: 35.219% (+0.1%) from 35.095%
20382848403

push

github

web-flow
feat: Upgrade to handle ArrayBufferLike (#3271)

1190 of 2002 branches covered (59.44%)

Branch coverage included in aggregate %.

157 of 269 new or added lines in 41 files covered. (58.36%)

3 existing lines in 3 files now uncovered.

37536 of 107957 relevant lines covered (34.77%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.95
/modules/core/src/lib/api/select-loader.ts
1
// loaders.gl
1✔
2
// SPDX-License-Identifier: MIT
1✔
3
// Copyright (c) vis.gl contributors
1✔
4

1✔
5
import type {LoaderContext, LoaderOptions, Loader, DataType} from '@loaders.gl/loader-utils';
1✔
6
import {
1✔
7
  compareArrayBuffers,
1✔
8
  path,
1✔
9
  log,
1✔
10
  isBlob,
1✔
11
  ensureArrayBuffer,
1✔
12
  isArrayBufferLike
1✔
13
} from '@loaders.gl/loader-utils';
1✔
14
import {TypedArray} from '@loaders.gl/schema';
1✔
15
import {normalizeLoader} from '../loader-utils/normalize-loader';
1✔
16
import {normalizeLoaderOptions} from '../loader-utils/option-utils';
1✔
17
import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
1✔
18
import {compareMIMETypes} from '../utils/mime-type-utils';
1✔
19
import {getRegisteredLoaders} from './register-loaders';
1✔
20
import {stripQueryString} from '../utils/url-utils';
1✔
21

1✔
22
const EXT_PATTERN = /\.([^.]+)$/;
1✔
23

1✔
24
// TODO - Need a variant that peeks at streams for parseInBatches
1✔
25
// TODO - Detect multiple matching loaders? Use heuristics to grade matches?
1✔
26
// TODO - Allow apps to pass context to disambiguate between multiple matches (e.g. multiple .json formats)?
1✔
27

1✔
28
/**
1✔
29
 * Find a loader that matches file extension and/or initial file content
1✔
30
 * Search the loaders array argument for a loader that matches url extension or initial data
1✔
31
 * Returns: a normalized loader
1✔
32
 * @param data data to assist
1✔
33
 * @param loaders
1✔
34
 * @param options
1✔
35
 * @param context used internally, applications should not provide this parameter
1✔
36
 */
1✔
37
export async function selectLoader(
2✔
38
  data: DataType,
2✔
39
  loaders: Loader[] | Loader = [],
2✔
40
  options?: LoaderOptions,
2✔
41
  context?: LoaderContext
2✔
42
): Promise<Loader | null> {
2✔
43
  if (!validHTTPResponse(data)) {
2!
44
    return null;
×
45
  }
×
46

2✔
47
  const normalizedOptions = normalizeLoaderOptions(options || {});
2!
48
  normalizedOptions.core ||= {};
2✔
49

2✔
50
  // First make a sync attempt, disabling exceptions
2✔
51
  let loader = selectLoaderSync(
2✔
52
    data,
2✔
53
    loaders,
2✔
54
    {...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
2✔
55
    context
2✔
56
  );
2✔
57
  if (loader) {
2✔
58
    return loader;
2✔
59
  }
2!
60

×
61
  // For Blobs and Files, try to asynchronously read a small initial slice and test again with that
×
62
  // to see if we can detect by initial content
×
63
  if (isBlob(data)) {
×
64
    data = await data.slice(0, 10).arrayBuffer();
×
65
    loader = selectLoaderSync(data, loaders, normalizedOptions, context);
×
66
  }
×
67

×
68
  // no loader available
×
69
  if (!loader && !normalizedOptions.core.nothrow) {
2!
70
    throw new Error(getNoValidLoaderMessage(data));
×
71
  }
×
72

×
73
  return loader;
×
74
}
×
75

1✔
76
/**
1✔
77
 * Find a loader that matches file extension and/or initial file content
1✔
78
 * Search the loaders array argument for a loader that matches url extension or initial data
1✔
79
 * Returns: a normalized loader
1✔
80
 * @param data data to assist
1✔
81
 * @param loaders
1✔
82
 * @param options
1✔
83
 * @param context used internally, applications should not provide this parameter
1✔
84
 */
1✔
85
export function selectLoaderSync(
1✔
86
  data: DataType,
2✔
87
  loaders: Loader[] | Loader = [],
2✔
88
  options?: LoaderOptions,
2✔
89
  context?: LoaderContext
2✔
90
): Loader | null {
2✔
91
  if (!validHTTPResponse(data)) {
2!
92
    return null;
×
93
  }
×
94

2✔
95
  const normalizedOptions = normalizeLoaderOptions(options || {});
2!
96
  normalizedOptions.core ||= {};
2✔
97

2✔
98
  // eslint-disable-next-line complexity
2✔
99
  // if only a single loader was provided (not as array), force its use
2✔
100
  // TODO - Should this behavior be kept and documented?
2✔
101
  if (loaders && !Array.isArray(loaders)) {
2✔
102
    // TODO - remove support for legacy loaders
2✔
103
    return normalizeLoader(loaders);
2✔
104
  }
2!
105

×
106
  // Build list of candidate loaders that will be searched in order for a match
×
107
  let candidateLoaders: Loader[] = [];
×
108
  // First search supplied loaders
×
109
  if (loaders) {
×
110
    candidateLoaders = candidateLoaders.concat(loaders);
×
111
  }
×
112
  // Then fall back to registered loaders
×
113
  if (!normalizedOptions.core.ignoreRegisteredLoaders) {
×
114
    candidateLoaders.push(...getRegisteredLoaders());
×
115
  }
×
116

×
117
  // TODO - remove support for legacy loaders
×
118
  normalizeLoaders(candidateLoaders);
×
119

×
120
  const loader = selectLoaderInternal(data, candidateLoaders, normalizedOptions, context);
×
121

×
122
  // no loader available
×
123
  if (!loader && !normalizedOptions.core.nothrow) {
2!
124
    throw new Error(getNoValidLoaderMessage(data));
×
125
  }
×
126

×
127
  return loader;
×
128
}
×
129

1✔
130
/** Implements loaders selection logic */
1✔
131
// eslint-disable-next-line complexity
1✔
132
function selectLoaderInternal(
×
133
  data: DataType,
×
134
  loaders: Loader[],
×
135
  options?: LoaderOptions,
×
136
  context?: LoaderContext
×
137
) {
×
138
  const url = getResourceUrl(data);
×
139
  const type = getResourceMIMEType(data);
×
140

×
141
  const testUrl = stripQueryString(url) || context?.url;
×
142

×
143
  let loader: Loader | null = null;
×
144
  let reason: string = '';
×
145

×
146
  // if options.mimeType is supplied, it takes precedence
×
147
  if (options?.core?.mimeType) {
×
148
    loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
×
149
    reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
×
150
  }
×
151

×
152
  // Look up loader by url
×
153
  loader = loader || findLoaderByUrl(loaders, testUrl);
×
154
  reason = reason || (loader ? `matched url ${testUrl}` : '');
×
155

×
156
  // Look up loader by mime type
×
157
  loader = loader || findLoaderByMIMEType(loaders, type);
×
158
  reason = reason || (loader ? `matched MIME type ${type}` : '');
×
159

×
160
  // Look for loader via initial bytes (Note: not always accessible (e.g. Response, stream, async iterator)
×
161
  // @ts-ignore Blob | Response
×
162
  loader = loader || findLoaderByInitialBytes(loaders, data);
×
163
  // @ts-ignore Blob | Response
×
164
  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
×
165

×
166
  // Look up loader by fallback mime type
×
167
  if (options?.core?.fallbackMimeType) {
×
168
    loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
×
169
    reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
×
170
  }
×
171

×
172
  if (reason) {
×
173
    log.log(1, `selectLoader selected ${loader?.name}: ${reason}.`);
×
174
  }
×
175

×
176
  return loader;
×
177
}
×
178

1✔
179
/** Check HTTP Response */
1✔
180
function validHTTPResponse(data: unknown): boolean {
4✔
181
  // HANDLE HTTP status
4✔
182
  if (data instanceof Response) {
4!
183
    // 204 - NO CONTENT. This handles cases where e.g. a tile server responds with 204 for a missing tile
×
184
    if (data.status === 204) {
×
185
      return false;
×
186
    }
×
187
  }
×
188
  return true;
4✔
189
}
4✔
190

1✔
191
/** Generate a helpful message to help explain why loader selection failed. */
1✔
192
function getNoValidLoaderMessage(data: DataType): string {
×
193
  const url = getResourceUrl(data);
×
194
  const type = getResourceMIMEType(data);
×
195

×
196
  let message = 'No valid loader found (';
×
197
  message += url ? `${path.filename(url)}, ` : 'no url provided, ';
×
198
  message += `MIME type: ${type ? `"${type}"` : 'not provided'}, `;
×
199
  // First characters are only accessible when called on data (string or arrayBuffer).
×
200
  // @ts-ignore Blob | Response
×
201
  const firstCharacters: string = data ? getFirstCharacters(data) : '';
×
202
  message += firstCharacters ? ` first bytes: "${firstCharacters}"` : 'first bytes: not available';
×
203
  message += ')';
×
204
  return message;
×
205
}
×
206

1✔
207
function normalizeLoaders(loaders: Loader[]): void {
×
208
  for (const loader of loaders) {
×
209
    normalizeLoader(loader);
×
210
  }
×
211
}
×
212

1✔
213
// TODO - Would be nice to support http://example.com/file.glb?parameter=1
1✔
214
// E.g: x = new URL('http://example.com/file.glb?load=1'; x.pathname
1✔
215
function findLoaderByUrl(loaders: Loader[], url?: string): Loader | null {
×
216
  // Get extension
×
217
  const match = url && EXT_PATTERN.exec(url);
×
218
  const extension = match && match[1];
×
219
  return extension ? findLoaderByExtension(loaders, extension) : null;
×
220
}
×
221

1✔
222
function findLoaderByExtension(loaders: Loader[], extension: string): Loader | null {
×
223
  extension = extension.toLowerCase();
×
224

×
225
  for (const loader of loaders) {
×
226
    for (const loaderExtension of loader.extensions) {
×
227
      if (loaderExtension.toLowerCase() === extension) {
×
228
        return loader;
×
229
      }
×
230
    }
×
231
  }
×
232
  return null;
×
233
}
×
234

1✔
235
function findLoaderByMIMEType(loaders: Loader[], mimeType: string): Loader | null {
×
236
  for (const loader of loaders) {
×
237
    if (loader.mimeTypes?.some((mimeType1) => compareMIMETypes(mimeType, mimeType1))) {
×
238
      return loader;
×
239
    }
×
240

×
241
    // Support referring to loaders using the "unregistered tree"
×
242
    // https://en.wikipedia.org/wiki/Media_type#Unregistered_tree
×
243
    if (compareMIMETypes(mimeType, `application/x.${loader.id}`)) {
×
244
      return loader;
×
245
    }
×
246
  }
×
247
  return null;
×
248
}
×
249

1✔
250
function findLoaderByInitialBytes(loaders: Loader[], data: string | ArrayBuffer): Loader | null {
×
251
  if (!data) {
×
252
    return null;
×
253
  }
×
254

×
255
  for (const loader of loaders) {
×
256
    if (typeof data === 'string') {
×
257
      if (testDataAgainstText(data, loader)) {
×
258
        return loader;
×
259
      }
×
260
    } else if (ArrayBuffer.isView(data)) {
×
261
      // Typed Arrays can have offsets into underlying buffer
×
262
      if (testDataAgainstBinary(data.buffer, data.byteOffset, loader)) {
×
263
        return loader;
×
264
      }
×
265
    } else if (data instanceof ArrayBuffer) {
×
266
      const byteOffset = 0;
×
267
      if (testDataAgainstBinary(data, byteOffset, loader)) {
×
268
        return loader;
×
269
      }
×
270
    }
×
271
    // TODO Handle streaming case (requires creating a new AsyncIterator)
×
272
  }
×
273
  return null;
×
274
}
×
275

1✔
276
function testDataAgainstText(data: string, loader: Loader): boolean {
×
277
  if (loader.testText) {
×
278
    return loader.testText(data);
×
279
  }
×
280

×
281
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
×
282
  return tests.some((test) => data.startsWith(test as string));
×
283
}
×
284

1✔
NEW
285
function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
×
286
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
×
287
  return tests.some((test) => testBinary(data, byteOffset, loader, test));
×
288
}
×
289

1✔
290
function testBinary(
×
NEW
291
  data: ArrayBufferLike,
×
292
  byteOffset: number,
×
293
  loader: Loader,
×
294
  test?: ArrayBuffer | string | ((b: ArrayBuffer) => boolean)
×
295
): boolean {
×
NEW
296
  if (isArrayBufferLike(test)) {
×
297
    return compareArrayBuffers(test, data, test.byteLength);
×
298
  }
×
299
  switch (typeof test) {
×
300
    case 'function':
×
NEW
301
      return test(ensureArrayBuffer(data));
×
302

×
303
    case 'string':
×
304
      // Magic bytes check: If `test` is a string, check if binary data starts with that strings
×
305
      const magic = getMagicString(data, byteOffset, test.length);
×
306
      return test === magic;
×
307

×
308
    default:
×
309
      return false;
×
310
  }
×
311
}
×
312

1✔
313
function getFirstCharacters(data: string | ArrayBuffer | TypedArray, length: number = 5) {
×
314
  if (typeof data === 'string') {
×
315
    return data.slice(0, length);
×
316
  } else if (ArrayBuffer.isView(data)) {
×
317
    // Typed Arrays can have offsets into underlying buffer
×
318
    return getMagicString(data.buffer, data.byteOffset, length);
×
319
  } else if (data instanceof ArrayBuffer) {
×
320
    const byteOffset = 0;
×
321
    return getMagicString(data, byteOffset, length);
×
322
  }
×
323
  return '';
×
324
}
×
325

1✔
NEW
326
function getMagicString(arrayBuffer: ArrayBufferLike, byteOffset: number, length: number): string {
×
327
  if (arrayBuffer.byteLength < byteOffset + length) {
×
328
    return '';
×
329
  }
×
330
  const dataView = new DataView(arrayBuffer);
×
331
  let magic = '';
×
332
  for (let i = 0; i < length; i++) {
×
333
    magic += String.fromCharCode(dataView.getUint8(byteOffset + i));
×
334
  }
×
335
  return magic;
×
336
}
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc