24369398061

Committed 13 Apr 2026 10:07PM UTC coverage: 55.756% (+0.004%) from 55.752%

Build # 24369398061

Build Type

push

github

Committed by

web-flow

Commit Message

chore: Warn if core is imported by loader module (#3379)

Coverage Stats

9478 of 18401 branches covered (51.51%)

Branch coverage included in aggregate %.

85 of 136 new or added lines in 33 files covered. (62.5%)

4 existing lines in 4 files now uncovered.

19703 of 33936 relevant lines covered (58.06%)

4979.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/modules/zip/src/parse-zip/zip-composition.ts

import {
  concatenateArrayBuffers,
  path,
  NodeFilesystem,
  NodeFile,
  toArrayBuffer
} from '@loaders.gl/loader-utils';
import {ZipEoCDRecord, generateEoCD, parseEoCDRecord, updateEoCD} from './end-of-central-directory';
import {CRC32Hash} from '@loaders.gl/crypto';
import {generateLocalHeader} from './local-file-header';
import {generateCDHeader} from './cd-file-header';
import {readRange} from './readable-file-utils';

/**
 * cut off CD and EoCD records from zip file
 * @param provider zip file
 * @returns tuple with three values: CD, EoCD record, EoCD information
 */
async function cutTheTailOff(
  provider: NodeFile
): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {
  // define where the body ends
  const oldEoCDinfo = await parseEoCDRecord(provider);
  const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
  const providerSize = (await provider.stat()).bigsize;

  // define cd length
  const oldCDLength = Number(
    oldEoCDinfo.offsets.zip64EoCDOffset
      ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
      : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset
  );

  // cut off everything except of archieve body
  const zipEnding = await readRange(provider, oldCDStartOffset, providerSize);
  await provider.truncate(Number(oldCDStartOffset));

  // divide cd body and eocd record
  const oldCDBody = zipEnding.slice(0, oldCDLength);
  const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);

  return [oldCDBody, eocdBody, oldEoCDinfo];
}

/**
 * generates CD and local headers for the file
 * @param fileName name of the file
 * @param fileToAdd buffer with the file
 * @param localFileHeaderOffset offset of the file local header
 * @returns tuple with two values: local header and file body, cd header
 */
async function generateFileHeaders(
  fileName: string,
  fileToAdd: ArrayBuffer,
  localFileHeaderOffset: bigint
): Promise<[Uint8Array, Uint8Array]> {
  // generating CRC32 of the content
  const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);

  // generate local header for the file
  const newFileLocalHeader = generateLocalHeader({
    crc32: newFileCRC322,
    fileName,
    length: fileToAdd.byteLength
  });

  // generate hash file cd header
  const newFileCDHeader = generateCDHeader({
    crc32: newFileCRC322,
    fileName,
    offset: localFileHeaderOffset,
    length: fileToAdd.byteLength
  });
  return [
    new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
    new Uint8Array(newFileCDHeader)
  ];
}

/**
 * adds one file in the end of the archieve
 * @param zipUrl path to the file
 * @param fileToAdd new file body
 * @param fileName new file name
 */
export async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {
  // init file handler
  const provider = new NodeFile(zipUrl, 'a+');

  const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);

  let currentOffset = (await provider.stat()).bigsize;

  // remember the new file local header start offset
  const newFileOffset = currentOffset;

  const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);

  // write down the file local header
  await provider.append(localPart);
  currentOffset += BigInt(localPart.byteLength);

  // add the file CD header to the CD
  const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);

  // remember the CD start offset
  const newCDStartOffset = currentOffset;

  // write down new CD
  await provider.append(new Uint8Array(newCDBody));
  currentOffset += BigInt(newCDBody.byteLength);

  // remember where eocd starts
  const eocdOffset = currentOffset;

  const updatedEoCD = updateEoCD(
    eocdBody,
    oldEoCDinfo.offsets,
    newCDStartOffset,
    eocdOffset,
    oldEoCDinfo.cdRecordsNumber + 1n
  );

  await provider.append(updatedEoCD);
  currentOffset += BigInt(updatedEoCD.byteLength);
}

/**
 * creates zip archive with no compression
 * @note This is a node specific function that works on files
 * @param inputPath path where files for the achive are stored
 * @param outputPath path where zip archive will be placed
 */
export async function createZip(
  inputPath: string,
  outputPath: string,
  createAdditionalData?: (
    fileList: {fileName: string; localHeaderOffset: bigint}[]
  ) => Promise<{path: string; file: ArrayBuffer}>
) {
  const fileIterator = getFileIterator(inputPath);

  const resFile = new NodeFile(outputPath, 'w');
  const fileList: {fileName: string; localHeaderOffset: bigint}[] = [];

  const cdArray: ArrayBuffer[] = [];
  for await (const file of fileIterator) {
    await addFile(file, resFile, cdArray, fileList);
  }
  if (createAdditionalData) {
    const additionaldata = await createAdditionalData(fileList);
    await addFile(additionaldata, resFile, cdArray);
  }
  const cdOffset = (await resFile.stat()).bigsize;
  const cd = concatenateArrayBuffers(...cdArray);
  await resFile.append(new Uint8Array(cd));
  const eoCDStart = (await resFile.stat()).bigsize;
  await resFile.append(
    new Uint8Array(
      generateEoCD({recordsNumber: cdArray.length, cdSize: cd.byteLength, cdOffset, eoCDStart})
    )
  );
}

/**
 * Adds file to zip parts
 * @param file file to add
 * @param resFile zip file body
 * @param cdArray zip file central directory
 * @param fileList list of file offsets
 */
async function addFile(
  file: {path: string; file: ArrayBuffer},
  resFile: NodeFile,
  cdArray: ArrayBuffer[],
  fileList?: {fileName: string; localHeaderOffset: bigint}[]
) {
  const size = (await resFile.stat()).bigsize;
  fileList?.push({fileName: file.path, localHeaderOffset: size});
  const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size);
  await resFile.append(localPart);
  cdArray.push(toArrayBuffer(cdHeaderPart));
}

/**
 * creates iterator providing buffer with file content and path to every file in the input folder
 * @param inputPath path to the input folder
 * @returns iterator
 */
export function getFileIterator(
  inputPath: string
): AsyncIterable<{path: string; file: ArrayBuffer}> {
  async function* iterable() {
    const fileList = await getAllFiles(inputPath);
    for (const filePath of fileList) {
      const provider = new NodeFile(path.join(inputPath, filePath), 'r');
      const {bigsize} = await provider.stat();
      const file = await provider.read(0, Number(bigsize));
      await provider.close();
      yield {path: filePath, file};
    }
  }
  return iterable();
}

/**
 * creates a list of relative paths to all files in the provided folder
 * @param basePath path of the root folder
 * @param subfolder relative path from the root folder.
 * @returns list of paths
 */
export async function getAllFiles(
  basePath: string,
  subfolder: string = '',
  fsPassed?: NodeFilesystem
): Promise<string[]> {
  const fs = fsPassed ? fsPassed : new NodeFilesystem({});
  const files = await fs.readdir(pathJoin(basePath, subfolder));

  const arrayOfFiles: string[] = [];

  for (const file of files) {
    const fullPath = pathJoin(basePath, subfolder, file);
    if ((await fs.stat(fullPath)).isDirectory) {
      const files = await getAllFiles(basePath, pathJoin(subfolder, file));
      arrayOfFiles.push(...files);
    } else {
      arrayOfFiles.push(pathJoin(subfolder, file));
    }
  }

  return arrayOfFiles;
}

/**
 * removes empty parts from path array and joins it
 * @param paths paths to join
 * @returns joined path
 */
function pathJoin(...paths: string[]): string {
  const resPaths: string[] = paths.filter(val => val.length);
  return path.join(...resPaths);
}

1	import {
2	concatenateArrayBuffers,
3	path,
4	NodeFilesystem,
5	NodeFile,
6	toArrayBuffer
7	} from '@loaders.gl/loader-utils';
8	import {ZipEoCDRecord, generateEoCD, parseEoCDRecord, updateEoCD} from './end-of-central-directory';
9	import {CRC32Hash} from '@loaders.gl/crypto';
10	import {generateLocalHeader} from './local-file-header';
11	import {generateCDHeader} from './cd-file-header';
12	import {readRange} from './readable-file-utils';
13
14	/**
15	* cut off CD and EoCD records from zip file
16	* @param provider zip file
17	* @returns tuple with three values: CD, EoCD record, EoCD information
18	*/
19	async function cutTheTailOff(
20	provider: NodeFile
21	): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {
22	// define where the body ends
23	const oldEoCDinfo = await parseEoCDRecord(provider);	×
24	const oldCDStartOffset = oldEoCDinfo.cdStartOffset;	×
25	const providerSize = (await provider.stat()).bigsize;	×
26
27	// define cd length
28	const oldCDLength = Number(	×
29	oldEoCDinfo.offsets.zip64EoCDOffset	×
30	? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
31	: oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset
32	);
33
34	// cut off everything except of archieve body
35	const zipEnding = await readRange(provider, oldCDStartOffset, providerSize);	×
36	await provider.truncate(Number(oldCDStartOffset));	×
37
38	// divide cd body and eocd record
39	const oldCDBody = zipEnding.slice(0, oldCDLength);	×
40	const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);	×
41
42	return [oldCDBody, eocdBody, oldEoCDinfo];	×
43	}
44
45	/**
46	* generates CD and local headers for the file
47	* @param fileName name of the file
48	* @param fileToAdd buffer with the file
49	* @param localFileHeaderOffset offset of the file local header
50	* @returns tuple with two values: local header and file body, cd header
51	*/
52	async function generateFileHeaders(
53	fileName: string,
54	fileToAdd: ArrayBuffer,
55	localFileHeaderOffset: bigint
56	): Promise<[Uint8Array, Uint8Array]> {
57	// generating CRC32 of the content
58	const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);	×
59
60	// generate local header for the file
61	const newFileLocalHeader = generateLocalHeader({	×
62	crc32: newFileCRC322,
63	fileName,
64	length: fileToAdd.byteLength
65	});
66
67	// generate hash file cd header
68	const newFileCDHeader = generateCDHeader({	×
69	crc32: newFileCRC322,
70	fileName,
71	offset: localFileHeaderOffset,
72	length: fileToAdd.byteLength
73	});
74	return [	×
75	new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
76	new Uint8Array(newFileCDHeader)
77	];
78	}
79
80	/**
81	* adds one file in the end of the archieve
82	* @param zipUrl path to the file
83	* @param fileToAdd new file body
84	* @param fileName new file name
85	*/
86	export async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {
87	// init file handler
88	const provider = new NodeFile(zipUrl, 'a+');	×
89
90	const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);	×
91
92	let currentOffset = (await provider.stat()).bigsize;	×
93
94	// remember the new file local header start offset
95	const newFileOffset = currentOffset;	×
96
97	const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);	×
98
99	// write down the file local header
100	await provider.append(localPart);	×
101	currentOffset += BigInt(localPart.byteLength);	×
102
103	// add the file CD header to the CD
104	const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);	×
105
106	// remember the CD start offset
107	const newCDStartOffset = currentOffset;	×
108
109	// write down new CD
110	await provider.append(new Uint8Array(newCDBody));	×
111	currentOffset += BigInt(newCDBody.byteLength);	×
112
113	// remember where eocd starts
114	const eocdOffset = currentOffset;	×
115
116	const updatedEoCD = updateEoCD(	×
117	eocdBody,
118	oldEoCDinfo.offsets,
119	newCDStartOffset,
120	eocdOffset,
121	oldEoCDinfo.cdRecordsNumber + 1n
122	);
123
124	await provider.append(updatedEoCD);	×
125	currentOffset += BigInt(updatedEoCD.byteLength);	×
126	}
127
128	/**
129	* creates zip archive with no compression
130	* @note This is a node specific function that works on files
131	* @param inputPath path where files for the achive are stored
132	* @param outputPath path where zip archive will be placed
133	*/
134	export async function createZip(
135	inputPath: string,
136	outputPath: string,
137	createAdditionalData?: (
138	fileList: {fileName: string; localHeaderOffset: bigint}[]
139	) => Promise<{path: string; file: ArrayBuffer}>
140	) {
141	const fileIterator = getFileIterator(inputPath);	×
142
143	const resFile = new NodeFile(outputPath, 'w');	×
144	const fileList: {fileName: string; localHeaderOffset: bigint}[] = [];	×
145
146	const cdArray: ArrayBuffer[] = [];	×
147	for await (const file of fileIterator) {	×
148	await addFile(file, resFile, cdArray, fileList);	×
149	}
150	if (createAdditionalData) {	×
151	const additionaldata = await createAdditionalData(fileList);	×
152	await addFile(additionaldata, resFile, cdArray);	×
153	}
154	const cdOffset = (await resFile.stat()).bigsize;	×
155	const cd = concatenateArrayBuffers(...cdArray);	×
156	await resFile.append(new Uint8Array(cd));	×
157	const eoCDStart = (await resFile.stat()).bigsize;	×
158	await resFile.append(	×
159	new Uint8Array(
160	generateEoCD({recordsNumber: cdArray.length, cdSize: cd.byteLength, cdOffset, eoCDStart})
161	)
162	);
163	}
164
165	/**
166	* Adds file to zip parts
167	* @param file file to add
168	* @param resFile zip file body
169	* @param cdArray zip file central directory
170	* @param fileList list of file offsets
171	*/
172	async function addFile(
173	file: {path: string; file: ArrayBuffer},
174	resFile: NodeFile,
175	cdArray: ArrayBuffer[],
176	fileList?: {fileName: string; localHeaderOffset: bigint}[]
177	) {
178	const size = (await resFile.stat()).bigsize;	×
179	fileList?.push({fileName: file.path, localHeaderOffset: size});	×
180	const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size);	×
181	await resFile.append(localPart);	×
182	cdArray.push(toArrayBuffer(cdHeaderPart));	×
183	}
184
185	/**
186	* creates iterator providing buffer with file content and path to every file in the input folder
187	* @param inputPath path to the input folder
188	* @returns iterator
189	*/
190	export function getFileIterator(
191	inputPath: string
192	): AsyncIterable<{path: string; file: ArrayBuffer}> {
193	async function* iterable() {
194	const fileList = await getAllFiles(inputPath);	×
195	for (const filePath of fileList) {	×
NEW 196	const provider = new NodeFile(path.join(inputPath, filePath), 'r');	×
NEW 197	const {bigsize} = await provider.stat();	×
NEW 198	const file = await provider.read(0, Number(bigsize));	×
NEW 199	await provider.close();	×
UNCOV 200	yield {path: filePath, file};	×
201	}
202	}
203	return iterable();	×
204	}
205
206	/**
207	* creates a list of relative paths to all files in the provided folder
208	* @param basePath path of the root folder
209	* @param subfolder relative path from the root folder.
210	* @returns list of paths
211	*/
212	export async function getAllFiles(
213	basePath: string,
214	subfolder: string = '',	×
215	fsPassed?: NodeFilesystem
216	): Promise<string[]> {
217	const fs = fsPassed ? fsPassed : new NodeFilesystem({});	×
218	const files = await fs.readdir(pathJoin(basePath, subfolder));	×
219
220	const arrayOfFiles: string[] = [];	×
221
222	for (const file of files) {	×
223	const fullPath = pathJoin(basePath, subfolder, file);	×
224	if ((await fs.stat(fullPath)).isDirectory) {	×
225	const files = await getAllFiles(basePath, pathJoin(subfolder, file));	×
226	arrayOfFiles.push(...files);	×
227	} else {
228	arrayOfFiles.push(pathJoin(subfolder, file));	×
229	}
230	}
231
232	return arrayOfFiles;	×
233	}
234
235	/**
236	* removes empty parts from path array and joins it
237	* @param paths paths to join
238	* @returns joined path
239	*/
240	function pathJoin(...paths: string[]): string {
241	const resPaths: string[] = paths.filter(val => val.length);	×
242	return path.join(...resPaths);	×
243	}

visgl / loaders.gl / 24369398061

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous