24340918156

Committed 13 Apr 2026 11:28AM UTC coverage: 55.752% (+0.1%) from 55.643%

Build # 24340918156

Build Type

push

github

Committed by

web-flow

Commit Message

feat(shapefile) ShapefileArrowLoader (#3375)

Coverage Stats

9457 of 18361 branches covered (51.51%)

Branch coverage included in aggregate %.

122 of 158 new or added lines in 2 files covered. (77.22%)

19661 of 33867 relevant lines covered (58.05%)

4989.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.55

/modules/shapefile/src/shapefile-arrow-loader.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {LoaderContext, LoaderWithParser} from '@loaders.gl/loader-utils';
import {parseFromContext, parseInBatchesFromContext, toArrayBufferIterator} from '@loaders.gl/loader-utils';
import type {
  ArrowTable,
  ArrowTableBatch,
  BinaryGeometry,
  Field,
  Geometry,
  Schema as TableSchema,
  Feature
} from '@loaders.gl/schema';
import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
import {convertBinaryGeometryToGeometry, convertGeometryToWKB, transformGeoJsonCoords} from '@loaders.gl/gis';
import {Proj4Projection} from '@math.gl/proj4';
import {SHP_MAGIC_NUMBER, SHPLoader} from './shp-loader';
import {DBFArrowLoader} from './dbf-arrow-loader';
import {DBFLoader} from './dbf-loader';
import type {ShapefileLoaderOptions} from './shapefile-loader';
import type {SHPHeader} from './lib/parsers/parse-shp-header';
import {loadShapefileSidecarFiles, replaceExtension} from './lib/parsers/parse-shapefile';

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

const GEOMETRY_COLUMN_NAME = 'geometry';

/** Options for `ShapefileArrowLoader`. */
export type ShapefileArrowLoaderOptions = ShapefileLoaderOptions;

/**
 * Shapefile loader that returns properties and geometry as an Arrow table.
 *
 * The loader preserves DBF attributes as Arrow columns and appends a WKB
 * `geometry` column annotated with geospatial schema metadata.
 */
export const ShapefileArrowLoader = {
  name: 'Shapefile Arrow',
  id: 'shapefile-arrow',
  module: 'shapefile',
  version: VERSION,
  category: 'geometry',
  extensions: ['shp'],
  mimeTypes: ['application/octet-stream'],
  tests: [new Uint8Array(SHP_MAGIC_NUMBER).buffer],
  options: {
    shapefile: {
      shape: 'v3'
    },
    shp: {
      _maxDimensions: 4
    }
  },
  parse: parseShapefileToArrow,
  parseInBatches: parseShapefileToArrowInBatches
} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, ShapefileArrowLoaderOptions>;

/** Parses a shapefile and returns an Arrow table with a WKB geometry column. */
async function parseShapefileToArrow(
  arrayBuffer: ArrayBuffer,
  options?: ShapefileArrowLoaderOptions,
  context?: LoaderContext
): Promise<ArrowTable> {
  const {header, geometries} = await parseFromContext(arrayBuffer, SHPLoader, options, context!);
  const {cpg, prj} = await loadShapefileSidecarFiles(options, context);

  const geometryObjects = parseGeometries(geometries);
  const features = maybeReprojectFeatures(
    geometryObjects.map(geometry => ({type: 'Feature', geometry, properties: {}})),
    prj,
    options
  );

  let propertySchema: TableSchema | null = null;
  let propertyRows: Record<string, unknown>[] = [];

  const dbfResponse = await context?.fetch(replaceExtension(context?.url || '', 'dbf'));
  if (dbfResponse?.ok) {
    const table = await parseFromContext(
      dbfResponse as any,
      DBFArrowLoader,
      {
        ...options,
        dbf: {
          ...options?.dbf,
          encoding: cpg || 'latin1'
        }
      },
      context!
    );
    propertySchema = table.schema || null;
    propertyRows = getRowsFromArrowTable(table);
  }

  const schema = buildOutputSchema(propertySchema, features.map(feature => feature.geometry), header);
  const tableBuilder = new ArrowTableBuilder(schema);

  const rowCount = Math.max(features.length, propertyRows.length);
  for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
    tableBuilder.addObjectRow(makeArrowRow(propertyRows[rowIndex], features[rowIndex]?.geometry, header));
  }

  return tableBuilder.finishTable();
}

/** Parses a shapefile into Arrow batches while keeping DBF-derived schema stable. */
async function* parseShapefileToArrowInBatches(
  asyncIterator:
    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
    | Iterable<ArrayBufferLike | ArrayBufferView>,
  options?: ShapefileArrowLoaderOptions,
  context?: LoaderContext
): AsyncIterable<ArrowTableBatch> {
  const {cpg, prj} = await loadShapefileSidecarFiles(options, context);

  const shapeIterable = await parseInBatchesFromContext(
    toArrayBufferIterator(asyncIterator),
    SHPLoader,
    options,
    context!
  );
  const shapeIterator = getAsyncIterator(shapeIterable);

  const shapeHeader = await getNextNonMetadataValue(shapeIterator);
  const header = shapeHeader as SHPHeader;

  let propertyIterator: AsyncIterator<any> | null = null;
  let propertySchema: TableSchema | null = null;

  const dbfResponse = await context?.fetch(replaceExtension(context?.url || '', 'dbf'));
  if (dbfResponse?.ok) {
    const dbfOptions = {
      ...options,
      dbf: {
        ...options?.dbf,
        shape: 'object-row-table' as const,
        encoding: cpg || 'latin1'
      }
    };
    const schemaResponse = 'clone' in dbfResponse ? dbfResponse.clone() : await context?.fetch(replaceExtension(context?.url || '', 'dbf'));
    const propertyTable = await parseFromContext(schemaResponse as any, DBFLoader, dbfOptions, context!);
    propertySchema = propertyTable?.schema || null;

    const propertyIterable = await parseInBatchesFromContext(
      dbfResponse,
      DBFLoader,
      dbfOptions,
      context!
    );
    propertyIterator = getAsyncIterator(propertyIterable);

    const outputSchema = buildOutputSchema(propertySchema, [], header);
    const propertyQueue: Record<string, unknown>[] = [];
    const geometryQueue: Geometry[] = [];
    let yieldedDataBatch = false;

    const firstPropertyBatch = await getNextPropertyBatch(propertyIterator);
    if (firstPropertyBatch) {
      propertyQueue.push(...firstPropertyBatch);
    }

    let shapeDone = false;
    let propertyDone = false;
    while (!shapeDone || !propertyDone || geometryQueue.length > 0 || propertyQueue.length > 0) {
      if (!shapeDone && geometryQueue.length === 0) {
        const shapeBatch = await shapeIterator.next();
        if (shapeBatch.done) {
          shapeDone = true;
        } else if (shapeBatch.value?.batchType !== 'metadata') {
          geometryQueue.push(...parseGeometries(shapeBatch.value as BinaryGeometry[]));
        }
      }

      if (!propertyDone && propertyQueue.length < geometryQueue.length) {
        const propertyBatch = await propertyIterator.next();
        if (propertyBatch.done) {
          propertyDone = true;
        } else if (Array.isArray(propertyBatch.value)) {
          propertyQueue.push(...propertyBatch.value);
        }
      }

      const rowCount = Math.min(geometryQueue.length, propertyQueue.length);
      if (rowCount === 0) {
        if ((shapeDone && geometryQueue.length === 0) || (propertyDone && propertyQueue.length === 0)) {
          break;
        }
        continue;
      }

      const features = maybeReprojectFeatures(
        geometryQueue.splice(0, rowCount).map(geometry => ({type: 'Feature', geometry, properties: {}})),
        prj,
        options
      );
      const propertyRows = propertyQueue.splice(0, rowCount);
      const batchBuilder = new ArrowTableBuilder(outputSchema);
      for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
        batchBuilder.addObjectRow(makeArrowRow(propertyRows[rowIndex], features[rowIndex]?.geometry, header));
      }
      const batch = batchBuilder.finishBatch();
      if (batch) {
        yieldedDataBatch = true;
        yield batch;
      }
    }
    if (!yieldedDataBatch) {
      yield makeEmptyArrowBatch(outputSchema);
    }
    return;
  }

  const outputSchema = buildOutputSchema(null, [], header);
  let yieldedDataBatch = false;

  while (true) {
    const shapeBatch = await shapeIterator.next();
    if (shapeBatch.done) {
      break;
    }
    if (shapeBatch.value?.batchType === 'metadata') {
      continue;
    }
    const features = maybeReprojectFeatures(
      parseGeometries(shapeBatch.value as BinaryGeometry[]).map(geometry => ({
        type: 'Feature',
        geometry,
        properties: {}
      })),
      prj,
      options
    );
    const batchBuilder = new ArrowTableBuilder(outputSchema);
    for (const feature of features) {
      batchBuilder.addObjectRow(makeArrowRow(undefined, feature.geometry, header));
    }
    const batch = batchBuilder.finishBatch();
    if (batch) {
      yieldedDataBatch = true;
      yield batch;
    }
  }
  if (!yieldedDataBatch) {
    yield makeEmptyArrowBatch(outputSchema);
  }
}

/** Creates the output Arrow schema by appending the WKB geometry column to DBF fields. */
function buildOutputSchema(
  propertySchema: TableSchema | null,
  geometries: Geometry[],
  header?: SHPHeader
): TableSchema {
  const geometryTypes = inferGeometryTypes(geometries, header);
  const metadata = {
    geo: JSON.stringify({
      version: '1.1.0',
      primary_column: GEOMETRY_COLUMN_NAME,
      columns: {
        [GEOMETRY_COLUMN_NAME]: {
          encoding: 'wkb',
          geometry_types: geometryTypes
        }
      }
    })
  };
  const geometryField: Field = {
    name: GEOMETRY_COLUMN_NAME,
    type: 'binary',
    nullable: true,
    metadata: {}
  };

  return {
    fields: [...(propertySchema?.fields || []), geometryField],
    metadata: {
      ...(propertySchema?.metadata || {}),
      ...metadata
    }
  };
}

/** Combines one property row and one geometry into an Arrow-builder friendly object row. */
function makeArrowRow(
  propertyRow: Record<string, unknown> | undefined,
  geometry: Geometry | undefined,
  header?: SHPHeader
): Record<string, unknown> {
  return {
    ...(propertyRow || {}),
    [GEOMETRY_COLUMN_NAME]: geometry ? new Uint8Array(convertGeometryToWKB(geometry, getWKBOptions(geometry, header))) : null
  };
}

/** Materializes Arrow rows as plain objects for row-wise joining with SHP geometry output. */
function getRowsFromArrowTable(table: ArrowTable | ArrowTableBatch): Record<string, unknown>[] {
  const rows: Record<string, unknown>[] = [];
  for (let rowIndex = 0; rowIndex < table.data.numRows; rowIndex++) {
    rows.push(table.data.get(rowIndex)?.toJSON() || {});
  }
  return rows;
}

/** Converts binary SHP geometries to GeoJSON geometries. */
function parseGeometries(geometries: BinaryGeometry[]): Geometry[] {
  return geometries.map(geometry => convertBinaryGeometryToGeometry(geometry));
}

/** Reprojects features when requested through standard shapefile GIS options. */
function maybeReprojectFeatures(
  features: Feature[],
  sourceCrs: string | undefined,
  options?: ShapefileArrowLoaderOptions
): Feature[] {
  const {reproject = false, _targetCrs = 'WGS84'} = options?.gis || {};
  if (!reproject) {
    return features;
  }
  const projection = new Proj4Projection({from: sourceCrs || 'WGS84', to: _targetCrs || 'WGS84'});
  return transformGeoJsonCoords(features, coord => projection.project(coord));
}

/** Selects WKB dimensional flags from the shapefile header and parsed coordinate dimensionality. */
function getWKBOptions(geometry: Geometry, header?: SHPHeader): {hasZ?: boolean; hasM?: boolean} {
  const dimensions = getCoordinateDimensions(getGeometrySampleCoordinates(geometry));
  switch (header?.type) {
    case 11:
    case 13:
    case 15:
    case 18:
      return {hasZ: dimensions > 2, hasM: dimensions > 3};
    case 21:
    case 23:
    case 25:
    case 28:
      return {hasM: dimensions > 2};
    default:
      return {hasZ: dimensions > 2, hasM: dimensions > 3};
  }
}

/** Returns the coordinate dimensionality of the first coordinate tuple in a geometry. */
function getCoordinateDimensions(coordinates: unknown): number {
  if (!Array.isArray(coordinates)) {
    return 2;
  }
  if (typeof coordinates[0] === 'number') {
    return coordinates.length;
  }
  if (coordinates.length === 0) {
    return 2;
  }
  return getCoordinateDimensions(coordinates[0]);
}

/** Infers GeoParquet geometry type metadata from parsed geometries or the SHP header. */
function inferGeometryTypes(geometries: Geometry[], header?: SHPHeader): string[] {
  const geometryTypes = new Set<string>();
  for (const geometry of geometries) {
    const dimensions = getCoordinateDimensions(getGeometrySampleCoordinates(geometry));
    geometryTypes.add(dimensions > 2 ? `${geometry.type} Z` : geometry.type);
  }
  if (geometryTypes.size > 0) {
    return [...geometryTypes];
  }

  const fallbackType = getGeometryTypeFromHeader(header?.type);
  return fallbackType ? [fallbackType] : [];
}

/** Maps SHP header geometry type codes to GeoParquet geometry type strings. */
function getGeometryTypeFromHeader(type?: number): string | null {
  switch (type) {
    case 1:
    case 11:
    case 21:
      return type === 11 ? 'Point Z' : 'Point';
    case 3:
    case 13:
    case 23:
      return type === 13 ? 'LineString Z' : 'LineString';
    case 5:
    case 15:
    case 25:
      return type === 15 ? 'Polygon Z' : 'Polygon';
    case 8:
    case 18:
    case 28:
      return type === 18 ? 'MultiPoint Z' : 'MultiPoint';
    default:
      return null;
  }
}

/** Extracts a representative coordinate array from any GeoJSON geometry. */
function getGeometrySampleCoordinates(geometry: Geometry): unknown {
  if ('coordinates' in geometry) {
    return geometry.coordinates;
  }
  if ('geometries' in geometry && geometry.geometries.length > 0) {
    return getGeometrySampleCoordinates(geometry.geometries[0]);
  }
  return undefined;
}

/** Normalizes sync or async iterables to a single async iterator interface. */
function getAsyncIterator(iterable: AsyncIterable<any> | Iterable<any>): AsyncIterator<any> {
  const iterator = iterable[Symbol.asyncIterator]?.() || iterable[Symbol.iterator]?.();
  return iterator as AsyncIterator<any>;
}

/** Reads the next non-metadata value from a parser iterator. */
async function getNextNonMetadataValue(iterator: AsyncIterator<any>): Promise<any> {
  while (true) {
    const result = await iterator.next();
    if (result.done) {
      return null;
    }
    if (result.value?.batchType !== 'metadata') {
      return result.value;
    }
  }
}

/** Reads the next DBF row batch, skipping header objects. */
async function getNextPropertyBatch(
  iterator: AsyncIterator<any>
): Promise<Record<string, unknown>[] | null> {
  while (true) {
    const result = await iterator.next();
    if (result.done) {
      return null;
    }
    if (Array.isArray(result.value)) {
      return result.value;
    }
  }
}

/** Creates an explicit empty Arrow batch so zero-row shapefiles still expose schema in batch mode. */
function makeEmptyArrowBatch(schema: TableSchema): ArrowTableBatch {
  const table = new ArrowTableBuilder(schema).finishTable();
  return {
    shape: 'arrow-table',
    batchType: 'data',
    length: 0,
    schema,
    data: table.data
  };
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {LoaderContext, LoaderWithParser} from '@loaders.gl/loader-utils';
6	import {parseFromContext, parseInBatchesFromContext, toArrayBufferIterator} from '@loaders.gl/loader-utils';
7	import type {
8	ArrowTable,
9	ArrowTableBatch,
10	BinaryGeometry,
11	Field,
12	Geometry,
13	Schema as TableSchema,
14	Feature
15	} from '@loaders.gl/schema';
16	import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
17	import {convertBinaryGeometryToGeometry, convertGeometryToWKB, transformGeoJsonCoords} from '@loaders.gl/gis';
18	import {Proj4Projection} from '@math.gl/proj4';
19	import {SHP_MAGIC_NUMBER, SHPLoader} from './shp-loader';
20	import {DBFArrowLoader} from './dbf-arrow-loader';
21	import {DBFLoader} from './dbf-loader';
22	import type {ShapefileLoaderOptions} from './shapefile-loader';
23	import type {SHPHeader} from './lib/parsers/parse-shp-header';
24	import {loadShapefileSidecarFiles, replaceExtension} from './lib/parsers/parse-shapefile';
25
26	// __VERSION__ is injected by babel-plugin-version-inline
27	// @ts-ignore TS2304: Cannot find name '__VERSION__'.
28	const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';	9!
29
30	const GEOMETRY_COLUMN_NAME = 'geometry';	9✔
31
32	/** Options for `ShapefileArrowLoader`. */
33	export type ShapefileArrowLoaderOptions = ShapefileLoaderOptions;
34
35	/**
36	* Shapefile loader that returns properties and geometry as an Arrow table.
37	*
38	* The loader preserves DBF attributes as Arrow columns and appends a WKB
39	* `geometry` column annotated with geospatial schema metadata.
40	*/
41	export const ShapefileArrowLoader = {	9✔
42	name: 'Shapefile Arrow',
43	id: 'shapefile-arrow',
44	module: 'shapefile',
45	version: VERSION,
46	category: 'geometry',
47	extensions: ['shp'],
48	mimeTypes: ['application/octet-stream'],
49	tests: [new Uint8Array(SHP_MAGIC_NUMBER).buffer],
50	options: {
51	shapefile: {
52	shape: 'v3'
53	},
54	shp: {
55	_maxDimensions: 4
56	}
57	},
58	parse: parseShapefileToArrow,
59	parseInBatches: parseShapefileToArrowInBatches
60	} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, ShapefileArrowLoaderOptions>;
61
62	/** Parses a shapefile and returns an Arrow table with a WKB geometry column. */
63	async function parseShapefileToArrow(
64	arrayBuffer: ArrayBuffer,
65	options?: ShapefileArrowLoaderOptions,
66	context?: LoaderContext
67	): Promise<ArrowTable> {
68	const {header, geometries} = await parseFromContext(arrayBuffer, SHPLoader, options, context!);	8✔
69	const {cpg, prj} = await loadShapefileSidecarFiles(options, context);	7✔
70
71	const geometryObjects = parseGeometries(geometries);	7✔
72	const features = maybeReprojectFeatures(	7✔
73	geometryObjects.map(geometry => ({type: 'Feature', geometry, properties: {}})),	27✔
74	prj,
75	options
76	);
77
78	let propertySchema: TableSchema \| null = null;	7✔
79	let propertyRows: Record<string, unknown>[] = [];	7✔
80
81	const dbfResponse = await context?.fetch(replaceExtension(context?.url \|\| '', 'dbf'));	7!
82	if (dbfResponse?.ok) {	7!
83	const table = await parseFromContext(	7✔
84	dbfResponse as any,
85	DBFArrowLoader,
86	{
87	...options,
88	dbf: {
89	...options?.dbf,
90	encoding: cpg \|\| 'latin1'	13✔
91	}
92	},
93	context!
94	);
95	propertySchema = table.schema \|\| null;	7!
96	propertyRows = getRowsFromArrowTable(table);	7✔
97	}
98
99	const schema = buildOutputSchema(propertySchema, features.map(feature => feature.geometry), header);	27✔
100	const tableBuilder = new ArrowTableBuilder(schema);	7✔
101
102	const rowCount = Math.max(features.length, propertyRows.length);	7✔
103	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {	7✔
104	tableBuilder.addObjectRow(makeArrowRow(propertyRows[rowIndex], features[rowIndex]?.geometry, header));	27✔
105	}
106
107	return tableBuilder.finishTable();	7✔
108	}
109
110	/** Parses a shapefile into Arrow batches while keeping DBF-derived schema stable. */
111	async function* parseShapefileToArrowInBatches(
112	asyncIterator:
113	\| AsyncIterable<ArrayBufferLike \| ArrayBufferView>
114	\| Iterable<ArrayBufferLike \| ArrayBufferView>,
115	options?: ShapefileArrowLoaderOptions,
116	context?: LoaderContext
117	): AsyncIterable<ArrowTableBatch> {
118	const {cpg, prj} = await loadShapefileSidecarFiles(options, context);	6✔
119
120	const shapeIterable = await parseInBatchesFromContext(	6✔
121	toArrayBufferIterator(asyncIterator),
122	SHPLoader,
123	options,
124	context!
125	);
126	const shapeIterator = getAsyncIterator(shapeIterable);	6✔
127
128	const shapeHeader = await getNextNonMetadataValue(shapeIterator);	6✔
129	const header = shapeHeader as SHPHeader;	6✔
130
131	let propertyIterator: AsyncIterator<any> \| null = null;	6✔
132	let propertySchema: TableSchema \| null = null;	6✔
133
134	const dbfResponse = await context?.fetch(replaceExtension(context?.url \|\| '', 'dbf'));	6!
135	if (dbfResponse?.ok) {	6!
136	const dbfOptions = {	6✔
137	...options,
138	dbf: {
139	...options?.dbf,
140	shape: 'object-row-table' as const,
141	encoding: cpg \|\| 'latin1'	11✔
142	}
143	};
144	const schemaResponse = 'clone' in dbfResponse ? dbfResponse.clone() : await context?.fetch(replaceExtension(context?.url \|\| '', 'dbf'));	6!
NEW 145	const propertyTable = await parseFromContext(schemaResponse as any, DBFLoader, dbfOptions, context!);	×
146	propertySchema = propertyTable?.schema \|\| null;	6!
147
148	const propertyIterable = await parseInBatchesFromContext(	6✔
149	dbfResponse,
150	DBFLoader,
151	dbfOptions,
152	context!
153	);
154	propertyIterator = getAsyncIterator(propertyIterable);	6✔
155
156	const outputSchema = buildOutputSchema(propertySchema, [], header);	6✔
157	const propertyQueue: Record<string, unknown>[] = [];	6✔
158	const geometryQueue: Geometry[] = [];	6✔
159	let yieldedDataBatch = false;	6✔
160
161	const firstPropertyBatch = await getNextPropertyBatch(propertyIterator);	6✔
162	if (firstPropertyBatch) {	6✔
163	propertyQueue.push(...firstPropertyBatch);	5✔
164	}
165
166	let shapeDone = false;	6✔
167	let propertyDone = false;	6✔
168	while (!shapeDone \|\| !propertyDone \|\| geometryQueue.length > 0 \|\| propertyQueue.length > 0) {	6!
169	if (!shapeDone && geometryQueue.length === 0) {	11!
170	const shapeBatch = await shapeIterator.next();	11✔
171	if (shapeBatch.done) {	11✔
172	shapeDone = true;	6✔
173	} else if (shapeBatch.value?.batchType !== 'metadata') {	5!
174	geometryQueue.push(...parseGeometries(shapeBatch.value as BinaryGeometry[]));	5✔
175	}
176	}
177
178	if (!propertyDone && propertyQueue.length < geometryQueue.length) {	11!
NEW 179	const propertyBatch = await propertyIterator.next();	×
NEW 180	if (propertyBatch.done) {	×
NEW 181	propertyDone = true;	×
NEW 182	} else if (Array.isArray(propertyBatch.value)) {	×
NEW 183	propertyQueue.push(...propertyBatch.value);	×
184	}
185	}
186
187	const rowCount = Math.min(geometryQueue.length, propertyQueue.length);	11✔
188	if (rowCount === 0) {	11✔
189	if ((shapeDone && geometryQueue.length === 0) \|\| (propertyDone && propertyQueue.length === 0)) {	6!
190	break;	6✔
191	}
NEW 192	continue;	×
193	}
194
195	const features = maybeReprojectFeatures(	5✔
196	geometryQueue.splice(0, rowCount).map(geometry => ({type: 'Feature', geometry, properties: {}})),	18✔
197	prj,
198	options
199	);
200	const propertyRows = propertyQueue.splice(0, rowCount);	5✔
201	const batchBuilder = new ArrowTableBuilder(outputSchema);	5✔
202	for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {	5✔
203	batchBuilder.addObjectRow(makeArrowRow(propertyRows[rowIndex], features[rowIndex]?.geometry, header));	18✔
204	}
205	const batch = batchBuilder.finishBatch();	5✔
206	if (batch) {	5!
207	yieldedDataBatch = true;	5✔
208	yield batch;	5✔
209	}
210	}
211	if (!yieldedDataBatch) {	6✔
212	yield makeEmptyArrowBatch(outputSchema);	1✔
213	}
214	return;	6✔
215	}
216
NEW 217	const outputSchema = buildOutputSchema(null, [], header);	×
NEW 218	let yieldedDataBatch = false;	×
219
NEW 220	while (true) {	×
NEW 221	const shapeBatch = await shapeIterator.next();	×
NEW 222	if (shapeBatch.done) {	×
NEW 223	break;	×
224	}
NEW 225	if (shapeBatch.value?.batchType === 'metadata') {	×
NEW 226	continue;	×
227	}
NEW 228	const features = maybeReprojectFeatures(	×
NEW 229	parseGeometries(shapeBatch.value as BinaryGeometry[]).map(geometry => ({	×
230	type: 'Feature',
231	geometry,
232	properties: {}
233	})),
234	prj,
235	options
236	);
NEW 237	const batchBuilder = new ArrowTableBuilder(outputSchema);	×
NEW 238	for (const feature of features) {	×
NEW 239	batchBuilder.addObjectRow(makeArrowRow(undefined, feature.geometry, header));	×
240	}
NEW 241	const batch = batchBuilder.finishBatch();	×
NEW 242	if (batch) {	×
NEW 243	yieldedDataBatch = true;	×
NEW 244	yield batch;	×
245	}
246	}
NEW 247	if (!yieldedDataBatch) {	×
NEW 248	yield makeEmptyArrowBatch(outputSchema);	×
249	}
250	}
251
252	/** Creates the output Arrow schema by appending the WKB geometry column to DBF fields. */
253	function buildOutputSchema(
254	propertySchema: TableSchema \| null,
255	geometries: Geometry[],
256	header?: SHPHeader
257	): TableSchema {
258	const geometryTypes = inferGeometryTypes(geometries, header);	13✔
259	const metadata = {	13✔
260	geo: JSON.stringify({
261	version: '1.1.0',
262	primary_column: GEOMETRY_COLUMN_NAME,
263	columns: {
264	[GEOMETRY_COLUMN_NAME]: {
265	encoding: 'wkb',
266	geometry_types: geometryTypes
267	}
268	}
269	})
270	};
271	const geometryField: Field = {	13✔
272	name: GEOMETRY_COLUMN_NAME,
273	type: 'binary',
274	nullable: true,
275	metadata: {}
276	};
277
278	return {	13✔
279	fields: [...(propertySchema?.fields \|\| []), geometryField],	13!
280	metadata: {
281	...(propertySchema?.metadata \|\| {}),	13!
282	...metadata
283	}
284	};
285	}
286
287	/** Combines one property row and one geometry into an Arrow-builder friendly object row. */
288	function makeArrowRow(
289	propertyRow: Record<string, unknown> \| undefined,
290	geometry: Geometry \| undefined,
291	header?: SHPHeader
292	): Record<string, unknown> {
293	return {	45✔
294	...(propertyRow \|\| {}),	45!
295	[GEOMETRY_COLUMN_NAME]: geometry ? new Uint8Array(convertGeometryToWKB(geometry, getWKBOptions(geometry, header))) : null	45!
296	};
297	}
298
299	/** Materializes Arrow rows as plain objects for row-wise joining with SHP geometry output. */
300	function getRowsFromArrowTable(table: ArrowTable \| ArrowTableBatch): Record<string, unknown>[] {
301	const rows: Record<string, unknown>[] = [];	7✔
302	for (let rowIndex = 0; rowIndex < table.data.numRows; rowIndex++) {	7✔
303	rows.push(table.data.get(rowIndex)?.toJSON() \|\| {});	27!
304	}
305	return rows;	7✔
306	}
307
308	/** Converts binary SHP geometries to GeoJSON geometries. */
309	function parseGeometries(geometries: BinaryGeometry[]): Geometry[] {
310	return geometries.map(geometry => convertBinaryGeometryToGeometry(geometry));	45✔
311	}
312
313	/** Reprojects features when requested through standard shapefile GIS options. */
314	function maybeReprojectFeatures(
315	features: Feature[],
316	sourceCrs: string \| undefined,
317	options?: ShapefileArrowLoaderOptions
318	): Feature[] {
319	const {reproject = false, _targetCrs = 'WGS84'} = options?.gis \|\| {};	12✔
320	if (!reproject) {	12✔
321	return features;	11✔
322	}
323	const projection = new Proj4Projection({from: sourceCrs \|\| 'WGS84', to: _targetCrs \|\| 'WGS84'});	1!
324	return transformGeoJsonCoords(features, coord => projection.project(coord));	12✔
325	}
326
327	/** Selects WKB dimensional flags from the shapefile header and parsed coordinate dimensionality. */
328	function getWKBOptions(geometry: Geometry, header?: SHPHeader): {hasZ?: boolean; hasM?: boolean} {
329	const dimensions = getCoordinateDimensions(getGeometrySampleCoordinates(geometry));	45✔
330	switch (header?.type) {	45!
331	case 11:
332	case 13:
333	case 15:
334	case 18:
NEW 335	return {hasZ: dimensions > 2, hasM: dimensions > 3};	×
336	case 21:
337	case 23:
338	case 25:
339	case 28:
NEW 340	return {hasM: dimensions > 2};	×
341	default:
342	return {hasZ: dimensions > 2, hasM: dimensions > 3};	45✔
343	}
344	}
345
346	/** Returns the coordinate dimensionality of the first coordinate tuple in a geometry. */
347	function getCoordinateDimensions(coordinates: unknown): number {
348	if (!Array.isArray(coordinates)) {	102!
NEW 349	return 2;	×
350	}
351	if (typeof coordinates[0] === 'number') {	102✔
352	return coordinates.length;	72✔
353	}
354	if (coordinates.length === 0) {	30!
NEW 355	return 2;	×
356	}
357	return getCoordinateDimensions(coordinates[0]);	30✔
358	}
359
360	/** Infers GeoParquet geometry type metadata from parsed geometries or the SHP header. */
361	function inferGeometryTypes(geometries: Geometry[], header?: SHPHeader): string[] {
362	const geometryTypes = new Set<string>();	13✔
363	for (const geometry of geometries) {	13✔
364	const dimensions = getCoordinateDimensions(getGeometrySampleCoordinates(geometry));	27✔
365	geometryTypes.add(dimensions > 2 ? `${geometry.type} Z` : geometry.type);	27!
366	}
367	if (geometryTypes.size > 0) {	13✔
368	return [...geometryTypes];	6✔
369	}
370
371	const fallbackType = getGeometryTypeFromHeader(header?.type);	7✔
372	return fallbackType ? [fallbackType] : [];	13!
373	}
374
375	/** Maps SHP header geometry type codes to GeoParquet geometry type strings. */
376	function getGeometryTypeFromHeader(type?: number): string \| null {
377	switch (type) {	7!
378	case 1:
379	case 11:
380	case 21:
381	return type === 11 ? 'Point Z' : 'Point';	3!
382	case 3:
383	case 13:
384	case 23:
385	return type === 13 ? 'LineString Z' : 'LineString';	3!
386	case 5:
387	case 15:
388	case 25:
389	return type === 15 ? 'Polygon Z' : 'Polygon';	1!
390	case 8:
391	case 18:
392	case 28:
NEW 393	return type === 18 ? 'MultiPoint Z' : 'MultiPoint';	×
394	default:
NEW 395	return null;	×
396	}
397	}
398
399	/** Extracts a representative coordinate array from any GeoJSON geometry. */
400	function getGeometrySampleCoordinates(geometry: Geometry): unknown {
401	if ('coordinates' in geometry) {	72!
402	return geometry.coordinates;	72✔
403	}
NEW 404	if ('geometries' in geometry && geometry.geometries.length > 0) {	×
NEW 405	return getGeometrySampleCoordinates(geometry.geometries[0]);	×
406	}
NEW 407	return undefined;	×
408	}
409
410	/** Normalizes sync or async iterables to a single async iterator interface. */
411	function getAsyncIterator(iterable: AsyncIterable<any> \| Iterable<any>): AsyncIterator<any> {
412	const iterator = iterable[Symbol.asyncIterator]?.() \|\| iterable[Symbol.iterator]?.();	12!
413	return iterator as AsyncIterator<any>;	12✔
414	}
415
416	/** Reads the next non-metadata value from a parser iterator. */
417	async function getNextNonMetadataValue(iterator: AsyncIterator<any>): Promise<any> {
418	while (true) {	6✔
419	const result = await iterator.next();	12✔
420	if (result.done) {	12!
NEW 421	return null;	×
422	}
423	if (result.value?.batchType !== 'metadata') {	12✔
424	return result.value;	6✔
425	}
426	}
427	}
428
429	/** Reads the next DBF row batch, skipping header objects. */
430	async function getNextPropertyBatch(
431	iterator: AsyncIterator<any>
432	): Promise<Record<string, unknown>[] \| null> {
433	while (true) {	6✔
434	const result = await iterator.next();	18✔
435	if (result.done) {	18✔
436	return null;	1✔
437	}
438	if (Array.isArray(result.value)) {	17✔
439	return result.value;	5✔
440	}
441	}
442	}
443
444	/** Creates an explicit empty Arrow batch so zero-row shapefiles still expose schema in batch mode. */
445	function makeEmptyArrowBatch(schema: TableSchema): ArrowTableBatch {
446	const table = new ArrowTableBuilder(schema).finishTable();	1✔
447	return {	1✔
448	shape: 'arrow-table',
449	batchType: 'data',
450	length: 0,
451	schema,
452	data: table.data
453	};
454	}

visgl / loaders.gl / 24340918156

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous