12941437080

Committed 24 Jan 2025 01:29AM UTC coverage: 66.52% (+0.1%) from 66.413%

Build # 12941437080

Build Type

Pull #2798

github

Committed by

web-flow

Commit Message

Merge fdf04530c into 4be4b6987

Pull Request Pull Request #2798: [feat] duckdb plugin

Run Details

5983 of 10500 branches covered (56.98%)

Branch coverage included in aggregate %.

6 of 11 new or added lines in 4 files covered. (54.55%)

1 existing line in 1 file now uncovered.

12312 of 17003 relevant lines covered (72.41%)

89.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

5.78

/src/layers/src/layer-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import * as arrow from 'apache-arrow';
import {Feature, BBox} from 'geojson';
import {getGeoMetadata} from '@loaders.gl/gis';

import {
  Field,
  ProtoDatasetField,
  FieldPair,
  SupportedColumnMode,
  LayerColumn
} from '@kepler.gl/types';
import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
import {
  getBinaryGeometriesFromArrow,
  parseGeometryFromArrow,
  BinaryGeometriesFromArrowOptions,
  updateBoundsFromGeoArrowSamples
} from '@loaders.gl/arrow';
import {EXTENSION_NAME} from '@kepler.gl/deckgl-arrow-layers';

import {WKBLoader} from '@loaders.gl/wkt';
import {geojsonToBinary} from '@loaders.gl/gis';
import {
  BinaryFeatureCollection,
  Geometry,
  BinaryPointFeature,
  BinaryLineFeature,
  BinaryPolygonFeature
} from '@loaders.gl/schema';

import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';

export function assignPointPairToLayerColumn(pair: FieldPair, hasAlt: boolean) {
  const {lat, lng, altitude} = pair.pair;
  if (!hasAlt) {
    return {lat, lng};
  }

  const defaultAltColumn = {value: null, fieldIdx: -1, optional: true};

  return {
    lat,
    lng,
    altitude: altitude ? {...defaultAltColumn, ...altitude} : defaultAltColumn
  };
}

export type GeojsonLayerMetaProps = {
  dataToFeature: GeojsonDataMaps;
  featureTypes: DeckGlGeoTypes;
  bounds: BBox | null;
  fixedRadius: boolean;
  centroids?: Array<number[] | null>;
};

/**
 * Converts a geoarrow.wkb vector into an array of BinaryFeatureCollections.
 * @param geoColumn A vector column with geoarrow.wkb extension.
 * @param options Options for geometry transformation.
 * @returns
 */
function getBinaryGeometriesFromWKBArrow(
  geoColumn: arrow.Vector,
  options: {chunkIndex?: number; chunkOffset?: number}
): GeojsonLayerMetaProps {
  const dataToFeature: BinaryFeatureCollection[] = [];
  const featureTypes: GeojsonLayerMetaProps['featureTypes'] = {
    point: false,
    line: false,
    polygon: false
  };

  const chunks =
    options?.chunkIndex !== undefined && options?.chunkIndex >= 0
      ? [geoColumn.data[options?.chunkIndex]]
      : geoColumn.data;
  const globalFeatureIdOffset = options?.chunkOffset || 0;
  let featureIndex = globalFeatureIdOffset;
  let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];

  chunks.forEach(chunk => {
    const geojsonFeatures: Feature[] = [];
    for (let i = 0; i < chunk.length; ++i) {
      // ignore features without any geometry
      if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {
        const valuesSlice = chunk.values.slice(chunk.valueOffsets[i], chunk.valueOffsets[i + 1]);

        const geometry = WKBLoader?.parseSync?.(valuesSlice.buffer, {
          wkb: {shape: 'geojson-geometry'}
        }) as Geometry;
        const feature: Feature = {
          type: 'Feature',
          geometry,
          properties: {index: featureIndex}
        };
        geojsonFeatures.push(feature);

        const {type} = geometry;
        featureTypes.polygon = type === 'Polygon' || type === 'MultiPolygon';
        featureTypes.point = type === 'Point' || type === 'MultiPoint';
        featureTypes.line = type === 'LineString' || type === 'MultiLineString';
      }

      featureIndex++;
    }

    const geojsonToBinaryOptions = {
      triangulate: true,
      fixRingWinding: true
    };
    const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);

    // Need to update globalFeatureIds, to take into account previous batches,
    // as geojsonToBinary doesn't have such option.
    const featureTypesArr = ['points', 'lines', 'polygons'];
    featureTypesArr.forEach(prop => {
      const features = binaryFeatures[prop] as
        | BinaryPointFeature
        | BinaryLineFeature
        | BinaryPolygonFeature;
      if (features) {
        bounds = updateBoundsFromGeoArrowSamples(
          features.positions.value as Float64Array,
          features.positions.size,
          bounds
        );

        const {globalFeatureIds, numericProps} = features;
        const {index} = numericProps;
        const len = globalFeatureIds.value.length;
        for (let i = 0; i < len; ++i) {
          globalFeatureIds.value[i] = index.value[i];
        }
      }
    });

    dataToFeature.push(binaryFeatures);
  });

  return {
    dataToFeature: dataToFeature,
    featureTypes: featureTypes,
    bounds,
    fixedRadius: false
  };
}

export function getGeojsonLayerMetaFromArrow({
  dataContainer,
  geoColumn,
  geoField,
  chunkIndex
}: {
  dataContainer: DataContainerInterface;
  geoColumn: arrow.Vector;
  geoField: ProtoDatasetField;
  chunkIndex?: number;
}): GeojsonLayerMetaProps {
  const encoding = geoField?.metadata?.get('ARROW:extension:name');
  const options: BinaryGeometriesFromArrowOptions = {
    ...(chunkIndex !== undefined && chunkIndex >= 0
      ? {
          chunkIndex,
          chunkOffset: geoColumn.data[0].length * chunkIndex
        }
      : {}),
    triangulate: true,
    calculateMeanCenters: true
  };

  // getBinaryGeometriesFromArrow doesn't support geoarrow.wkb
  // TODO why EXTENSION_NAME.WKB is undefined?
  if (encoding === 'geoarrow.wkb' /* EXTENSION_NAME.WKB*/) {
    return getBinaryGeometriesFromWKBArrow(geoColumn, options);
  }

  // create binary data from arrow data for GeoJsonLayer
  const {binaryGeometries, featureTypes, bounds, meanCenters} = getBinaryGeometriesFromArrow(
    // @ts-ignore
    geoColumn,
    encoding,
    options
  );

  // since there is no feature.properties.radius, we set fixedRadius to false
  const fixedRadius = false;

  return {
    dataToFeature: binaryGeometries,
    featureTypes,
    bounds,
    fixedRadius,
    centroids: meanCenters
  };
}

export function isLayerHoveredFromArrow(objectInfo, layerId: string): boolean {
  // there could be multiple deck.gl layers created from multiple chunks in arrow table
  // the objectInfo.layer id should be `${this.id}-${i}`
  if (objectInfo?.picked) {
    const deckLayerId = objectInfo?.layer?.props?.id;
    return deckLayerId.startsWith(layerId);
  }
  return false;
}

export function getHoveredObjectFromArrow(
  objectInfo,
  dataContainer,
  layerId,
  columnAccessor,
  fieldAccessor
): Feature | null {
  // hover object returns the index of the object in the data array
  // NOTE: this could be done in Deck.gl getPickingInfo(params) and binaryToGeojson()
  if (isLayerHoveredFromArrow(objectInfo, layerId) && objectInfo.index >= 0 && dataContainer) {
    const col = columnAccessor(dataContainer);
    const rawGeometry = col?.get(objectInfo.index);

    const field = fieldAccessor(dataContainer);
    const encoding = field?.metadata?.get('ARROW:extension:name');

    const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);

    const properties = dataContainer.rowAsArray(objectInfo.index).reduce((prev, cur, i) => {
      const fieldName = dataContainer?.getField?.(i).name;
      if (fieldName !== field.name) {
        prev[fieldName] = cur;
      }
      return prev;
    }, {});

    return hoveredFeature
      ? {
          type: 'Feature',
          geometry: hoveredFeature,
          properties: {
            ...properties,
            index: objectInfo.index
          }
        }
      : null;
  }
  return null;
}

/**
 * find requiredColumns of supported column mode based on column mode
 */
export function getColumnModeRequiredColumns(
  supportedColumnModes: SupportedColumnMode[] | null,
  columnMode?: string
): string[] | undefined {
  return supportedColumnModes?.find(({key}) => key === columnMode)?.requiredColumns;
}

/**
 * Returns geoarrow fields with ARROW:extension:name POINT metadata
 * @param fields Any fields
 * @returns geoarrow fields with ARROW:extension:name POINT metadata
 */
export function getGeoArrowPointFields(fields: Field[]): Field[] {
  return fields.filter(field => {
    return (
      field.type === 'geoarrow' &&
      field.metadata?.get('ARROW:extension:name') === EXTENSION_NAME.POINT
    );
  });
}

/**
 * Builds an arrow vector compatible with ARROW:extension:name geoarrow.point.
 * @param getPosition Position accessor.
 * @param numElements Number of elements in the vector.
 * @returns An arrow vector compatible with ARROW:extension:name geoarrow.point.
 */
export function createGeoArrowPointVector(
  dataContainer: ArrowDataContainer,
  getPosition: ({index}: {index: number}) => number[]
): arrow.Vector {
  // TODO update/resize existing vector?
  // TODO find an easier way to create point geo columns
  // in a correct arrow format, as this approach seems too excessive for just a simple interleaved buffer.

  const numElements = dataContainer.numRows();
  const table = dataContainer.getTable();

  const numCoords = numElements > 0 ? getPosition({index: 0}).length : 2;
  const precision = 2;

  const metadata = new Map();
  metadata.set('ARROW:extension:name', EXTENSION_NAME.POINT);

  const childField = new arrow.Field('xyz', new arrow.Float(precision), false, metadata);
  const fixedSizeList = new arrow.FixedSizeList(numCoords, childField);
  const floatBuilder = new arrow.FloatBuilder({type: new arrow.Float(precision)});
  const fixedSizeListBuilder = new arrow.FixedSizeListBuilder({type: fixedSizeList});
  fixedSizeListBuilder.addChild(floatBuilder);

  const assembledBatches: arrow.Data[] = [];
  const indexData = {index: 0};
  for (let batchIndex = 0; batchIndex < table.batches.length; ++batchIndex) {
    const numRowsInBatch = table.batches[batchIndex].numRows;

    for (let i = 0; i < numRowsInBatch; ++i) {
      const pos = getPosition(indexData);
      fixedSizeListBuilder.append(pos);

      ++indexData.index;
    }
    assembledBatches.push(fixedSizeListBuilder.flush());
  }

  return arrow.makeVector(assembledBatches);
}

/**
 * Builds a filtered index suitable for FilterArrowExtension.
 * @param numElements Size for filtered index array.
 * @param visibleIndices An array with indices of elements that aren't filtered out.
 * @returns filteredIndex [0|1] array for GPU filtering
 */
export function getFilteredIndex(
  numElements: number,
  visibleIndices: number[],
  existingFilteredIndex: Uint8ClampedArray | null
) {
  // contents are initialized with zeros by default, meaning not visible
  const filteredIndex =
    existingFilteredIndex && existingFilteredIndex.length === numElements
      ? existingFilteredIndex
      : new Uint8ClampedArray(numElements);
  filteredIndex.fill(0);

  if (visibleIndices) {
    for (let i = 0; i < visibleIndices.length; ++i) {
      filteredIndex[visibleIndices[i]] = 1;
    }
  }
  return filteredIndex;
}

/**
 * Returns an array of neighbors to the specified index.
 * @param neighborsField LayerColumn field with information about neighbors.
 * @param dataContainer Data container.
 * @param index Index of interest.
 * @param getPosition Position accessor.
 * @returns An array with information about neighbors.
 */
export function getNeighbors(
  neighborsField: LayerColumn | undefined,
  dataContainer: DataContainerInterface,
  index: number,
  getPosition: ({index}: {index: number}) => number[]
): {index: number; position: number[]}[] {
  if (!neighborsField || neighborsField.fieldIdx < 0) return [];

  let neighborIndices = dataContainer.valueAt(index, neighborsField.fieldIdx);
  // In case of arrow column with an array of indices.
  if (neighborIndices.toArray) {
    neighborIndices = Array.from(neighborIndices.toArray());
  }
  if (!Array.isArray(neighborIndices)) return [];

  // find neighbor
  const neighborsData = neighborIndices.map(idx => ({
    index: idx,
    position: getPosition({index: idx})
  }));

  return neighborsData;
}

/**
 * Returns bounds from a geoarrow field.
 * TODO: refactor once metadata extraction from parquet to arrow vectors is in place.
 * @param layerColumn Layer columns for which to check for a bounding box.
 * @param dataContainer Data container with geoarrow metadata.
 * @returns Returns bounding box if exists.
 */
export function getBoundsFromArrowMetadata(
  layerColumn: LayerColumn,
  dataContainer: ArrowDataContainer
): [number, number, number, number] | false {
  try {
    const field = dataContainer.getField(layerColumn.fieldIdx);
    const table = dataContainer.getTable();

    const geoMetadata = getGeoMetadata({
      metadata: {
        // @ts-expect-error
        geo: table.schema.metadata.get('geo')
      }
    });

    if (geoMetadata) {
      const fieldMetadata = geoMetadata.columns[field.name];
      if (fieldMetadata) {
        const boundsFromMetadata = fieldMetadata['bbox'];
        if (Array.isArray(boundsFromMetadata) && boundsFromMetadata.length === 4) {
          return boundsFromMetadata;
        }
      }
    }
  } catch (error) {
    // ignore for now
  }

  return false;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import * as arrow from 'apache-arrow';
5	import {Feature, BBox} from 'geojson';
6	import {getGeoMetadata} from '@loaders.gl/gis';
7
8	import {
9	Field,
10	ProtoDatasetField,
11	FieldPair,
12	SupportedColumnMode,
13	LayerColumn
14	} from '@kepler.gl/types';
15	import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
16	import {
17	getBinaryGeometriesFromArrow,
18	parseGeometryFromArrow,
19	BinaryGeometriesFromArrowOptions,
20	updateBoundsFromGeoArrowSamples
21	} from '@loaders.gl/arrow';
22	import {EXTENSION_NAME} from '@kepler.gl/deckgl-arrow-layers';
23
24	import {WKBLoader} from '@loaders.gl/wkt';
25	import {geojsonToBinary} from '@loaders.gl/gis';
26	import {
27	BinaryFeatureCollection,
28	Geometry,
29	BinaryPointFeature,
30	BinaryLineFeature,
31	BinaryPolygonFeature
32	} from '@loaders.gl/schema';
33
34	import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';
35
36	export function assignPointPairToLayerColumn(pair: FieldPair, hasAlt: boolean) {
37	const {lat, lng, altitude} = pair.pair;	79✔
38	if (!hasAlt) {	79!
39	return {lat, lng};	×
40	}
41
42	const defaultAltColumn = {value: null, fieldIdx: -1, optional: true};	79✔
43
44	return {	79✔
45	lat,
46	lng,
47	altitude: altitude ? {...defaultAltColumn, ...altitude} : defaultAltColumn	79✔
48	};
49	}
50
51	export type GeojsonLayerMetaProps = {
52	dataToFeature: GeojsonDataMaps;
53	featureTypes: DeckGlGeoTypes;
54	bounds: BBox \| null;
55	fixedRadius: boolean;
56	centroids?: Array<number[] \| null>;
57	};
58
59	/**
60	* Converts a geoarrow.wkb vector into an array of BinaryFeatureCollections.
61	* @param geoColumn A vector column with geoarrow.wkb extension.
62	* @param options Options for geometry transformation.
63	* @returns
64	*/
65	function getBinaryGeometriesFromWKBArrow(
66	geoColumn: arrow.Vector,
67	options: {chunkIndex?: number; chunkOffset?: number}
68	): GeojsonLayerMetaProps {
69	const dataToFeature: BinaryFeatureCollection[] = [];	×
70	const featureTypes: GeojsonLayerMetaProps['featureTypes'] = {	×
71	point: false,
72	line: false,
73	polygon: false
74	};
75
76	const chunks =
77	options?.chunkIndex !== undefined && options?.chunkIndex >= 0	×
78	? [geoColumn.data[options?.chunkIndex]]
79	: geoColumn.data;
80	const globalFeatureIdOffset = options?.chunkOffset \|\| 0;	×
81	let featureIndex = globalFeatureIdOffset;	×
82	let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];	×
83
84	chunks.forEach(chunk => {	×
85	const geojsonFeatures: Feature[] = [];	×
86	for (let i = 0; i < chunk.length; ++i) {	×
87	// ignore features without any geometry
88	if (chunk.valueOffsets[i + 1] - chunk.valueOffsets[i] > 0) {	×
89	const valuesSlice = chunk.values.slice(chunk.valueOffsets[i], chunk.valueOffsets[i + 1]);	×
90
91	const geometry = WKBLoader?.parseSync?.(valuesSlice.buffer, {	×
92	wkb: {shape: 'geojson-geometry'}
93	}) as Geometry;
94	const feature: Feature = {	×
95	type: 'Feature',
96	geometry,
97	properties: {index: featureIndex}
98	};
99	geojsonFeatures.push(feature);	×
100
101	const {type} = geometry;	×
102	featureTypes.polygon = type === 'Polygon' \|\| type === 'MultiPolygon';	×
103	featureTypes.point = type === 'Point' \|\| type === 'MultiPoint';	×
104	featureTypes.line = type === 'LineString' \|\| type === 'MultiLineString';	×
105	}
106
107	featureIndex++;	×
108	}
109
110	const geojsonToBinaryOptions = {	×
111	triangulate: true,
112	fixRingWinding: true
113	};
114	const binaryFeatures = geojsonToBinary(geojsonFeatures, geojsonToBinaryOptions);	×
115
116	// Need to update globalFeatureIds, to take into account previous batches,
117	// as geojsonToBinary doesn't have such option.
118	const featureTypesArr = ['points', 'lines', 'polygons'];	×
119	featureTypesArr.forEach(prop => {	×
120	const features = binaryFeatures[prop] as	×
121	\| BinaryPointFeature
122	\| BinaryLineFeature
123	\| BinaryPolygonFeature;
124	if (features) {	×
125	bounds = updateBoundsFromGeoArrowSamples(	×
126	features.positions.value as Float64Array,
127	features.positions.size,
128	bounds
129	);
130
131	const {globalFeatureIds, numericProps} = features;	×
132	const {index} = numericProps;	×
133	const len = globalFeatureIds.value.length;	×
134	for (let i = 0; i < len; ++i) {	×
135	globalFeatureIds.value[i] = index.value[i];	×
136	}
137	}
138	});
139
140	dataToFeature.push(binaryFeatures);	×
141	});
142
143	return {	×
144	dataToFeature: dataToFeature,
145	featureTypes: featureTypes,
146	bounds,
147	fixedRadius: false
148	};
149	}
150
151	export function getGeojsonLayerMetaFromArrow({
152	dataContainer,
153	geoColumn,
154	geoField,
155	chunkIndex
156	}: {
157	dataContainer: DataContainerInterface;
158	geoColumn: arrow.Vector;
159	geoField: ProtoDatasetField;
160	chunkIndex?: number;
161	}): GeojsonLayerMetaProps {
162	const encoding = geoField?.metadata?.get('ARROW:extension:name');	×
163	const options: BinaryGeometriesFromArrowOptions = {	×
164	...(chunkIndex !== undefined && chunkIndex >= 0	×
165	? {
166	chunkIndex,
167	chunkOffset: geoColumn.data[0].length * chunkIndex
168	}
169	: {}),
170	triangulate: true,
171	calculateMeanCenters: true
172	};
173
174	// getBinaryGeometriesFromArrow doesn't support geoarrow.wkb
175	// TODO why EXTENSION_NAME.WKB is undefined?
NEW 176	if (encoding === 'geoarrow.wkb' /* EXTENSION_NAME.WKB*/) {	×
UNCOV 177	return getBinaryGeometriesFromWKBArrow(geoColumn, options);	×
178	}
179
180	// create binary data from arrow data for GeoJsonLayer
181	const {binaryGeometries, featureTypes, bounds, meanCenters} = getBinaryGeometriesFromArrow(	×
182	// @ts-ignore
183	geoColumn,
184	encoding,
185	options
186	);
187
188	// since there is no feature.properties.radius, we set fixedRadius to false
189	const fixedRadius = false;	×
190
191	return {	×
192	dataToFeature: binaryGeometries,
193	featureTypes,
194	bounds,
195	fixedRadius,
196	centroids: meanCenters
197	};
198	}
199
200	export function isLayerHoveredFromArrow(objectInfo, layerId: string): boolean {
201	// there could be multiple deck.gl layers created from multiple chunks in arrow table
202	// the objectInfo.layer id should be `${this.id}-${i}`
203	if (objectInfo?.picked) {	22!
204	const deckLayerId = objectInfo?.layer?.props?.id;	×
205	return deckLayerId.startsWith(layerId);	×
206	}
207	return false;	22✔
208	}
209
210	export function getHoveredObjectFromArrow(
211	objectInfo,
212	dataContainer,
213	layerId,
214	columnAccessor,
215	fieldAccessor
216	): Feature \| null {
217	// hover object returns the index of the object in the data array
218	// NOTE: this could be done in Deck.gl getPickingInfo(params) and binaryToGeojson()
219	if (isLayerHoveredFromArrow(objectInfo, layerId) && objectInfo.index >= 0 && dataContainer) {	×
220	const col = columnAccessor(dataContainer);	×
221	const rawGeometry = col?.get(objectInfo.index);	×
222
223	const field = fieldAccessor(dataContainer);	×
224	const encoding = field?.metadata?.get('ARROW:extension:name');	×
225
226	const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);	×
227
228	const properties = dataContainer.rowAsArray(objectInfo.index).reduce((prev, cur, i) => {	×
229	const fieldName = dataContainer?.getField?.(i).name;	×
230	if (fieldName !== field.name) {	×
231	prev[fieldName] = cur;	×
232	}
233	return prev;	×
234	}, {});
235
236	return hoveredFeature	×
237	? {
238	type: 'Feature',
239	geometry: hoveredFeature,
240	properties: {
241	...properties,
242	index: objectInfo.index
243	}
244	}
245	: null;
246	}
247	return null;	×
248	}
249
250	/**
251	* find requiredColumns of supported column mode based on column mode
252	*/
253	export function getColumnModeRequiredColumns(
254	supportedColumnModes: SupportedColumnMode[] \| null,
255	columnMode?: string
256	): string[] \| undefined {
257	return supportedColumnModes?.find(({key}) => key === columnMode)?.requiredColumns;	×
258	}
259
260	/**
261	* Returns geoarrow fields with ARROW:extension:name POINT metadata
262	* @param fields Any fields
263	* @returns geoarrow fields with ARROW:extension:name POINT metadata
264	*/
265	export function getGeoArrowPointFields(fields: Field[]): Field[] {
266	return fields.filter(field => {	×
267	return (	×
268	field.type === 'geoarrow' &&	×
269	field.metadata?.get('ARROW:extension:name') === EXTENSION_NAME.POINT
270	);
271	});
272	}
273
274	/**
275	* Builds an arrow vector compatible with ARROW:extension:name geoarrow.point.
276	* @param getPosition Position accessor.
277	* @param numElements Number of elements in the vector.
278	* @returns An arrow vector compatible with ARROW:extension:name geoarrow.point.
279	*/
280	export function createGeoArrowPointVector(
281	dataContainer: ArrowDataContainer,
282	getPosition: ({index}: {index: number}) => number[]
283	): arrow.Vector {
284	// TODO update/resize existing vector?
285	// TODO find an easier way to create point geo columns
286	// in a correct arrow format, as this approach seems too excessive for just a simple interleaved buffer.
287
288	const numElements = dataContainer.numRows();	×
289	const table = dataContainer.getTable();	×
290
291	const numCoords = numElements > 0 ? getPosition({index: 0}).length : 2;	×
292	const precision = 2;	×
293
294	const metadata = new Map();	×
295	metadata.set('ARROW:extension:name', EXTENSION_NAME.POINT);	×
296
297	const childField = new arrow.Field('xyz', new arrow.Float(precision), false, metadata);	×
298	const fixedSizeList = new arrow.FixedSizeList(numCoords, childField);	×
299	const floatBuilder = new arrow.FloatBuilder({type: new arrow.Float(precision)});	×
300	const fixedSizeListBuilder = new arrow.FixedSizeListBuilder({type: fixedSizeList});	×
301	fixedSizeListBuilder.addChild(floatBuilder);	×
302
303	const assembledBatches: arrow.Data[] = [];	×
304	const indexData = {index: 0};	×
305	for (let batchIndex = 0; batchIndex < table.batches.length; ++batchIndex) {	×
306	const numRowsInBatch = table.batches[batchIndex].numRows;	×
307
308	for (let i = 0; i < numRowsInBatch; ++i) {	×
309	const pos = getPosition(indexData);	×
310	fixedSizeListBuilder.append(pos);	×
311
312	++indexData.index;	×
313	}
314	assembledBatches.push(fixedSizeListBuilder.flush());	×
315	}
316
317	return arrow.makeVector(assembledBatches);	×
318	}
319
320	/**
321	* Builds a filtered index suitable for FilterArrowExtension.
322	* @param numElements Size for filtered index array.
323	* @param visibleIndices An array with indices of elements that aren't filtered out.
324	* @returns filteredIndex [0\|1] array for GPU filtering
325	*/
326	export function getFilteredIndex(
327	numElements: number,
328	visibleIndices: number[],
329	existingFilteredIndex: Uint8ClampedArray \| null
330	) {
331	// contents are initialized with zeros by default, meaning not visible
332	const filteredIndex =
333	existingFilteredIndex && existingFilteredIndex.length === numElements	×
334	? existingFilteredIndex
335	: new Uint8ClampedArray(numElements);
336	filteredIndex.fill(0);	×
337
338	if (visibleIndices) {	×
339	for (let i = 0; i < visibleIndices.length; ++i) {	×
340	filteredIndex[visibleIndices[i]] = 1;	×
341	}
342	}
343	return filteredIndex;	×
344	}
345
346	/**
347	* Returns an array of neighbors to the specified index.
348	* @param neighborsField LayerColumn field with information about neighbors.
349	* @param dataContainer Data container.
350	* @param index Index of interest.
351	* @param getPosition Position accessor.
352	* @returns An array with information about neighbors.
353	*/
354	export function getNeighbors(
355	neighborsField: LayerColumn \| undefined,
356	dataContainer: DataContainerInterface,
357	index: number,
358	getPosition: ({index}: {index: number}) => number[]
359	): {index: number; position: number[]}[] {
360	if (!neighborsField \|\| neighborsField.fieldIdx < 0) return [];	×
361
362	let neighborIndices = dataContainer.valueAt(index, neighborsField.fieldIdx);	×
363	// In case of arrow column with an array of indices.
364	if (neighborIndices.toArray) {	×
365	neighborIndices = Array.from(neighborIndices.toArray());	×
366	}
367	if (!Array.isArray(neighborIndices)) return [];	×
368
369	// find neighbor
370	const neighborsData = neighborIndices.map(idx => ({	×
371	index: idx,
372	position: getPosition({index: idx})
373	}));
374
375	return neighborsData;	×
376	}
377
378	/**
379	* Returns bounds from a geoarrow field.
380	* TODO: refactor once metadata extraction from parquet to arrow vectors is in place.
381	* @param layerColumn Layer columns for which to check for a bounding box.
382	* @param dataContainer Data container with geoarrow metadata.
383	* @returns Returns bounding box if exists.
384	*/
385	export function getBoundsFromArrowMetadata(
386	layerColumn: LayerColumn,
387	dataContainer: ArrowDataContainer
388	): [number, number, number, number] \| false {
389	try {	×
390	const field = dataContainer.getField(layerColumn.fieldIdx);	×
391	const table = dataContainer.getTable();	×
392
393	const geoMetadata = getGeoMetadata({	×
394	metadata: {
395	// @ts-expect-error
396	geo: table.schema.metadata.get('geo')
397	}
398	});
399
400	if (geoMetadata) {	×
401	const fieldMetadata = geoMetadata.columns[field.name];	×
402	if (fieldMetadata) {	×
403	const boundsFromMetadata = fieldMetadata['bbox'];	×
404	if (Array.isArray(boundsFromMetadata) && boundsFromMetadata.length === 4) {	×
405	return boundsFromMetadata;	×
406	}
407	}
408	}
409	} catch (error) {
410	// ignore for now
411	}
412
413	return false;	×
414	}

keplergl / kepler.gl / 12941437080

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous