19670167673

Committed 25 Nov 2025 12:52PM UTC coverage: 61.76% (-0.007%) from 61.767%

Build # 19670167673

Build Type

push

github

Committed by

web-flow

Commit Message

fix: Allow passing arrow tables to ArrowDataContainer (#3242)

* fix: copy geometry when geometry is of binary format (#3236)

* fix: copy geometry when geometry is of binary format

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* nit

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

* fix: Allow passing arrow tables to ArrowDataContainer

Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

* only add arrowTable prop to CREATE_TABLE_TASK when set

Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>
Co-authored-by: Igor Dykhta <igorDykhta@users.noreply.github.com>

Run Details

6349 of 12189 branches covered (52.09%)

Branch coverage included in aggregate %.

2 of 5 new or added lines in 2 files covered. (40.0%)

48 existing lines in 2 files now uncovered.

13043 of 19210 relevant lines covered (67.9%)

81.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.55

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash/uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {
  DatasetType,
  RasterTileDatasetMetadata,
  PMTilesType,
  RemoteTileFormat,
  VectorTileDatasetMetadata,
  WMSDatasetMetadata
} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';

import {load} from '@loaders.gl/core';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {WMSCapabilities, WMSCapabilitiesLoader} from '@loaders.gl/wms';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseRasterMetadata} from './tileset/raster-tile-utils';
import {
  parseVectorMetadata,
  getFieldsFromTile,
  VectorTileMetadata
} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets | null {
  const TableClass = getApplicationConfig().table ?? KeplerTable;
  let dataValidator = validateInputData;
  if (typeof TableClass.getInputDataValidator === 'function') {
    dataValidator = TableClass.getInputDataValidator();
  }

  const validatedData = dataValidator(data);
  if (!validatedData) {
    return null;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: data.arrowTable ? {...validatedData, arrowTable: data.arrowTable} : validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    if (metadata.fields) {
      data.fields = metadata.fields;
    }
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  try {
    await table.importData({data});
  } catch (error) {
    console.error('Failed to create table', error);
    throw error;
  }

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object | null> {
  const {type} = datasetInfo.info;
  switch (type) {
    case DatasetType.VECTOR_TILE:
      return await refreshVectorTileMetadata(datasetInfo);
    case DatasetType.RASTER_TILE:
      return await refreshRasterTileMetadata(datasetInfo);
    case DatasetType.WMS_TILE:
      return await refreshWMSMetadata(datasetInfo);
    default:
      return null;
  }
}

async function refreshVectorTileMetadata(
  datasetInfo: CreateTableProps
): Promise<VectorTileMetadata | null> {
  const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
    typeof tilesetMetadataUrl !== 'string' ||
    typeof tilesetDataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      const metadata = parseVectorMetadata(rawMetadata);

      await getFieldsFromTile({
        remoteTileFormat,
        tilesetUrl: tilesetDataUrl,
        metadataUrl: tilesetMetadataUrl,
        metadata
      });

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) || {};

  if (typeof metadataUrl !== 'string') {
    return null;
  }

  try {
    if (pmtilesType === PMTilesType.RASTER) {
      const tileSource = PMTilesSource.createDataSource(metadataUrl, {});
      const rawMetadata: PMTilesMetadata = await tileSource.metadata;

      if (rawMetadata) {
        return parseVectorMetadata(rawMetadata);
      }
    } else {
      // it's stac raster tiles
      const response = await fetch(metadataUrl);
      if (!response.ok) {
        throw new Error(`Failed Fetch ${metadataUrl}`);
      }
      const rawMetadata = await response.json();

      const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});
      if (metadata instanceof Error) {
        throw new Error(`Failed to parse metadata ${metadata.message}`);
      }

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshWMSMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {remoteTileFormat, tilesetDataUrl} =
    (datasetInfo.opts.metadata as WMSDatasetMetadata) || {};

  if (remoteTileFormat !== RemoteTileFormat.WMS || typeof tilesetDataUrl !== 'string') {
    return null;
  }

  try {
    const data = await getWMSCapabilities(tilesetDataUrl);
    return wmsCapabilitiesToDatasetMetadata(data);
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

export async function getWMSCapabilities(wsmUrl: string): Promise<WMSCapabilities> {
  return (await load(
    `${wsmUrl}?service=WMS&request=GetCapabilities`,
    WMSCapabilitiesLoader
  )) as WMSCapabilities;
}

export function wmsCapabilitiesToDatasetMetadata(capabilities: WMSCapabilities): any | null {
  // Flatten layers if they are nested
  const layers = capabilities.layers.flatMap(layer => {
    if (layer.layers && layer.layers.length > 0) {
      return layer.layers;
    }
    return layer;
  });

  let availableLayers: WMSDatasetMetadata['layers'] = [];
  if (Array.isArray(layers)) {
    availableLayers = layers.map((layer: any) => {
      const bb = layer.geographicBoundingBox;

      let boundingBox: number[] | null = null;
      if (Array.isArray(bb) && Array.isArray(bb[0]) && Array.isArray(bb[1])) {
        boundingBox = [bb[0][0], bb[0][1], bb[1][0], bb[1][1]];
      }

      return {
        name: layer.name,
        title: layer.title || layer.name,
        boundingBox,
        queryable: layer.queryable
      };
    });
  }

  return {
    layers: availableLayers,
    version: capabilities.version || '1.3.0'
  };
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash/uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {
10	DatasetType,
11	RasterTileDatasetMetadata,
12	PMTilesType,
13	RemoteTileFormat,
14	VectorTileDatasetMetadata,
15	WMSDatasetMetadata
16	} from '@kepler.gl/constants';
17	import {
18	hexToRgb,
19	validateInputData,
20	datasetColorMaker,
21	getApplicationConfig
22	} from '@kepler.gl/utils';
23
24	import {load} from '@loaders.gl/core';
25	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
26	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
27	import {WMSCapabilities, WMSCapabilitiesLoader} from '@loaders.gl/wms';
28
29	import {getMVTMetadata} from './tileset/tileset-utils';
30	import {parseRasterMetadata} from './tileset/raster-tile-utils';
31	import {
32	parseVectorMetadata,
33	getFieldsFromTile,
34	VectorTileMetadata
35	} from './tileset/vector-tile-utils';
36
37	// apply a color for each dataset
38	// to use as label colors
39	const datasetColors = [	13✔
40	'#8F2FBF',
41	'#005CFF',
42	'#C06C84',
43	'#F8B195',
44	'#547A82',
45	'#3EACA8',
46	'#A2D4AB'
47	].map(hexToRgb);
48
49	export function getNewDatasetColor(datasets: Datasets): RGBColor {
50	const presetColors = datasetColors.map(String);	100✔
51	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
52	presetColors.includes(c)	25✔
53	);
54
55	if (usedColors.length === presetColors.length) {	100!
56	// if we already depleted the pool of color
57	return datasetColorMaker.next().value;	×
58	}
59
60	let color = datasetColorMaker.next().value;	100✔
61	while (usedColors.includes(String(color))) {	100✔
62	color = datasetColorMaker.next().value;	2✔
63	}
64
65	return color;	100✔
66	}
67
68	/**
69	* Take datasets payload from addDataToMap, create datasets entry save to visState
70	*/
71	export function createNewDataEntry(
72	{info, data, ...opts}: ProtoDataset,
73	datasets: Datasets = {}	15✔
74	): Datasets \| null {
75	const TableClass = getApplicationConfig().table ?? KeplerTable;	154✔
76	let dataValidator = validateInputData;	154✔
77	if (typeof TableClass.getInputDataValidator === 'function') {	154!
78	dataValidator = TableClass.getInputDataValidator();	×
79	}
80
81	const validatedData = dataValidator(data);	154✔
82	if (!validatedData) {	154✔
83	return null;	3✔
84	}
85
86	// check if dataset already exists, and update it when loading data by batches incrementally
87	if (info && info.id && datasets[info.id]) {	151!
88	// get keplerTable from datasets
89	const keplerTable = datasets[info.id];	×
90	// update the data in keplerTable
91	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
92	}
93
94	info = info \|\| {};	151!
95	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
96
97	return CREATE_TABLE_TASK({	151✔
98	info,
99	color,
100	opts,
101	data: data.arrowTable ? {...validatedData, arrowTable: data.arrowTable} : validatedData	151!
102	});
103	}
104
105	async function updateTable({table, data}) {
106	const updated = await table.update(data); // Assuming `table` has an `update` method	×
107	return updated;	×
108	}
109
110	type CreateTableProps = {
111	info: any;
112	color: RGBColor;
113	opts: {
114	metadata?: Record<string, unknown>;
115	};
116	data: any;
117	};
118
119	async function createTable(datasetInfo: CreateTableProps) {
120	const {info, color, opts, data} = datasetInfo;	14✔
121
122	// update metadata for remote tiled datasets
123	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
124	let metadata = opts.metadata;	14✔
125	if (refreshedMetadata) {	14!
126	metadata = {...opts.metadata, ...refreshedMetadata};	×
127	if (metadata.fields) {	×
128	data.fields = metadata.fields;	×
129	}
130	}
131
132	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
133	const table = new TableClass({	14✔
134	info,
135	color,
136	...opts,
137	metadata
138	});
139	try {	14✔
140	await table.importData({data});	14✔
141	} catch (error) {
NEW 142	console.error('Failed to create table', error);	×
NEW 143	throw error;	×
144	}
145
146	return table;	14✔
147	}
148	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
149	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
150
151	/**
152	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
153	* @param datasetInfo
154	* @returns
155	*/
156	async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object \| null> {
157	const {type} = datasetInfo.info;	14✔
158	switch (type) {	14!
159	case DatasetType.VECTOR_TILE:
160	return await refreshVectorTileMetadata(datasetInfo);	×
161	case DatasetType.RASTER_TILE:
162	return await refreshRasterTileMetadata(datasetInfo);	×
163	case DatasetType.WMS_TILE:
164	return await refreshWMSMetadata(datasetInfo);	×
165	default:
166	return null;	14✔
167	}
168	}
169
170	async function refreshVectorTileMetadata(
171	datasetInfo: CreateTableProps
172	): Promise<VectorTileMetadata \| null> {
173	const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
174	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
175
176	if (	×
177	!(remoteTileFormat === RemoteTileFormat.PMTILES \|\| remoteTileFormat === RemoteTileFormat.MVT) \|\|	×
178	typeof tilesetMetadataUrl !== 'string' \|\|
179	typeof tilesetDataUrl !== 'string'
180	) {
181	return null;	×
182	}
183
184	try {	×
185	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
186	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
187	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
188	} else {
189	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
190	rawMetadata = await tileSource.metadata;	×
191	}
192
193	if (rawMetadata) {	×
194	const metadata = parseVectorMetadata(rawMetadata);	×
195
196	await getFieldsFromTile({	×
197	remoteTileFormat,
198	tilesetUrl: tilesetDataUrl,
199	metadataUrl: tilesetMetadataUrl,
200	metadata
201	});
202
203	return metadata;	×
204	}
205	} catch (err) {
206	// ignore for now, and use old metadata
207	}
208	return null;	×
209	}
210
211	async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
212	const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) \|\| {};	×
213
214	if (typeof metadataUrl !== 'string') {	×
215	return null;	×
216	}
217
218	try {	×
219	if (pmtilesType === PMTilesType.RASTER) {	×
220	const tileSource = PMTilesSource.createDataSource(metadataUrl, {});	×
221	const rawMetadata: PMTilesMetadata = await tileSource.metadata;	×
222
223	if (rawMetadata) {	×
224	return parseVectorMetadata(rawMetadata);	×
225	}
226	} else {
227	// it's stac raster tiles
228	const response = await fetch(metadataUrl);	×
229	if (!response.ok) {	×
230	throw new Error(`Failed Fetch ${metadataUrl}`);	×
231	}
232	const rawMetadata = await response.json();	×
233
234	const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});	×
235	if (metadata instanceof Error) {	×
236	throw new Error(`Failed to parse metadata ${metadata.message}`);	×
237	}
238
239	return metadata;	×
240	}
241	} catch (err) {
242	// ignore for now, and use old metadata
243	}
244	return null;	×
245	}
246
247	async function refreshWMSMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
248	const {remoteTileFormat, tilesetDataUrl} =
249	(datasetInfo.opts.metadata as WMSDatasetMetadata) \|\| {};	×
250
251	if (remoteTileFormat !== RemoteTileFormat.WMS \|\| typeof tilesetDataUrl !== 'string') {	×
252	return null;	×
253	}
254
255	try {	×
256	const data = await getWMSCapabilities(tilesetDataUrl);	×
257	return wmsCapabilitiesToDatasetMetadata(data);	×
258	} catch (err) {
259	// ignore for now, and use old metadata
260	}
261	return null;	×
262	}
263
264	export async function getWMSCapabilities(wsmUrl: string): Promise<WMSCapabilities> {
265	return (await load(	×
266	`${wsmUrl}?service=WMS&request=GetCapabilities`,
267	WMSCapabilitiesLoader
268	)) as WMSCapabilities;
269	}
270
271	export function wmsCapabilitiesToDatasetMetadata(capabilities: WMSCapabilities): any \| null {
272	// Flatten layers if they are nested
273	const layers = capabilities.layers.flatMap(layer => {	×
274	if (layer.layers && layer.layers.length > 0) {	×
275	return layer.layers;	×
276	}
277	return layer;	×
278	});
279
280	let availableLayers: WMSDatasetMetadata['layers'] = [];	×
281	if (Array.isArray(layers)) {	×
282	availableLayers = layers.map((layer: any) => {	×
283	const bb = layer.geographicBoundingBox;	×
284
285	let boundingBox: number[] \| null = null;	×
286	if (Array.isArray(bb) && Array.isArray(bb[0]) && Array.isArray(bb[1])) {	×
287	boundingBox = [bb[0][0], bb[0][1], bb[1][0], bb[1][1]];	×
288	}
289
290	return {	×
291	name: layer.name,
292	title: layer.title \|\| layer.name,	×
293	boundingBox,
294	queryable: layer.queryable
295	};
296	});
297	}
298
299	return {	×
300	layers: availableLayers,
301	version: capabilities.version \|\| '1.3.0'	×
302	};
303	}

keplergl / kepler.gl / 19670167673

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous