23965535588

Committed 03 Apr 2026 11:07PM UTC coverage: 59.873% (-1.8%) from 61.699%

Build # 23965535588

Build Type

push

github

Committed by

web-flow

Commit Message

chore: deck.gl 9.2 upgrade & loaders.gl, luma.gl upgrades (#3271)

* chore: upgrade to deckgl 9.2.11

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix lint follow up

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix blending

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix geojson layer

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* more fixes for aggregation layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix h3 layer

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* potential fix for issues with geoarrow in point and line layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix postprocessing effects

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for light and shadow effect

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* shadow and light effect - restore uniform shadow during the nighttime

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* don't hide line and arc layers when layer type changed

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* restore filters for aggregation layers

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix aggregation layers - hightlight outlines

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for raster tile layer - raster pmtiles related

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fixes for raster tiles shader modules updates

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* fix lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* more lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix CI lint

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix CI tests

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* install webgpu

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* try to fix Ci test env setup

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* cont fix setup browser env

Signed-o... (continued)

Coverage Stats

6517 of 12977 branches covered (50.22%)

Branch coverage included in aggregate %.

324 of 1031 new or added lines in 58 files covered. (31.43%)

123 existing lines in 18 files now uncovered.

13313 of 20143 relevant lines covered (66.09%)

78.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.21

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash/uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {
  DatasetType,
  RasterTileDatasetMetadata,
  PMTilesType,
  RemoteTileFormat,
  VectorTileDatasetMetadata,
  WMSDatasetMetadata
} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';

import {load} from '@loaders.gl/core';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {WMSCapabilities, WMSCapabilitiesLoader} from '@loaders.gl/wms';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseRasterMetadata} from './tileset/raster-tile-utils';
import {
  parseVectorMetadata,
  getFieldsFromTile,
  VectorTileMetadata
} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets | null {
  const TableClass = getApplicationConfig().table ?? KeplerTable;
  let dataValidator = validateInputData;
  if (typeof TableClass.getInputDataValidator === 'function') {
    dataValidator = TableClass.getInputDataValidator();
  }

  const validatedData = dataValidator(data);
  if (!validatedData) {
    return null;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: data.arrowTable ? {...validatedData, arrowTable: data.arrowTable} : validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    if (metadata.fields) {
      data.fields = metadata.fields;
    }
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  try {
    await table.importData({data});
  } catch (error) {
    console.error('Failed to create table', error);
    throw error;
  }

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object | null> {
  const {type} = datasetInfo.info;
  switch (type) {
    case DatasetType.VECTOR_TILE:
      return await refreshVectorTileMetadata(datasetInfo);
    case DatasetType.RASTER_TILE:
      return await refreshRasterTileMetadata(datasetInfo);
    case DatasetType.WMS_TILE:
      return await refreshWMSMetadata(datasetInfo);
    case DatasetType.TILE_3D:
      return null;
    default:
      return null;
  }
}

async function refreshVectorTileMetadata(
  datasetInfo: CreateTableProps
): Promise<VectorTileMetadata | null> {
  const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
    typeof tilesetMetadataUrl !== 'string' ||
    typeof tilesetDataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      const metadata = parseVectorMetadata(rawMetadata);

      await getFieldsFromTile({
        remoteTileFormat,
        tilesetUrl: tilesetDataUrl,
        metadataUrl: tilesetMetadataUrl,
        metadata
      });

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) || {};

  if (typeof metadataUrl !== 'string') {
    return null;
  }

  try {
    if (pmtilesType === PMTilesType.RASTER) {
      const tileSource = PMTilesSource.createDataSource(metadataUrl, {});
      const rawMetadata: PMTilesMetadata = await tileSource.metadata;

      if (rawMetadata) {
        return parseVectorMetadata(rawMetadata);
      }
    } else {
      // it's stac raster tiles
      const response = await fetch(metadataUrl);
      if (!response.ok) {
        throw new Error(`Failed Fetch ${metadataUrl}`);
      }
      const rawMetadata = await response.json();

      const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});
      if (metadata instanceof Error) {
        throw new Error(`Failed to parse metadata ${metadata.message}`);
      }

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshWMSMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {remoteTileFormat, tilesetDataUrl} =
    (datasetInfo.opts.metadata as WMSDatasetMetadata) || {};

  if (remoteTileFormat !== RemoteTileFormat.WMS || typeof tilesetDataUrl !== 'string') {
    return null;
  }

  try {
    const data = await getWMSCapabilities(tilesetDataUrl);
    return wmsCapabilitiesToDatasetMetadata(data);
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

export async function getWMSCapabilities(wsmUrl: string): Promise<WMSCapabilities> {
  return (await load(
    `${wsmUrl}?service=WMS&request=GetCapabilities`,
    WMSCapabilitiesLoader
  )) as WMSCapabilities;
}

export function wmsCapabilitiesToDatasetMetadata(capabilities: WMSCapabilities): any | null {
  // Flatten layers if they are nested
  const layers = capabilities.layers.flatMap(layer => {
    if (layer.layers && layer.layers.length > 0) {
      return layer.layers;
    }
    return layer;
  });

  let availableLayers: WMSDatasetMetadata['layers'] = [];
  if (Array.isArray(layers)) {
    availableLayers = layers.map((layer: any) => {
      const bb = layer.geographicBoundingBox;

      let boundingBox: number[] | null = null;
      if (Array.isArray(bb) && Array.isArray(bb[0]) && Array.isArray(bb[1])) {
        boundingBox = [bb[0][0], bb[0][1], bb[1][0], bb[1][1]];
      }

      return {
        name: layer.name,
        title: layer.title || layer.name,
        boundingBox,
        queryable: layer.queryable
      };
    });
  }

  return {
    layers: availableLayers,
    version: capabilities.version || '1.3.0'
  };
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash/uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {
10	DatasetType,
11	RasterTileDatasetMetadata,
12	PMTilesType,
13	RemoteTileFormat,
14	VectorTileDatasetMetadata,
15	WMSDatasetMetadata
16	} from '@kepler.gl/constants';
17	import {
18	hexToRgb,
19	validateInputData,
20	datasetColorMaker,
21	getApplicationConfig
22	} from '@kepler.gl/utils';
23
24	import {load} from '@loaders.gl/core';
25	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
26	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
27	import {WMSCapabilities, WMSCapabilitiesLoader} from '@loaders.gl/wms';
28
29	import {getMVTMetadata} from './tileset/tileset-utils';
30	import {parseRasterMetadata} from './tileset/raster-tile-utils';
31	import {
32	parseVectorMetadata,
33	getFieldsFromTile,
34	VectorTileMetadata
35	} from './tileset/vector-tile-utils';
36
37	// apply a color for each dataset
38	// to use as label colors
39	const datasetColors = [	13✔
40	'#8F2FBF',
41	'#005CFF',
42	'#C06C84',
43	'#F8B195',
44	'#547A82',
45	'#3EACA8',
46	'#A2D4AB'
47	].map(hexToRgb);
48
49	export function getNewDatasetColor(datasets: Datasets): RGBColor {
50	const presetColors = datasetColors.map(String);	100✔
51	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
52	presetColors.includes(c)	25✔
53	);
54
55	if (usedColors.length === presetColors.length) {	100!
56	// if we already depleted the pool of color
57	return datasetColorMaker.next().value;	×
58	}
59
60	let color = datasetColorMaker.next().value;	100✔
61	while (usedColors.includes(String(color))) {	100✔
62	color = datasetColorMaker.next().value;	2✔
63	}
64
65	return color;	100✔
66	}
67
68	/**
69	* Take datasets payload from addDataToMap, create datasets entry save to visState
70	*/
71	export function createNewDataEntry(
72	{info, data, ...opts}: ProtoDataset,
73	datasets: Datasets = {}	15✔
74	): Datasets \| null {
75	const TableClass = getApplicationConfig().table ?? KeplerTable;	154✔
76	let dataValidator = validateInputData;	154✔
77	if (typeof TableClass.getInputDataValidator === 'function') {	154!
78	dataValidator = TableClass.getInputDataValidator();	×
79	}
80
81	const validatedData = dataValidator(data);	154✔
82	if (!validatedData) {	154✔
83	return null;	3✔
84	}
85
86	// check if dataset already exists, and update it when loading data by batches incrementally
87	if (info && info.id && datasets[info.id]) {	151!
88	// get keplerTable from datasets
89	const keplerTable = datasets[info.id];	×
90	// update the data in keplerTable
91	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
92	}
93
94	info = info \|\| {};	151!
95	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
96
97	return CREATE_TABLE_TASK({	151✔
98	info,
99	color,
100	opts,
101	data: data.arrowTable ? {...validatedData, arrowTable: data.arrowTable} : validatedData	151!
102	});
103	}
104
105	async function updateTable({table, data}) {
106	const updated = await table.update(data); // Assuming `table` has an `update` method	×
107	return updated;	×
108	}
109
110	type CreateTableProps = {
111	info: any;
112	color: RGBColor;
113	opts: {
114	metadata?: Record<string, unknown>;
115	};
116	data: any;
117	};
118
119	async function createTable(datasetInfo: CreateTableProps) {
120	const {info, color, opts, data} = datasetInfo;	14✔
121
122	// update metadata for remote tiled datasets
123	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
124	let metadata = opts.metadata;	14✔
125	if (refreshedMetadata) {	14!
126	metadata = {...opts.metadata, ...refreshedMetadata};	×
127	if (metadata.fields) {	×
128	data.fields = metadata.fields;	×
129	}
130	}
131
132	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
133	const table = new TableClass({	14✔
134	info,
135	color,
136	...opts,
137	metadata
138	});
139	try {	14✔
140	await table.importData({data});	14✔
141	} catch (error) {
142	console.error('Failed to create table', error);	×
143	throw error;	×
144	}
145
146	return table;	14✔
147	}
148	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
149	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
150
151	/**
152	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
153	* @param datasetInfo
154	* @returns
155	*/
156	async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object \| null> {
157	const {type} = datasetInfo.info;	14✔
158	switch (type) {	14!
159	case DatasetType.VECTOR_TILE:
160	return await refreshVectorTileMetadata(datasetInfo);	×
161	case DatasetType.RASTER_TILE:
162	return await refreshRasterTileMetadata(datasetInfo);	×
163	case DatasetType.WMS_TILE:
164	return await refreshWMSMetadata(datasetInfo);	×
165	case DatasetType.TILE_3D:
NEW 166	return null;	×
167	default:
168	return null;	14✔
169	}
170	}
171
172	async function refreshVectorTileMetadata(
173	datasetInfo: CreateTableProps
174	): Promise<VectorTileMetadata \| null> {
175	const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
176	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
177
178	if (	×
179	!(remoteTileFormat === RemoteTileFormat.PMTILES \|\| remoteTileFormat === RemoteTileFormat.MVT) \|\|	×
180	typeof tilesetMetadataUrl !== 'string' \|\|
181	typeof tilesetDataUrl !== 'string'
182	) {
183	return null;	×
184	}
185
186	try {	×
187	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
188	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
189	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
190	} else {
191	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
192	rawMetadata = await tileSource.metadata;	×
193	}
194
195	if (rawMetadata) {	×
196	const metadata = parseVectorMetadata(rawMetadata);	×
197
198	await getFieldsFromTile({	×
199	remoteTileFormat,
200	tilesetUrl: tilesetDataUrl,
201	metadataUrl: tilesetMetadataUrl,
202	metadata
203	});
204
205	return metadata;	×
206	}
207	} catch (err) {
208	// ignore for now, and use old metadata
209	}
210	return null;	×
211	}
212
213	async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
214	const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) \|\| {};	×
215
216	if (typeof metadataUrl !== 'string') {	×
217	return null;	×
218	}
219
220	try {	×
221	if (pmtilesType === PMTilesType.RASTER) {	×
222	const tileSource = PMTilesSource.createDataSource(metadataUrl, {});	×
223	const rawMetadata: PMTilesMetadata = await tileSource.metadata;	×
224
225	if (rawMetadata) {	×
226	return parseVectorMetadata(rawMetadata);	×
227	}
228	} else {
229	// it's stac raster tiles
230	const response = await fetch(metadataUrl);	×
231	if (!response.ok) {	×
232	throw new Error(`Failed Fetch ${metadataUrl}`);	×
233	}
234	const rawMetadata = await response.json();	×
235
236	const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});	×
237	if (metadata instanceof Error) {	×
238	throw new Error(`Failed to parse metadata ${metadata.message}`);	×
239	}
240
241	return metadata;	×
242	}
243	} catch (err) {
244	// ignore for now, and use old metadata
245	}
246	return null;	×
247	}
248
249	async function refreshWMSMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
250	const {remoteTileFormat, tilesetDataUrl} =
251	(datasetInfo.opts.metadata as WMSDatasetMetadata) \|\| {};	×
252
253	if (remoteTileFormat !== RemoteTileFormat.WMS \|\| typeof tilesetDataUrl !== 'string') {	×
254	return null;	×
255	}
256
257	try {	×
258	const data = await getWMSCapabilities(tilesetDataUrl);	×
259	return wmsCapabilitiesToDatasetMetadata(data);	×
260	} catch (err) {
261	// ignore for now, and use old metadata
262	}
263	return null;	×
264	}
265
266	export async function getWMSCapabilities(wsmUrl: string): Promise<WMSCapabilities> {
267	return (await load(	×
268	`${wsmUrl}?service=WMS&request=GetCapabilities`,
269	WMSCapabilitiesLoader
270	)) as WMSCapabilities;
271	}
272
273	export function wmsCapabilitiesToDatasetMetadata(capabilities: WMSCapabilities): any \| null {
274	// Flatten layers if they are nested
275	const layers = capabilities.layers.flatMap(layer => {	×
276	if (layer.layers && layer.layers.length > 0) {	×
277	return layer.layers;	×
278	}
279	return layer;	×
280	});
281
282	let availableLayers: WMSDatasetMetadata['layers'] = [];	×
283	if (Array.isArray(layers)) {	×
284	availableLayers = layers.map((layer: any) => {	×
285	const bb = layer.geographicBoundingBox;	×
286
287	let boundingBox: number[] \| null = null;	×
288	if (Array.isArray(bb) && Array.isArray(bb[0]) && Array.isArray(bb[1])) {	×
289	boundingBox = [bb[0][0], bb[0][1], bb[1][0], bb[1][1]];	×
290	}
291
292	return {	×
293	name: layer.name,
294	title: layer.title \|\| layer.name,	×
295	boundingBox,
296	queryable: layer.queryable
297	};
298	});
299	}
300
301	return {	×
302	layers: availableLayers,
303	version: capabilities.version \|\| '1.3.0'	×
304	};
305	}

keplergl / kepler.gl / 23965535588

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous