16021825421

Committed 02 Jul 2025 09:48AM UTC coverage: 61.364% (-0.3%) from 61.63%

Build # 16021825421

Build Type

push

github

Committed by

web-flow

Commit Message

[fix] WMS layer fixes 1 (#3148)

* [fix] WMS layer fixes and updates

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

* wmp fixes

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

---------

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

Coverage Stats

6100 of 11820 branches covered (51.61%)

Branch coverage included in aggregate %.

1 of 11 new or added lines in 3 files covered. (9.09%)

65 existing lines in 4 files now uncovered.

12664 of 18758 relevant lines covered (67.51%)

82.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.06

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash/uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {
  DatasetType,
  RasterTileDatasetMetadata,
  PMTilesType,
  RemoteTileFormat,
  VectorTileDatasetMetadata
} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseRasterMetadata} from './tileset/raster-tile-utils';
import {
  parseVectorMetadata,
  getFieldsFromTile,
  VectorTileMetadata
} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets | null {
  const TableClass = getApplicationConfig().table ?? KeplerTable;
  let dataValidator = validateInputData;
  if (typeof TableClass.getInputDataValidator === 'function') {
    dataValidator = TableClass.getInputDataValidator();
  }

  const validatedData = dataValidator(data);
  if (!validatedData) {
    return null;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    if (metadata.fields) {
      data.fields = metadata.fields;
    }
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  await table.importData({data});

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object | null> {
  const {type} = datasetInfo.info;
  switch (type) {
    case DatasetType.VECTOR_TILE:
      return await refreshVectorTileMetadata(datasetInfo);
    case DatasetType.RASTER_TILE:
      return await refreshRasterTileMetadata(datasetInfo);
    default:
      return null;
  }
}

async function refreshVectorTileMetadata(
  datasetInfo: CreateTableProps
): Promise<VectorTileMetadata | null> {
  const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(
      remoteTileFormat === RemoteTileFormat.PMTILES ||
      remoteTileFormat === RemoteTileFormat.MVT ||
      remoteTileFormat === RemoteTileFormat.WMS
    ) ||
    typeof tilesetMetadataUrl !== 'string' ||
    typeof tilesetDataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      const metadata = parseVectorMetadata(rawMetadata);

      await getFieldsFromTile({
        remoteTileFormat,
        tilesetUrl: tilesetDataUrl,
        metadataUrl: tilesetMetadataUrl,
        metadata
      });

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) || {};

  if (typeof metadataUrl !== 'string') {
    return null;
  }

  try {
    if (pmtilesType === PMTilesType.RASTER) {
      const tileSource = PMTilesSource.createDataSource(metadataUrl, {});
      const rawMetadata: PMTilesMetadata = await tileSource.metadata;

      if (rawMetadata) {
        return parseVectorMetadata(rawMetadata);
      }
    } else {
      // it's stac raster tiles
      const response = await fetch(metadataUrl);
      if (!response.ok) {
        throw new Error(`Failed Fetch ${metadataUrl}`);
      }
      const rawMetadata = await response.json();

      const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});
      if (metadata instanceof Error) {
        throw new Error(`Failed to parse metadata ${metadata.message}`);
      }

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash/uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {
10	DatasetType,
11	RasterTileDatasetMetadata,
12	PMTilesType,
13	RemoteTileFormat,
14	VectorTileDatasetMetadata
15	} from '@kepler.gl/constants';
16	import {
17	hexToRgb,
18	validateInputData,
19	datasetColorMaker,
20	getApplicationConfig
21	} from '@kepler.gl/utils';
22	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
23	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
24
25	import {getMVTMetadata} from './tileset/tileset-utils';
26	import {parseRasterMetadata} from './tileset/raster-tile-utils';
27	import {
28	parseVectorMetadata,
29	getFieldsFromTile,
30	VectorTileMetadata
31	} from './tileset/vector-tile-utils';
32
33	// apply a color for each dataset
34	// to use as label colors
35	const datasetColors = [	13✔
36	'#8F2FBF',
37	'#005CFF',
38	'#C06C84',
39	'#F8B195',
40	'#547A82',
41	'#3EACA8',
42	'#A2D4AB'
43	].map(hexToRgb);
44
45	export function getNewDatasetColor(datasets: Datasets): RGBColor {
46	const presetColors = datasetColors.map(String);	100✔
47	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
48	presetColors.includes(c)	25✔
49	);
50
51	if (usedColors.length === presetColors.length) {	100!
52	// if we already depleted the pool of color
53	return datasetColorMaker.next().value;	×
54	}
55
56	let color = datasetColorMaker.next().value;	100✔
57	while (usedColors.includes(String(color))) {	100✔
58	color = datasetColorMaker.next().value;	2✔
59	}
60
61	return color;	100✔
62	}
63
64	/**
65	* Take datasets payload from addDataToMap, create datasets entry save to visState
66	*/
67	export function createNewDataEntry(
68	{info, data, ...opts}: ProtoDataset,
69	datasets: Datasets = {}	15✔
70	): Datasets \| null {
71	const TableClass = getApplicationConfig().table ?? KeplerTable;	154✔
72	let dataValidator = validateInputData;	154✔
73	if (typeof TableClass.getInputDataValidator === 'function') {	154!
74	dataValidator = TableClass.getInputDataValidator();	×
75	}
76
77	const validatedData = dataValidator(data);	154✔
78	if (!validatedData) {	154✔
79	return null;	3✔
80	}
81
82	// check if dataset already exists, and update it when loading data by batches incrementally
83	if (info && info.id && datasets[info.id]) {	151!
84	// get keplerTable from datasets
85	const keplerTable = datasets[info.id];	×
86	// update the data in keplerTable
87	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
88	}
89
90	info = info \|\| {};	151!
91	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
92
93	return CREATE_TABLE_TASK({	151✔
94	info,
95	color,
96	opts,
97	data: validatedData
98	});
99	}
100
101	async function updateTable({table, data}) {
102	const updated = await table.update(data); // Assuming `table` has an `update` method	×
103	return updated;	×
104	}
105
106	type CreateTableProps = {
107	info: any;
108	color: RGBColor;
109	opts: {
110	metadata?: Record<string, unknown>;
111	};
112	data: any;
113	};
114
115	async function createTable(datasetInfo: CreateTableProps) {
116	const {info, color, opts, data} = datasetInfo;	14✔
117
118	// update metadata for remote tiled datasets
119	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
120	let metadata = opts.metadata;	14✔
121	if (refreshedMetadata) {	14!
122	metadata = {...opts.metadata, ...refreshedMetadata};	×
123	if (metadata.fields) {	×
124	data.fields = metadata.fields;	×
125	}
126	}
127
128	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
129	const table = new TableClass({	14✔
130	info,
131	color,
132	...opts,
133	metadata
134	});
135	await table.importData({data});	14✔
136
137	return table;	14✔
138	}
139	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
140	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
141
142	/**
143	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
144	* @param datasetInfo
145	* @returns
146	*/
147	async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object \| null> {
148	const {type} = datasetInfo.info;	14✔
149	switch (type) {	14!
150	case DatasetType.VECTOR_TILE:
151	return await refreshVectorTileMetadata(datasetInfo);	×
152	case DatasetType.RASTER_TILE:
153	return await refreshRasterTileMetadata(datasetInfo);	×
154	default:
155	return null;	14✔
156	}
157	}
158
159	async function refreshVectorTileMetadata(
160	datasetInfo: CreateTableProps
161	): Promise<VectorTileMetadata \| null> {
162	const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
163	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
164
165	if (	×
166	!(	×
167	remoteTileFormat === RemoteTileFormat.PMTILES \|\|	×
168	remoteTileFormat === RemoteTileFormat.MVT \|\|
169	remoteTileFormat === RemoteTileFormat.WMS
170	) \|\|
171	typeof tilesetMetadataUrl !== 'string' \|\|
172	typeof tilesetDataUrl !== 'string'
173	) {
174	return null;	×
175	}
176
UNCOV 177	try {	×
178	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
179	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
UNCOV 180	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
181	} else {
182	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
183	rawMetadata = await tileSource.metadata;	×
184	}
185
UNCOV 186	if (rawMetadata) {	×
UNCOV 187	const metadata = parseVectorMetadata(rawMetadata);	×
188
UNCOV 189	await getFieldsFromTile({	×
190	remoteTileFormat,
191	tilesetUrl: tilesetDataUrl,
192	metadataUrl: tilesetMetadataUrl,
193	metadata
194	});
195
UNCOV 196	return metadata;	×
197	}
198	} catch (err) {
199	// ignore for now, and use old metadata
200	}
201	return null;	×
202	}
203
204	async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
UNCOV 205	const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) \|\| {};	×
206
207	if (typeof metadataUrl !== 'string') {	×
208	return null;	×
209	}
210
UNCOV 211	try {	×
212	if (pmtilesType === PMTilesType.RASTER) {	×
213	const tileSource = PMTilesSource.createDataSource(metadataUrl, {});	×
UNCOV 214	const rawMetadata: PMTilesMetadata = await tileSource.metadata;	×
215
UNCOV 216	if (rawMetadata) {	×
217	return parseVectorMetadata(rawMetadata);	×
218	}
219	} else {
220	// it's stac raster tiles
221	const response = await fetch(metadataUrl);	×
UNCOV 222	if (!response.ok) {	×
223	throw new Error(`Failed Fetch ${metadataUrl}`);	×
224	}
225	const rawMetadata = await response.json();	×
226
UNCOV 227	const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});	×
228	if (metadata instanceof Error) {	×
UNCOV 229	throw new Error(`Failed to parse metadata ${metadata.message}`);	×
230	}
231
UNCOV 232	return metadata;	×
233	}
234	} catch (err) {
235	// ignore for now, and use old metadata
236	}
UNCOV 237	return null;	×
238	}

keplergl / kepler.gl / 16021825421

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous