14430112347

Committed 13 Apr 2025 01:53PM UTC coverage: 61.717% (-4.4%) from 66.129%

Build # 14430112347

Build Type

Pull #3048

github

Committed by

web-flow

Commit Message

Merge 4d33fb563 into 9de30e2ba

Pull Request Pull Request #3048: [feat] Raster Tile Layer

Run Details

6066 of 11656 branches covered (52.04%)

Branch coverage included in aggregate %.

136 of 1263 new or added lines in 45 files covered. (10.77%)

5 existing lines in 3 files now uncovered.

12551 of 18509 relevant lines covered (67.81%)

82.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.37

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash.uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {
  DatasetType,
  RasterTileDatasetMetadata,
  PMTilesType,
  RemoteTileFormat,
  VectorTileDatasetMetadata
} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseRasterMetadata} from './tileset/raster-tile-utils';
import {
  parseVectorMetadata,
  getFieldsFromTile,
  VectorTileMetadata
} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets | null {
  const TableClass = getApplicationConfig().table ?? KeplerTable;
  let dataValidator = validateInputData;
  if (typeof TableClass.getInputDataValidator === 'function') {
    dataValidator = TableClass.getInputDataValidator();
  }

  const validatedData = dataValidator(data);
  if (!validatedData) {
    return null;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    if (metadata.fields) {
      data.fields = metadata.fields;
    }
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  await table.importData({data});

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object | null> {
  const {type} = datasetInfo.info;
  switch (type) {
    case DatasetType.VECTOR_TILE:
      return await refreshVectorTileMetadata(datasetInfo);
    case DatasetType.RASTER_TILE:
      return await refreshRasterTileMetadata(datasetInfo);
    default:
      return null;
  }
}

async function refreshVectorTileMetadata(
  datasetInfo: CreateTableProps
): Promise<VectorTileMetadata | null> {
  const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
    typeof tilesetMetadataUrl !== 'string' ||
    typeof tilesetDataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      const metadata = parseVectorMetadata(rawMetadata);

      await getFieldsFromTile({
        remoteTileFormat,
        tilesetUrl: tilesetDataUrl,
        metadataUrl: tilesetMetadataUrl,
        metadata
      });

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any | null> {
  const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) || {};

  if (typeof metadataUrl !== 'string') {
    return null;
  }

  try {
    if (pmtilesType === PMTilesType.RASTER) {
      const tileSource = PMTilesSource.createDataSource(metadataUrl, {});
      const rawMetadata: PMTilesMetadata = await tileSource.metadata;

      if (rawMetadata) {
        return parseVectorMetadata(rawMetadata);
      }
    } else {
      // it's stac raster tiles
      const response = await fetch(metadataUrl);
      if (!response.ok) {
        throw new Error(`Failed Fetch ${metadataUrl}`);
      }
      const rawMetadata = await response.json();

      const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});
      if (metadata instanceof Error) {
        throw new Error(`Failed to parse metadata ${metadata.message}`);
      }

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata
  }
  return null;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash.uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {
10	DatasetType,
11	RasterTileDatasetMetadata,
12	PMTilesType,
13	RemoteTileFormat,
14	VectorTileDatasetMetadata
15	} from '@kepler.gl/constants';
16	import {
17	hexToRgb,
18	validateInputData,
19	datasetColorMaker,
20	getApplicationConfig
21	} from '@kepler.gl/utils';
22	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
23	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
24
25	import {getMVTMetadata} from './tileset/tileset-utils';
26	import {parseRasterMetadata} from './tileset/raster-tile-utils';
27	import {
28	parseVectorMetadata,
29	getFieldsFromTile,
30	VectorTileMetadata
31	} from './tileset/vector-tile-utils';
32
33	// apply a color for each dataset
34	// to use as label colors
35	const datasetColors = [	13✔
36	'#8F2FBF',
37	'#005CFF',
38	'#C06C84',
39	'#F8B195',
40	'#547A82',
41	'#3EACA8',
42	'#A2D4AB'
43	].map(hexToRgb);
44
45	export function getNewDatasetColor(datasets: Datasets): RGBColor {
46	const presetColors = datasetColors.map(String);	100✔
47	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
48	presetColors.includes(c)	25✔
49	);
50
51	if (usedColors.length === presetColors.length) {	100!
52	// if we already depleted the pool of color
53	return datasetColorMaker.next().value;	×
54	}
55
56	let color = datasetColorMaker.next().value;	100✔
57	while (usedColors.includes(String(color))) {	100✔
58	color = datasetColorMaker.next().value;	2✔
59	}
60
61	return color;	100✔
62	}
63
64	/**
65	* Take datasets payload from addDataToMap, create datasets entry save to visState
66	*/
67	export function createNewDataEntry(
68	{info, data, ...opts}: ProtoDataset,
69	datasets: Datasets = {}	15✔
70	): Datasets \| null {
71	const TableClass = getApplicationConfig().table ?? KeplerTable;	154✔
72	let dataValidator = validateInputData;	154✔
73	if (typeof TableClass.getInputDataValidator === 'function') {	154!
74	dataValidator = TableClass.getInputDataValidator();	×
75	}
76
77	const validatedData = dataValidator(data);	154✔
78	if (!validatedData) {	154✔
79	return null;	3✔
80	}
81
82	// check if dataset already exists, and update it when loading data by batches incrementally
83	if (info && info.id && datasets[info.id]) {	151!
84	// get keplerTable from datasets
85	const keplerTable = datasets[info.id];	×
86	// update the data in keplerTable
87	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
88	}
89
90	info = info \|\| {};	151!
91	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
92
93	return CREATE_TABLE_TASK({	151✔
94	info,
95	color,
96	opts,
97	data: validatedData
98	});
99	}
100
101	async function updateTable({table, data}) {
102	const updated = await table.update(data); // Assuming `table` has an `update` method	×
103	return updated;	×
104	}
105
106	type CreateTableProps = {
107	info: any;
108	color: RGBColor;
109	opts: {
110	metadata?: Record<string, unknown>;
111	};
112	data: any;
113	};
114
115	async function createTable(datasetInfo: CreateTableProps) {
116	const {info, color, opts, data} = datasetInfo;	14✔
117
118	// update metadata for remote tiled datasets
119	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
120	let metadata = opts.metadata;	14✔
121	if (refreshedMetadata) {	14!
122	metadata = {...opts.metadata, ...refreshedMetadata};	×
NEW 123	if (metadata.fields) {	×
NEW 124	data.fields = metadata.fields;	×
125	}
126	}
127
128	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
129	const table = new TableClass({	14✔
130	info,
131	color,
132	...opts,
133	metadata
134	});
135	await table.importData({data});	14✔
136
137	return table;	14✔
138	}
139	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
140	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
141
142	/**
143	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
144	* @param datasetInfo
145	* @returns
146	*/
147	async function refreshRemoteData(datasetInfo: CreateTableProps): Promise<object \| null> {
148	const {type} = datasetInfo.info;	14✔
149	switch (type) {	14!
150	case DatasetType.VECTOR_TILE:
NEW 151	return await refreshVectorTileMetadata(datasetInfo);	×
152	case DatasetType.RASTER_TILE:
NEW 153	return await refreshRasterTileMetadata(datasetInfo);	×
154	default:
155	return null;	14✔
156	}
157	}
158
159	async function refreshVectorTileMetadata(
160	datasetInfo: CreateTableProps
161	): Promise<VectorTileMetadata \| null> {
162	const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
163	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
164
165	if (	×
166	!(remoteTileFormat === RemoteTileFormat.PMTILES \|\| remoteTileFormat === RemoteTileFormat.MVT) \|\|	×
167	typeof tilesetMetadataUrl !== 'string' \|\|
168	typeof tilesetDataUrl !== 'string'
169	) {
170	return null;	×
171	}
172
173	try {	×
174	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
175	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
176	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
177	} else {
178	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
179	rawMetadata = await tileSource.metadata;	×
180	}
181
182	if (rawMetadata) {	×
183	const metadata = parseVectorMetadata(rawMetadata);	×
184
185	await getFieldsFromTile({	×
186	remoteTileFormat,
187	tilesetUrl: tilesetDataUrl,
188	metadataUrl: tilesetMetadataUrl,
189	metadata
190	});
191
192	return metadata;	×
193	}
194	} catch (err) {
195	// ignore for now, and use old metadata
196	}
NEW 197	return null;	×
198	}
199
200	async function refreshRasterTileMetadata(datasetInfo: CreateTableProps): Promise<any \| null> {
NEW 201	const {metadataUrl, pmtilesType} = (datasetInfo.opts.metadata as RasterTileDatasetMetadata) \|\| {};	×
202
NEW 203	if (typeof metadataUrl !== 'string') {	×
NEW 204	return null;	×
205	}
206
NEW 207	try {	×
NEW 208	if (pmtilesType === PMTilesType.RASTER) {	×
NEW 209	const tileSource = PMTilesSource.createDataSource(metadataUrl, {});	×
NEW 210	const rawMetadata: PMTilesMetadata = await tileSource.metadata;	×
211
NEW 212	if (rawMetadata) {	×
NEW 213	return parseVectorMetadata(rawMetadata);	×
214	}
215	} else {
216	// it's stac raster tiles
NEW 217	const response = await fetch(metadataUrl);	×
NEW 218	if (!response.ok) {	×
NEW 219	throw new Error(`Failed Fetch ${metadataUrl}`);	×
220	}
NEW 221	const rawMetadata = await response.json();	×
222
NEW 223	const metadata = parseRasterMetadata(rawMetadata, {allowCollections: true});	×
NEW 224	if (metadata instanceof Error) {	×
NEW 225	throw new Error(`Failed to parse metadata ${metadata.message}`);	×
226	}
227
NEW 228	return metadata;	×
229	}
230	} catch (err) {
231	// ignore for now, and use old metadata
232	}
UNCOV 233	return null;	×
234	}

keplergl / kepler.gl / 14430112347

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous