13707252193

Committed 06 Mar 2025 08:09PM UTC coverage: 66.122% (-0.2%) from 66.286%

Build # 13707252193

Build Type

Pull #3015

github

Committed by

web-flow

Commit Message

Merge 20d148ea6 into 7107e4177

Pull Request Pull Request #3015: [fix] fix for potential freeze during add data

Run Details

6051 of 10678 branches covered (56.67%)

Branch coverage included in aggregate %.

11 of 12 new or added lines in 2 files covered. (91.67%)

290 existing lines in 7 files now uncovered.

12415 of 17249 relevant lines covered (71.98%)

87.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.44

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash.uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseVectorMetadata, getFieldsFromTile} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets | null {
  const TableClass = getApplicationConfig().table ?? KeplerTable;
  let dataValidator = validateInputData;
  if (typeof TableClass.getInputDataValidator === 'function') {
    dataValidator = TableClass.getInputDataValidator();
  }

  const validatedData = dataValidator(data);
  if (!validatedData) {
    return null;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    data.fields = metadata?.fields;
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  await table.importData({data});

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps) {
  // so far only vector tile layers should refresh metadata
  if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
    return null;
  }

  const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
    typeof tilesetMetadataUrl !== 'string' ||
    typeof tilesetDataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      const metadata = parseVectorMetadata(rawMetadata);

      await getFieldsFromTile({
        remoteTileFormat,
        tilesetUrl: tilesetDataUrl,
        metadataUrl: tilesetMetadataUrl,
        metadata
      });

      return metadata;
    }
  } catch (err) {
    // ignore for now, and use old metadata?
  }

  return null;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash.uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
10	import {
11	hexToRgb,
12	validateInputData,
13	datasetColorMaker,
14	getApplicationConfig
15	} from '@kepler.gl/utils';
16	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
17	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
18
19	import {getMVTMetadata} from './tileset/tileset-utils';
20	import {parseVectorMetadata, getFieldsFromTile} from './tileset/vector-tile-utils';
21
22	// apply a color for each dataset
23	// to use as label colors
24	const datasetColors = [	13✔
25	'#8F2FBF',
26	'#005CFF',
27	'#C06C84',
28	'#F8B195',
29	'#547A82',
30	'#3EACA8',
31	'#A2D4AB'
32	].map(hexToRgb);
33
34	export function getNewDatasetColor(datasets: Datasets): RGBColor {
35	const presetColors = datasetColors.map(String);	100✔
36	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
37	presetColors.includes(c)	25✔
38	);
39
40	if (usedColors.length === presetColors.length) {	100!
41	// if we already depleted the pool of color
42	return datasetColorMaker.next().value;	×
43	}
44
45	let color = datasetColorMaker.next().value;	100✔
46	while (usedColors.includes(String(color))) {	100✔
47	color = datasetColorMaker.next().value;	2✔
48	}
49
50	return color;	100✔
51	}
52
53	/**
54	* Take datasets payload from addDataToMap, create datasets entry save to visState
55	*/
56	export function createNewDataEntry(
57	{info, data, ...opts}: ProtoDataset,
58	datasets: Datasets = {}	15✔
59	): Datasets \| null {
60	const TableClass = getApplicationConfig().table ?? KeplerTable;	154✔
61	let dataValidator = validateInputData;	154✔
62	if (typeof TableClass.getInputDataValidator === 'function') {	154!
63	dataValidator = TableClass.getInputDataValidator();	×
64	}
65
66	const validatedData = dataValidator(data);	154✔
67	if (!validatedData) {	154✔
68	return null;	3✔
69	}
70
71	// check if dataset already exists, and update it when loading data by batches incrementally
72	if (info && info.id && datasets[info.id]) {	151!
73	// get keplerTable from datasets
74	const keplerTable = datasets[info.id];	×
75	// update the data in keplerTable
76	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
77	}
78
79	info = info \|\| {};	151!
80	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
81
82	return CREATE_TABLE_TASK({	151✔
83	info,
84	color,
85	opts,
86	data: validatedData
87	});
88	}
89
90	async function updateTable({table, data}) {
91	const updated = await table.update(data); // Assuming `table` has an `update` method	×
92	return updated;	×
93	}
94
95	type CreateTableProps = {
96	info: any;
97	color: RGBColor;
98	opts: {
99	metadata?: Record<string, unknown>;
100	};
101	data: any;
102	};
103
104	async function createTable(datasetInfo: CreateTableProps) {
105	const {info, color, opts, data} = datasetInfo;	14✔
106
107	// update metadata for remote tiled datasets
108	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
109	let metadata = opts.metadata;	14✔
110	if (refreshedMetadata) {	14!
111	metadata = {...opts.metadata, ...refreshedMetadata};	×
112	data.fields = metadata?.fields;	×
113	}
114
115	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
116	const table = new TableClass({	14✔
117	info,
118	color,
119	...opts,
120	metadata
121	});
122	await table.importData({data});	14✔
123
124	return table;	14✔
125	}
126	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
127	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
128
129	/**
130	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
131	* @param datasetInfo
132	* @returns
133	*/
134	async function refreshRemoteData(datasetInfo: CreateTableProps) {
135	// so far only vector tile layers should refresh metadata
136	if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {	14!
137	return null;	14✔
138	}
139
140	const {remoteTileFormat, tilesetMetadataUrl, tilesetDataUrl} =
141	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
142
143	if (	×
144	!(remoteTileFormat === RemoteTileFormat.PMTILES \|\| remoteTileFormat === RemoteTileFormat.MVT) \|\|	×
145	typeof tilesetMetadataUrl !== 'string' \|\|
146	typeof tilesetDataUrl !== 'string'
147	) {
UNCOV 148	return null;	×
149	}
150
151	try {	×
152	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
153	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
UNCOV 154	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
155	} else {
156	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
UNCOV 157	rawMetadata = await tileSource.metadata;	×
158	}
159
160	if (rawMetadata) {	×
UNCOV 161	const metadata = parseVectorMetadata(rawMetadata);	×
162
UNCOV 163	await getFieldsFromTile({	×
164	remoteTileFormat,
165	tilesetUrl: tilesetDataUrl,
166	metadataUrl: tilesetMetadataUrl,
167	metadata
168	});
169
UNCOV 170	return metadata;	×
171	}
172	} catch (err) {
173	// ignore for now, and use old metadata?
174	}
175
UNCOV 176	return null;	×
177	}

keplergl / kepler.gl / 13707252193

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous