12717940823

Committed 10 Jan 2025 10:06PM UTC coverage: 66.515% (-0.2%) from 66.757%

Build # 12717940823

Build Type

push

github

Committed by

web-flow

Commit Message

[feat] Vector Tile layer fixes (#2899)

- show Vector Tile layer tab
- changes to radius controls
- fix highlighted filled polygons
- hide show data table icon
- display loading spinner while we are loading metadata
- display errors metadata loading failed
- fix for custom ordinal crash when all colors are deleted
- collect dataset attributions from metadata and show in the lower right corner 
- refetch tile dataset metadata createNewDataEntry

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

Run Details

5976 of 10465 branches covered (57.1%)

Branch coverage included in aggregate %.

23 of 83 new or added lines in 13 files covered. (27.71%)

3 existing lines in 3 files now uncovered.

12261 of 16953 relevant lines covered (72.32%)

89.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.26

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash.uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {DatasetType, VectorTileType} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseVectorMetadata} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets {
  const validatedData = validateInputData(data);
  if (!validatedData) {
    return {};
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(dataasetInfo: CreateTableProps) {
  const {info, color, opts, data} = dataasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshMetadata(dataasetInfo);
  const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  await table.importData({data});

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshMetadata(datasetInfo: CreateTableProps) {
  // so far only vector tile layers should refresh metadata
  if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
    return null;
  }

  const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata || {};

  if (
    !(type === VectorTileType.PMTILES || type === VectorTileType.MVT) ||
    typeof tilesetMetadataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (type === VectorTileType.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      return parseVectorMetadata(rawMetadata);
    }
  } catch (err) {
    // ignore for now, and use old metadata?
  }

  return null;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash.uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {DatasetType, VectorTileType} from '@kepler.gl/constants';
10	import {
11	hexToRgb,
12	validateInputData,
13	datasetColorMaker,
14	getApplicationConfig
15	} from '@kepler.gl/utils';
16	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
17	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
18
19	import {getMVTMetadata} from './tileset/tileset-utils';
20	import {parseVectorMetadata} from './tileset/vector-tile-utils';
21
22	// apply a color for each dataset
23	// to use as label colors
24	const datasetColors = [	13✔
25	'#8F2FBF',
26	'#005CFF',
27	'#C06C84',
28	'#F8B195',
29	'#547A82',
30	'#3EACA8',
31	'#A2D4AB'
32	].map(hexToRgb);
33
34	export function getNewDatasetColor(datasets: Datasets): RGBColor {
35	const presetColors = datasetColors.map(String);	100✔
36	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	100✔
37	presetColors.includes(c)	25✔
38	);
39
40	if (usedColors.length === presetColors.length) {	100!
41	// if we already depleted the pool of color
42	return datasetColorMaker.next().value;	×
43	}
44
45	let color = datasetColorMaker.next().value;	100✔
46	while (usedColors.includes(String(color))) {	100✔
47	color = datasetColorMaker.next().value;	2✔
48	}
49
50	return color;	100✔
51	}
52
53	/**
54	* Take datasets payload from addDataToMap, create datasets entry save to visState
55	*/
56	export function createNewDataEntry(
57	{info, data, ...opts}: ProtoDataset,
58	datasets: Datasets = {}	15✔
59	): Datasets {
60	const validatedData = validateInputData(data);	154✔
61	if (!validatedData) {	154✔
62	return {};	3✔
63	}
64
65	// check if dataset already exists, and update it when loading data by batches incrementally
66	if (info && info.id && datasets[info.id]) {	151!
67	// get keplerTable from datasets
68	const keplerTable = datasets[info.id];	×
69	// update the data in keplerTable
70	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
71	}
72
73	info = info \|\| {};	151!
74	const color = info.color \|\| getNewDatasetColor(datasets);	151✔
75
76	return CREATE_TABLE_TASK({	151✔
77	info,
78	color,
79	opts,
80	data: validatedData
81	});
82	}
83
84	async function updateTable({table, data}) {
85	const updated = await table.update(data); // Assuming `table` has an `update` method	×
86	return updated;	×
87	}
88
89	type CreateTableProps = {
90	info: any;
91	color: RGBColor;
92	opts: {
93	metadata?: Record<string, unknown>;
94	};
95	data: any;
96	};
97
98	async function createTable(dataasetInfo: CreateTableProps) {
99	const {info, color, opts, data} = dataasetInfo;	14✔
100
101	// update metadata for remote tiled datasets
102	const refreshedMetadata = await refreshMetadata(dataasetInfo);	14✔
103	const metadata = refreshedMetadata ? {...opts.metadata, ...refreshedMetadata} : opts.metadata;	14!
104
105	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
106	const table = new TableClass({	14✔
107	info,
108	color,
109	...opts,
110	metadata
111	});
112	await table.importData({data});	14✔
113
114	return table;	14✔
115	}
116	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
117	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
118
119	/**
120	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
121	* @param datasetInfo
122	* @returns
123	*/
124	async function refreshMetadata(datasetInfo: CreateTableProps) {
125	// so far only vector tile layers should refresh metadata
126	if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {	14!
127	return null;	14✔
128	}
129
NEW 130	const {type, tilesetMetadataUrl} = datasetInfo.opts.metadata \|\| {};	×
131
NEW 132	if (	×
133	!(type === VectorTileType.PMTILES \|\| type === VectorTileType.MVT) \|\|	×
134	typeof tilesetMetadataUrl !== 'string'
135	) {
NEW 136	return null;	×
137	}
138
NEW 139	try {	×
NEW 140	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
NEW 141	if (type === VectorTileType.MVT) {	×
NEW 142	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
143	} else {
NEW 144	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
NEW 145	rawMetadata = await tileSource.metadata;	×
146	}
147
NEW 148	if (rawMetadata) {	×
NEW 149	return parseVectorMetadata(rawMetadata);	×
150	}
151	} catch (err) {
152	// ignore for now, and use old metadata?
153	}
154
NEW 155	return null;	×
156	}

keplergl / kepler.gl / 12717940823

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous