12987250949

Committed 27 Jan 2025 10:46AM UTC coverage: 66.524% (+0.1%) from 66.408%

Build # 12987250949

Build Type

Pull #2798

github

Committed by

web-flow

Commit Message

Merge 8165b22d0 into 029bcc548

Pull Request Pull Request #2798: [feat] duckdb plugin

Run Details

5989 of 10510 branches covered (56.98%)

Branch coverage included in aggregate %.

6 of 8 new or added lines in 3 files covered. (75.0%)

12 existing lines in 1 file now uncovered.

12327 of 17023 relevant lines covered (72.41%)

89.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.7

/src/table/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import uniq from 'lodash.uniq';
import KeplerTable, {Datasets} from './kepler-table';
import {ProtoDataset, RGBColor} from '@kepler.gl/types';
import Task from 'react-palm/tasks';

import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
import {
  hexToRgb,
  validateInputData,
  datasetColorMaker,
  getApplicationConfig
} from '@kepler.gl/utils';
import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';

import {getMVTMetadata} from './tileset/tileset-utils';
import {parseVectorMetadata} from './tileset/vector-tile-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

export function getNewDatasetColor(datasets: Datasets): RGBColor {
  const presetColors = datasetColors.map(String);
  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
    presetColors.includes(c)
  );

  if (usedColors.length === presetColors.length) {
    // if we already depleted the pool of color
    return datasetColorMaker.next().value;
  }

  let color = datasetColorMaker.next().value;
  while (usedColors.includes(String(color))) {
    color = datasetColorMaker.next().value;
  }

  return color;
}

/**
 * Take datasets payload from addDataToMap, create datasets entry save to visState
 */
export function createNewDataEntry(
  {info, data, ...opts}: ProtoDataset,
  datasets: Datasets = {}
): Datasets {
  let validatedData = validateInputData(data);
  if (!validatedData) {
    // For DuckDb plugin return original data?
    // return {};
    validatedData = data;
  }

  // check if dataset already exists, and update it when loading data by batches incrementally
  if (info && info.id && datasets[info.id]) {
    // get keplerTable from datasets
    const keplerTable = datasets[info.id];
    // update the data in keplerTable
    return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});
  }

  info = info || {};
  const color = info.color || getNewDatasetColor(datasets);

  return CREATE_TABLE_TASK({
    info,
    color,
    opts,
    data: validatedData
  });
}

async function updateTable({table, data}) {
  const updated = await table.update(data); // Assuming `table` has an `update` method
  return updated;
}

type CreateTableProps = {
  info: any;
  color: RGBColor;
  opts: {
    metadata?: Record<string, unknown>;
  };
  data: any;
};

async function createTable(datasetInfo: CreateTableProps) {
  const {info, color, opts, data} = datasetInfo;

  // update metadata for remote tiled datasets
  const refreshedMetadata = await refreshRemoteData(datasetInfo);
  let metadata = opts.metadata;
  if (refreshedMetadata) {
    metadata = {...opts.metadata, ...refreshedMetadata};
    data.fields = metadata?.fields;
  }

  const TableClass = getApplicationConfig().table ?? KeplerTable;
  const table = new TableClass({
    info,
    color,
    ...opts,
    metadata
  });
  await table.importData({data});

  return table;
}
const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');
const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');

/**
 * Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
 * @param datasetInfo
 * @returns
 */
async function refreshRemoteData(datasetInfo: CreateTableProps) {
  // so far only vector tile layers should refresh metadata
  if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {
    return null;
  }

  const {remoteTileFormat, tilesetMetadataUrl} =
    (datasetInfo.opts.metadata as VectorTileDatasetMetadata) || {};

  if (
    !(remoteTileFormat === RemoteTileFormat.PMTILES || remoteTileFormat === RemoteTileFormat.MVT) ||
    typeof tilesetMetadataUrl !== 'string'
  ) {
    return null;
  }

  try {
    let rawMetadata: PMTilesMetadata | TileJSON | null = null;
    if (remoteTileFormat === RemoteTileFormat.MVT) {
      rawMetadata = await getMVTMetadata(tilesetMetadataUrl);
    } else {
      const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});
      rawMetadata = await tileSource.metadata;
    }

    if (rawMetadata) {
      return parseVectorMetadata(rawMetadata);
    }
  } catch (err) {
    // ignore for now, and use old metadata?
  }

  return null;
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import uniq from 'lodash.uniq';
5	import KeplerTable, {Datasets} from './kepler-table';
6	import {ProtoDataset, RGBColor} from '@kepler.gl/types';
7	import Task from 'react-palm/tasks';
8
9	import {DatasetType, RemoteTileFormat, VectorTileDatasetMetadata} from '@kepler.gl/constants';
10	import {
11	hexToRgb,
12	validateInputData,
13	datasetColorMaker,
14	getApplicationConfig
15	} from '@kepler.gl/utils';
16	import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
17	import {/* MVTSource,*/ TileJSON} from '@loaders.gl/mvt';
18
19	import {getMVTMetadata} from './tileset/tileset-utils';
20	import {parseVectorMetadata} from './tileset/vector-tile-utils';
21
22	// apply a color for each dataset
23	// to use as label colors
24	const datasetColors = [	13✔
25	'#8F2FBF',
26	'#005CFF',
27	'#C06C84',
28	'#F8B195',
29	'#547A82',
30	'#3EACA8',
31	'#A2D4AB'
32	].map(hexToRgb);
33
34	export function getNewDatasetColor(datasets: Datasets): RGBColor {
35	const presetColors = datasetColors.map(String);	103✔
36	const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>	103✔
37	presetColors.includes(c)	25✔
38	);
39
40	if (usedColors.length === presetColors.length) {	103!
41	// if we already depleted the pool of color
42	return datasetColorMaker.next().value;	×
43	}
44
45	let color = datasetColorMaker.next().value;	103✔
46	while (usedColors.includes(String(color))) {	103✔
47	color = datasetColorMaker.next().value;	2✔
48	}
49
50	return color;	103✔
51	}
52
53	/**
54	* Take datasets payload from addDataToMap, create datasets entry save to visState
55	*/
56	export function createNewDataEntry(
57	{info, data, ...opts}: ProtoDataset,
58	datasets: Datasets = {}	15✔
59	): Datasets {
60	let validatedData = validateInputData(data);	154✔
61	if (!validatedData) {	154✔
62	// For DuckDb plugin return original data?
63	// return {};
64	validatedData = data;	3✔
65	}
66
67	// check if dataset already exists, and update it when loading data by batches incrementally
68	if (info && info.id && datasets[info.id]) {	154!
69	// get keplerTable from datasets
UNCOV 70	const keplerTable = datasets[info.id];	×
71	// update the data in keplerTable
UNCOV 72	return UPDATE_TABLE_TASK({table: keplerTable, data: validatedData});	×
73	}
74
75	info = info \|\| {};	154✔
76	const color = info.color \|\| getNewDatasetColor(datasets);	154✔
77
78	return CREATE_TABLE_TASK({	154✔
79	info,
80	color,
81	opts,
82	data: validatedData
83	});
84	}
85
86	async function updateTable({table, data}) {
UNCOV 87	const updated = await table.update(data); // Assuming `table` has an `update` method	×
UNCOV 88	return updated;	×
89	}
90
91	type CreateTableProps = {
92	info: any;
93	color: RGBColor;
94	opts: {
95	metadata?: Record<string, unknown>;
96	};
97	data: any;
98	};
99
100	async function createTable(datasetInfo: CreateTableProps) {
101	const {info, color, opts, data} = datasetInfo;	14✔
102
103	// update metadata for remote tiled datasets
104	const refreshedMetadata = await refreshRemoteData(datasetInfo);	14✔
105	let metadata = opts.metadata;	14✔
106	if (refreshedMetadata) {	14!
UNCOV 107	metadata = {...opts.metadata, ...refreshedMetadata};	×
UNCOV 108	data.fields = metadata?.fields;	×
109	}
110
111	const TableClass = getApplicationConfig().table ?? KeplerTable;	14✔
112	const table = new TableClass({	14✔
113	info,
114	color,
115	...opts,
116	metadata
117	});
118	await table.importData({data});	14✔
119
120	return table;	14✔
121	}
122	const UPDATE_TABLE_TASK = Task.fromPromise(updateTable, 'UPDATE_TABLE_TASK');	13✔
123	const CREATE_TABLE_TASK = Task.fromPromise(createTable, 'CREATE_TABLE_TASK');	13✔
124
125	/**
126	* Fetch metadata for vector tile layers using tilesetMetadataUrl from metadata
127	* @param datasetInfo
128	* @returns
129	*/
130	async function refreshRemoteData(datasetInfo: CreateTableProps) {
131	// so far only vector tile layers should refresh metadata
132	if (datasetInfo.info.type !== DatasetType.VECTOR_TILE) {	14!
133	return null;	14✔
134	}
135
136	const {remoteTileFormat, tilesetMetadataUrl} =
UNCOV 137	(datasetInfo.opts.metadata as VectorTileDatasetMetadata) \|\| {};	×
138
UNCOV 139	if (	×
140	!(remoteTileFormat === RemoteTileFormat.PMTILES \|\| remoteTileFormat === RemoteTileFormat.MVT) \|\|	×
141	typeof tilesetMetadataUrl !== 'string'
142	) {
143	return null;	×
144	}
145
UNCOV 146	try {	×
147	let rawMetadata: PMTilesMetadata \| TileJSON \| null = null;	×
UNCOV 148	if (remoteTileFormat === RemoteTileFormat.MVT) {	×
UNCOV 149	rawMetadata = await getMVTMetadata(tilesetMetadataUrl);	×
150	} else {
151	const tileSource = PMTilesSource.createDataSource(tilesetMetadataUrl, {});	×
152	rawMetadata = await tileSource.metadata;	×
153	}
154
155	if (rawMetadata) {	×
156	return parseVectorMetadata(rawMetadata);	×
157	}
158	} catch (err) {
159	// ignore for now, and use old metadata?
160	}
161
UNCOV 162	return null;	×
163	}

keplergl / kepler.gl / 12987250949

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous