12031095165

Committed 26 Nov 2024 12:57PM UTC coverage: 69.321% (+22.9%) from 46.466%

Build # 12031095165

Build Type

push

github

Committed by

web-flow

Commit Message

[feat] create new dataset action (#2778)

* [feat] create new dataset action

- createNewDataEntry now returns a react-palm task to create or update a dataset asynchronously.
- updateVisDataUpdater now returns tasks to create or update a dataset asynchronously, and once done triggers createNewDatasetSuccess action.
- refactoring of demo-app App and Container to functional components

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
Co-authored-by: Shan He <heshan0131@gmail.com>

Run Details

5436 of 9079 branches covered (59.87%)

Branch coverage included in aggregate %.

91 of 111 new or added lines in 13 files covered. (81.98%)

8 existing lines in 3 files now uncovered.

11368 of 15162 relevant lines covered (74.98%)

95.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.36

/src/utils/src/dataset-utils.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import {
  ALL_FIELD_TYPES,
  FIELD_OPTS,
  TOOLTIP_FORMATS,
  TOOLTIP_FORMAT_TYPES
} from '@kepler.gl/constants';
import {getSampleForTypeAnalyze, getFieldsFromData} from '@kepler.gl/common-utils';
import {Analyzer} from 'type-analyzer';
import assert from 'assert';

import {
  ProcessorResult,
  RGBColor,
  Field,
  FieldPair,
  TimeLabelFormat,
  TooltipFields,
  ProtoDataset
} from '@kepler.gl/types';
import {TooltipFormat} from '@kepler.gl/constants';
import {notNullorUndefined} from '@kepler.gl/common-utils';

import {isPlainObject} from './utils';
import {getFormatter} from './data-utils';
import {getFormatValue} from './format';
import {hexToRgb} from './color-utils';

// apply a color for each dataset
// to use as label colors
const datasetColors = [
  '#8F2FBF',
  '#005CFF',
  '#C06C84',
  '#F8B195',
  '#547A82',
  '#3EACA8',
  '#A2D4AB'
].map(hexToRgb);

/**
 * Random color generator
 */
function* generateColor(): Generator<RGBColor> {
  let index = 0;
  while (index < datasetColors.length + 1) {
    if (index === datasetColors.length) {
      index = 0;
    }
    yield datasetColors[index++];
  }
}

export const datasetColorMaker = generateColor();

/**
 * Field name prefixes and suffixes which should not be considered
 * as metrics. Fields will still be included if a 'metric word'
 * is found on the field name, however.
 */
const EXCLUDED_DEFAULT_FIELDS = [
  // Serial numbers and identification numbers
  '_id',
  'id',
  'index',
  'uuid',
  'guid',
  'uid',
  'gid',
  'serial',
  // Geographic IDs are unlikely to be interesting to color
  'zip',
  'code',
  'post',
  'region',
  'fips',
  'cbgs',
  'h3',
  's2',
  // Geographic coords (but not z/elevation/altitude
  // since that might be a metric)
  'lat',
  'lon',
  'lng',
  'latitude',
  'longitude',
  '_x',
  '_y'
];

/**
 * Prefixes and suffixes that indicate a field is a metric.
 *
 * Note that these are in order of preference, first being
 * most preferred.
 */
const METRIC_DEFAULT_FIELDS = [
  'metric',
  'value',
  'sum',
  'count',
  'unique',
  'mean',
  'mode',
  'median',
  'max',
  'min',
  'deviation',
  'variance',
  'p99',
  'p95',
  'p75',
  'p50',
  'p25',
  'p05',
  // Abbreviations are less preferred
  'cnt',
  'val'
];

/**
 * Choose a field to use as the default color field of a layer.
 *
 * The heuristic is:
 *
 * First, exclude fields that are on the exclusion list and don't
 * have names that suggest they contain metrics. Also exclude
 * field names that are blank.
 *
 * Next, look for a field that is of real type and contains one
 * of the preferred names (in order of the preferred names).
 *
 * Next, look for a field that is of integer type and contains
 * one of the preferred names (in order of the preferred names).
 *
 * Next, look for the first field that is of real type (in order
 * of field index).
 *
 * Next, look for the first field that is of integer type (in
 * order of field index).
 *
 * It's possible no field will be chosen (i.e. because all fields
 * are strings.)
 *
 * @param dataset
 */
export function findDefaultColorField({
  fields,
  fieldPairs = []
}: {
  fields: Field[];
  fieldPairs: FieldPair[];
}): null | Field {
  const fieldsWithoutExcluded = fields.filter(field => {
    if (field.type !== ALL_FIELD_TYPES.real && field.type !== ALL_FIELD_TYPES.integer) {
      // Only select numeric fields.
      return false;
    }
    if (
      fieldPairs.find(
        pair => pair.pair.lat.value === field.name || pair.pair.lng.value === field.name
      )
    ) {
      // Do not permit lat, lon fields
      return false;
    }

    const normalizedFieldName = field.name.toLowerCase();
    if (normalizedFieldName === '') {
      // Special case excluded name when the name is blank.
      return false;
    }
    const hasExcluded = EXCLUDED_DEFAULT_FIELDS.find(
      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
    );
    const hasInclusion = METRIC_DEFAULT_FIELDS.find(
      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
    );
    return !hasExcluded || hasInclusion;
  });

  const sortedFields = fieldsWithoutExcluded.sort((left, right) => {
    const normalizedLeft = left.name.toLowerCase();
    const normalizedRight = right.name.toLowerCase();
    const leftHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
      f => normalizedLeft.startsWith(f) || normalizedLeft.endsWith(f)
    );
    const rightHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
      f => normalizedRight.startsWith(f) || normalizedRight.endsWith(f)
    );
    if (leftHasInclusion !== rightHasInclusion) {
      if (leftHasInclusion === -1) {
        // Elements that do not have the inclusion list should go after those that do.
        return 1;
      } else if (rightHasInclusion === -1) {
        // Elements that do have the inclusion list should go before those that don't.
        return -1;
      }
      // Compare based on order in the inclusion list
      return leftHasInclusion - rightHasInclusion;
    }

    // Compare based on type
    if (left.type !== right.type) {
      if (left.type === ALL_FIELD_TYPES.real) {
        return -1;
      }
      // left is an integer and right is not
      // and reals come before integers
      return 1;
    }

    // Finally, order based on the order in the datasets columns
    // @ts-expect-error
    return left.index - right.index;
  });

  if (sortedFields.length) {
    // There was a best match
    return sortedFields[0];
  }
  // No matches
  return null;
}

/**
 * Validate input data, adding missing field types, rename duplicate columns
 */
export function validateInputData(data: ProtoDataset['data']): ProcessorResult {
  if (!isPlainObject(data)) {
    assert('addDataToMap Error: dataset.data cannot be null');
    return null;
  } else if (!Array.isArray(data.fields)) {
    assert('addDataToMap Error: expect dataset.data.fields to be an array');
    return null;
  } else if (!Array.isArray(data.rows)) {
    assert('addDataToMap Error: expect dataset.data.rows to be an array');
    return null;
  }

  const {fields, rows, cols} = data;

  // check if all fields has name, format and type
  const allValid = fields.every((f, i) => {
    if (!isPlainObject(f)) {
      assert(`fields needs to be an array of object, but find ${typeof f}`);
      fields[i] = {name: `column_${i}`, type: ALL_FIELD_TYPES.string};
    }

    if (!f.name) {
      assert(`field.name is required but missing in ${JSON.stringify(f)}`);
      // assign a name
      fields[i].name = `column_${i}`;
    }

    if (!f.type || !ALL_FIELD_TYPES[f.type]) {
      assert(`unknown field type ${f.type}`);
      return false;
    }

    if (!fields.every(field => field.analyzerType)) {
      assert('field missing analyzerType');
      return false;
    }

    // check time format is correct based on first 10 not empty element
    if (f.type === ALL_FIELD_TYPES.timestamp) {
      const sample = findNonEmptyRowsAtField(rows, i, 10).map(r => ({ts: r[i]}));
      const analyzedType = Analyzer.computeColMeta(sample)[0];
      return analyzedType && analyzedType.category === 'TIME' && analyzedType.format === f.format;
    }

    return true;
  });

  if (allValid) {
    return {rows, fields, cols};
  }

  // if any field has missing type, recalculate it for everyone
  // because we simply lost faith in humanity
  const sampleData = getSampleForTypeAnalyze({
    fields: fields.map(f => f.name),
    rows
  });
  const fieldOrder = fields.map(f => f.name);
  const meta = getFieldsFromData(sampleData, fieldOrder);
  const updatedFields = fields.map((f, i) => ({
    ...f,
    type: meta[i].type,
    format: meta[i].format,
    analyzerType: meta[i].analyzerType
  }));

  return {fields: updatedFields, rows};
}

function findNonEmptyRowsAtField(rows: unknown[][], fieldIdx: number, total: number): any[] {
  const sample: any[] = [];
  let i = 0;
  while (sample.length < total && i < rows.length) {
    if (notNullorUndefined(rows[i]?.[fieldIdx])) {
      sample.push(rows[i]);
    }
    i++;
  }
  return sample;
}

const TIME_DISPLAY = '2020-05-11 14:00';

export const addTimeLabel = (formats: TimeLabelFormat[]) =>
  formats.map(f => ({
    ...f,
    label:
      f.type === TOOLTIP_FORMAT_TYPES.DATE_TIME || f.type === TOOLTIP_FORMAT_TYPES.DATE
        ? getFormatter(getFormatValue(f))(TIME_DISPLAY)
        : f.label
  }));

export function getFieldFormatLabels(fieldType?: string): TooltipFormat[] {
  const tooltipTypes = (fieldType && FIELD_OPTS[fieldType].format.tooltip) || [];
  const formatLabels: TimeLabelFormat[] = Object.values(TOOLTIP_FORMATS).filter(t =>
    tooltipTypes.includes(t.type)
  );
  return addTimeLabel(formatLabels);
}

export const getFormatLabels = (fields: TooltipFields[], fieldName: string): TooltipFormat[] => {
  const fieldType = fields.find(f => f.name === fieldName)?.type;
  return getFieldFormatLabels(fieldType);
};

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import {
5	ALL_FIELD_TYPES,
6	FIELD_OPTS,
7	TOOLTIP_FORMATS,
8	TOOLTIP_FORMAT_TYPES
9	} from '@kepler.gl/constants';
10	import {getSampleForTypeAnalyze, getFieldsFromData} from '@kepler.gl/common-utils';
11	import {Analyzer} from 'type-analyzer';
12	import assert from 'assert';
13
14	import {
15	ProcessorResult,
16	RGBColor,
17	Field,
18	FieldPair,
19	TimeLabelFormat,
20	TooltipFields,
21	ProtoDataset
22	} from '@kepler.gl/types';
23	import {TooltipFormat} from '@kepler.gl/constants';
24	import {notNullorUndefined} from '@kepler.gl/common-utils';
25
26	import {isPlainObject} from './utils';
27	import {getFormatter} from './data-utils';
28	import {getFormatValue} from './format';
29	import {hexToRgb} from './color-utils';
30
31	// apply a color for each dataset
32	// to use as label colors
33	const datasetColors = [	11✔
34	'#8F2FBF',
35	'#005CFF',
36	'#C06C84',
37	'#F8B195',
38	'#547A82',
39	'#3EACA8',
40	'#A2D4AB'
41	].map(hexToRgb);
42
43	/**
44	* Random color generator
45	*/
46	function* generateColor(): Generator<RGBColor> {
47	let index = 0;	3✔
48	while (index < datasetColors.length + 1) {	3✔
49	if (index === datasetColors.length) {	118✔
50	index = 0;	15✔
51	}
52	yield datasetColors[index++];	118✔
53	}
54	}
55
56	export const datasetColorMaker = generateColor();	11✔
57
58	/**
59	* Field name prefixes and suffixes which should not be considered
60	* as metrics. Fields will still be included if a 'metric word'
61	* is found on the field name, however.
62	*/
63	const EXCLUDED_DEFAULT_FIELDS = [	11✔
64	// Serial numbers and identification numbers
65	'_id',
66	'id',
67	'index',
68	'uuid',
69	'guid',
70	'uid',
71	'gid',
72	'serial',
73	// Geographic IDs are unlikely to be interesting to color
74	'zip',
75	'code',
76	'post',
77	'region',
78	'fips',
79	'cbgs',
80	'h3',
81	's2',
82	// Geographic coords (but not z/elevation/altitude
83	// since that might be a metric)
84	'lat',
85	'lon',
86	'lng',
87	'latitude',
88	'longitude',
89	'_x',
90	'_y'
91	];
92
93	/**
94	* Prefixes and suffixes that indicate a field is a metric.
95	*
96	* Note that these are in order of preference, first being
97	* most preferred.
98	*/
99	const METRIC_DEFAULT_FIELDS = [	11✔
100	'metric',
101	'value',
102	'sum',
103	'count',
104	'unique',
105	'mean',
106	'mode',
107	'median',
108	'max',
109	'min',
110	'deviation',
111	'variance',
112	'p99',
113	'p95',
114	'p75',
115	'p50',
116	'p25',
117	'p05',
118	// Abbreviations are less preferred
119	'cnt',
120	'val'
121	];
122
123	/**
124	* Choose a field to use as the default color field of a layer.
125	*
126	* The heuristic is:
127	*
128	* First, exclude fields that are on the exclusion list and don't
129	* have names that suggest they contain metrics. Also exclude
130	* field names that are blank.
131	*
132	* Next, look for a field that is of real type and contains one
133	* of the preferred names (in order of the preferred names).
134	*
135	* Next, look for a field that is of integer type and contains
136	* one of the preferred names (in order of the preferred names).
137	*
138	* Next, look for the first field that is of real type (in order
139	* of field index).
140	*
141	* Next, look for the first field that is of integer type (in
142	* order of field index).
143	*
144	* It's possible no field will be chosen (i.e. because all fields
145	* are strings.)
146	*
147	* @param dataset
148	*/
149	export function findDefaultColorField({
150	fields,
151	fieldPairs = []	×
152	}: {
153	fields: Field[];
154	fieldPairs: FieldPair[];
155	}): null \| Field {
156	const fieldsWithoutExcluded = fields.filter(field => {	69✔
157	if (field.type !== ALL_FIELD_TYPES.real && field.type !== ALL_FIELD_TYPES.integer) {	585✔
158	// Only select numeric fields.
159	return false;	309✔
160	}
161	if (	276✔
162	fieldPairs.find(
163	pair => pair.pair.lat.value === field.name \|\| pair.pair.lng.value === field.name	382✔
164	)
165	) {
166	// Do not permit lat, lon fields
167	return false;	216✔
168	}
169
170	const normalizedFieldName = field.name.toLowerCase();	60✔
171	if (normalizedFieldName === '') {	60!
172	// Special case excluded name when the name is blank.
173	return false;	×
174	}
175	const hasExcluded = EXCLUDED_DEFAULT_FIELDS.find(	60✔
176	f => normalizedFieldName.startsWith(f) \|\| normalizedFieldName.endsWith(f)	645✔
177	);
178	const hasInclusion = METRIC_DEFAULT_FIELDS.find(	60✔
179	f => normalizedFieldName.startsWith(f) \|\| normalizedFieldName.endsWith(f)	1,072✔
180	);
181	return !hasExcluded \|\| hasInclusion;	60✔
182	});
183
184	const sortedFields = fieldsWithoutExcluded.sort((left, right) => {	69✔
185	const normalizedLeft = left.name.toLowerCase();	10✔
186	const normalizedRight = right.name.toLowerCase();	10✔
187	const leftHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(	10✔
188	f => normalizedLeft.startsWith(f) \|\| normalizedLeft.endsWith(f)	200✔
189	);
190	const rightHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(	10✔
191	f => normalizedRight.startsWith(f) \|\| normalizedRight.endsWith(f)	72✔
192	);
193	if (leftHasInclusion !== rightHasInclusion) {	10✔
194	if (leftHasInclusion === -1) {	8!
195	// Elements that do not have the inclusion list should go after those that do.
196	return 1;	8✔
197	} else if (rightHasInclusion === -1) {	×
198	// Elements that do have the inclusion list should go before those that don't.
199	return -1;	×
200	}
201	// Compare based on order in the inclusion list
202	return leftHasInclusion - rightHasInclusion;	×
203	}
204
205	// Compare based on type
206	if (left.type !== right.type) {	2!
UNCOV 207	if (left.type === ALL_FIELD_TYPES.real) {	×
UNCOV 208	return -1;	×
209	}
210	// left is an integer and right is not
211	// and reals come before integers
UNCOV 212	return 1;	×
213	}
214
215	// Finally, order based on the order in the datasets columns
216	// @ts-expect-error
217	return left.index - right.index;	2✔
218	});
219
220	if (sortedFields.length) {	69✔
221	// There was a best match
222	return sortedFields[0];	15✔
223	}
224	// No matches
225	return null;	54✔
226	}
227
228	/**
229	* Validate input data, adding missing field types, rename duplicate columns
230	*/
231	export function validateInputData(data: ProtoDataset['data']): ProcessorResult {
232	if (!isPlainObject(data)) {	181✔
233	assert('addDataToMap Error: dataset.data cannot be null');	2✔
234	return null;	2✔
235	} else if (!Array.isArray(data.fields)) {	179✔
236	assert('addDataToMap Error: expect dataset.data.fields to be an array');	3✔
237	return null;	3✔
238	} else if (!Array.isArray(data.rows)) {	176✔
239	assert('addDataToMap Error: expect dataset.data.rows to be an array');	1✔
240	return null;	1✔
241	}
242
243	const {fields, rows, cols} = data;	175✔
244
245	// check if all fields has name, format and type
246	const allValid = fields.every((f, i) => {	175✔
247	if (!isPlainObject(f)) {	1,401✔
248	assert(`fields needs to be an array of object, but find ${typeof f}`);	1✔
249	fields[i] = {name: `column_${i}`, type: ALL_FIELD_TYPES.string};	1✔
250	}
251
252	if (!f.name) {	1,401✔
253	assert(`field.name is required but missing in ${JSON.stringify(f)}`);	2✔
254	// assign a name
255	fields[i].name = `column_${i}`;	2✔
256	}
257
258	if (!f.type \|\| !ALL_FIELD_TYPES[f.type]) {	1,401✔
259	assert(`unknown field type ${f.type}`);	2✔
260	return false;	2✔
261	}
262
263	if (!fields.every(field => field.analyzerType)) {	14,061✔
264	assert('field missing analyzerType');	15✔
265	return false;	15✔
266	}
267
268	// check time format is correct based on first 10 not empty element
269	if (f.type === ALL_FIELD_TYPES.timestamp) {	1,384✔
270	const sample = findNonEmptyRowsAtField(rows, i, 10).map(r => ({ts: r[i]}));	2,749✔
271	const analyzedType = Analyzer.computeColMeta(sample)[0];	283✔
272	return analyzedType && analyzedType.category === 'TIME' && analyzedType.format === f.format;	283✔
273	}
274
275	return true;	1,101✔
276	});
277
278	if (allValid) {	175✔
279	return {rows, fields, cols};	158✔
280	}
281
282	// if any field has missing type, recalculate it for everyone
283	// because we simply lost faith in humanity
284	const sampleData = getSampleForTypeAnalyze({	17✔
285	fields: fields.map(f => f.name),	84✔
286	rows
287	});
288	const fieldOrder = fields.map(f => f.name);	84✔
289	const meta = getFieldsFromData(sampleData, fieldOrder);	17✔
290	const updatedFields = fields.map((f, i) => ({	84✔
291	...f,
292	type: meta[i].type,
293	format: meta[i].format,
294	analyzerType: meta[i].analyzerType
295	}));
296
297	return {fields: updatedFields, rows};	17✔
298	}
299
300	function findNonEmptyRowsAtField(rows: unknown[][], fieldIdx: number, total: number): any[] {
301	const sample: any[] = [];	283✔
302	let i = 0;	283✔
303	while (sample.length < total && i < rows.length) {	283✔
304	if (notNullorUndefined(rows[i]?.[fieldIdx])) {	3,862✔
305	sample.push(rows[i]);	2,749✔
306	}
307	i++;	3,862✔
308	}
309	return sample;	283✔
310	}
311
312	const TIME_DISPLAY = '2020-05-11 14:00';	11✔
313
314	export const addTimeLabel = (formats: TimeLabelFormat[]) =>	11✔
315	formats.map(f => ({	1,548✔
316	...f,
317	label:
318	f.type === TOOLTIP_FORMAT_TYPES.DATE_TIME \|\| f.type === TOOLTIP_FORMAT_TYPES.DATE	4,392✔
319	? getFormatter(getFormatValue(f))(TIME_DISPLAY)
320	: f.label
321	}));
322
323	export function getFieldFormatLabels(fieldType?: string): TooltipFormat[] {
324	const tooltipTypes = (fieldType && FIELD_OPTS[fieldType].format.tooltip) \|\| [];	138!
325	const formatLabels: TimeLabelFormat[] = Object.values(TOOLTIP_FORMATS).filter(t =>	138✔
326	tooltipTypes.includes(t.type)	4,278✔
327	);
328	return addTimeLabel(formatLabels);	138✔
329	}
330
331	export const getFormatLabels = (fields: TooltipFields[], fieldName: string): TooltipFormat[] => {	11✔
332	const fieldType = fields.find(f => f.name === fieldName)?.type;	154✔
333	return getFieldFormatLabels(fieldType);	37✔
334	};

keplergl / kepler.gl / 12031095165

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous