• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

keplergl / kepler.gl / 12941219541

24 Jan 2025 01:10AM UTC coverage: 66.52% (+0.1%) from 66.413%
12941219541

Pull #2798

github

web-flow
Merge 8e06ed466 into 4be4b6987
Pull Request #2798: [feat] duckdb plugin

5983 of 10500 branches covered (56.98%)

Branch coverage included in aggregate %.

6 of 11 new or added lines in 4 files covered. (54.55%)

1 existing line in 1 file now uncovered.

12312 of 17003 relevant lines covered (72.41%)

89.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.42
/src/utils/src/data-utils.ts
1
// SPDX-License-Identifier: MIT
2
// Copyright contributors to the kepler.gl project
3

4
import assert from 'assert';
5
import {format as d3Format} from 'd3-format';
6
import moment from 'moment-timezone';
7

8
import {
9
  ALL_FIELD_TYPES,
10
  TOOLTIP_FORMATS,
11
  TOOLTIP_FORMAT_TYPES,
12
  TOOLTIP_KEY,
13
  TooltipFormat
14
} from '@kepler.gl/constants';
15
import {notNullorUndefined} from '@kepler.gl/common-utils';
16
import {Field, Millisecond} from '@kepler.gl/types';
17

18
import {snapToMarks} from './plot';
19
import {isPlainObject} from './utils';
20

21
export type FieldFormatter = (value: any) => string;
22

23
// We need threat latitude differently otherwise marcator project view throws
24
// a projection matrix error
25
// Uncaught Error: Pixel project matrix not invertible
26
// at WebMercatorViewport16.Viewport6 (viewport.js:81:13)
27
export const MAX_LATITUDE = 89.9;
15✔
28
export const MIN_LATITUDE = -89.9;
15✔
29
export const MAX_LONGITUDE = 180;
15✔
30
export const MIN_LONGITUDE = -180;
15✔
31

32
/**
33
 * Validates a latitude value.
34
 * Ensures that the latitude is within the defined minimum and maximum latitude bounds.
35
 * If the value is out of bounds, it returns the nearest bound value.
36
 * @param latitude - The latitude value to validate.
37
 * @returns The validated latitude value.
38
 */
39
export function validateLatitude(latitude: number | undefined): number {
40
  return validateCoordinate(latitude ?? 0, MIN_LATITUDE, MAX_LATITUDE);
260!
41
}
42

43
/**
44
 * Validates a longitude value.
45
 * Ensures that the longitude is within the defined minimum and maximum longitude bounds.
46
 * If the value is out of bounds, it returns the nearest bound value.
47
 * @param longitude - The longitude value to validate.
48
 * @returns The validated longitude value.
49
 */
50
export function validateLongitude(longitude: number | undefined): number {
51
  return validateCoordinate(longitude ?? 0, MIN_LONGITUDE, MAX_LONGITUDE);
260!
52
}
53

54
/**
55
 * Validates a coordinate value.
56
 * Ensures that the value is within the specified minimum and maximum bounds.
57
 * If the value is out of bounds, it returns the nearest bound value.
58
 * @param value - The coordinate value to validate.
59
 * @param minValue - The minimum bound for the value.
60
 * @param maxValue - The maximum bound for the value.
61
 * @returns The validated coordinate value.
62
 */
63
export function validateCoordinate(value: number, minValue: number, maxValue: number): number {
64
  if (value <= minValue) {
520✔
65
    return minValue;
7✔
66
  }
67
  if (value >= maxValue) {
513!
68
    return maxValue;
×
69
  }
70

71
  return value;
513✔
72
}
73

74
/**
75
 * simple getting unique values of an array
76
 *
77
 * @param values
78
 * @returns unique values
79
 */
80
export function unique<T>(values: T[]) {
81
  const results: T[] = [];
57✔
82
  const uniqueSet = new Set(values);
57✔
83
  uniqueSet.forEach(v => {
57✔
84
    if (notNullorUndefined(v)) {
210✔
85
      results.push(v);
182✔
86
    }
87
  });
88
  return results;
57✔
89
}
90

91
export function getLatLngBounds(
92
  points: number[][],
93
  idx: number,
94
  limit: [number, number]
95
): [number, number] | null {
96
  const lats = points
666✔
97
    .map(d => Number(Array.isArray(d)) && d[idx])
6,756✔
98
    .filter(Number.isFinite)
99
    .sort(numberSort);
100

101
  if (!lats.length) {
666!
102
    return null;
×
103
  }
104

105
  // clamp to limit
106
  return [Math.max(lats[0], limit[0]), Math.min(lats[lats.length - 1], limit[1])];
666✔
107
}
108

109
export function clamp([min, max]: [number, number], val = 0): number {
×
110
  return val <= min ? min : val >= max ? max : val;
25✔
111
}
112

113
export function getSampleData(data, sampleSize = 500, getValue = d => d) {
×
114
  const sampleStep = Math.max(Math.floor(data.length / sampleSize), 1);
×
115
  const output: any[] = [];
×
116
  for (let i = 0; i < data.length; i += sampleStep) {
×
117
    output.push(getValue(data[i]));
×
118
  }
119

120
  return output;
×
121
}
122

123
/**
124
 * Convert different time format to unix milliseconds
125
 */
126
export function timeToUnixMilli(value: string | number | Date, format: string): Millisecond | null {
127
  if (notNullorUndefined(value)) {
2,342✔
128
    if (typeof value === 'string') {
2,289✔
129
      return moment.utc(value, format).valueOf();
648✔
130
    }
131
    if (typeof value === 'number') {
1,641!
132
      return format === 'x' ? value * 1000 : value;
1,641✔
133
    }
134
    if (value instanceof Date) {
×
135
      return value.valueOf();
×
136
    }
137
  }
138
  return null;
53✔
139
}
140

141
/**
142
 * Whether d is a number, this filtered out NaN as well
143
 */
144
export function isNumber(d: unknown): d is number {
145
  return Number.isFinite(d);
805✔
146
}
147

148
/**
149
 * whether object has property
150
 * @param {string} prop
151
 * @returns {boolean} - yes or no
152
 */
153
export function hasOwnProperty<X extends object, Y extends PropertyKey>(
154
  obj: X,
155
  prop: Y
156
): obj is X & Record<Y, unknown> {
157
  return Object.prototype.hasOwnProperty.call(obj, prop);
9✔
158
}
159

160
export function numberSort(a: number, b: number): number {
161
  return a - b;
17,076✔
162
}
163

164
export function getSortingFunction(fieldType: string): typeof numberSort | undefined {
165
  switch (fieldType) {
131✔
166
    case ALL_FIELD_TYPES.real:
167
    case ALL_FIELD_TYPES.integer:
168
    case ALL_FIELD_TYPES.timestamp:
169
      return numberSort;
96✔
170
    default:
171
      return undefined;
35✔
172
  }
173
}
174

175
/**
176
 * round number with exact number of decimals
177
 * return as a string
178
 */
179
export function preciseRound(num: number, decimals: number): string {
180
  const t = Math.pow(10, decimals);
82✔
181
  return (
82✔
182
    Math.round(
183
      num * t + (decimals > 0 ? 1 : 0) * (Math.sign(num) * (10 / Math.pow(100, decimals)))
82✔
184
    ) / t
185
  ).toFixed(decimals);
186
}
187

188
/**
189
 * round a giving number at most 4 decimal places
190
 * e.g. 10 -> 10, 1.12345 -> 1.2345, 2.0 -> 2
191
 */
192
export function roundToFour(num: number): number {
193
  // @ts-expect-error
194
  return Number(`${Math.round(`${num}e+4`)}e-4`);
5✔
195
}
196
/**
197
 * get number of decimals to round to for slider from step
198
 * @param step
199
 * @returns- number of decimal
200
 */
201
export function getRoundingDecimalFromStep(step: number): number {
202
  if (isNaN(step)) {
30!
203
    assert('step is not a number');
×
204
    assert(step);
×
205
  }
206

207
  const stepStr = step.toString();
30✔
208

209
  // in case the step is a very small number e.g. 1e-7, return decimal e.g. 7 directly
210
  const splitExponential = stepStr.split('e-');
30✔
211
  if (splitExponential.length === 2) {
30✔
212
    const coeffZero = splitExponential[0].split('.');
2✔
213
    const coeffDecimal = coeffZero.length === 1 ? 0 : coeffZero[1].length;
2✔
214
    return parseInt(splitExponential[1], 10) + coeffDecimal;
2✔
215
  }
216

217
  const splitZero = stepStr.split('.');
28✔
218
  if (splitZero.length === 1) {
28✔
219
    return 0;
9✔
220
  }
221
  return splitZero[1].length;
19✔
222
}
223

224
/**
225
 * If marks is provided, snap to marks, if not normalize to step
226
 * @param val
227
 * @param minValue
228
 * @param step
229
 * @param marks
230
 */
231
export function normalizeSliderValue(
232
  val: number,
233
  minValue: number | undefined,
234
  step: number,
235
  marks?: number[] | null
236
): number {
237
  if (marks && marks.length) {
8✔
238
    // Use in slider, given a number and an array of numbers, return the nears number from the array
239
    return snapToMarks(val, marks);
1✔
240
  }
241

242
  return roundValToStep(minValue, step, val);
7✔
243
}
244

245
/**
246
 * round the value to step for the slider
247
 * @param minValue
248
 * @param step
249
 * @param val
250
 * @returns - rounded number
251
 */
252
export function roundValToStep(minValue: number | undefined, step: number, val: number): number {
253
  if (!isNumber(step) || !isNumber(minValue)) {
22✔
254
    return val;
2✔
255
  }
256

257
  const decimal = getRoundingDecimalFromStep(step);
20✔
258
  const steps = Math.floor((val - minValue) / step);
20✔
259
  let remain = val - (steps * step + minValue);
20✔
260

261
  // has to round because javascript turns 0.1 into 0.9999999999999987
262
  remain = Number(preciseRound(remain, 8));
20✔
263

264
  let closest: number;
265
  if (remain === 0) {
20✔
266
    closest = val;
5✔
267
  } else if (remain < step / 2) {
15✔
268
    closest = steps * step + minValue;
4✔
269
  } else {
270
    closest = (steps + 1) * step + minValue;
11✔
271
  }
272

273
  // precise round return a string rounded to the defined decimal
274
  const rounded = preciseRound(closest, decimal);
20✔
275

276
  return Number(rounded);
20✔
277
}
278

279
/**
280
 * Get the value format based on field and format options
281
 * Used in render tooltip value
282
 */
283
export const defaultFormatter: FieldFormatter = v => (notNullorUndefined(v) ? String(v) : '');
1,713✔
284

285
export const floatFormatter = v => (isNumber(v) ? String(roundToFour(v)) : '');
15!
286

287
export const FIELD_DISPLAY_FORMAT: {
288
  [key: string]: FieldFormatter;
289
} = {
15✔
290
  [ALL_FIELD_TYPES.string]: defaultFormatter,
291
  [ALL_FIELD_TYPES.timestamp]: defaultFormatter,
292
  [ALL_FIELD_TYPES.integer]: defaultFormatter,
293
  [ALL_FIELD_TYPES.real]: defaultFormatter,
294
  [ALL_FIELD_TYPES.boolean]: defaultFormatter,
295
  [ALL_FIELD_TYPES.date]: defaultFormatter,
296
  [ALL_FIELD_TYPES.geojson]: d =>
297
    typeof d === 'string'
23✔
298
      ? d
299
      : isPlainObject(d)
20!
300
      ? JSON.stringify(d)
301
      : Array.isArray(d)
×
302
      ? `[${String(d)}]`
303
      : '',
304
  [ALL_FIELD_TYPES.geoarrow]: d => d,
×
305
  [ALL_FIELD_TYPES.object]: (value: any) => {
306
    try {
5✔
307
      return JSON.stringify(value);
5✔
308
    } catch (e) {
NEW
309
      if (typeof value?.toString === 'function') {
×
NEW
310
        return value.toString();
×
311
      }
NEW
312
      return String(value);
×
313
    }
314
  },
315
  [ALL_FIELD_TYPES.array]: JSON.stringify,
316
  [ALL_FIELD_TYPES.h3]: defaultFormatter
317
};
318

319
/**
320
 * Parse field value and type and return a string representation
321
 */
322
export const parseFieldValue = (value: any, type: string): string => {
15✔
323
  if (!notNullorUndefined(value)) {
665✔
324
    return '';
107✔
325
  }
326
  // BigInt values cannot be serialized with JSON.stringify() directly
327
  // We need to explicitly convert them to strings using .toString()
328
  // See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt#use_within_json
329
  if (typeof value === 'bigint') {
558!
NEW
330
    return value.toString();
×
331
  }
332
  return FIELD_DISPLAY_FORMAT[type] ? FIELD_DISPLAY_FORMAT[type](value) : String(value);
558!
333
};
334

335
/**
336
 * Get the value format based on field and format options
337
 * Used in render tooltip value
338
 * @param format
339
 * @param field
340
 */
341
export function getFormatter(
342
  format?: string | Record<string, string> | null,
343
  field?: Field
344
): FieldFormatter {
345
  if (!format) {
473✔
346
    return defaultFormatter;
1✔
347
  }
348
  const tooltipFormat = Object.values(TOOLTIP_FORMATS).find(f => f[TOOLTIP_KEY] === format);
11,470✔
349

350
  if (tooltipFormat) {
472✔
351
    return applyDefaultFormat(tooltipFormat as TooltipFormat);
465✔
352
  } else if (typeof format === 'string' && field) {
7✔
353
    return applyCustomFormat(format, field);
6✔
354
  }
355

356
  return defaultFormatter;
1✔
357
}
358

359
export function getColumnFormatter(
360
  field: Pick<Field, 'type'> & Partial<Pick<Field, 'format' | 'displayFormat'>>
361
): FieldFormatter {
362
  const {format, displayFormat} = field;
1,566✔
363

364
  if (!format && !displayFormat) {
1,566✔
365
    return FIELD_DISPLAY_FORMAT[field.type];
621✔
366
  }
367
  const tooltipFormat = Object.values(TOOLTIP_FORMATS).find(f => f[TOOLTIP_KEY] === displayFormat);
27,540✔
368

369
  if (tooltipFormat) {
945✔
370
    return applyDefaultFormat(tooltipFormat);
81✔
371
  } else if (typeof displayFormat === 'string' && field) {
864!
372
    return applyCustomFormat(displayFormat, field);
×
373
  } else if (typeof displayFormat === 'object') {
864!
374
    return applyValueMap(displayFormat);
×
375
  }
376

377
  return defaultFormatter;
864✔
378
}
379

380
export function applyValueMap(format) {
381
  return v => format[v];
×
382
}
383

384
export function applyDefaultFormat(tooltipFormat: TooltipFormat): (v: any) => string {
385
  if (!tooltipFormat || !tooltipFormat.format) {
546!
386
    return defaultFormatter;
×
387
  }
388

389
  switch (tooltipFormat.type) {
546!
390
    case TOOLTIP_FORMAT_TYPES.DECIMAL:
391
      return d3Format(tooltipFormat.format);
83✔
392
    case TOOLTIP_FORMAT_TYPES.DATE:
393
    case TOOLTIP_FORMAT_TYPES.DATE_TIME:
394
      return datetimeFormatter(null)(tooltipFormat.format);
458✔
395
    case TOOLTIP_FORMAT_TYPES.PERCENTAGE:
396
      return v => `${d3Format(TOOLTIP_FORMATS.DECIMAL_DECIMAL_FIXED_2.format)(v)}%`;
1✔
397
    case TOOLTIP_FORMAT_TYPES.BOOLEAN:
398
      return getBooleanFormatter(tooltipFormat.format);
4✔
399
    default:
400
      return defaultFormatter;
×
401
  }
402
}
403

404
export function getBooleanFormatter(format: string): FieldFormatter {
405
  switch (format) {
4!
406
    case '01':
407
      return (v: boolean) => (v ? '1' : '0');
2✔
408
    case 'yn':
409
      return (v: boolean) => (v ? 'yes' : 'no');
2✔
410
    default:
411
      return defaultFormatter;
×
412
  }
413
}
414
// Allow user to specify custom tooltip format via config
415
export function applyCustomFormat(format, field: {type?: string}): FieldFormatter {
416
  switch (field.type) {
6!
417
    case ALL_FIELD_TYPES.real:
418
    case ALL_FIELD_TYPES.integer:
419
      return d3Format(format);
5✔
420
    case ALL_FIELD_TYPES.date:
421
    case ALL_FIELD_TYPES.timestamp:
422
      return datetimeFormatter(null)(format);
1✔
423
    default:
424
      return v => v;
×
425
  }
426
}
427

428
function formatLargeNumber(n) {
429
  // SI-prefix with 4 significant digits
430
  return d3Format('.4~s')(n);
24✔
431
}
432

433
export function formatNumber(n: number, type?: string): string {
434
  switch (type) {
217✔
435
    case ALL_FIELD_TYPES.integer:
436
      if (n < 0) {
137✔
437
        return `-${formatNumber(-n, 'integer')}`;
1✔
438
      }
439
      if (n < 1000) {
136✔
440
        return `${Math.round(n)}`;
36✔
441
      }
442
      if (n < 10 * 1000) {
100✔
443
        return d3Format(',')(Math.round(n));
77✔
444
      }
445
      return formatLargeNumber(n);
23✔
446
    case ALL_FIELD_TYPES.real:
447
      if (n < 0) {
77!
448
        return `-${formatNumber(-n, 'number')}`;
×
449
      }
450
      if (n < 1000) {
77✔
451
        return d3Format('.4~r')(n);
52✔
452
      }
453
      if (n < 10 * 1000) {
25✔
454
        return d3Format(',.2~f')(n);
24✔
455
      }
456
      return formatLargeNumber(n);
1✔
457

458
    default:
459
      return formatNumber(n, 'real');
3✔
460
  }
461
}
462

463
const transformation = {
15✔
464
  Y: Math.pow(10, 24),
465
  Z: Math.pow(10, 21),
466
  E: Math.pow(10, 18),
467
  P: Math.pow(10, 15),
468
  T: Math.pow(10, 12),
469
  G: Math.pow(10, 9),
470
  M: Math.pow(10, 6),
471
  k: Math.pow(10, 3),
472
  h: Math.pow(10, 2),
473
  da: Math.pow(10, 1),
474
  d: Math.pow(10, -1),
475
  c: Math.pow(10, -2),
476
  m: Math.pow(10, -3),
477
  μ: Math.pow(10, -6),
478
  n: Math.pow(10, -9),
479
  p: Math.pow(10, -12),
480
  f: Math.pow(10, -15),
481
  a: Math.pow(10, -18),
482
  z: Math.pow(10, -21),
483
  y: Math.pow(10, -24)
484
};
485

486
/**
487
 * Convert a formatted number from string back to number
488
 */
489
export function reverseFormatNumber(str: string): number {
490
  let returnValue: number | null = null;
110✔
491
  const strNum = str.trim().replace(/,/g, '');
110✔
492
  Object.entries(transformation).forEach(d => {
110✔
493
    if (strNum.includes(d[0])) {
2,200✔
494
      returnValue = parseFloat(strNum) * d[1];
21✔
495
      return true;
21✔
496
    }
497
    return false;
2,179✔
498
  });
499

500
  // if no transformer found, convert to nuber regardless
501
  return returnValue === null ? Number(strNum) : returnValue;
110✔
502
}
503

504
/**
505
 * Format epoch milliseconds with a format string
506
 * @type timezone
507
 */
508
export function datetimeFormatter(
509
  timezone?: string | null
510
): (format?: string) => (ts: number) => string {
511
  return timezone
480✔
512
    ? format => ts => moment.utc(ts).tz(timezone).format(format)
18✔
513
    : // return empty string instead of 'Invalid date' if ts is undefined/null
514
      format => ts => ts ? moment.utc(ts).format(format) : '';
493✔
515
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc