• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

keplergl / kepler.gl / 12201097896

06 Dec 2024 03:04PM UTC coverage: 69.287% (-0.03%) from 69.313%
12201097896

push

github

web-flow
[fix] Updated plot when changing cross filters (#2801)

* [fix] Updated plot when changing cross filters

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

5442 of 9100 branches covered (59.8%)

Branch coverage included in aggregate %.

7 of 8 new or added lines in 1 file covered. (87.5%)

1 existing line in 1 file now uncovered.

11378 of 15176 relevant lines covered (74.97%)

95.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.66
/src/utils/src/plot.ts
1
// SPDX-License-Identifier: MIT
2
// Copyright contributors to the kepler.gl project
3

4
import {bisectLeft, bisector, extent, histogram as d3Histogram, ticks} from 'd3-array';
5
import isEqual from 'lodash.isequal';
6
import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
7
import moment from 'moment';
8
import {
9
  Bin,
10
  TimeBins,
11
  Millisecond,
12
  TimeRangeFilter,
13
  RangeFilter,
14
  PlotType,
15
  Filter,
16
  LineChart,
17
  Field,
18
  ValueOf,
19
  LineDatum
20
} from '@kepler.gl/types';
21
import {notNullorUndefined} from '@kepler.gl/common-utils';
22
import {
23
  ANIMATION_WINDOW,
24
  BINS,
25
  durationDay,
26
  TIME_AGGREGATION,
27
  AGGREGATION_TYPES,
28
  PLOT_TYPES,
29
  AggregationTypes
30
} from '@kepler.gl/constants';
31
import {VisState} from '@kepler.gl/schemas';
32

33
import {roundValToStep} from './data-utils';
34
import {aggregate, AGGREGATION_NAME} from './aggregation';
35
import {capitalizeFirstLetter} from './strings';
36
import {getDefaultTimeFormat} from './format';
37
import {rgbToHex} from './color-utils';
38
import {DataContainerInterface} from '.';
39
import {KeplerTableModel} from './types';
40

41
// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
42
type Datasets = any;
43

44
/**
45
 *
46
 * @param thresholds
47
 * @param values
48
 * @param indexes
49
 */
50
export function histogramFromThreshold(
51
  thresholds: number[],
52
  values: number[],
53
  indexes: number[]
54
): Bin[] {
55
  const bins = d3Histogram()
52✔
56
    .value(idx => values[idx])
902✔
57
    .domain([thresholds[0], thresholds[thresholds.length - 1]])
58
    .thresholds(thresholds)(indexes)
59
    .map(bin => ({
3,373✔
60
      count: bin.length,
61
      indexes: bin,
62
      x0: bin.x0,
63
      x1: bin.x1
64
    }));
65

66
  // d3-histogram ignores threshold values outside the domain
67
  // The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.
68

69
  // bins[0].x0 = thresholds[0];
70
  // bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];
71

72
  // @ts-ignore
73
  // filter out bins with 0 counts
74
  return bins.filter(b => b.count > 0);
3,373✔
75
  // return bins;
76
}
77

78
/**
79
 *
80
 * @param domain
81
 * @param values
82
 * @param numBins
83
 * @param valueAccessor
84
 */
85
export function histogramFromDomain(
86
  domain: [number, number],
87
  values: (Millisecond | null | number)[],
88
  numBins: number,
89
  valueAccessor?: (d: unknown) => number
90
): Bin[] {
91
  const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);
32✔
92
  if (valueAccessor) {
32✔
93
    getBins.value(valueAccessor);
24✔
94
  }
95

96
  // @ts-ignore d3-array types doesn't match
97
  return getBins(values).map(bin => ({
1,023✔
98
    count: bin.length,
99
    indexes: bin,
100
    x0: bin.x0,
101
    x1: bin.x1
102
  }));
103
}
104

105
/**
106
 * @param filter
107
 * @param datasets
108
 * @param interval
109
 */
110
export function getTimeBins(
111
  filter: TimeRangeFilter,
112
  datasets: Datasets,
113
  interval: PlotType['interval']
114
): TimeBins {
115
  let bins = filter.timeBins || {};
61✔
116

117
  filter.dataId.forEach((dataId, dataIdIdx) => {
61✔
118
    // reuse bins if filterData did not change
119
    if (bins[dataId] && bins[dataId][interval]) {
68✔
120
      return;
26✔
121
    }
122
    const dataset = datasets[dataId];
42✔
123

124
    // do not apply current filter
125
    const indexes = runGpuFilterForPlot(dataset, filter);
42✔
126

127
    bins = {
42✔
128
      ...bins,
129
      [dataId]: {
130
        ...bins[dataId],
131
        [interval]: binByTime(indexes, dataset, interval, filter)
132
      }
133
    };
134
  });
135

136
  return bins;
61✔
137
}
138

139
export function binByTime(indexes, dataset, interval, filter) {
140
  // gpuFilters need to be apply to filteredIndex
141
  const mappedValue = getFilterMappedValue(dataset, filter);
42✔
142
  if (!mappedValue) {
42!
143
    return null;
×
144
  }
145
  const intervalBins = getBinThresholds(interval, filter.domain);
42✔
146

147
  const bins = histogramFromThreshold(intervalBins, mappedValue, indexes);
42✔
148

149
  return bins;
42✔
150
}
151

152
export function getBinThresholds(interval: string, domain: number[]): number[] {
153
  const timeInterval = intervalToFunction(interval);
52✔
154
  const [t0, t1] = domain;
52✔
155
  const floor = timeInterval.floor(t0).getTime();
52✔
156
  const ceiling = timeInterval.ceil(t1).getTime();
52✔
157

158
  if (!timeInterval) {
52!
159
    // if time interval is not defined
160
    // this should not happen
161
    return [t0, t0 + durationDay];
×
162
  }
163
  const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());
3,349✔
164
  const lastStep = binThresholds[binThresholds.length - 1];
52✔
165
  if (lastStep === t1) {
52✔
166
    // when last step equal to domain max, add one more step
167
    binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());
24✔
168
  }
169

170
  return binThresholds;
52✔
171
}
172

173
/**
174
 * Run GPU filter on current filter result to generate indexes for ploting chart
175
 * Skip ruuning for the same field
176
 * @param dataset
177
 * @param filter
178
 */
179
export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
180
  dataset: K,
181
  filter?: Filter
182
): number[] {
183
  const skipIndexes = getSkipIndexes(dataset, filter);
66✔
184

185
  const {
186
    gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
187
    filteredIndex
188
  } = dataset;
66✔
189
  const getFilterValue = filterValueAccessor(dataset.dataContainer)();
66✔
190

191
  const allChannels = Object.keys(filterValueUpdateTriggers)
66✔
192
    .map((_, i) => i)
264✔
193
    .filter(i => Object.values(filterValueUpdateTriggers)[i]);
264✔
194
  const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;
66✔
195
  if (skipAll) {
66✔
196
    return filteredIndex;
63✔
197
  }
198

199
  const filterData = getFilterDataFunc(
3✔
200
    filterRange,
201
    getFilterValue,
202
    dataset.dataContainer,
203
    skipIndexes
204
  );
205

206
  return filteredIndex.filter(filterData);
3✔
207
}
208

209
function getSkipIndexes(dataset, filter) {
210
  // array of gpu filter names
211
  if (!filter) {
66!
212
    return [];
×
213
  }
214
  const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({
66✔
215
    name: string;
216
  } | null)[];
217
  const valueIndex = filter.dataId.findIndex(id => id === dataset.id);
70✔
218
  const filterColumn = filter.name[valueIndex];
66✔
219

220
  return gpuFilters.reduce((accu, item, idx) => {
66✔
221
    if (item && filterColumn === item.name) {
264✔
222
      accu.push(idx);
32✔
223
    }
224
    return accu;
264✔
225
  }, [] as number[]);
226
}
227

228
export function getFilterDataFunc(
229
  filterRange,
230
  getFilterValue,
231
  dataContainer: DataContainerInterface,
232
  skips
233
) {
234
  return index =>
3✔
235
    getFilterValue({index}).every(
30✔
236
      (val, i) => skips.includes(i) || (val >= filterRange[i][0] && val <= filterRange[i][1])
48✔
237
    );
238
}
239

240
export function validBin(b) {
241
  return b.x0 !== undefined && b.x1 !== undefined;
×
242
}
243

244
/**
245
 * Use in slider, given a number and an array of numbers, return the nears number from the array.
246
 * Takes a value, timesteps and return the actual step.
247
 * @param value
248
 * @param marks
249
 */
250
export function snapToMarks(value: number, marks: number[]): number {
251
  // always use bin x0
252
  if (!marks.length) {
9!
253
    // @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
254
    return null;
×
255
  }
256
  const i = bisectLeft(marks, value);
9✔
257
  if (i === 0) {
9✔
258
    return marks[i];
2✔
259
  } else if (i === marks.length) {
7✔
260
    return marks[i - 1];
2✔
261
  }
262
  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
5✔
263
  return marks[idx];
5✔
264
}
265

266
export function normalizeValue(val, minValue, step, marks) {
267
  if (marks && marks.length) {
×
268
    return snapToMarks(val, marks);
×
269
  }
270

271
  return roundValToStep(minValue, step, val);
×
272
}
273

274
export function isPercentField(field) {
275
  return field.metadata && field.metadata.numerator && field.metadata.denominator;
44!
276
}
277

278
export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
279
  // shouldn't apply sum to percent fiele type
280
  // default aggregation is average
281
  return field && isPercentField(field)
×
282
    ? AGGREGATION_TYPES.average
283
    : aggregation || AGGREGATION_TYPES.average;
×
284
}
285

286
const getAgregationType = (field, aggregation) => {
11✔
287
  if (isPercentField(field)) {
22!
288
    return 'mean_of_percent';
×
289
  }
290
  return aggregation;
22✔
291
};
292

293
const getAgregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {
11✔
294
  if (isPercentField(field)) {
22!
295
    const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);
×
296
    const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);
×
297

298
    return {
×
299
      getNumerator: i => dataContainer.valueAt(i, numeratorIdx),
×
300
      getDenominator: i => dataContainer.valueAt(i, denominatorIdx)
×
301
    };
302
  }
303

304
  return i => field.valueAccessor({index: i});
24✔
305
};
306

307
export const getValueAggrFunc = (
11✔
308
  field: Field | string | null,
309
  aggregation: string,
310
  dataset: KeplerTableModel<any, any>
311
): ((bin: Bin) => number) => {
312
  const {dataContainer, fields} = dataset;
1✔
313
  return field && aggregation
1!
314
    ? bin =>
315
        aggregate(
22✔
316
          bin.indexes,
317
          getAgregationType(field, aggregation),
318
          // @ts-expect-error can return {getNumerator, getDenominator}
319
          getAgregationAccessor(field, dataContainer, fields)
320
        )
321
    : bin => bin.count;
×
322
};
323

324
export const getAggregationOptiosnBasedOnField = field => {
11✔
325
  if (isPercentField(field)) {
×
326
    // don't show sum
327
    return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);
×
328
  }
329
  return TIME_AGGREGATION;
×
330
};
331

332
function getDelta(
333
  bins: LineDatum[],
334
  y: number,
335
  interval: PlotType['interval']
336
): Partial<LineDatum> & {delta: 'last'; pct: number | null} {
337
  // if (WOW[interval]) return getWow(bins, y, interval);
338
  const lastBin = bins[bins.length - 1];
22✔
339

340
  return {
22✔
341
    delta: 'last',
342
    pct: lastBin ? getPctChange(y, lastBin.y) : null
22✔
343
  };
344
}
345

346
export function getPctChange(y, y0) {
347
  if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {
21✔
348
    return (y - y0) / y0;
12✔
349
  }
350
  return null;
9✔
351
}
352

353
/**
354
 *
355
 * @param datasets
356
 * @param filter
357
 */
358
export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
359
  const {dataId, yAxis, plotType, lineChart} = filter;
1✔
360
  const {aggregation, interval} = plotType;
1✔
361
  const seriesDataId = dataId[0];
1✔
362
  const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];
1✔
363

364
  if (
1!
365
    lineChart &&
1!
366
    lineChart.aggregation === aggregation &&
367
    lineChart.interval === interval &&
368
    lineChart.yAxis === yAxis?.name &&
369
    // we need to make sure we validate bins because of cross filter data changes
370
    isEqual(bins, lineChart?.bins)
371
  ) {
372
    // don't update lineChart if plotType hasn't change
NEW
373
    return lineChart;
×
374
  }
375

376
  const dataset = datasets[seriesDataId];
1✔
377
  const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);
1✔
378

379
  const init: LineDatum[] = [];
1✔
380
  const series = (bins || []).reduce((accu, bin, i) => {
1!
381
    const y = getYValue(bin);
22✔
382
    const delta = getDelta(accu, y, interval);
22✔
383
    accu.push({
22✔
384
      x: bin.x0,
385
      y,
386
      ...delta
387
    });
388
    return accu;
22✔
389
  }, init);
390

391
  const yDomain = extent<{y: any}>(series, d => d.y);
22✔
392
  const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];
1!
393

394
  // treat missing data as another series
395
  const split = splitSeries(series);
1✔
396
  const aggrName = AGGREGATION_NAME[aggregation];
1✔
397

398
  return {
1✔
399
    // @ts-ignore
400
    yDomain,
401
    // @ts-ignore
402
    xDomain,
403
    interval,
404
    aggregation,
405
    // @ts-ignore
406
    series: split,
407
    title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,
1!
408
    fieldType: yAxis ? yAxis.type : 'integer',
1!
409
    yAxis: yAxis ? yAxis.name : null,
1!
410
    allTime: {
411
      title: `All Time Average`,
412
      value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)
22✔
413
    },
414
    // @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
415
    bins
416
  };
417
}
418

419
// split into multiple series when see missing data
420
export function splitSeries(series) {
421
  const lines: any[] = [];
1✔
422
  let temp: any[] = [];
1✔
423
  for (let i = 0; i < series.length; i++) {
1✔
424
    const d = series[i];
22✔
425
    if (!notNullorUndefined(d.y) && temp.length) {
22!
426
      // ends temp
427
      lines.push(temp);
×
428
      temp = [];
×
429
    } else if (notNullorUndefined(d.y)) {
22!
430
      temp.push(d);
22✔
431
    }
432

433
    if (i === series.length - 1 && temp.length) {
22✔
434
      lines.push(temp);
1✔
435
    }
436
  }
437

438
  const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];
1!
439

440
  return {lines, markers};
1✔
441
}
442

443
export function filterSeriesByRange(series, range) {
444
  if (!series) {
×
445
    return [];
×
446
  }
447
  const [start, end] = range;
×
448
  const inRange: any[] = [];
×
449

450
  for (const serie of series) {
×
451
    if (!serie.length) {
×
452
      // eslint-disable-next-line no-console, no-undef
453
      console.warn('Serie shouldnt be empty', series);
×
454
    }
455

456
    const i0 = bisector((s: {x: any}) => s.x).left(serie, start);
×
457
    const i1 = bisector((s: {x: any}) => s.x).right(serie, end);
×
458
    const sliced = serie.slice(i0, i1);
×
459
    if (sliced.length) inRange.push(sliced);
×
460
  }
461

462
  return inRange;
×
463
}
464

465
export function adjustValueToAnimationWindow(state: VisState, filter: TimeRangeFilter) {
466
  const {
467
    plotType,
468
    value: [value0, value1],
469
    animationWindow
470
  } = filter;
1✔
471

472
  const interval = plotType.interval || getInitialInterval(filter, state.datasets);
1!
473
  const bins = getTimeBins(filter, state.datasets, interval);
1✔
474
  const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];
1✔
475
  const thresholds = (datasetBins || []).map(b => b.x0);
1!
476

477
  let val0 = value0;
1✔
478
  let val1 = value1;
1✔
479
  let idx;
480
  if (animationWindow === ANIMATION_WINDOW.interval) {
1!
481
    val0 = snapToMarks(value1, thresholds);
1✔
482
    idx = thresholds.indexOf(val0);
1✔
483
    val1 = idx > -1 ? datasetBins[idx].x1 : NaN;
1!
484
  } else {
485
    // fit current value to window
486
    val0 = snapToMarks(value0, thresholds);
×
487
    val1 = snapToMarks(value1, thresholds);
×
488

489
    if (val0 === val1) {
×
490
      idx = thresholds.indexOf(val0);
×
491
      if (idx === thresholds.length - 1) {
×
492
        val0 = thresholds[idx - 1];
×
493
      } else {
494
        val1 = thresholds[idx + 1];
×
495
      }
496
    }
497
  }
498

499
  const updatedFilter = {
1✔
500
    ...filter,
501
    plotType: {
502
      ...filter.plotType,
503
      interval
504
    },
505
    timeBins: bins,
506
    value: [val0, val1]
507
  };
508

509
  return updatedFilter;
1✔
510
}
511

512
/**
513
 * Create or update colors for a filter plot
514
 * @param filter
515
 * @param datasets
516
 * @param oldColorsByDataId
517
 */
518
function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
519
  let colorsByDataId = oldColorsByDataId || {};
7✔
520
  for (const dataId of filter.dataId) {
7✔
521
    if (!colorsByDataId[dataId] && datasets[dataId]) {
14✔
522
      colorsByDataId = {
6✔
523
        ...colorsByDataId,
524
        [dataId]: rgbToHex(datasets[dataId].color)
525
      };
526
    }
527
  }
528
  return colorsByDataId;
7✔
529
}
530

531
/**
532
 *
533
 * @param filter
534
 * @param plotType
535
 * @param datasets
536
 * @param dataId
537
 */
538
export function updateTimeFilterPlotType(
539
  filter: TimeRangeFilter,
540
  plotType: TimeRangeFilter['plotType'],
541
  datasets: Datasets,
542
  dataId?: string
543
): TimeRangeFilter {
544
  let nextFilter = filter;
60✔
545
  let nextPlotType = plotType;
60✔
546
  if (typeof nextPlotType !== 'object' || !nextPlotType.aggregation || !nextPlotType.interval) {
60✔
547
    nextPlotType = getDefaultPlotType(filter, datasets);
28✔
548
  }
549

550
  if (filter.dataId.length > 1) {
60✔
551
    nextPlotType = {
7✔
552
      ...nextPlotType,
553
      colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
554
    };
555
  }
556
  nextFilter = {
60✔
557
    ...nextFilter,
558
    plotType: nextPlotType
559
  };
560

561
  const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);
60✔
562

563
  nextFilter = {
60✔
564
    ...nextFilter,
565
    timeBins: bins
566
  };
567

568
  if (plotType.type === PLOT_TYPES.histogram) {
60✔
569
    // Histogram is calculated and memoized in the chart itself
570
  } else if (plotType.type === PLOT_TYPES.lineChart) {
1!
571
    // we should be able to move this into its own component so react will do the shallow comparison for us.
572
    nextFilter = {
1✔
573
      ...nextFilter,
574
      lineChart: getLineChart(datasets, nextFilter)
575
    };
576
  }
577

578
  return nextFilter;
60✔
579
}
580

581
export function getRangeFilterBins(filter, datasets, numBins) {
582
  const {domain} = filter;
36✔
583
  if (!filter.dataId) return null;
36!
584

585
  return filter.dataId.reduce((acc, dataId, datasetIdx) => {
36✔
586
    if (filter.bins?.[dataId]) {
36✔
587
      // don't recalculate bins
588
      acc[dataId] = filter.bins[dataId];
12✔
589
      return acc;
12✔
590
    }
591
    const fieldName = filter.name[datasetIdx];
24✔
592
    if (dataId && fieldName) {
24!
593
      const dataset = datasets[dataId];
24✔
594
      const field = dataset?.getColumnField(fieldName);
24✔
595
      if (dataset && field) {
24!
596
        const indexes = runGpuFilterForPlot(dataset, filter);
24✔
597
        const valueAccessor = index => field.valueAccessor({index});
211✔
598
        acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);
24✔
599
      }
600
    }
601
    return acc;
24✔
602
  }, {});
603
}
604

605
export function updateRangeFilterPlotType(
606
  filter: RangeFilter,
607
  plotType: RangeFilter['plotType'],
608
  datasets: Datasets,
609
  dataId?: string
610
): RangeFilter {
611
  const nextFilter = {
36✔
612
    ...filter,
613
    plotType
614
  };
615

616
  // if (dataId) {
617
  //   // clear bins
618
  //   nextFilter = {
619
  //     ...nextFilter,
620
  //     bins: {
621
  //       ...nextFilter.bins,
622
  //       [dataId]: null
623
  //     }
624
  //   };
625
  // }
626

627
  return {
36✔
628
    ...filter,
629
    plotType,
630
    bins: getRangeFilterBins(nextFilter, datasets, BINS)
631
  };
632
}
633

634
export function getChartTitle(yAxis, plotType: {aggregation: string}) {
635
  const yAxisName = yAxis?.displayName;
×
636
  const {aggregation} = plotType;
×
637

638
  if (yAxisName) {
×
639
    return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);
×
640
  }
641

642
  return `Count of Rows over Time`;
×
643
}
644

645
export function getDefaultPlotType(filter, datasets) {
646
  const interval = getInitialInterval(filter, datasets);
28✔
647
  const defaultTimeFormat = getDefaultTimeFormat(interval);
28✔
648
  return {
28✔
649
    interval,
650
    defaultTimeFormat,
651
    type: PLOT_TYPES.histogram,
652
    aggregation: AGGREGATION_TYPES.sum
653
  };
654
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc