12201097896

Committed 06 Dec 2024 03:04PM UTC coverage: 69.287% (-0.03%) from 69.313%

Build # 12201097896

Build Type

push

github

Committed by

web-flow

Commit Message

[fix] Updated plot when changing cross filters (#2801)

* [fix] Updated plot when changing cross filters

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

Run Details

5442 of 9100 branches covered (59.8%)

Branch coverage included in aggregate %.

7 of 8 new or added lines in 1 file covered. (87.5%)

1 existing line in 1 file now uncovered.

11378 of 15176 relevant lines covered (74.97%)

95.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.66

/src/utils/src/plot.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import {bisectLeft, bisector, extent, histogram as d3Histogram, ticks} from 'd3-array';
import isEqual from 'lodash.isequal';
import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
import moment from 'moment';
import {
  Bin,
  TimeBins,
  Millisecond,
  TimeRangeFilter,
  RangeFilter,
  PlotType,
  Filter,
  LineChart,
  Field,
  ValueOf,
  LineDatum
} from '@kepler.gl/types';
import {notNullorUndefined} from '@kepler.gl/common-utils';
import {
  ANIMATION_WINDOW,
  BINS,
  durationDay,
  TIME_AGGREGATION,
  AGGREGATION_TYPES,
  PLOT_TYPES,
  AggregationTypes
} from '@kepler.gl/constants';
import {VisState} from '@kepler.gl/schemas';

import {roundValToStep} from './data-utils';
import {aggregate, AGGREGATION_NAME} from './aggregation';
import {capitalizeFirstLetter} from './strings';
import {getDefaultTimeFormat} from './format';
import {rgbToHex} from './color-utils';
import {DataContainerInterface} from '.';
import {KeplerTableModel} from './types';

// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
type Datasets = any;

/**
 *
 * @param thresholds
 * @param values
 * @param indexes
 */
export function histogramFromThreshold(
  thresholds: number[],
  values: number[],
  indexes: number[]
): Bin[] {
  const bins = d3Histogram()
    .value(idx => values[idx])
    .domain([thresholds[0], thresholds[thresholds.length - 1]])
    .thresholds(thresholds)(indexes)
    .map(bin => ({
      count: bin.length,
      indexes: bin,
      x0: bin.x0,
      x1: bin.x1
    }));

  // d3-histogram ignores threshold values outside the domain
  // The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.

  // bins[0].x0 = thresholds[0];
  // bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];

  // @ts-ignore
  // filter out bins with 0 counts
  return bins.filter(b => b.count > 0);
  // return bins;
}

/**
 *
 * @param domain
 * @param values
 * @param numBins
 * @param valueAccessor
 */
export function histogramFromDomain(
  domain: [number, number],
  values: (Millisecond | null | number)[],
  numBins: number,
  valueAccessor?: (d: unknown) => number
): Bin[] {
  const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);
  if (valueAccessor) {
    getBins.value(valueAccessor);
  }

  // @ts-ignore d3-array types doesn't match
  return getBins(values).map(bin => ({
    count: bin.length,
    indexes: bin,
    x0: bin.x0,
    x1: bin.x1
  }));
}

/**
 * @param filter
 * @param datasets
 * @param interval
 */
export function getTimeBins(
  filter: TimeRangeFilter,
  datasets: Datasets,
  interval: PlotType['interval']
): TimeBins {
  let bins = filter.timeBins || {};

  filter.dataId.forEach((dataId, dataIdIdx) => {
    // reuse bins if filterData did not change
    if (bins[dataId] && bins[dataId][interval]) {
      return;
    }
    const dataset = datasets[dataId];

    // do not apply current filter
    const indexes = runGpuFilterForPlot(dataset, filter);

    bins = {
      ...bins,
      [dataId]: {
        ...bins[dataId],
        [interval]: binByTime(indexes, dataset, interval, filter)
      }
    };
  });

  return bins;
}

export function binByTime(indexes, dataset, interval, filter) {
  // gpuFilters need to be apply to filteredIndex
  const mappedValue = getFilterMappedValue(dataset, filter);
  if (!mappedValue) {
    return null;
  }
  const intervalBins = getBinThresholds(interval, filter.domain);

  const bins = histogramFromThreshold(intervalBins, mappedValue, indexes);

  return bins;
}

export function getBinThresholds(interval: string, domain: number[]): number[] {
  const timeInterval = intervalToFunction(interval);
  const [t0, t1] = domain;
  const floor = timeInterval.floor(t0).getTime();
  const ceiling = timeInterval.ceil(t1).getTime();

  if (!timeInterval) {
    // if time interval is not defined
    // this should not happen
    return [t0, t0 + durationDay];
  }
  const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());
  const lastStep = binThresholds[binThresholds.length - 1];
  if (lastStep === t1) {
    // when last step equal to domain max, add one more step
    binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());
  }

  return binThresholds;
}

/**
 * Run GPU filter on current filter result to generate indexes for ploting chart
 * Skip ruuning for the same field
 * @param dataset
 * @param filter
 */
export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
  dataset: K,
  filter?: Filter
): number[] {
  const skipIndexes = getSkipIndexes(dataset, filter);

  const {
    gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
    filteredIndex
  } = dataset;
  const getFilterValue = filterValueAccessor(dataset.dataContainer)();

  const allChannels = Object.keys(filterValueUpdateTriggers)
    .map((_, i) => i)
    .filter(i => Object.values(filterValueUpdateTriggers)[i]);
  const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;
  if (skipAll) {
    return filteredIndex;
  }

  const filterData = getFilterDataFunc(
    filterRange,
    getFilterValue,
    dataset.dataContainer,
    skipIndexes
  );

  return filteredIndex.filter(filterData);
}

function getSkipIndexes(dataset, filter) {
  // array of gpu filter names
  if (!filter) {
    return [];
  }
  const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({
    name: string;
  } | null)[];
  const valueIndex = filter.dataId.findIndex(id => id === dataset.id);
  const filterColumn = filter.name[valueIndex];

  return gpuFilters.reduce((accu, item, idx) => {
    if (item && filterColumn === item.name) {
      accu.push(idx);
    }
    return accu;
  }, [] as number[]);
}

export function getFilterDataFunc(
  filterRange,
  getFilterValue,
  dataContainer: DataContainerInterface,
  skips
) {
  return index =>
    getFilterValue({index}).every(
      (val, i) => skips.includes(i) || (val >= filterRange[i][0] && val <= filterRange[i][1])
    );
}

export function validBin(b) {
  return b.x0 !== undefined && b.x1 !== undefined;
}

/**
 * Use in slider, given a number and an array of numbers, return the nears number from the array.
 * Takes a value, timesteps and return the actual step.
 * @param value
 * @param marks
 */
export function snapToMarks(value: number, marks: number[]): number {
  // always use bin x0
  if (!marks.length) {
    // @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
    return null;
  }
  const i = bisectLeft(marks, value);
  if (i === 0) {
    return marks[i];
  } else if (i === marks.length) {
    return marks[i - 1];
  }
  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
  return marks[idx];
}

export function normalizeValue(val, minValue, step, marks) {
  if (marks && marks.length) {
    return snapToMarks(val, marks);
  }

  return roundValToStep(minValue, step, val);
}

export function isPercentField(field) {
  return field.metadata && field.metadata.numerator && field.metadata.denominator;
}

export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
  // shouldn't apply sum to percent fiele type
  // default aggregation is average
  return field && isPercentField(field)
    ? AGGREGATION_TYPES.average
    : aggregation || AGGREGATION_TYPES.average;
}

const getAgregationType = (field, aggregation) => {
  if (isPercentField(field)) {
    return 'mean_of_percent';
  }
  return aggregation;
};

const getAgregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {
  if (isPercentField(field)) {
    const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);
    const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);

    return {
      getNumerator: i => dataContainer.valueAt(i, numeratorIdx),
      getDenominator: i => dataContainer.valueAt(i, denominatorIdx)
    };
  }

  return i => field.valueAccessor({index: i});
};

export const getValueAggrFunc = (
  field: Field | string | null,
  aggregation: string,
  dataset: KeplerTableModel<any, any>
): ((bin: Bin) => number) => {
  const {dataContainer, fields} = dataset;
  return field && aggregation
    ? bin =>
        aggregate(
          bin.indexes,
          getAgregationType(field, aggregation),
          // @ts-expect-error can return {getNumerator, getDenominator}
          getAgregationAccessor(field, dataContainer, fields)
        )
    : bin => bin.count;
};

export const getAggregationOptiosnBasedOnField = field => {
  if (isPercentField(field)) {
    // don't show sum
    return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);
  }
  return TIME_AGGREGATION;
};

function getDelta(
  bins: LineDatum[],
  y: number,
  interval: PlotType['interval']
): Partial<LineDatum> & {delta: 'last'; pct: number | null} {
  // if (WOW[interval]) return getWow(bins, y, interval);
  const lastBin = bins[bins.length - 1];

  return {
    delta: 'last',
    pct: lastBin ? getPctChange(y, lastBin.y) : null
  };
}

export function getPctChange(y, y0) {
  if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {
    return (y - y0) / y0;
  }
  return null;
}

/**
 *
 * @param datasets
 * @param filter
 */
export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
  const {dataId, yAxis, plotType, lineChart} = filter;
  const {aggregation, interval} = plotType;
  const seriesDataId = dataId[0];
  const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];

  if (
    lineChart &&
    lineChart.aggregation === aggregation &&
    lineChart.interval === interval &&
    lineChart.yAxis === yAxis?.name &&
    // we need to make sure we validate bins because of cross filter data changes
    isEqual(bins, lineChart?.bins)
  ) {
    // don't update lineChart if plotType hasn't change
    return lineChart;
  }

  const dataset = datasets[seriesDataId];
  const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);

  const init: LineDatum[] = [];
  const series = (bins || []).reduce((accu, bin, i) => {
    const y = getYValue(bin);
    const delta = getDelta(accu, y, interval);
    accu.push({
      x: bin.x0,
      y,
      ...delta
    });
    return accu;
  }, init);

  const yDomain = extent<{y: any}>(series, d => d.y);
  const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];

  // treat missing data as another series
  const split = splitSeries(series);
  const aggrName = AGGREGATION_NAME[aggregation];

  return {
    // @ts-ignore
    yDomain,
    // @ts-ignore
    xDomain,
    interval,
    aggregation,
    // @ts-ignore
    series: split,
    title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,
    fieldType: yAxis ? yAxis.type : 'integer',
    yAxis: yAxis ? yAxis.name : null,
    allTime: {
      title: `All Time Average`,
      value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)
    },
    // @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
    bins
  };
}

// split into multiple series when see missing data
export function splitSeries(series) {
  const lines: any[] = [];
  let temp: any[] = [];
  for (let i = 0; i < series.length; i++) {
    const d = series[i];
    if (!notNullorUndefined(d.y) && temp.length) {
      // ends temp
      lines.push(temp);
      temp = [];
    } else if (notNullorUndefined(d.y)) {
      temp.push(d);
    }

    if (i === series.length - 1 && temp.length) {
      lines.push(temp);
    }
  }

  const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];

  return {lines, markers};
}

export function filterSeriesByRange(series, range) {
  if (!series) {
    return [];
  }
  const [start, end] = range;
  const inRange: any[] = [];

  for (const serie of series) {
    if (!serie.length) {
      // eslint-disable-next-line no-console, no-undef
      console.warn('Serie shouldnt be empty', series);
    }

    const i0 = bisector((s: {x: any}) => s.x).left(serie, start);
    const i1 = bisector((s: {x: any}) => s.x).right(serie, end);
    const sliced = serie.slice(i0, i1);
    if (sliced.length) inRange.push(sliced);
  }

  return inRange;
}

export function adjustValueToAnimationWindow(state: VisState, filter: TimeRangeFilter) {
  const {
    plotType,
    value: [value0, value1],
    animationWindow
  } = filter;

  const interval = plotType.interval || getInitialInterval(filter, state.datasets);
  const bins = getTimeBins(filter, state.datasets, interval);
  const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];
  const thresholds = (datasetBins || []).map(b => b.x0);

  let val0 = value0;
  let val1 = value1;
  let idx;
  if (animationWindow === ANIMATION_WINDOW.interval) {
    val0 = snapToMarks(value1, thresholds);
    idx = thresholds.indexOf(val0);
    val1 = idx > -1 ? datasetBins[idx].x1 : NaN;
  } else {
    // fit current value to window
    val0 = snapToMarks(value0, thresholds);
    val1 = snapToMarks(value1, thresholds);

    if (val0 === val1) {
      idx = thresholds.indexOf(val0);
      if (idx === thresholds.length - 1) {
        val0 = thresholds[idx - 1];
      } else {
        val1 = thresholds[idx + 1];
      }
    }
  }

  const updatedFilter = {
    ...filter,
    plotType: {
      ...filter.plotType,
      interval
    },
    timeBins: bins,
    value: [val0, val1]
  };

  return updatedFilter;
}

/**
 * Create or update colors for a filter plot
 * @param filter
 * @param datasets
 * @param oldColorsByDataId
 */
function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
  let colorsByDataId = oldColorsByDataId || {};
  for (const dataId of filter.dataId) {
    if (!colorsByDataId[dataId] && datasets[dataId]) {
      colorsByDataId = {
        ...colorsByDataId,
        [dataId]: rgbToHex(datasets[dataId].color)
      };
    }
  }
  return colorsByDataId;
}

/**
 *
 * @param filter
 * @param plotType
 * @param datasets
 * @param dataId
 */
export function updateTimeFilterPlotType(
  filter: TimeRangeFilter,
  plotType: TimeRangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): TimeRangeFilter {
  let nextFilter = filter;
  let nextPlotType = plotType;
  if (typeof nextPlotType !== 'object' || !nextPlotType.aggregation || !nextPlotType.interval) {
    nextPlotType = getDefaultPlotType(filter, datasets);
  }

  if (filter.dataId.length > 1) {
    nextPlotType = {
      ...nextPlotType,
      colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
    };
  }
  nextFilter = {
    ...nextFilter,
    plotType: nextPlotType
  };

  const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);

  nextFilter = {
    ...nextFilter,
    timeBins: bins
  };

  if (plotType.type === PLOT_TYPES.histogram) {
    // Histogram is calculated and memoized in the chart itself
  } else if (plotType.type === PLOT_TYPES.lineChart) {
    // we should be able to move this into its own component so react will do the shallow comparison for us.
    nextFilter = {
      ...nextFilter,
      lineChart: getLineChart(datasets, nextFilter)
    };
  }

  return nextFilter;
}

export function getRangeFilterBins(filter, datasets, numBins) {
  const {domain} = filter;
  if (!filter.dataId) return null;

  return filter.dataId.reduce((acc, dataId, datasetIdx) => {
    if (filter.bins?.[dataId]) {
      // don't recalculate bins
      acc[dataId] = filter.bins[dataId];
      return acc;
    }
    const fieldName = filter.name[datasetIdx];
    if (dataId && fieldName) {
      const dataset = datasets[dataId];
      const field = dataset?.getColumnField(fieldName);
      if (dataset && field) {
        const indexes = runGpuFilterForPlot(dataset, filter);
        const valueAccessor = index => field.valueAccessor({index});
        acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);
      }
    }
    return acc;
  }, {});
}

export function updateRangeFilterPlotType(
  filter: RangeFilter,
  plotType: RangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): RangeFilter {
  const nextFilter = {
    ...filter,
    plotType
  };

  // if (dataId) {
  //   // clear bins
  //   nextFilter = {
  //     ...nextFilter,
  //     bins: {
  //       ...nextFilter.bins,
  //       [dataId]: null
  //     }
  //   };
  // }

  return {
    ...filter,
    plotType,
    bins: getRangeFilterBins(nextFilter, datasets, BINS)
  };
}

export function getChartTitle(yAxis, plotType: {aggregation: string}) {
  const yAxisName = yAxis?.displayName;
  const {aggregation} = plotType;

  if (yAxisName) {
    return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);
  }

  return `Count of Rows over Time`;
}

export function getDefaultPlotType(filter, datasets) {
  const interval = getInitialInterval(filter, datasets);
  const defaultTimeFormat = getDefaultTimeFormat(interval);
  return {
    interval,
    defaultTimeFormat,
    type: PLOT_TYPES.histogram,
    aggregation: AGGREGATION_TYPES.sum
  };
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import {bisectLeft, bisector, extent, histogram as d3Histogram, ticks} from 'd3-array';
5	import isEqual from 'lodash.isequal';
6	import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
7	import moment from 'moment';
8	import {
9	Bin,
10	TimeBins,
11	Millisecond,
12	TimeRangeFilter,
13	RangeFilter,
14	PlotType,
15	Filter,
16	LineChart,
17	Field,
18	ValueOf,
19	LineDatum
20	} from '@kepler.gl/types';
21	import {notNullorUndefined} from '@kepler.gl/common-utils';
22	import {
23	ANIMATION_WINDOW,
24	BINS,
25	durationDay,
26	TIME_AGGREGATION,
27	AGGREGATION_TYPES,
28	PLOT_TYPES,
29	AggregationTypes
30	} from '@kepler.gl/constants';
31	import {VisState} from '@kepler.gl/schemas';
32
33	import {roundValToStep} from './data-utils';
34	import {aggregate, AGGREGATION_NAME} from './aggregation';
35	import {capitalizeFirstLetter} from './strings';
36	import {getDefaultTimeFormat} from './format';
37	import {rgbToHex} from './color-utils';
38	import {DataContainerInterface} from '.';
39	import {KeplerTableModel} from './types';
40
41	// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
42	type Datasets = any;
43
44	/**
45	*
46	* @param thresholds
47	* @param values
48	* @param indexes
49	*/
50	export function histogramFromThreshold(
51	thresholds: number[],
52	values: number[],
53	indexes: number[]
54	): Bin[] {
55	const bins = d3Histogram()	52✔
56	.value(idx => values[idx])	902✔
57	.domain([thresholds[0], thresholds[thresholds.length - 1]])
58	.thresholds(thresholds)(indexes)
59	.map(bin => ({	3,373✔
60	count: bin.length,
61	indexes: bin,
62	x0: bin.x0,
63	x1: bin.x1
64	}));
65
66	// d3-histogram ignores threshold values outside the domain
67	// The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.
68
69	// bins[0].x0 = thresholds[0];
70	// bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];
71
72	// @ts-ignore
73	// filter out bins with 0 counts
74	return bins.filter(b => b.count > 0);	3,373✔
75	// return bins;
76	}
77
78	/**
79	*
80	* @param domain
81	* @param values
82	* @param numBins
83	* @param valueAccessor
84	*/
85	export function histogramFromDomain(
86	domain: [number, number],
87	values: (Millisecond \| null \| number)[],
88	numBins: number,
89	valueAccessor?: (d: unknown) => number
90	): Bin[] {
91	const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);	32✔
92	if (valueAccessor) {	32✔
93	getBins.value(valueAccessor);	24✔
94	}
95
96	// @ts-ignore d3-array types doesn't match
97	return getBins(values).map(bin => ({	1,023✔
98	count: bin.length,
99	indexes: bin,
100	x0: bin.x0,
101	x1: bin.x1
102	}));
103	}
104
105	/**
106	* @param filter
107	* @param datasets
108	* @param interval
109	*/
110	export function getTimeBins(
111	filter: TimeRangeFilter,
112	datasets: Datasets,
113	interval: PlotType['interval']
114	): TimeBins {
115	let bins = filter.timeBins \|\| {};	61✔
116
117	filter.dataId.forEach((dataId, dataIdIdx) => {	61✔
118	// reuse bins if filterData did not change
119	if (bins[dataId] && bins[dataId][interval]) {	68✔
120	return;	26✔
121	}
122	const dataset = datasets[dataId];	42✔
123
124	// do not apply current filter
125	const indexes = runGpuFilterForPlot(dataset, filter);	42✔
126
127	bins = {	42✔
128	...bins,
129	[dataId]: {
130	...bins[dataId],
131	[interval]: binByTime(indexes, dataset, interval, filter)
132	}
133	};
134	});
135
136	return bins;	61✔
137	}
138
139	export function binByTime(indexes, dataset, interval, filter) {
140	// gpuFilters need to be apply to filteredIndex
141	const mappedValue = getFilterMappedValue(dataset, filter);	42✔
142	if (!mappedValue) {	42!
143	return null;	×
144	}
145	const intervalBins = getBinThresholds(interval, filter.domain);	42✔
146
147	const bins = histogramFromThreshold(intervalBins, mappedValue, indexes);	42✔
148
149	return bins;	42✔
150	}
151
152	export function getBinThresholds(interval: string, domain: number[]): number[] {
153	const timeInterval = intervalToFunction(interval);	52✔
154	const [t0, t1] = domain;	52✔
155	const floor = timeInterval.floor(t0).getTime();	52✔
156	const ceiling = timeInterval.ceil(t1).getTime();	52✔
157
158	if (!timeInterval) {	52!
159	// if time interval is not defined
160	// this should not happen
161	return [t0, t0 + durationDay];	×
162	}
163	const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());	3,349✔
164	const lastStep = binThresholds[binThresholds.length - 1];	52✔
165	if (lastStep === t1) {	52✔
166	// when last step equal to domain max, add one more step
167	binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());	24✔
168	}
169
170	return binThresholds;	52✔
171	}
172
173	/**
174	* Run GPU filter on current filter result to generate indexes for ploting chart
175	* Skip ruuning for the same field
176	* @param dataset
177	* @param filter
178	*/
179	export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
180	dataset: K,
181	filter?: Filter
182	): number[] {
183	const skipIndexes = getSkipIndexes(dataset, filter);	66✔
184
185	const {
186	gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
187	filteredIndex
188	} = dataset;	66✔
189	const getFilterValue = filterValueAccessor(dataset.dataContainer)();	66✔
190
191	const allChannels = Object.keys(filterValueUpdateTriggers)	66✔
192	.map((_, i) => i)	264✔
193	.filter(i => Object.values(filterValueUpdateTriggers)[i]);	264✔
194	const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;	66✔
195	if (skipAll) {	66✔
196	return filteredIndex;	63✔
197	}
198
199	const filterData = getFilterDataFunc(	3✔
200	filterRange,
201	getFilterValue,
202	dataset.dataContainer,
203	skipIndexes
204	);
205
206	return filteredIndex.filter(filterData);	3✔
207	}
208
209	function getSkipIndexes(dataset, filter) {
210	// array of gpu filter names
211	if (!filter) {	66!
212	return [];	×
213	}
214	const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({	66✔
215	name: string;
216	} \| null)[];
217	const valueIndex = filter.dataId.findIndex(id => id === dataset.id);	70✔
218	const filterColumn = filter.name[valueIndex];	66✔
219
220	return gpuFilters.reduce((accu, item, idx) => {	66✔
221	if (item && filterColumn === item.name) {	264✔
222	accu.push(idx);	32✔
223	}
224	return accu;	264✔
225	}, [] as number[]);
226	}
227
228	export function getFilterDataFunc(
229	filterRange,
230	getFilterValue,
231	dataContainer: DataContainerInterface,
232	skips
233	) {
234	return index =>	3✔
235	getFilterValue({index}).every(	30✔
236	(val, i) => skips.includes(i) \|\| (val >= filterRange[i][0] && val <= filterRange[i][1])	48✔
237	);
238	}
239
240	export function validBin(b) {
241	return b.x0 !== undefined && b.x1 !== undefined;	×
242	}
243
244	/**
245	* Use in slider, given a number and an array of numbers, return the nears number from the array.
246	* Takes a value, timesteps and return the actual step.
247	* @param value
248	* @param marks
249	*/
250	export function snapToMarks(value: number, marks: number[]): number {
251	// always use bin x0
252	if (!marks.length) {	9!
253	// @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
254	return null;	×
255	}
256	const i = bisectLeft(marks, value);	9✔
257	if (i === 0) {	9✔
258	return marks[i];	2✔
259	} else if (i === marks.length) {	7✔
260	return marks[i - 1];	2✔
261	}
262	const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;	5✔
263	return marks[idx];	5✔
264	}
265
266	export function normalizeValue(val, minValue, step, marks) {
267	if (marks && marks.length) {	×
268	return snapToMarks(val, marks);	×
269	}
270
271	return roundValToStep(minValue, step, val);	×
272	}
273
274	export function isPercentField(field) {
275	return field.metadata && field.metadata.numerator && field.metadata.denominator;	44!
276	}
277
278	export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
279	// shouldn't apply sum to percent fiele type
280	// default aggregation is average
281	return field && isPercentField(field)	×
282	? AGGREGATION_TYPES.average
283	: aggregation \|\| AGGREGATION_TYPES.average;	×
284	}
285
286	const getAgregationType = (field, aggregation) => {	11✔
287	if (isPercentField(field)) {	22!
288	return 'mean_of_percent';	×
289	}
290	return aggregation;	22✔
291	};
292
293	const getAgregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {	11✔
294	if (isPercentField(field)) {	22!
295	const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);	×
296	const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);	×
297
298	return {	×
299	getNumerator: i => dataContainer.valueAt(i, numeratorIdx),	×
300	getDenominator: i => dataContainer.valueAt(i, denominatorIdx)	×
301	};
302	}
303
304	return i => field.valueAccessor({index: i});	24✔
305	};
306
307	export const getValueAggrFunc = (	11✔
308	field: Field \| string \| null,
309	aggregation: string,
310	dataset: KeplerTableModel<any, any>
311	): ((bin: Bin) => number) => {
312	const {dataContainer, fields} = dataset;	1✔
313	return field && aggregation	1!
314	? bin =>
315	aggregate(	22✔
316	bin.indexes,
317	getAgregationType(field, aggregation),
318	// @ts-expect-error can return {getNumerator, getDenominator}
319	getAgregationAccessor(field, dataContainer, fields)
320	)
321	: bin => bin.count;	×
322	};
323
324	export const getAggregationOptiosnBasedOnField = field => {	11✔
325	if (isPercentField(field)) {	×
326	// don't show sum
327	return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);	×
328	}
329	return TIME_AGGREGATION;	×
330	};
331
332	function getDelta(
333	bins: LineDatum[],
334	y: number,
335	interval: PlotType['interval']
336	): Partial<LineDatum> & {delta: 'last'; pct: number \| null} {
337	// if (WOW[interval]) return getWow(bins, y, interval);
338	const lastBin = bins[bins.length - 1];	22✔
339
340	return {	22✔
341	delta: 'last',
342	pct: lastBin ? getPctChange(y, lastBin.y) : null	22✔
343	};
344	}
345
346	export function getPctChange(y, y0) {
347	if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {	21✔
348	return (y - y0) / y0;	12✔
349	}
350	return null;	9✔
351	}
352
353	/**
354	*
355	* @param datasets
356	* @param filter
357	*/
358	export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
359	const {dataId, yAxis, plotType, lineChart} = filter;	1✔
360	const {aggregation, interval} = plotType;	1✔
361	const seriesDataId = dataId[0];	1✔
362	const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];	1✔
363
364	if (	1!
365	lineChart &&	1!
366	lineChart.aggregation === aggregation &&
367	lineChart.interval === interval &&
368	lineChart.yAxis === yAxis?.name &&
369	// we need to make sure we validate bins because of cross filter data changes
370	isEqual(bins, lineChart?.bins)
371	) {
372	// don't update lineChart if plotType hasn't change
NEW 373	return lineChart;	×
374	}
375
376	const dataset = datasets[seriesDataId];	1✔
377	const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);	1✔
378
379	const init: LineDatum[] = [];	1✔
380	const series = (bins \|\| []).reduce((accu, bin, i) => {	1!
381	const y = getYValue(bin);	22✔
382	const delta = getDelta(accu, y, interval);	22✔
383	accu.push({	22✔
384	x: bin.x0,
385	y,
386	...delta
387	});
388	return accu;	22✔
389	}, init);
390
391	const yDomain = extent<{y: any}>(series, d => d.y);	22✔
392	const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];	1!
393
394	// treat missing data as another series
395	const split = splitSeries(series);	1✔
396	const aggrName = AGGREGATION_NAME[aggregation];	1✔
397
398	return {	1✔
399	// @ts-ignore
400	yDomain,
401	// @ts-ignore
402	xDomain,
403	interval,
404	aggregation,
405	// @ts-ignore
406	series: split,
407	title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,	1!
408	fieldType: yAxis ? yAxis.type : 'integer',	1!
409	yAxis: yAxis ? yAxis.name : null,	1!
410	allTime: {
411	title: `All Time Average`,
412	value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)	22✔
413	},
414	// @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
415	bins
416	};
417	}
418
419	// split into multiple series when see missing data
420	export function splitSeries(series) {
421	const lines: any[] = [];	1✔
422	let temp: any[] = [];	1✔
423	for (let i = 0; i < series.length; i++) {	1✔
424	const d = series[i];	22✔
425	if (!notNullorUndefined(d.y) && temp.length) {	22!
426	// ends temp
427	lines.push(temp);	×
428	temp = [];	×
429	} else if (notNullorUndefined(d.y)) {	22!
430	temp.push(d);	22✔
431	}
432
433	if (i === series.length - 1 && temp.length) {	22✔
434	lines.push(temp);	1✔
435	}
436	}
437
438	const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];	1!
439
440	return {lines, markers};	1✔
441	}
442
443	export function filterSeriesByRange(series, range) {
444	if (!series) {	×
445	return [];	×
446	}
447	const [start, end] = range;	×
448	const inRange: any[] = [];	×
449
450	for (const serie of series) {	×
451	if (!serie.length) {	×
452	// eslint-disable-next-line no-console, no-undef
453	console.warn('Serie shouldnt be empty', series);	×
454	}
455
456	const i0 = bisector((s: {x: any}) => s.x).left(serie, start);	×
457	const i1 = bisector((s: {x: any}) => s.x).right(serie, end);	×
458	const sliced = serie.slice(i0, i1);	×
459	if (sliced.length) inRange.push(sliced);	×
460	}
461
462	return inRange;	×
463	}
464
465	export function adjustValueToAnimationWindow(state: VisState, filter: TimeRangeFilter) {
466	const {
467	plotType,
468	value: [value0, value1],
469	animationWindow
470	} = filter;	1✔
471
472	const interval = plotType.interval \|\| getInitialInterval(filter, state.datasets);	1!
473	const bins = getTimeBins(filter, state.datasets, interval);	1✔
474	const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];	1✔
475	const thresholds = (datasetBins \|\| []).map(b => b.x0);	1!
476
477	let val0 = value0;	1✔
478	let val1 = value1;	1✔
479	let idx;
480	if (animationWindow === ANIMATION_WINDOW.interval) {	1!
481	val0 = snapToMarks(value1, thresholds);	1✔
482	idx = thresholds.indexOf(val0);	1✔
483	val1 = idx > -1 ? datasetBins[idx].x1 : NaN;	1!
484	} else {
485	// fit current value to window
486	val0 = snapToMarks(value0, thresholds);	×
487	val1 = snapToMarks(value1, thresholds);	×
488
489	if (val0 === val1) {	×
490	idx = thresholds.indexOf(val0);	×
491	if (idx === thresholds.length - 1) {	×
492	val0 = thresholds[idx - 1];	×
493	} else {
494	val1 = thresholds[idx + 1];	×
495	}
496	}
497	}
498
499	const updatedFilter = {	1✔
500	...filter,
501	plotType: {
502	...filter.plotType,
503	interval
504	},
505	timeBins: bins,
506	value: [val0, val1]
507	};
508
509	return updatedFilter;	1✔
510	}
511
512	/**
513	* Create or update colors for a filter plot
514	* @param filter
515	* @param datasets
516	* @param oldColorsByDataId
517	*/
518	function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
519	let colorsByDataId = oldColorsByDataId \|\| {};	7✔
520	for (const dataId of filter.dataId) {	7✔
521	if (!colorsByDataId[dataId] && datasets[dataId]) {	14✔
522	colorsByDataId = {	6✔
523	...colorsByDataId,
524	[dataId]: rgbToHex(datasets[dataId].color)
525	};
526	}
527	}
528	return colorsByDataId;	7✔
529	}
530
531	/**
532	*
533	* @param filter
534	* @param plotType
535	* @param datasets
536	* @param dataId
537	*/
538	export function updateTimeFilterPlotType(
539	filter: TimeRangeFilter,
540	plotType: TimeRangeFilter['plotType'],
541	datasets: Datasets,
542	dataId?: string
543	): TimeRangeFilter {
544	let nextFilter = filter;	60✔
545	let nextPlotType = plotType;	60✔
546	if (typeof nextPlotType !== 'object' \|\| !nextPlotType.aggregation \|\| !nextPlotType.interval) {	60✔
547	nextPlotType = getDefaultPlotType(filter, datasets);	28✔
548	}
549
550	if (filter.dataId.length > 1) {	60✔
551	nextPlotType = {	7✔
552	...nextPlotType,
553	colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
554	};
555	}
556	nextFilter = {	60✔
557	...nextFilter,
558	plotType: nextPlotType
559	};
560
561	const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);	60✔
562
563	nextFilter = {	60✔
564	...nextFilter,
565	timeBins: bins
566	};
567
568	if (plotType.type === PLOT_TYPES.histogram) {	60✔
569	// Histogram is calculated and memoized in the chart itself
570	} else if (plotType.type === PLOT_TYPES.lineChart) {	1!
571	// we should be able to move this into its own component so react will do the shallow comparison for us.
572	nextFilter = {	1✔
573	...nextFilter,
574	lineChart: getLineChart(datasets, nextFilter)
575	};
576	}
577
578	return nextFilter;	60✔
579	}
580
581	export function getRangeFilterBins(filter, datasets, numBins) {
582	const {domain} = filter;	36✔
583	if (!filter.dataId) return null;	36!
584
585	return filter.dataId.reduce((acc, dataId, datasetIdx) => {	36✔
586	if (filter.bins?.[dataId]) {	36✔
587	// don't recalculate bins
588	acc[dataId] = filter.bins[dataId];	12✔
589	return acc;	12✔
590	}
591	const fieldName = filter.name[datasetIdx];	24✔
592	if (dataId && fieldName) {	24!
593	const dataset = datasets[dataId];	24✔
594	const field = dataset?.getColumnField(fieldName);	24✔
595	if (dataset && field) {	24!
596	const indexes = runGpuFilterForPlot(dataset, filter);	24✔
597	const valueAccessor = index => field.valueAccessor({index});	211✔
598	acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);	24✔
599	}
600	}
601	return acc;	24✔
602	}, {});
603	}
604
605	export function updateRangeFilterPlotType(
606	filter: RangeFilter,
607	plotType: RangeFilter['plotType'],
608	datasets: Datasets,
609	dataId?: string
610	): RangeFilter {
611	const nextFilter = {	36✔
612	...filter,
613	plotType
614	};
615
616	// if (dataId) {
617	// // clear bins
618	// nextFilter = {
619	// ...nextFilter,
620	// bins: {
621	// ...nextFilter.bins,
622	// [dataId]: null
623	// }
624	// };
625	// }
626
627	return {	36✔
628	...filter,
629	plotType,
630	bins: getRangeFilterBins(nextFilter, datasets, BINS)
631	};
632	}
633
634	export function getChartTitle(yAxis, plotType: {aggregation: string}) {
635	const yAxisName = yAxis?.displayName;	×
636	const {aggregation} = plotType;	×
637
638	if (yAxisName) {	×
639	return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);	×
640	}
641
642	return `Count of Rows over Time`;	×
643	}
644
645	export function getDefaultPlotType(filter, datasets) {
646	const interval = getInitialInterval(filter, datasets);	28✔
647	const defaultTimeFormat = getDefaultTimeFormat(interval);	28✔
648	return {	28✔
649	interval,
650	defaultTimeFormat,
651	type: PLOT_TYPES.histogram,
652	aggregation: AGGREGATION_TYPES.sum
653	};
654	}

keplergl / kepler.gl / 12201097896

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous