12221701043

Committed 08 Dec 2024 12:17PM UTC coverage: 69.289%. Remained the same

Build # 12221701043

Build Type

Pull #2827

github

Committed by

web-flow

Commit Message

Merge 1a57ab755 into f476a1c4c

Pull Request Pull Request #2827: [chore] ts fixes

Run Details

5448 of 9109 branches covered (59.81%)

Branch coverage included in aggregate %.

4 of 9 new or added lines in 3 files covered. (44.44%)

5 existing lines in 2 files now uncovered.

11381 of 15179 relevant lines covered (74.98%)

95.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.07

/src/utils/src/plot.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import {bisectLeft, bisector, extent, histogram as d3Histogram, ticks} from 'd3-array';
import isEqual from 'lodash.isequal';
import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
import moment from 'moment';
import {
  Bin,
  TimeBins,
  Millisecond,
  TimeRangeFilter,
  RangeFilter,
  PlotType,
  Filter,
  LineChart,
  Field,
  ValueOf,
  LineDatum
} from '@kepler.gl/types';
import {notNullorUndefined} from '@kepler.gl/common-utils';
import {
  ANIMATION_WINDOW,
  BINS,
  durationDay,
  TIME_AGGREGATION,
  AGGREGATION_TYPES,
  PLOT_TYPES,
  AggregationTypes
} from '@kepler.gl/constants';

import {roundValToStep} from './data-utils';
import {aggregate, AGGREGATION_NAME} from './aggregation';
import {capitalizeFirstLetter} from './strings';
import {getDefaultTimeFormat} from './format';
import {rgbToHex} from './color-utils';
import {DataContainerInterface} from '.';
import {KeplerTableModel} from './types';

// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
type Datasets = any;

/**
 *
 * @param thresholds
 * @param values
 * @param indexes
 */
export function histogramFromThreshold(
  thresholds: number[],
  values: number[],
  indexes: number[]
): Bin[] {
  const bins = d3Histogram()
    .value(idx => values[idx])
    .domain([thresholds[0], thresholds[thresholds.length - 1]])
    .thresholds(thresholds)(indexes)
    .map(bin => ({
      count: bin.length,
      indexes: bin,
      x0: bin.x0,
      x1: bin.x1
    }));

  // d3-histogram ignores threshold values outside the domain
  // The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.

  // bins[0].x0 = thresholds[0];
  // bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];

  // @ts-ignore
  // filter out bins with 0 counts
  return bins.filter(b => b.count > 0);
  // return bins;
}

/**
 *
 * @param domain
 * @param values
 * @param numBins
 * @param valueAccessor
 */
export function histogramFromDomain(
  domain: [number, number],
  values: (Millisecond | null | number)[],
  numBins: number,
  valueAccessor?: (d: unknown) => number
): Bin[] {
  const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);
  if (valueAccessor) {
    getBins.value(valueAccessor);
  }

  // @ts-ignore d3-array types doesn't match
  return getBins(values).map(bin => ({
    count: bin.length,
    indexes: bin,
    x0: bin.x0,
    x1: bin.x1
  }));
}

/**
 * @param filter
 * @param datasets
 * @param interval
 */
export function getTimeBins(
  filter: TimeRangeFilter,
  datasets: Datasets,
  interval: PlotType['interval']
): TimeBins {
  let bins = filter.timeBins || {};

  filter.dataId.forEach((dataId, dataIdIdx) => {
    // reuse bins if filterData did not change
    if (bins[dataId] && bins[dataId][interval]) {
      return;
    }
    const dataset = datasets[dataId];

    // do not apply current filter
    const indexes = runGpuFilterForPlot(dataset, filter);

    bins = {
      ...bins,
      [dataId]: {
        ...bins[dataId],
        [interval]: binByTime(indexes, dataset, interval, filter)
      }
    };
  });

  return bins;
}

export function binByTime(indexes, dataset, interval, filter) {
  // gpuFilters need to be apply to filteredIndex
  const mappedValue = getFilterMappedValue(dataset, filter);
  if (!mappedValue) {
    return null;
  }
  const intervalBins = getBinThresholds(interval, filter.domain);

  const bins = histogramFromThreshold(intervalBins, mappedValue, indexes);

  return bins;
}

export function getBinThresholds(interval: string, domain: number[]): number[] {
  const timeInterval = intervalToFunction(interval);
  const [t0, t1] = domain;
  const floor = timeInterval.floor(t0).getTime();
  const ceiling = timeInterval.ceil(t1).getTime();

  if (!timeInterval) {
    // if time interval is not defined
    // this should not happen
    return [t0, t0 + durationDay];
  }
  const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());
  const lastStep = binThresholds[binThresholds.length - 1];
  if (lastStep === t1) {
    // when last step equal to domain max, add one more step
    binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());
  }

  return binThresholds;
}

/**
 * Run GPU filter on current filter result to generate indexes for ploting chart
 * Skip ruuning for the same field
 * @param dataset
 * @param filter
 */
export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
  dataset: K,
  filter?: Filter
): number[] {
  const skipIndexes = getSkipIndexes(dataset, filter);

  const {
    gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
    filteredIndex
  } = dataset;
  const getFilterValue = filterValueAccessor(dataset.dataContainer)();

  const allChannels = Object.keys(filterValueUpdateTriggers)
    .map((_, i) => i)
    .filter(i => Object.values(filterValueUpdateTriggers)[i]);
  const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;
  if (skipAll) {
    return filteredIndex;
  }

  const filterData = getFilterDataFunc(
    filterRange,
    getFilterValue,
    dataset.dataContainer,
    skipIndexes
  );

  return filteredIndex.filter(filterData);
}

function getSkipIndexes(dataset, filter) {
  // array of gpu filter names
  if (!filter) {
    return [];
  }
  const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({
    name: string;
  } | null)[];
  const valueIndex = filter.dataId.findIndex(id => id === dataset.id);
  const filterColumn = filter.name[valueIndex];

  return gpuFilters.reduce((accu, item, idx) => {
    if (item && filterColumn === item.name) {
      accu.push(idx);
    }
    return accu;
  }, [] as number[]);
}

export function getFilterDataFunc(
  filterRange,
  getFilterValue,
  dataContainer: DataContainerInterface,
  skips
) {
  return index =>
    getFilterValue({index}).every(
      (val, i) => skips.includes(i) || (val >= filterRange[i][0] && val <= filterRange[i][1])
    );
}

export function validBin(b) {
  return b.x0 !== undefined && b.x1 !== undefined;
}

/**
 * Use in slider, given a number and an array of numbers, return the nears number from the array.
 * Takes a value, timesteps and return the actual step.
 * @param value
 * @param marks
 */
export function snapToMarks(value: number, marks: number[]): number {
  // always use bin x0
  if (!marks.length) {
    // @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
    return null;
  }
  const i = bisectLeft(marks, value);
  if (i === 0) {
    return marks[i];
  } else if (i === marks.length) {
    return marks[i - 1];
  }
  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
  return marks[idx];
}

export function normalizeValue(val, minValue, step, marks) {
  if (marks && marks.length) {
    return snapToMarks(val, marks);
  }

  return roundValToStep(minValue, step, val);
}

export function isPercentField(field) {
  return field.metadata && field.metadata.numerator && field.metadata.denominator;
}

export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
  // shouldn't apply sum to percent fiele type
  // default aggregation is average
  return field && isPercentField(field)
    ? AGGREGATION_TYPES.average
    : aggregation || AGGREGATION_TYPES.average;
}

const getAgregationType = (field, aggregation) => {
  if (isPercentField(field)) {
    return 'mean_of_percent';
  }
  return aggregation;
};

const getAggregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {
  if (isPercentField(field)) {
    const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);
    const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);

    return {
      getNumerator: i => dataContainer.valueAt(i, numeratorIdx),
      getDenominator: i => dataContainer.valueAt(i, denominatorIdx)
    };
  }

  return i => field.valueAccessor({index: i});
};

export const getValueAggrFunc = (
  field: Field | string | null,
  aggregation: string,
  dataset: KeplerTableModel<any, any>
): ((bin: Bin) => number) => {
  const {dataContainer, fields} = dataset;

  // The passed-in field might not have all the fields set (e.g. valueAccessor)
  const datasetField = fields.find(
    f => field && (f.name === field || f.name === (field as Field).name)
  );

  return datasetField && aggregation
    ? bin =>
        aggregate(
          bin.indexes,
          getAgregationType(datasetField, aggregation),
          // @ts-expect-error can return {getNumerator, getDenominator}
          getAggregationAccessor(datasetField, dataContainer, fields)
        )
    : bin => bin.count;
};

export const getAggregationOptiosnBasedOnField = field => {
  if (isPercentField(field)) {
    // don't show sum
    return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);
  }
  return TIME_AGGREGATION;
};

function getDelta(
  bins: LineDatum[],
  y: number,
  interval: PlotType['interval']
): Partial<LineDatum> & {delta: 'last'; pct: number | null} {
  // if (WOW[interval]) return getWow(bins, y, interval);
  const lastBin = bins[bins.length - 1];

  return {
    delta: 'last',
    pct: lastBin ? getPctChange(y, lastBin.y) : null
  };
}

export function getPctChange(y: unknown, y0: unknown): number | null {
  if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {
    return ((y as number) - (y0 as number)) / (y0 as number);
  }
  return null;
}

/**
 *
 * @param datasets
 * @param filter
 */
export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
  const {dataId, yAxis, plotType, lineChart} = filter;
  const {aggregation, interval} = plotType;
  const seriesDataId = dataId[0];
  const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];

  if (
    lineChart &&
    lineChart.aggregation === aggregation &&
    lineChart.interval === interval &&
    lineChart.yAxis === yAxis?.name &&
    // we need to make sure we validate bins because of cross filter data changes
    isEqual(bins, lineChart?.bins)
  ) {
    // don't update lineChart if plotType hasn't change
    return lineChart;
  }

  const dataset = datasets[seriesDataId];
  const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);

  const init: LineDatum[] = [];
  const series = (bins || []).reduce((accu, bin, i) => {
    const y = getYValue(bin);
    const delta = getDelta(accu, y, interval);
    accu.push({
      x: bin.x0,
      y,
      ...delta
    });
    return accu;
  }, init);

  const yDomain = extent<{y: any}>(series, d => d.y);
  const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];

  // treat missing data as another series
  const split = splitSeries(series);
  const aggrName = AGGREGATION_NAME[aggregation];

  return {
    // @ts-ignore
    yDomain,
    // @ts-ignore
    xDomain,
    interval,
    aggregation,
    // @ts-ignore
    series: split,
    title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,
    fieldType: yAxis ? yAxis.type : 'integer',
    yAxis: yAxis ? yAxis.name : null,
    allTime: {
      title: `All Time Average`,
      value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)
    },
    // @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
    bins
  };
}

// split into multiple series when see missing data
export function splitSeries(series) {
  const lines: any[] = [];
  let temp: any[] = [];
  for (let i = 0; i < series.length; i++) {
    const d = series[i];
    if (!notNullorUndefined(d.y) && temp.length) {
      // ends temp
      lines.push(temp);
      temp = [];
    } else if (notNullorUndefined(d.y)) {
      temp.push(d);
    }

    if (i === series.length - 1 && temp.length) {
      lines.push(temp);
    }
  }

  const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];

  return {lines, markers};
}

export function filterSeriesByRange(series, range) {
  if (!series) {
    return [];
  }
  const [start, end] = range;
  const inRange: any[] = [];

  for (const serie of series) {
    if (!serie.length) {
      // eslint-disable-next-line no-console, no-undef
      console.warn('Serie shouldnt be empty', series);
    }

    const i0 = bisector((s: {x: any}) => s.x).left(serie, start);
    const i1 = bisector((s: {x: any}) => s.x).right(serie, end);
    const sliced = serie.slice(i0, i1);
    if (sliced.length) inRange.push(sliced);
  }

  return inRange;
}

type MinVisStateForAnimationWindow = {
  datasets: Datasets;
};

export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWindow>(
  state: S,
  filter: TimeRangeFilter
) {
  const {
    plotType,
    value: [value0, value1],
    animationWindow
  } = filter;

  const interval = plotType.interval || getInitialInterval(filter, state.datasets);
  const bins = getTimeBins(filter, state.datasets, interval);
  const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];
  const thresholds = (datasetBins || []).map(b => b.x0);

  let val0 = value0;
  let val1 = value1;
  let idx;
  if (animationWindow === ANIMATION_WINDOW.interval) {
    val0 = snapToMarks(value1, thresholds);
    idx = thresholds.indexOf(val0);
    val1 = idx > -1 ? datasetBins[idx].x1 : NaN;
  } else {
    // fit current value to window
    val0 = snapToMarks(value0, thresholds);
    val1 = snapToMarks(value1, thresholds);

    if (val0 === val1) {
      idx = thresholds.indexOf(val0);
      if (idx === thresholds.length - 1) {
        val0 = thresholds[idx - 1];
      } else {
        val1 = thresholds[idx + 1];
      }
    }
  }

  const updatedFilter = {
    ...filter,
    plotType: {
      ...filter.plotType,
      interval
    },
    timeBins: bins,
    value: [val0, val1]
  };

  return updatedFilter;
}

/**
 * Create or update colors for a filter plot
 * @param filter
 * @param datasets
 * @param oldColorsByDataId
 */
function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
  let colorsByDataId = oldColorsByDataId || {};
  for (const dataId of filter.dataId) {
    if (!colorsByDataId[dataId] && datasets[dataId]) {
      colorsByDataId = {
        ...colorsByDataId,
        [dataId]: rgbToHex(datasets[dataId].color)
      };
    }
  }
  return colorsByDataId;
}

/**
 *
 * @param filter
 * @param plotType
 * @param datasets
 * @param dataId
 */
export function updateTimeFilterPlotType(
  filter: TimeRangeFilter,
  plotType: TimeRangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): TimeRangeFilter {
  let nextFilter = filter;
  let nextPlotType = plotType;
  if (typeof nextPlotType !== 'object' || !nextPlotType.aggregation || !nextPlotType.interval) {
    nextPlotType = getDefaultPlotType(filter, datasets);
  }

  if (filter.dataId.length > 1) {
    nextPlotType = {
      ...nextPlotType,
      colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
    };
  }
  nextFilter = {
    ...nextFilter,
    plotType: nextPlotType
  };

  const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);

  nextFilter = {
    ...nextFilter,
    timeBins: bins
  };

  if (plotType.type === PLOT_TYPES.histogram) {
    // Histogram is calculated and memoized in the chart itself
  } else if (plotType.type === PLOT_TYPES.lineChart) {
    // we should be able to move this into its own component so react will do the shallow comparison for us.
    nextFilter = {
      ...nextFilter,
      lineChart: getLineChart(datasets, nextFilter)
    };
  }

  return nextFilter;
}

export function getRangeFilterBins(filter, datasets, numBins) {
  const {domain} = filter;
  if (!filter.dataId) return null;

  return filter.dataId.reduce((acc, dataId, datasetIdx) => {
    if (filter.bins?.[dataId]) {
      // don't recalculate bins
      acc[dataId] = filter.bins[dataId];
      return acc;
    }
    const fieldName = filter.name[datasetIdx];
    if (dataId && fieldName) {
      const dataset = datasets[dataId];
      const field = dataset?.getColumnField(fieldName);
      if (dataset && field) {
        const indexes = runGpuFilterForPlot(dataset, filter);
        const valueAccessor = index => field.valueAccessor({index});
        acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);
      }
    }
    return acc;
  }, {});
}

export function updateRangeFilterPlotType(
  filter: RangeFilter,
  plotType: RangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): RangeFilter {
  const nextFilter = {
    ...filter,
    plotType
  };

  // if (dataId) {
  //   // clear bins
  //   nextFilter = {
  //     ...nextFilter,
  //     bins: {
  //       ...nextFilter.bins,
  //       [dataId]: null
  //     }
  //   };
  // }

  return {
    ...filter,
    plotType,
    bins: getRangeFilterBins(nextFilter, datasets, BINS)
  };
}

export function getChartTitle(yAxis, plotType: {aggregation: string}) {
  const yAxisName = yAxis?.displayName;
  const {aggregation} = plotType;

  if (yAxisName) {
    return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);
  }

  return `Count of Rows over Time`;
}

export function getDefaultPlotType(filter, datasets) {
  const interval = getInitialInterval(filter, datasets);
  const defaultTimeFormat = getDefaultTimeFormat(interval);
  return {
    interval,
    defaultTimeFormat,
    type: PLOT_TYPES.histogram,
    aggregation: AGGREGATION_TYPES.sum
  };
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import {bisectLeft, bisector, extent, histogram as d3Histogram, ticks} from 'd3-array';
5	import isEqual from 'lodash.isequal';
6	import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
7	import moment from 'moment';
8	import {
9	Bin,
10	TimeBins,
11	Millisecond,
12	TimeRangeFilter,
13	RangeFilter,
14	PlotType,
15	Filter,
16	LineChart,
17	Field,
18	ValueOf,
19	LineDatum
20	} from '@kepler.gl/types';
21	import {notNullorUndefined} from '@kepler.gl/common-utils';
22	import {
23	ANIMATION_WINDOW,
24	BINS,
25	durationDay,
26	TIME_AGGREGATION,
27	AGGREGATION_TYPES,
28	PLOT_TYPES,
29	AggregationTypes
30	} from '@kepler.gl/constants';
31
32	import {roundValToStep} from './data-utils';
33	import {aggregate, AGGREGATION_NAME} from './aggregation';
34	import {capitalizeFirstLetter} from './strings';
35	import {getDefaultTimeFormat} from './format';
36	import {rgbToHex} from './color-utils';
37	import {DataContainerInterface} from '.';
38	import {KeplerTableModel} from './types';
39
40	// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
41	type Datasets = any;
42
43	/**
44	*
45	* @param thresholds
46	* @param values
47	* @param indexes
48	*/
49	export function histogramFromThreshold(
50	thresholds: number[],
51	values: number[],
52	indexes: number[]
53	): Bin[] {
54	const bins = d3Histogram()	52✔
55	.value(idx => values[idx])	902✔
56	.domain([thresholds[0], thresholds[thresholds.length - 1]])
57	.thresholds(thresholds)(indexes)
58	.map(bin => ({	3,373✔
59	count: bin.length,
60	indexes: bin,
61	x0: bin.x0,
62	x1: bin.x1
63	}));
64
65	// d3-histogram ignores threshold values outside the domain
66	// The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.
67
68	// bins[0].x0 = thresholds[0];
69	// bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];
70
71	// @ts-ignore
72	// filter out bins with 0 counts
73	return bins.filter(b => b.count > 0);	3,373✔
74	// return bins;
75	}
76
77	/**
78	*
79	* @param domain
80	* @param values
81	* @param numBins
82	* @param valueAccessor
83	*/
84	export function histogramFromDomain(
85	domain: [number, number],
86	values: (Millisecond \| null \| number)[],
87	numBins: number,
88	valueAccessor?: (d: unknown) => number
89	): Bin[] {
90	const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);	32✔
91	if (valueAccessor) {	32✔
92	getBins.value(valueAccessor);	24✔
93	}
94
95	// @ts-ignore d3-array types doesn't match
96	return getBins(values).map(bin => ({	1,023✔
97	count: bin.length,
98	indexes: bin,
99	x0: bin.x0,
100	x1: bin.x1
101	}));
102	}
103
104	/**
105	* @param filter
106	* @param datasets
107	* @param interval
108	*/
109	export function getTimeBins(
110	filter: TimeRangeFilter,
111	datasets: Datasets,
112	interval: PlotType['interval']
113	): TimeBins {
114	let bins = filter.timeBins \|\| {};	61✔
115
116	filter.dataId.forEach((dataId, dataIdIdx) => {	61✔
117	// reuse bins if filterData did not change
118	if (bins[dataId] && bins[dataId][interval]) {	68✔
119	return;	26✔
120	}
121	const dataset = datasets[dataId];	42✔
122
123	// do not apply current filter
124	const indexes = runGpuFilterForPlot(dataset, filter);	42✔
125
126	bins = {	42✔
127	...bins,
128	[dataId]: {
129	...bins[dataId],
130	[interval]: binByTime(indexes, dataset, interval, filter)
131	}
132	};
133	});
134
135	return bins;	61✔
136	}
137
138	export function binByTime(indexes, dataset, interval, filter) {
139	// gpuFilters need to be apply to filteredIndex
140	const mappedValue = getFilterMappedValue(dataset, filter);	42✔
141	if (!mappedValue) {	42!
142	return null;	×
143	}
144	const intervalBins = getBinThresholds(interval, filter.domain);	42✔
145
146	const bins = histogramFromThreshold(intervalBins, mappedValue, indexes);	42✔
147
148	return bins;	42✔
149	}
150
151	export function getBinThresholds(interval: string, domain: number[]): number[] {
152	const timeInterval = intervalToFunction(interval);	55✔
153	const [t0, t1] = domain;	55✔
154	const floor = timeInterval.floor(t0).getTime();	55✔
155	const ceiling = timeInterval.ceil(t1).getTime();	55✔
156
157	if (!timeInterval) {	55!
158	// if time interval is not defined
159	// this should not happen
160	return [t0, t0 + durationDay];	×
161	}
162	const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());	3,481✔
163	const lastStep = binThresholds[binThresholds.length - 1];	55✔
164	if (lastStep === t1) {	55✔
165	// when last step equal to domain max, add one more step
166	binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());	26✔
167	}
168
169	return binThresholds;	55✔
170	}
171
172	/**
173	* Run GPU filter on current filter result to generate indexes for ploting chart
174	* Skip ruuning for the same field
175	* @param dataset
176	* @param filter
177	*/
178	export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
179	dataset: K,
180	filter?: Filter
181	): number[] {
182	const skipIndexes = getSkipIndexes(dataset, filter);	66✔
183
184	const {
185	gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
186	filteredIndex
187	} = dataset;	66✔
188	const getFilterValue = filterValueAccessor(dataset.dataContainer)();	66✔
189
190	const allChannels = Object.keys(filterValueUpdateTriggers)	66✔
191	.map((_, i) => i)	264✔
192	.filter(i => Object.values(filterValueUpdateTriggers)[i]);	264✔
193	const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;	66✔
194	if (skipAll) {	66✔
195	return filteredIndex;	63✔
196	}
197
198	const filterData = getFilterDataFunc(	3✔
199	filterRange,
200	getFilterValue,
201	dataset.dataContainer,
202	skipIndexes
203	);
204
205	return filteredIndex.filter(filterData);	3✔
206	}
207
208	function getSkipIndexes(dataset, filter) {
209	// array of gpu filter names
210	if (!filter) {	66!
211	return [];	×
212	}
213	const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({	66✔
214	name: string;
215	} \| null)[];
216	const valueIndex = filter.dataId.findIndex(id => id === dataset.id);	70✔
217	const filterColumn = filter.name[valueIndex];	66✔
218
219	return gpuFilters.reduce((accu, item, idx) => {	66✔
220	if (item && filterColumn === item.name) {	264✔
221	accu.push(idx);	32✔
222	}
223	return accu;	264✔
224	}, [] as number[]);
225	}
226
227	export function getFilterDataFunc(
228	filterRange,
229	getFilterValue,
230	dataContainer: DataContainerInterface,
231	skips
232	) {
233	return index =>	3✔
234	getFilterValue({index}).every(	30✔
235	(val, i) => skips.includes(i) \|\| (val >= filterRange[i][0] && val <= filterRange[i][1])	48✔
236	);
237	}
238
239	export function validBin(b) {
240	return b.x0 !== undefined && b.x1 !== undefined;	×
241	}
242
243	/**
244	* Use in slider, given a number and an array of numbers, return the nears number from the array.
245	* Takes a value, timesteps and return the actual step.
246	* @param value
247	* @param marks
248	*/
249	export function snapToMarks(value: number, marks: number[]): number {
250	// always use bin x0
251	if (!marks.length) {	9!
252	// @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
253	return null;	×
254	}
255	const i = bisectLeft(marks, value);	9✔
256	if (i === 0) {	9✔
257	return marks[i];	2✔
258	} else if (i === marks.length) {	7✔
259	return marks[i - 1];	2✔
260	}
261	const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;	5✔
262	return marks[idx];	5✔
263	}
264
265	export function normalizeValue(val, minValue, step, marks) {
266	if (marks && marks.length) {	×
267	return snapToMarks(val, marks);	×
268	}
269
270	return roundValToStep(minValue, step, val);	×
271	}
272
273	export function isPercentField(field) {
274	return field.metadata && field.metadata.numerator && field.metadata.denominator;	44!
275	}
276
277	export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
278	// shouldn't apply sum to percent fiele type
279	// default aggregation is average
280	return field && isPercentField(field)	×
281	? AGGREGATION_TYPES.average
282	: aggregation \|\| AGGREGATION_TYPES.average;	×
283	}
284
285	const getAgregationType = (field, aggregation) => {	11✔
286	if (isPercentField(field)) {	22!
287	return 'mean_of_percent';	×
288	}
289	return aggregation;	22✔
290	};
291
292	const getAggregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {	11✔
293	if (isPercentField(field)) {	22!
294	const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);	×
295	const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);	×
296
297	return {	×
298	getNumerator: i => dataContainer.valueAt(i, numeratorIdx),	×
299	getDenominator: i => dataContainer.valueAt(i, denominatorIdx)	×
300	};
301	}
302
303	return i => field.valueAccessor({index: i});	24✔
304	};
305
306	export const getValueAggrFunc = (	11✔
307	field: Field \| string \| null,
308	aggregation: string,
309	dataset: KeplerTableModel<any, any>
310	): ((bin: Bin) => number) => {
311	const {dataContainer, fields} = dataset;	1✔
312
313	// The passed-in field might not have all the fields set (e.g. valueAccessor)
314	const datasetField = fields.find(	1✔
315	f => field && (f.name === field \|\| f.name === (field as Field).name)	7✔
316	);
317
318	return datasetField && aggregation	1!
319	? bin =>
320	aggregate(	22✔
321	bin.indexes,
322	getAgregationType(datasetField, aggregation),
323	// @ts-expect-error can return {getNumerator, getDenominator}
324	getAggregationAccessor(datasetField, dataContainer, fields)
325	)
326	: bin => bin.count;	×
327	};
328
329	export const getAggregationOptiosnBasedOnField = field => {	11✔
330	if (isPercentField(field)) {	×
331	// don't show sum
332	return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);	×
333	}
334	return TIME_AGGREGATION;	×
335	};
336
337	function getDelta(
338	bins: LineDatum[],
339	y: number,
340	interval: PlotType['interval']
341	): Partial<LineDatum> & {delta: 'last'; pct: number \| null} {
342	// if (WOW[interval]) return getWow(bins, y, interval);
343	const lastBin = bins[bins.length - 1];	22✔
344
345	return {	22✔
346	delta: 'last',
347	pct: lastBin ? getPctChange(y, lastBin.y) : null	22✔
348	};
349	}
350
351	export function getPctChange(y: unknown, y0: unknown): number \| null {
352	if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {	21✔
353	return ((y as number) - (y0 as number)) / (y0 as number);	12✔
354	}
355	return null;	9✔
356	}
357
358	/**
359	*
360	* @param datasets
361	* @param filter
362	*/
363	export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
364	const {dataId, yAxis, plotType, lineChart} = filter;	1✔
365	const {aggregation, interval} = plotType;	1✔
366	const seriesDataId = dataId[0];	1✔
367	const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];	1✔
368
369	if (	1!
370	lineChart &&	1!
371	lineChart.aggregation === aggregation &&
372	lineChart.interval === interval &&
373	lineChart.yAxis === yAxis?.name &&
374	// we need to make sure we validate bins because of cross filter data changes
375	isEqual(bins, lineChart?.bins)
376	) {
377	// don't update lineChart if plotType hasn't change
378	return lineChart;	×
379	}
380
381	const dataset = datasets[seriesDataId];	1✔
382	const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);	1✔
383
384	const init: LineDatum[] = [];	1✔
385	const series = (bins \|\| []).reduce((accu, bin, i) => {	1!
386	const y = getYValue(bin);	22✔
387	const delta = getDelta(accu, y, interval);	22✔
388	accu.push({	22✔
389	x: bin.x0,
390	y,
391	...delta
392	});
393	return accu;	22✔
394	}, init);
395
396	const yDomain = extent<{y: any}>(series, d => d.y);	22✔
397	const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];	1!
398
399	// treat missing data as another series
400	const split = splitSeries(series);	1✔
401	const aggrName = AGGREGATION_NAME[aggregation];	1✔
402
403	return {	1✔
404	// @ts-ignore
405	yDomain,
406	// @ts-ignore
407	xDomain,
408	interval,
409	aggregation,
410	// @ts-ignore
411	series: split,
412	title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,	1!
413	fieldType: yAxis ? yAxis.type : 'integer',	1!
414	yAxis: yAxis ? yAxis.name : null,	1!
415	allTime: {
416	title: `All Time Average`,
417	value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)	22✔
418	},
419	// @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
420	bins
421	};
422	}
423
424	// split into multiple series when see missing data
425	export function splitSeries(series) {
426	const lines: any[] = [];	1✔
427	let temp: any[] = [];	1✔
428	for (let i = 0; i < series.length; i++) {	1✔
429	const d = series[i];	22✔
430	if (!notNullorUndefined(d.y) && temp.length) {	22!
431	// ends temp
432	lines.push(temp);	×
433	temp = [];	×
434	} else if (notNullorUndefined(d.y)) {	22!
435	temp.push(d);	22✔
436	}
437
438	if (i === series.length - 1 && temp.length) {	22✔
439	lines.push(temp);	1✔
440	}
441	}
442
443	const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];	1!
444
445	return {lines, markers};	1✔
446	}
447
448	export function filterSeriesByRange(series, range) {
NEW 449	if (!series) {	×
NEW 450	return [];	×
451	}
NEW 452	const [start, end] = range;	×
NEW 453	const inRange: any[] = [];	×
454
NEW 455	for (const serie of series) {	×
456	if (!serie.length) {	×
457	// eslint-disable-next-line no-console, no-undef
UNCOV 458	console.warn('Serie shouldnt be empty', series);	×
459	}
460
UNCOV 461	const i0 = bisector((s: {x: any}) => s.x).left(serie, start);	×
462	const i1 = bisector((s: {x: any}) => s.x).right(serie, end);	×
463	const sliced = serie.slice(i0, i1);	×
UNCOV 464	if (sliced.length) inRange.push(sliced);	×
465	}
466
UNCOV 467	return inRange;	×
468	}
469
470	type MinVisStateForAnimationWindow = {
471	datasets: Datasets;
472	};
473
474	export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWindow>(
475	state: S,
476	filter: TimeRangeFilter
477	) {
478	const {
479	plotType,
480	value: [value0, value1],
481	animationWindow
482	} = filter;	1✔
483
484	const interval = plotType.interval \|\| getInitialInterval(filter, state.datasets);	1!
485	const bins = getTimeBins(filter, state.datasets, interval);	1✔
486	const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];	1✔
487	const thresholds = (datasetBins \|\| []).map(b => b.x0);	1!
488
489	let val0 = value0;	1✔
490	let val1 = value1;	1✔
491	let idx;
492	if (animationWindow === ANIMATION_WINDOW.interval) {	1!
493	val0 = snapToMarks(value1, thresholds);	1✔
494	idx = thresholds.indexOf(val0);	1✔
495	val1 = idx > -1 ? datasetBins[idx].x1 : NaN;	1!
496	} else {
497	// fit current value to window
498	val0 = snapToMarks(value0, thresholds);	×
499	val1 = snapToMarks(value1, thresholds);	×
500
501	if (val0 === val1) {	×
502	idx = thresholds.indexOf(val0);	×
503	if (idx === thresholds.length - 1) {	×
504	val0 = thresholds[idx - 1];	×
505	} else {
506	val1 = thresholds[idx + 1];	×
507	}
508	}
509	}
510
511	const updatedFilter = {	1✔
512	...filter,
513	plotType: {
514	...filter.plotType,
515	interval
516	},
517	timeBins: bins,
518	value: [val0, val1]
519	};
520
521	return updatedFilter;	1✔
522	}
523
524	/**
525	* Create or update colors for a filter plot
526	* @param filter
527	* @param datasets
528	* @param oldColorsByDataId
529	*/
530	function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
531	let colorsByDataId = oldColorsByDataId \|\| {};	7✔
532	for (const dataId of filter.dataId) {	7✔
533	if (!colorsByDataId[dataId] && datasets[dataId]) {	14✔
534	colorsByDataId = {	6✔
535	...colorsByDataId,
536	[dataId]: rgbToHex(datasets[dataId].color)
537	};
538	}
539	}
540	return colorsByDataId;	7✔
541	}
542
543	/**
544	*
545	* @param filter
546	* @param plotType
547	* @param datasets
548	* @param dataId
549	*/
550	export function updateTimeFilterPlotType(
551	filter: TimeRangeFilter,
552	plotType: TimeRangeFilter['plotType'],
553	datasets: Datasets,
554	dataId?: string
555	): TimeRangeFilter {
556	let nextFilter = filter;	60✔
557	let nextPlotType = plotType;	60✔
558	if (typeof nextPlotType !== 'object' \|\| !nextPlotType.aggregation \|\| !nextPlotType.interval) {	60✔
559	nextPlotType = getDefaultPlotType(filter, datasets);	28✔
560	}
561
562	if (filter.dataId.length > 1) {	60✔
563	nextPlotType = {	7✔
564	...nextPlotType,
565	colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
566	};
567	}
568	nextFilter = {	60✔
569	...nextFilter,
570	plotType: nextPlotType
571	};
572
573	const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);	60✔
574
575	nextFilter = {	60✔
576	...nextFilter,
577	timeBins: bins
578	};
579
580	if (plotType.type === PLOT_TYPES.histogram) {	60✔
581	// Histogram is calculated and memoized in the chart itself
582	} else if (plotType.type === PLOT_TYPES.lineChart) {	1!
583	// we should be able to move this into its own component so react will do the shallow comparison for us.
584	nextFilter = {	1✔
585	...nextFilter,
586	lineChart: getLineChart(datasets, nextFilter)
587	};
588	}
589
590	return nextFilter;	60✔
591	}
592
593	export function getRangeFilterBins(filter, datasets, numBins) {
594	const {domain} = filter;	36✔
595	if (!filter.dataId) return null;	36!
596
597	return filter.dataId.reduce((acc, dataId, datasetIdx) => {	36✔
598	if (filter.bins?.[dataId]) {	36✔
599	// don't recalculate bins
600	acc[dataId] = filter.bins[dataId];	12✔
601	return acc;	12✔
602	}
603	const fieldName = filter.name[datasetIdx];	24✔
604	if (dataId && fieldName) {	24!
605	const dataset = datasets[dataId];	24✔
606	const field = dataset?.getColumnField(fieldName);	24✔
607	if (dataset && field) {	24!
608	const indexes = runGpuFilterForPlot(dataset, filter);	24✔
609	const valueAccessor = index => field.valueAccessor({index});	211✔
610	acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);	24✔
611	}
612	}
613	return acc;	24✔
614	}, {});
615	}
616
617	export function updateRangeFilterPlotType(
618	filter: RangeFilter,
619	plotType: RangeFilter['plotType'],
620	datasets: Datasets,
621	dataId?: string
622	): RangeFilter {
623	const nextFilter = {	36✔
624	...filter,
625	plotType
626	};
627
628	// if (dataId) {
629	// // clear bins
630	// nextFilter = {
631	// ...nextFilter,
632	// bins: {
633	// ...nextFilter.bins,
634	// [dataId]: null
635	// }
636	// };
637	// }
638
639	return {	36✔
640	...filter,
641	plotType,
642	bins: getRangeFilterBins(nextFilter, datasets, BINS)
643	};
644	}
645
646	export function getChartTitle(yAxis, plotType: {aggregation: string}) {
647	const yAxisName = yAxis?.displayName;	×
648	const {aggregation} = plotType;	×
649
650	if (yAxisName) {	×
651	return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);	×
652	}
653
654	return `Count of Rows over Time`;	×
655	}
656
657	export function getDefaultPlotType(filter, datasets) {
658	const interval = getInitialInterval(filter, datasets);	28✔
659	const defaultTimeFormat = getDefaultTimeFormat(interval);	28✔
660	return {	28✔
661	interval,
662	defaultTimeFormat,
663	type: PLOT_TYPES.histogram,
664	aggregation: AGGREGATION_TYPES.sum
665	};
666	}

keplergl / kepler.gl / 12221701043

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous