12564978170

Committed 31 Dec 2024 11:33PM UTC coverage: 67.342% (-0.03%) from 67.376%

Build # 12564978170

Build Type

push

github

Committed by

web-flow

Commit Message

[fix] Custom Color Scale fixes (#2875)

- [Chore] add exports for scenegraph to layers/index
- [fix] Apply custom column format on color legend
- [feat] support custom color scale for layers using colorScale.byZoom
- [fix] disable custom ordinal scale
- [chore] add FSQ Cool Tone color palette

Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>

Run Details

5894 of 10159 branches covered (58.02%)

Branch coverage included in aggregate %.

14 of 17 new or added lines in 7 files covered. (82.35%)

6 existing lines in 2 files now uncovered.

12087 of 16542 relevant lines covered (73.07%)

89.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.66

/src/utils/src/plot.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import {bisectLeft, extent, histogram as d3Histogram, ticks} from 'd3-array';
import isEqual from 'lodash.isequal';
import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
import moment from 'moment';
import {
  Bin,
  TimeBins,
  Millisecond,
  TimeRangeFilter,
  RangeFilter,
  PlotType,
  Filter,
  LineChart,
  Field,
  ValueOf,
  LineDatum
} from '@kepler.gl/types';
import {notNullorUndefined} from '@kepler.gl/common-utils';
import {
  ANIMATION_WINDOW,
  BINS,
  durationDay,
  TIME_AGGREGATION,
  AGGREGATION_TYPES,
  PLOT_TYPES,
  AggregationTypes
} from '@kepler.gl/constants';

import {isNumber, roundValToStep} from './data-utils';
import {aggregate, AGGREGATION_NAME} from './aggregation';
import {capitalizeFirstLetter} from './strings';
import {getDefaultTimeFormat} from './format';
import {rgbToHex} from './color-utils';
import {DataContainerInterface} from '.';
import {KeplerTableModel} from './types';

// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
type Datasets = any;

/**
 *
 * @param thresholds
 * @param values
 * @param indexes
 */
export function histogramFromThreshold(
  thresholds: number[],
  values: number[],
  valueAccessor?: (d: unknown) => number,
  filterEmptyBins = true
): Bin[] {
  const getBins = d3Histogram()
    .domain([thresholds[0], thresholds[thresholds.length - 1]])
    .thresholds(thresholds);

  if (valueAccessor) {
    getBins.value(valueAccessor);
  }

  // @ts-ignore
  const bins = getBins(values).map(bin => ({
    count: bin.length,
    indexes: bin,
    x0: bin.x0,
    x1: bin.x1
  }));

  // d3-histogram ignores threshold values outside the domain
  // The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.

  // bins[0].x0 = thresholds[0];
  // bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];

  // @ts-ignore
  return filterEmptyBins ? bins.filter(b => b.count > 0) : bins;
}

/**
 *
 * @param values
 * @param numBins
 * @param valueAccessor
 */
export function histogramFromValues(
  values: (Millisecond | null | number)[],
  numBins: number,
  valueAccessor?: (d: number) => number
) {
  const getBins = d3Histogram().thresholds(numBins);

  if (valueAccessor) {
    getBins.value(valueAccessor);
  }

  // @ts-ignore d3-array types doesn't match
  return getBins(values)
    .map(bin => ({
      count: bin.length,
      indexes: bin,
      x0: bin.x0,
      x1: bin.x1
    }))
    .filter(b => {
      const {x0, x1} = b;
      return isNumber(x0) && isNumber(x1);
    });
}

export function histogramFromOrdinal(
  domain: [string],
  values: (Millisecond | null | number)[],
  valueAccessor?: (d: unknown) => string
): Bin[] {
  // @ts-expect-error to typed to expect strings
  const getBins = d3Histogram().thresholds(domain);
  if (valueAccessor) {
    // @ts-expect-error to typed to expect strings
    getBins.value(valueAccessor);
  }

  // @ts-expect-error null values aren't expected
  const bins = getBins(values);

  // @ts-ignore d3-array types doesn't match
  return bins.map(bin => ({
    count: bin.length,
    indexes: bin,
    x0: bin.x0,
    x1: bin.x0
  }));
}

/**
 *
 * @param domain
 * @param values
 * @param numBins
 * @param valueAccessor
 */
export function histogramFromDomain(
  domain: [number, number],
  values: (Millisecond | null | number)[],
  numBins: number,
  valueAccessor?: (d: unknown) => number
): Bin[] {
  const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);
  if (valueAccessor) {
    getBins.value(valueAccessor);
  }

  // @ts-ignore d3-array types doesn't match
  return getBins(values).map(bin => ({
    count: bin.length,
    indexes: bin,
    x0: bin.x0,
    x1: bin.x1
  }));
}

/**
 * @param filter
 * @param datasets
 * @param interval
 */
export function getTimeBins(
  filter: TimeRangeFilter,
  datasets: Datasets,
  interval: PlotType['interval']
): TimeBins {
  let bins = filter.timeBins || {};

  filter.dataId.forEach((dataId, dataIdIdx) => {
    // reuse bins if filterData did not change
    if (bins[dataId] && bins[dataId][interval]) {
      return;
    }
    const dataset = datasets[dataId];

    // do not apply current filter
    const indexes = runGpuFilterForPlot(dataset, filter);

    bins = {
      ...bins,
      [dataId]: {
        ...bins[dataId],
        [interval]: binByTime(indexes, dataset, interval, filter)
      }
    };
  });

  return bins;
}

export function binByTime(indexes, dataset, interval, filter) {
  // gpuFilters need to be apply to filteredIndex
  const mappedValue = getFilterMappedValue(dataset, filter);
  if (!mappedValue) {
    return null;
  }
  const intervalBins = getBinThresholds(interval, filter.domain);
  const valueAccessor = idx => mappedValue[idx];
  const bins = histogramFromThreshold(intervalBins, indexes, valueAccessor);

  return bins;
}

export function getBinThresholds(interval: string, domain: number[]): number[] {
  const timeInterval = intervalToFunction(interval);
  const [t0, t1] = domain;
  const floor = timeInterval.floor(t0).getTime();
  const ceiling = timeInterval.ceil(t1).getTime();

  if (!timeInterval) {
    // if time interval is not defined
    // this should not happen
    return [t0, t0 + durationDay];
  }
  const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());
  const lastStep = binThresholds[binThresholds.length - 1];
  if (lastStep === t1) {
    // when last step equal to domain max, add one more step
    binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());
  }

  return binThresholds;
}

/**
 * Run GPU filter on current filter result to generate indexes for ploting chart
 * Skip ruuning for the same field
 * @param dataset
 * @param filter
 */
export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
  dataset: K,
  filter?: Filter
): number[] {
  const skipIndexes = getSkipIndexes(dataset, filter);

  const {
    gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
    filteredIndex
  } = dataset;
  const getFilterValue = filterValueAccessor(dataset.dataContainer)();

  const allChannels = Object.keys(filterValueUpdateTriggers)
    .map((_, i) => i)
    .filter(i => Object.values(filterValueUpdateTriggers)[i]);
  const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;
  if (skipAll) {
    return filteredIndex;
  }

  const filterData = getFilterDataFunc(
    filterRange,
    getFilterValue,
    dataset.dataContainer,
    skipIndexes
  );

  return filteredIndex.filter(filterData);
}

function getSkipIndexes(dataset, filter) {
  // array of gpu filter names
  if (!filter) {
    return [];
  }
  const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({
    name: string;
  } | null)[];
  const valueIndex = filter.dataId.findIndex(id => id === dataset.id);
  const filterColumn = filter.name[valueIndex];

  return gpuFilters.reduce((accu, item, idx) => {
    if (item && filterColumn === item.name) {
      accu.push(idx);
    }
    return accu;
  }, [] as number[]);
}

export function getFilterDataFunc(
  filterRange,
  getFilterValue,
  dataContainer: DataContainerInterface,
  skips
) {
  return index =>
    getFilterValue({index}).every(
      (val, i) => skips.includes(i) || (val >= filterRange[i][0] && val <= filterRange[i][1])
    );
}

export function validBin(b) {
  return b.x0 !== undefined && b.x1 !== undefined;
}

/**
 * Use in slider, given a number and an array of numbers, return the nears number from the array.
 * Takes a value, timesteps and return the actual step.
 * @param value
 * @param marks
 */
export function snapToMarks(value: number, marks: number[]): number {
  // always use bin x0
  if (!marks.length) {
    // @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
    return null;
  }
  const i = bisectLeft(marks, value);
  if (i === 0) {
    return marks[i];
  } else if (i === marks.length) {
    return marks[i - 1];
  }
  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
  return marks[idx];
}

export function normalizeValue(val, minValue, step, marks) {
  if (marks && marks.length) {
    return snapToMarks(val, marks);
  }

  return roundValToStep(minValue, step, val);
}

export function isPercentField(field) {
  return field.metadata && field.metadata.numerator && field.metadata.denominator;
}

export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
  // shouldn't apply sum to percent fiele type
  // default aggregation is average
  return field && isPercentField(field)
    ? AGGREGATION_TYPES.average
    : aggregation || AGGREGATION_TYPES.average;
}

const getAgregationType = (field, aggregation) => {
  if (isPercentField(field)) {
    return 'mean_of_percent';
  }
  return aggregation;
};

const getAggregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {
  if (isPercentField(field)) {
    const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);
    const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);

    return {
      getNumerator: i => dataContainer.valueAt(i, numeratorIdx),
      getDenominator: i => dataContainer.valueAt(i, denominatorIdx)
    };
  }

  return i => field.valueAccessor({index: i});
};

export const getValueAggrFunc = (
  field: Field | string | null,
  aggregation: string,
  dataset: KeplerTableModel<any, any>
): ((bin: Bin) => number) => {
  const {dataContainer, fields} = dataset;

  // The passed-in field might not have all the fields set (e.g. valueAccessor)
  const datasetField = fields.find(
    f => field && (f.name === field || f.name === (field as Field).name)
  );

  return datasetField && aggregation
    ? bin =>
        aggregate(
          bin.indexes,
          getAgregationType(datasetField, aggregation),
          // @ts-expect-error can return {getNumerator, getDenominator}
          getAggregationAccessor(datasetField, dataContainer, fields)
        )
    : bin => bin.count;
};

export const getAggregationOptiosnBasedOnField = field => {
  if (isPercentField(field)) {
    // don't show sum
    return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);
  }
  return TIME_AGGREGATION;
};

function getDelta(
  bins: LineDatum[],
  y: number,
  interval: PlotType['interval']
): Partial<LineDatum> & {delta: 'last'; pct: number | null} {
  // if (WOW[interval]) return getWow(bins, y, interval);
  const lastBin = bins[bins.length - 1];

  return {
    delta: 'last',
    pct: lastBin ? getPctChange(y, lastBin.y) : null
  };
}

export function getPctChange(y: unknown, y0: unknown): number | null {
  if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {
    return ((y as number) - (y0 as number)) / (y0 as number);
  }
  return null;
}

/**
 *
 * @param datasets
 * @param filter
 */
export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
  const {dataId, yAxis, plotType, lineChart} = filter;
  const {aggregation, interval} = plotType;
  const seriesDataId = dataId[0];
  const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];

  if (
    lineChart &&
    lineChart.aggregation === aggregation &&
    lineChart.interval === interval &&
    lineChart.yAxis === yAxis?.name &&
    // we need to make sure we validate bins because of cross filter data changes
    isEqual(bins, lineChart?.bins)
  ) {
    // don't update lineChart if plotType hasn't change
    return lineChart;
  }

  const dataset = datasets[seriesDataId];
  const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);

  const init: LineDatum[] = [];
  const series = (bins || []).reduce((accu, bin, i) => {
    const y = getYValue(bin);
    const delta = getDelta(accu, y, interval);
    accu.push({
      x: bin.x0,
      y,
      ...delta
    });
    return accu;
  }, init);

  const yDomain = extent<{y: any}>(series, d => d.y);
  const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];

  // treat missing data as another series
  const split = splitSeries(series);
  const aggrName = AGGREGATION_NAME[aggregation];

  return {
    // @ts-ignore
    yDomain,
    // @ts-ignore
    xDomain,
    interval,
    aggregation,
    // @ts-ignore
    series: split,
    title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,
    fieldType: yAxis ? yAxis.type : 'integer',
    yAxis: yAxis ? yAxis.name : null,
    allTime: {
      title: `All Time Average`,
      value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)
    },
    // @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
    bins
  };
}

// split into multiple series when see missing data
export function splitSeries(series) {
  const lines: any[] = [];
  let temp: any[] = [];
  for (let i = 0; i < series.length; i++) {
    const d = series[i];
    if (!notNullorUndefined(d.y) && temp.length) {
      // ends temp
      lines.push(temp);
      temp = [];
    } else if (notNullorUndefined(d.y)) {
      temp.push(d);
    }

    if (i === series.length - 1 && temp.length) {
      lines.push(temp);
    }
  }

  const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];

  return {lines, markers};
}

type MinVisStateForAnimationWindow = {
  datasets: Datasets;
};

export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWindow>(
  state: S,
  filter: TimeRangeFilter
) {
  const {
    plotType,
    value: [value0, value1],
    animationWindow
  } = filter;

  const interval = plotType.interval || getInitialInterval(filter, state.datasets);
  const bins = getTimeBins(filter, state.datasets, interval);
  const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];
  const thresholds = (datasetBins || []).map(b => b.x0);

  let val0 = value0;
  let val1 = value1;
  let idx;
  if (animationWindow === ANIMATION_WINDOW.interval) {
    val0 = snapToMarks(value1, thresholds);
    idx = thresholds.indexOf(val0);
    val1 = idx > -1 ? datasetBins[idx].x1 : NaN;
  } else {
    // fit current value to window
    val0 = snapToMarks(value0, thresholds);
    val1 = snapToMarks(value1, thresholds);

    if (val0 === val1) {
      idx = thresholds.indexOf(val0);
      if (idx === thresholds.length - 1) {
        val0 = thresholds[idx - 1];
      } else {
        val1 = thresholds[idx + 1];
      }
    }
  }

  const updatedFilter = {
    ...filter,
    plotType: {
      ...filter.plotType,
      interval
    },
    timeBins: bins,
    value: [val0, val1]
  };

  return updatedFilter;
}

/**
 * Create or update colors for a filter plot
 * @param filter
 * @param datasets
 * @param oldColorsByDataId
 */
function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
  let colorsByDataId = oldColorsByDataId || {};
  for (const dataId of filter.dataId) {
    if (!colorsByDataId[dataId] && datasets[dataId]) {
      colorsByDataId = {
        ...colorsByDataId,
        [dataId]: rgbToHex(datasets[dataId].color)
      };
    }
  }
  return colorsByDataId;
}

/**
 *
 * @param filter
 * @param plotType
 * @param datasets
 * @param dataId
 */
export function updateTimeFilterPlotType(
  filter: TimeRangeFilter,
  plotType: TimeRangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): TimeRangeFilter {
  let nextFilter = filter;
  let nextPlotType = plotType;
  if (typeof nextPlotType !== 'object' || !nextPlotType.aggregation || !nextPlotType.interval) {
    nextPlotType = getDefaultPlotType(filter, datasets);
  }

  if (filter.dataId.length > 1) {
    nextPlotType = {
      ...nextPlotType,
      colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
    };
  }
  nextFilter = {
    ...nextFilter,
    plotType: nextPlotType
  };

  const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);

  nextFilter = {
    ...nextFilter,
    timeBins: bins
  };

  if (plotType.type === PLOT_TYPES.histogram) {
    // Histogram is calculated and memoized in the chart itself
  } else if (plotType.type === PLOT_TYPES.lineChart) {
    // we should be able to move this into its own component so react will do the shallow comparison for us.
    nextFilter = {
      ...nextFilter,
      lineChart: getLineChart(datasets, nextFilter)
    };
  }

  return nextFilter;
}

export function getRangeFilterBins(filter, datasets, numBins) {
  const {domain} = filter;
  if (!filter.dataId) return null;

  return filter.dataId.reduce((acc, dataId, datasetIdx) => {
    if (filter.bins?.[dataId]) {
      // don't recalculate bins
      acc[dataId] = filter.bins[dataId];
      return acc;
    }
    const fieldName = filter.name[datasetIdx];
    if (dataId && fieldName) {
      const dataset = datasets[dataId];
      const field = dataset?.getColumnField(fieldName);
      if (dataset && field) {
        const indexes = runGpuFilterForPlot(dataset, filter);
        const valueAccessor = index => field.valueAccessor({index});
        acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);
      }
    }
    return acc;
  }, {});
}

export function updateRangeFilterPlotType(
  filter: RangeFilter,
  plotType: RangeFilter['plotType'],
  datasets: Datasets,
  dataId?: string
): RangeFilter {
  const nextFilter = {
    ...filter,
    plotType
  };

  // if (dataId) {
  //   // clear bins
  //   nextFilter = {
  //     ...nextFilter,
  //     bins: {
  //       ...nextFilter.bins,
  //       [dataId]: null
  //     }
  //   };
  // }

  return {
    ...filter,
    plotType,
    bins: getRangeFilterBins(nextFilter, datasets, BINS)
  };
}

export function getChartTitle(yAxis: Field, plotType: PlotType): string {
  const yAxisName = yAxis?.displayName;
  const {aggregation} = plotType;

  if (yAxisName) {
    return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);
  }

  return `Count of Rows over Time`;
}

export function getDefaultPlotType(filter, datasets) {
  const interval = getInitialInterval(filter, datasets);
  const defaultTimeFormat = getDefaultTimeFormat(interval);
  return {
    interval,
    defaultTimeFormat,
    type: PLOT_TYPES.histogram,
    aggregation: AGGREGATION_TYPES.sum
  };
}

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import {bisectLeft, extent, histogram as d3Histogram, ticks} from 'd3-array';
5	import isEqual from 'lodash.isequal';
6	import {getFilterMappedValue, getInitialInterval, intervalToFunction} from './time';
7	import moment from 'moment';
8	import {
9	Bin,
10	TimeBins,
11	Millisecond,
12	TimeRangeFilter,
13	RangeFilter,
14	PlotType,
15	Filter,
16	LineChart,
17	Field,
18	ValueOf,
19	LineDatum
20	} from '@kepler.gl/types';
21	import {notNullorUndefined} from '@kepler.gl/common-utils';
22	import {
23	ANIMATION_WINDOW,
24	BINS,
25	durationDay,
26	TIME_AGGREGATION,
27	AGGREGATION_TYPES,
28	PLOT_TYPES,
29	AggregationTypes
30	} from '@kepler.gl/constants';
31
32	import {isNumber, roundValToStep} from './data-utils';
33	import {aggregate, AGGREGATION_NAME} from './aggregation';
34	import {capitalizeFirstLetter} from './strings';
35	import {getDefaultTimeFormat} from './format';
36	import {rgbToHex} from './color-utils';
37	import {DataContainerInterface} from '.';
38	import {KeplerTableModel} from './types';
39
40	// TODO kepler-table module isn't accessible from utils. Add compatible interface to types
41	type Datasets = any;
42
43	/**
44	*
45	* @param thresholds
46	* @param values
47	* @param indexes
48	*/
49	export function histogramFromThreshold(
50	thresholds: number[],
51	values: number[],
52	valueAccessor?: (d: unknown) => number,
53	filterEmptyBins = true	54✔
54	): Bin[] {
55	const getBins = d3Histogram()	55✔
56	.domain([thresholds[0], thresholds[thresholds.length - 1]])
57	.thresholds(thresholds);
58
59	if (valueAccessor) {	55✔
60	getBins.value(valueAccessor);	42✔
61	}
62
63	// @ts-ignore
64	const bins = getBins(values).map(bin => ({	3,445✔
65	count: bin.length,
66	indexes: bin,
67	x0: bin.x0,
68	x1: bin.x1
69	}));
70
71	// d3-histogram ignores threshold values outside the domain
72	// The first bin.x0 is always equal to the minimum domain value, and the last bin.x1 is always equal to the maximum domain value.
73
74	// bins[0].x0 = thresholds[0];
75	// bins[bins.length - 1].x1 = thresholds[thresholds.length - 1];
76
77	// @ts-ignore
78	return filterEmptyBins ? bins.filter(b => b.count > 0) : bins;	3,420✔
79	}
80
81	/**
82	*
83	* @param values
84	* @param numBins
85	* @param valueAccessor
86	*/
87	export function histogramFromValues(
88	values: (Millisecond \| null \| number)[],
89	numBins: number,
90	valueAccessor?: (d: number) => number
91	) {
92	const getBins = d3Histogram().thresholds(numBins);	9✔
93
94	if (valueAccessor) {	9✔
95	getBins.value(valueAccessor);	8✔
96	}
97
98	// @ts-ignore d3-array types doesn't match
99	return getBins(values)	9✔
100	.map(bin => ({	179✔
101	count: bin.length,
102	indexes: bin,
103	x0: bin.x0,
104	x1: bin.x1
105	}))
106	.filter(b => {
107	const {x0, x1} = b;	179✔
108	return isNumber(x0) && isNumber(x1);	179✔
109	});
110	}
111
112	export function histogramFromOrdinal(
113	domain: [string],
114	values: (Millisecond \| null \| number)[],
115	valueAccessor?: (d: unknown) => string
116	): Bin[] {
117	// @ts-expect-error to typed to expect strings
UNCOV 118	const getBins = d3Histogram().thresholds(domain);	×
UNCOV 119	if (valueAccessor) {	×
120	// @ts-expect-error to typed to expect strings
UNCOV 121	getBins.value(valueAccessor);	×
122	}
123
124	// @ts-expect-error null values aren't expected
UNCOV 125	const bins = getBins(values);	×
126
127	// @ts-ignore d3-array types doesn't match
UNCOV 128	return bins.map(bin => ({	×
129	count: bin.length,
130	indexes: bin,
131	x0: bin.x0,
132	x1: bin.x0
133	}));
134	}
135
136	/**
137	*
138	* @param domain
139	* @param values
140	* @param numBins
141	* @param valueAccessor
142	*/
143	export function histogramFromDomain(
144	domain: [number, number],
145	values: (Millisecond \| null \| number)[],
146	numBins: number,
147	valueAccessor?: (d: unknown) => number
148	): Bin[] {
149	const getBins = d3Histogram().thresholds(ticks(domain[0], domain[1], numBins)).domain(domain);	30✔
150	if (valueAccessor) {	30✔
151	getBins.value(valueAccessor);	23✔
152	}
153
154	// @ts-ignore d3-array types doesn't match
155	return getBins(values).map(bin => ({	957✔
156	count: bin.length,
157	indexes: bin,
158	x0: bin.x0,
159	x1: bin.x1
160	}));
161	}
162
163	/**
164	* @param filter
165	* @param datasets
166	* @param interval
167	*/
168	export function getTimeBins(
169	filter: TimeRangeFilter,
170	datasets: Datasets,
171	interval: PlotType['interval']
172	): TimeBins {
173	let bins = filter.timeBins \|\| {};	55✔
174
175	filter.dataId.forEach((dataId, dataIdIdx) => {	55✔
176	// reuse bins if filterData did not change
177	if (bins[dataId] && bins[dataId][interval]) {	62✔
178	return;	22✔
179	}
180	const dataset = datasets[dataId];	40✔
181
182	// do not apply current filter
183	const indexes = runGpuFilterForPlot(dataset, filter);	40✔
184
185	bins = {	40✔
186	...bins,
187	[dataId]: {
188	...bins[dataId],
189	[interval]: binByTime(indexes, dataset, interval, filter)
190	}
191	};
192	});
193
194	return bins;	55✔
195	}
196
197	export function binByTime(indexes, dataset, interval, filter) {
198	// gpuFilters need to be apply to filteredIndex
199	const mappedValue = getFilterMappedValue(dataset, filter);	40✔
200	if (!mappedValue) {	40!
201	return null;	×
202	}
203	const intervalBins = getBinThresholds(interval, filter.domain);	40✔
204	const valueAccessor = idx => mappedValue[idx];	646✔
205	const bins = histogramFromThreshold(intervalBins, indexes, valueAccessor);	40✔
206
207	return bins;	40✔
208	}
209
210	export function getBinThresholds(interval: string, domain: number[]): number[] {
211	const timeInterval = intervalToFunction(interval);	53✔
212	const [t0, t1] = domain;	53✔
213	const floor = timeInterval.floor(t0).getTime();	53✔
214	const ceiling = timeInterval.ceil(t1).getTime();	53✔
215
216	if (!timeInterval) {	53!
217	// if time interval is not defined
218	// this should not happen
219	return [t0, t0 + durationDay];	×
220	}
221	const binThresholds = timeInterval.range(floor, ceiling + 1).map(t => moment.utc(t).valueOf());	3,519✔
222	const lastStep = binThresholds[binThresholds.length - 1];	53✔
223	if (lastStep === t1) {	53✔
224	// when last step equal to domain max, add one more step
225	binThresholds.push(moment.utc(timeInterval.offset(lastStep)).valueOf());	22✔
226	}
227
228	return binThresholds;	53✔
229	}
230
231	/**
232	* Run GPU filter on current filter result to generate indexes for ploting chart
233	* Skip ruuning for the same field
234	* @param dataset
235	* @param filter
236	*/
237	export function runGpuFilterForPlot<K extends KeplerTableModel<K, L>, L>(
238	dataset: K,
239	filter?: Filter
240	): number[] {
241	const skipIndexes = getSkipIndexes(dataset, filter);	63✔
242
243	const {
244	gpuFilter: {filterValueUpdateTriggers, filterRange, filterValueAccessor},
245	filteredIndex
246	} = dataset;	63✔
247	const getFilterValue = filterValueAccessor(dataset.dataContainer)();	63✔
248
249	const allChannels = Object.keys(filterValueUpdateTriggers)	63✔
250	.map((_, i) => i)	252✔
251	.filter(i => Object.values(filterValueUpdateTriggers)[i]);	252✔
252	const skipAll = !allChannels.filter(i => !skipIndexes.includes(i)).length;	63✔
253	if (skipAll) {	63✔
254	return filteredIndex;	61✔
255	}
256
257	const filterData = getFilterDataFunc(	2✔
258	filterRange,
259	getFilterValue,
260	dataset.dataContainer,
261	skipIndexes
262	);
263
264	return filteredIndex.filter(filterData);	2✔
265	}
266
267	function getSkipIndexes(dataset, filter) {
268	// array of gpu filter names
269	if (!filter) {	63!
270	return [];	×
271	}
272	const gpuFilters = Object.values(dataset.gpuFilter.filterValueUpdateTriggers) as ({	63✔
273	name: string;
274	} \| null)[];
275	const valueIndex = filter.dataId.findIndex(id => id === dataset.id);	68✔
276	const filterColumn = filter.name[valueIndex];	63✔
277
278	return gpuFilters.reduce((accu, item, idx) => {	63✔
279	if (item && filterColumn === item.name) {	252✔
280	accu.push(idx);	29✔
281	}
282	return accu;	252✔
283	}, [] as number[]);
284	}
285
286	export function getFilterDataFunc(
287	filterRange,
288	getFilterValue,
289	dataContainer: DataContainerInterface,
290	skips
291	) {
292	return index =>	2✔
293	getFilterValue({index}).every(	20✔
294	(val, i) => skips.includes(i) \|\| (val >= filterRange[i][0] && val <= filterRange[i][1])	32✔
295	);
296	}
297
298	export function validBin(b) {
299	return b.x0 !== undefined && b.x1 !== undefined;	×
300	}
301
302	/**
303	* Use in slider, given a number and an array of numbers, return the nears number from the array.
304	* Takes a value, timesteps and return the actual step.
305	* @param value
306	* @param marks
307	*/
308	export function snapToMarks(value: number, marks: number[]): number {
309	// always use bin x0
310	if (!marks.length) {	9!
311	// @ts-expect-error looking at the usage null return value isn't expected and requires extra handling in a lot of places
312	return null;	×
313	}
314	const i = bisectLeft(marks, value);	9✔
315	if (i === 0) {	9✔
316	return marks[i];	2✔
317	} else if (i === marks.length) {	7✔
318	return marks[i - 1];	2✔
319	}
320	const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;	5✔
321	return marks[idx];	5✔
322	}
323
324	export function normalizeValue(val, minValue, step, marks) {
325	if (marks && marks.length) {	×
326	return snapToMarks(val, marks);	×
327	}
328
329	return roundValToStep(minValue, step, val);	×
330	}
331
332	export function isPercentField(field) {
333	return field.metadata && field.metadata.numerator && field.metadata.denominator;	44!
334	}
335
336	export function updateAggregationByField(field: Field, aggregation: ValueOf<AggregationTypes>) {
337	// shouldn't apply sum to percent fiele type
338	// default aggregation is average
339	return field && isPercentField(field)	×
340	? AGGREGATION_TYPES.average
341	: aggregation \|\| AGGREGATION_TYPES.average;	×
342	}
343
344	const getAgregationType = (field, aggregation) => {	14✔
345	if (isPercentField(field)) {	22!
346	return 'mean_of_percent';	×
347	}
348	return aggregation;	22✔
349	};
350
351	const getAggregationAccessor = (field, dataContainer: DataContainerInterface, fields) => {	14✔
352	if (isPercentField(field)) {	22!
353	const numeratorIdx = fields.findIndex(f => f.name === field.metadata.numerator);	×
354	const denominatorIdx = fields.findIndex(f => f.name === field.metadata.denominator);	×
355
356	return {	×
357	getNumerator: i => dataContainer.valueAt(i, numeratorIdx),	×
358	getDenominator: i => dataContainer.valueAt(i, denominatorIdx)	×
359	};
360	}
361
362	return i => field.valueAccessor({index: i});	24✔
363	};
364
365	export const getValueAggrFunc = (	14✔
366	field: Field \| string \| null,
367	aggregation: string,
368	dataset: KeplerTableModel<any, any>
369	): ((bin: Bin) => number) => {
370	const {dataContainer, fields} = dataset;	1✔
371
372	// The passed-in field might not have all the fields set (e.g. valueAccessor)
373	const datasetField = fields.find(	1✔
374	f => field && (f.name === field \|\| f.name === (field as Field).name)	7✔
375	);
376
377	return datasetField && aggregation	1!
378	? bin =>
379	aggregate(	22✔
380	bin.indexes,
381	getAgregationType(datasetField, aggregation),
382	// @ts-expect-error can return {getNumerator, getDenominator}
383	getAggregationAccessor(datasetField, dataContainer, fields)
384	)
385	: bin => bin.count;	×
386	};
387
388	export const getAggregationOptiosnBasedOnField = field => {	14✔
389	if (isPercentField(field)) {	×
390	// don't show sum
391	return TIME_AGGREGATION.filter(({id}) => id !== AGGREGATION_TYPES.sum);	×
392	}
393	return TIME_AGGREGATION;	×
394	};
395
396	function getDelta(
397	bins: LineDatum[],
398	y: number,
399	interval: PlotType['interval']
400	): Partial<LineDatum> & {delta: 'last'; pct: number \| null} {
401	// if (WOW[interval]) return getWow(bins, y, interval);
402	const lastBin = bins[bins.length - 1];	22✔
403
404	return {	22✔
405	delta: 'last',
406	pct: lastBin ? getPctChange(y, lastBin.y) : null	22✔
407	};
408	}
409
410	export function getPctChange(y: unknown, y0: unknown): number \| null {
411	if (Number.isFinite(y) && Number.isFinite(y0) && y0 !== 0) {	21✔
412	return ((y as number) - (y0 as number)) / (y0 as number);	12✔
413	}
414	return null;	9✔
415	}
416
417	/**
418	*
419	* @param datasets
420	* @param filter
421	*/
422	export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
423	const {dataId, yAxis, plotType, lineChart} = filter;	1✔
424	const {aggregation, interval} = plotType;	1✔
425	const seriesDataId = dataId[0];	1✔
426	const bins = (filter as TimeRangeFilter).timeBins?.[seriesDataId]?.[interval];	1✔
427
428	if (	1!
429	lineChart &&	1!
430	lineChart.aggregation === aggregation &&
431	lineChart.interval === interval &&
432	lineChart.yAxis === yAxis?.name &&
433	// we need to make sure we validate bins because of cross filter data changes
434	isEqual(bins, lineChart?.bins)
435	) {
436	// don't update lineChart if plotType hasn't change
437	return lineChart;	×
438	}
439
440	const dataset = datasets[seriesDataId];	1✔
441	const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);	1✔
442
443	const init: LineDatum[] = [];	1✔
444	const series = (bins \|\| []).reduce((accu, bin, i) => {	1!
445	const y = getYValue(bin);	22✔
446	const delta = getDelta(accu, y, interval);	22✔
447	accu.push({	22✔
448	x: bin.x0,
449	y,
450	...delta
451	});
452	return accu;	22✔
453	}, init);
454
455	const yDomain = extent<{y: any}>(series, d => d.y);	22✔
456	const xDomain = bins ? [bins[0].x0, bins[bins.length - 1].x1] : [];	1!
457
458	// treat missing data as another series
459	const split = splitSeries(series);	1✔
460	const aggrName = AGGREGATION_NAME[aggregation];	1✔
461
462	return {	1✔
463	// @ts-ignore
464	yDomain,
465	// @ts-ignore
466	xDomain,
467	interval,
468	aggregation,
469	// @ts-ignore
470	series: split,
471	title: `${aggrName}${' of '}${yAxis ? yAxis.name : 'Count'}`,	1!
472	fieldType: yAxis ? yAxis.type : 'integer',	1!
473	yAxis: yAxis ? yAxis.name : null,	1!
474	allTime: {
475	title: `All Time Average`,
476	value: aggregate(series, AGGREGATION_TYPES.average, d => d.y)	22✔
477	},
478	// @ts-expect-error bins is Bins[], not a Bins map. Refactor to use correct types.
479	bins
480	};
481	}
482
483	// split into multiple series when see missing data
484	export function splitSeries(series) {
485	const lines: any[] = [];	1✔
486	let temp: any[] = [];	1✔
487	for (let i = 0; i < series.length; i++) {	1✔
488	const d = series[i];	22✔
489	if (!notNullorUndefined(d.y) && temp.length) {	22!
490	// ends temp
491	lines.push(temp);	×
492	temp = [];	×
493	} else if (notNullorUndefined(d.y)) {	22!
494	temp.push(d);	22✔
495	}
496
497	if (i === series.length - 1 && temp.length) {	22✔
498	lines.push(temp);	1✔
499	}
500	}
501
502	const markers = lines.length > 1 ? series.filter(d => notNullorUndefined(d.y)) : [];	1!
503
504	return {lines, markers};	1✔
505	}
506
507	type MinVisStateForAnimationWindow = {
508	datasets: Datasets;
509	};
510
511	export function adjustValueToAnimationWindow<S extends MinVisStateForAnimationWindow>(
512	state: S,
513	filter: TimeRangeFilter
514	) {
515	const {
516	plotType,
517	value: [value0, value1],
518	animationWindow
519	} = filter;	1✔
520
521	const interval = plotType.interval \|\| getInitialInterval(filter, state.datasets);	1!
522	const bins = getTimeBins(filter, state.datasets, interval);	1✔
523	const datasetBins = bins && Object.keys(bins).length && Object.values(bins)[0][interval];	1✔
524	const thresholds = (datasetBins \|\| []).map(b => b.x0);	1!
525
526	let val0 = value0;	1✔
527	let val1 = value1;	1✔
528	let idx;
529	if (animationWindow === ANIMATION_WINDOW.interval) {	1!
530	val0 = snapToMarks(value1, thresholds);	1✔
531	idx = thresholds.indexOf(val0);	1✔
532	val1 = idx > -1 ? datasetBins[idx].x1 : NaN;	1!
533	} else {
534	// fit current value to window
535	val0 = snapToMarks(value0, thresholds);	×
536	val1 = snapToMarks(value1, thresholds);	×
537
538	if (val0 === val1) {	×
539	idx = thresholds.indexOf(val0);	×
540	if (idx === thresholds.length - 1) {	×
541	val0 = thresholds[idx - 1];	×
542	} else {
543	val1 = thresholds[idx + 1];	×
544	}
545	}
546	}
547
548	const updatedFilter = {	1✔
549	...filter,
550	plotType: {
551	...filter.plotType,
552	interval
553	},
554	timeBins: bins,
555	value: [val0, val1]
556	};
557
558	return updatedFilter;	1✔
559	}
560
561	/**
562	* Create or update colors for a filter plot
563	* @param filter
564	* @param datasets
565	* @param oldColorsByDataId
566	*/
567	function getFilterPlotColorsByDataId(filter, datasets, oldColorsByDataId) {
568	let colorsByDataId = oldColorsByDataId \|\| {};	6✔
569	for (const dataId of filter.dataId) {	6✔
570	if (!colorsByDataId[dataId] && datasets[dataId]) {	12✔
571	colorsByDataId = {	4✔
572	...colorsByDataId,
573	[dataId]: rgbToHex(datasets[dataId].color)
574	};
575	}
576	}
577	return colorsByDataId;	6✔
578	}
579
580	/**
581	*
582	* @param filter
583	* @param plotType
584	* @param datasets
585	* @param dataId
586	*/
587	export function updateTimeFilterPlotType(
588	filter: TimeRangeFilter,
589	plotType: TimeRangeFilter['plotType'],
590	datasets: Datasets,
591	dataId?: string
592	): TimeRangeFilter {
593	let nextFilter = filter;	53✔
594	let nextPlotType = plotType;	53✔
595	if (typeof nextPlotType !== 'object' \|\| !nextPlotType.aggregation \|\| !nextPlotType.interval) {	53✔
596	nextPlotType = getDefaultPlotType(filter, datasets);	23✔
597	}
598
599	if (filter.dataId.length > 1) {	53✔
600	nextPlotType = {	6✔
601	...nextPlotType,
602	colorsByDataId: getFilterPlotColorsByDataId(filter, datasets, nextPlotType.colorsByDataId)
603	};
604	}
605	nextFilter = {	53✔
606	...nextFilter,
607	plotType: nextPlotType
608	};
609
610	const bins = getTimeBins(nextFilter, datasets, nextPlotType.interval);	53✔
611
612	nextFilter = {	53✔
613	...nextFilter,
614	timeBins: bins
615	};
616
617	if (plotType.type === PLOT_TYPES.histogram) {	53✔
618	// Histogram is calculated and memoized in the chart itself
619	} else if (plotType.type === PLOT_TYPES.lineChart) {	1!
620	// we should be able to move this into its own component so react will do the shallow comparison for us.
621	nextFilter = {	1✔
622	...nextFilter,
623	lineChart: getLineChart(datasets, nextFilter)
624	};
625	}
626
627	return nextFilter;	53✔
628	}
629
630	export function getRangeFilterBins(filter, datasets, numBins) {
631	const {domain} = filter;	34✔
632	if (!filter.dataId) return null;	34!
633
634	return filter.dataId.reduce((acc, dataId, datasetIdx) => {	34✔
635	if (filter.bins?.[dataId]) {	34✔
636	// don't recalculate bins
637	acc[dataId] = filter.bins[dataId];	11✔
638	return acc;	11✔
639	}
640	const fieldName = filter.name[datasetIdx];	23✔
641	if (dataId && fieldName) {	23!
642	const dataset = datasets[dataId];	23✔
643	const field = dataset?.getColumnField(fieldName);	23✔
644	if (dataset && field) {	23!
645	const indexes = runGpuFilterForPlot(dataset, filter);	23✔
646	const valueAccessor = index => field.valueAccessor({index});	210✔
647	acc[dataId] = histogramFromDomain(domain, indexes, numBins, valueAccessor);	23✔
648	}
649	}
650	return acc;	23✔
651	}, {});
652	}
653
654	export function updateRangeFilterPlotType(
655	filter: RangeFilter,
656	plotType: RangeFilter['plotType'],
657	datasets: Datasets,
658	dataId?: string
659	): RangeFilter {
660	const nextFilter = {	34✔
661	...filter,
662	plotType
663	};
664
665	// if (dataId) {
666	// // clear bins
667	// nextFilter = {
668	// ...nextFilter,
669	// bins: {
670	// ...nextFilter.bins,
671	// [dataId]: null
672	// }
673	// };
674	// }
675
676	return {	34✔
677	...filter,
678	plotType,
679	bins: getRangeFilterBins(nextFilter, datasets, BINS)
680	};
681	}
682
683	export function getChartTitle(yAxis: Field, plotType: PlotType): string {
684	const yAxisName = yAxis?.displayName;	×
685	const {aggregation} = plotType;	×
686
687	if (yAxisName) {	×
688	return capitalizeFirstLetter(`${aggregation} ${yAxisName} over Time`);	×
689	}
690
691	return `Count of Rows over Time`;	×
692	}
693
694	export function getDefaultPlotType(filter, datasets) {
695	const interval = getInitialInterval(filter, datasets);	23✔
696	const defaultTimeFormat = getDefaultTimeFormat(interval);	23✔
697	return {	23✔
698	interval,
699	defaultTimeFormat,
700	type: PLOT_TYPES.histogram,
701	aggregation: AGGREGATION_TYPES.sum
702	};
703	}

keplergl / kepler.gl / 12564978170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous