12604159536

Committed 03 Jan 2025 09:26PM UTC coverage: 66.843% (-0.5%) from 67.344%

Build # 12604159536

Build Type

Pull #2892

github

Committed by

web-flow

Commit Message

Merge 894aa31a5 into 0b67c5409

Pull Request Pull Request #2892: [fix] Prevent infinite useEffects loop

Run Details

5959 of 10355 branches covered (57.55%)

Branch coverage included in aggregate %.

13 of 16 new or added lines in 1 file covered. (81.25%)

484 existing lines in 25 files now uncovered.

12215 of 16834 relevant lines covered (72.56%)

89.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.12

/src/table/src/kepler-table.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import Console from 'global/console';
import {ascending, descending} from 'd3-array';

import {
  TRIP_POINT_FIELDS,
  SORT_ORDER,
  ALL_FIELD_TYPES,
  ALTITUDE_FIELDS,
  SCALE_TYPES
} from '@kepler.gl/constants';
import {
  RGBColor,
  Field,
  FieldPair,
  FieldDomain,
  Filter,
  ProtoDataset,
  FilterRecord,
  FilterDatasetOpt,
  RangeFieldDomain,
  SelectFieldDomain,
  MultiSelectFieldDomain,
  TimeRangeFieldDomain
} from '@kepler.gl/types';

import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils';

import {
  getSortingFunction,
  timeToUnixMilli,
  createDataContainer,
  DataForm,
  diffFilters,
  filterDataByFilterTypes,
  FilterResult,
  getFilterFunction,
  getFilterProps,
  getFilterRecord,
  getNumericFieldDomain,
  getTimestampFieldDomain,
  getLinearDomain,
  getLogDomain,
  getOrdinalDomain,
  getQuantileDomain,
  DataContainerInterface,
  FilterChanged
} from '@kepler.gl/utils';
import {generateHashId, notNullorUndefined} from '@kepler.gl/common-utils';

// TODO isolate layer type, depends on @kepler.gl/layers
type Layer = any;

export type GpuFilter = {
  filterRange: number[][];
  filterValueUpdateTriggers: {
    [id: string]: {name: string; domain0: number} | null;
  };
  filterValueAccessor: (
    dc: DataContainerInterface
  ) => (
    getIndex?: (any) => number,
    getData?: (dc_: DataContainerInterface, d: any, fieldIndex: number) => any
  ) => (d: any, objectInfo?: {index: number}) => (number | number[])[];
};

export type FilterProps =
  | NumericFieldFilterProps
  | BooleanFieldFilterProps
  | StringFieldFilterProps
  | TimeFieldFilterProps;

export type NumericFieldFilterProps = RangeFieldDomain & {
  value: [number, number];
  type: string;
  typeOptions: string[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type BooleanFieldFilterProps = SelectFieldDomain & {
  type: string;
  value: boolean;
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type StringFieldFilterProps = MultiSelectFieldDomain & {
  type: string;
  value: string[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type TimeFieldFilterProps = TimeRangeFieldDomain & {
  type: string;
  view: Filter['view'];
  fixedDomain: boolean;
  value: number[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};

// Unique identifier of each field
const FID_KEY = 'name';

export function maybeToDate(
  isTime: boolean,
  fieldIdx: number,
  format: string,
  dc: DataContainerInterface,
  d: {index: number}
) {
  if (isTime) {
    return timeToUnixMilli(dc.valueAt(d.index, fieldIdx), format);
  }

  return dc.valueAt(d.index, fieldIdx);
}

class KeplerTable<F extends Field = Field> {
  readonly id: string;

  type?: string;
  label: string;
  color: RGBColor;

  // fields and data
  fields: F[] = [];

  dataContainer: DataContainerInterface;

  allIndexes: number[] = [];
  filteredIndex: number[] = [];
  filteredIdxCPU?: number[];
  filteredIndexForDomain: number[] = [];
  fieldPairs: FieldPair[] = [];
  gpuFilter: GpuFilter;
  filterRecord?: FilterRecord;
  filterRecordCPU?: FilterRecord;
  changedFilters?: FilterChanged;

  // table-injected metadata
  sortColumn?: {
    // column name: sorted idx
    [key: string]: string; // ASCENDING | DESCENDING | UNSORT
  };
  sortOrder?: number[] | null;

  pinnedColumns?: string[];
  supportedFilterTypes?: string[] | null;
  disableDataOperation?: boolean;

  // table-injected metadata
  metadata: object;

  constructor({
    info,
    color,
    metadata,
    supportedFilterTypes = null,
    disableDataOperation = false
  }: {
    info?: ProtoDataset['info'];
    color: RGBColor;
    metadata?: ProtoDataset['metadata'];
    supportedFilterTypes?: ProtoDataset['supportedFilterTypes'];
    disableDataOperation?: ProtoDataset['disableDataOperation'];
  }) {
    // TODO - what to do if validation fails? Can kepler handle exceptions?
    // const validatedData = validateInputData(data);
    // if (!validatedData) {
    //   return this;
    // }

    const datasetInfo = {
      id: generateHashId(4),
      label: 'new dataset',
      type: '',
      ...info
    };

    const defaultMetadata = {
      id: datasetInfo.id,
      // @ts-ignore
      format: datasetInfo.format || '',
      label: datasetInfo.label || ''
    };

    this.id = datasetInfo.id;
    this.type = datasetInfo.type;
    this.label = datasetInfo.label;
    this.color = color;
    this.metadata = {
      ...defaultMetadata,
      ...metadata
    };

    this.supportedFilterTypes = supportedFilterTypes;
    this.disableDataOperation = disableDataOperation;

    this.dataContainer = createDataContainer([]);
    this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);
  }

  importData({data}: {data: ProtoDataset['data']}) {
    const dataContainerData = data.cols ? data.cols : data.rows;
    const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;

    const dataContainer = createDataContainer(dataContainerData, {
      fields: data.fields,
      inputDataFormat
    });

    const fields: Field[] = data.fields.map((f, i) => ({
      ...f,
      fieldIdx: i,
      id: f.name,
      displayName: f.displayName || f.name,
      analyzerType: f.analyzerType || ALL_FIELD_TYPES.string,
      format: f.format || '',
      valueAccessor: getFieldValueAccessor(f, i, dataContainer)
    }));

    const allIndexes = dataContainer.getPlainIndex();
    this.dataContainer = dataContainer;
    this.allIndexes = allIndexes;
    this.filteredIndex = allIndexes;
    this.filteredIndexForDomain = allIndexes;
    this.fieldPairs = findPointFieldPairs(fields);
    // @ts-expect-error Make sure that fields satisfies F extends Field
    this.fields = fields;
    this.gpuFilter = getGpuFilterProps([], this.id, fields, undefined);
  }

  /**
   * update table with new data
   * @param data - new data e.g. the arrow data with new batches loaded
   */
  update(data: ProtoDataset['data']) {
    const dataContainerData = data.cols ? data.cols : data.rows;
    this.dataContainer.update?.(dataContainerData);
    this.allIndexes = this.dataContainer.getPlainIndex();
    this.filteredIndex = this.allIndexes;
    this.filteredIndexForDomain = this.allIndexes;

    return this;
  }

  get length() {
    return this.dataContainer.numRows();
  }

  /**
   * Get field
   * @param columnName
   */
  getColumnField(columnName: string): F | undefined {
    const field = this.fields.find(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, field);
    return field;
  }

  /**
   * Get fieldIdx
   * @param columnName
   */
  getColumnFieldIdx(columnName: string): number {
    const fieldIdx = this.fields.findIndex(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, Boolean(fieldIdx > -1));
    return fieldIdx;
  }

  /**
   * Get displayFormat
   * @param columnName
   */
  getColumnDisplayFormat(columnName) {
    const field = this.fields.find(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, field);
    return field?.displayFormat;
  }

  /**
   * Get the value of a cell
   */
  getValue(columnName: string, rowIdx: number): any {
    const field = this.getColumnField(columnName);
    return field ? field.valueAccessor({index: rowIdx}) : null;
  }

  /**
   * Updates existing field with a new object
   * @param fieldIdx
   * @param newField
   */
  updateColumnField(fieldIdx: number, newField: F): void {
    this.fields = Object.assign([...this.fields], {[fieldIdx]: newField});
  }

  /**
   * Update dataset color by custom color
   * @param newColor
   */
  updateTableColor(newColor: RGBColor): void {
    this.color = newColor;
  }

  /**
   * Save filterProps to field and retrieve it
   * @param columnName
   */
  getColumnFilterProps(columnName: string): F['filterProps'] | null | undefined {
    const fieldIdx = this.getColumnFieldIdx(columnName);
    if (fieldIdx < 0) {
      return null;
    }
    const field = this.fields[fieldIdx];
    if (Object.prototype.hasOwnProperty.call(field, 'filterProps')) {
      return field.filterProps;
    }

    const fieldDomain = this.getColumnFilterDomain(field);
    if (!fieldDomain) {
      return null;
    }

    const filterProps = getFilterProps(field, fieldDomain);
    const newField = {
      ...field,
      filterProps
    };

    this.updateColumnField(fieldIdx, newField);

    return filterProps;
  }

  /**
   * Apply filters to dataset, return the filtered dataset with updated `gpuFilter`, `filterRecord`, `filteredIndex`, `filteredIndexForDomain`
   * @param filters
   * @param layers
   * @param opt
   */
  filterTable(filters: Filter[], layers: Layer[], opt?: FilterDatasetOpt): KeplerTable<Field> {
    const {dataContainer, id: dataId, filterRecord: oldFilterRecord, fields} = this;

    // if there is no filters
    const filterRecord = getFilterRecord(dataId, filters, opt || {});

    this.filterRecord = filterRecord;
    this.gpuFilter = getGpuFilterProps(filters, dataId, fields, this.gpuFilter);

    this.changedFilters = diffFilters(filterRecord, oldFilterRecord);

    if (!filters.length) {
      this.filteredIndex = this.allIndexes;
      this.filteredIndexForDomain = this.allIndexes;
      return this;
    }

    // generate 2 sets of filter result
    // filteredIndex used to calculate layer data
    // filteredIndexForDomain used to calculate layer Domain
    const shouldCalDomain = Boolean(this.changedFilters.dynamicDomain);
    const shouldCalIndex = Boolean(this.changedFilters.cpu);

    let filterResult: FilterResult = {};
    if (shouldCalDomain || shouldCalIndex) {
      const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;
      const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;

      const filterFuncs = filters.reduce((acc, filter) => {
        const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);
        const field = fieldIndex !== -1 ? fields[fieldIndex] : null;

        return {
          ...acc,
          [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
        };
      }, {});

      filterResult = filterDataByFilterTypes(
        {dynamicDomainFilters, cpuFilters, filterFuncs},
        dataContainer
      );
    }

    this.filteredIndex = filterResult.filteredIndex || this.filteredIndex;
    this.filteredIndexForDomain =
      filterResult.filteredIndexForDomain || this.filteredIndexForDomain;

    return this;
  }

  /**
   * Apply filters to a dataset all on CPU, assign to `filteredIdxCPU`, `filterRecordCPU`
   * @param filters
   * @param layers
   */
  filterTableCPU(filters: Filter[], layers: Layer[]): KeplerTable<Field> {
    const opt = {
      cpuOnly: true,
      ignoreDomain: true
    };

    // no filter
    if (!filters.length) {
      this.filteredIdxCPU = this.allIndexes;
      this.filterRecordCPU = getFilterRecord(this.id, filters, opt);
      return this;
    }

    // no gpu filter
    if (!filters.find(f => f.gpu)) {
      this.filteredIdxCPU = this.filteredIndex;
      this.filterRecordCPU = getFilterRecord(this.id, filters, opt);
      return this;
    }

    // make a copy for cpu filtering
    const copied = copyTable(this);

    copied.filterRecord = this.filterRecordCPU;
    copied.filteredIndex = this.filteredIdxCPU || [];

    const filtered = copied.filterTable(filters, layers, opt);

    this.filteredIdxCPU = filtered.filteredIndex;
    this.filterRecordCPU = filtered.filterRecord;

    return this;
  }

  /**
   * Calculate field domain based on field type and data
   * for Filter
   */
  getColumnFilterDomain(field: F): FieldDomain {
    const {dataContainer} = this;
    const {valueAccessor} = field;

    let domain;

    switch (field.type) {
      case ALL_FIELD_TYPES.real:
      case ALL_FIELD_TYPES.integer:
        // calculate domain and step
        return getNumericFieldDomain(dataContainer, valueAccessor);

      case ALL_FIELD_TYPES.boolean:
        return {domain: [true, false]};

      case ALL_FIELD_TYPES.string:
      case ALL_FIELD_TYPES.h3:
      case ALL_FIELD_TYPES.date:
        domain = getOrdinalDomain(dataContainer, valueAccessor);
        return {domain};

      case ALL_FIELD_TYPES.timestamp:
        return getTimestampFieldDomain(dataContainer, valueAccessor);

      default:
        return {domain: getOrdinalDomain(dataContainer, valueAccessor)};
    }
  }

  /**
   *  Get the domain of this column based on scale type
   */
  getColumnLayerDomain(field: F, scaleType: string): number[] | string[] | [number, number] | null {
    const {dataContainer, filteredIndexForDomain} = this;

    if (!SCALE_TYPES[scaleType]) {
      Console.error(`scale type ${scaleType} not supported`);
      return null;
    }

    const {valueAccessor} = field;
    const indexValueAccessor = i => valueAccessor({index: i});
    const sortFunction = getSortingFunction(field.type);

    switch (scaleType) {
      case SCALE_TYPES.ordinal:
      case SCALE_TYPES.customOrdinal:
      case SCALE_TYPES.point:
        // do not recalculate ordinal domain based on filtered data
        // don't need to update ordinal domain every time
        return getOrdinalDomain(dataContainer, valueAccessor);

      case SCALE_TYPES.quantile:
        return getQuantileDomain(filteredIndexForDomain, indexValueAccessor, sortFunction);

      case SCALE_TYPES.log:
        return getLogDomain(filteredIndexForDomain, indexValueAccessor);

      case SCALE_TYPES.quantize:
      case SCALE_TYPES.linear:
      case SCALE_TYPES.sqrt:
      case SCALE_TYPES.custom:
      default:
        return getLinearDomain(filteredIndexForDomain, indexValueAccessor);
    }
  }

  /**
   * Get a sample of rows to calculate layer boundaries
   */
  // getSampleData(rows)

  /**
   * Parse cell value based on column type and return a string representation
   * Value the field value, type the field type
   */
  // parseFieldValue(value, type)

  // sortDatasetByColumn()

  /**
   * Assert whether field exist
   * @param fieldName
   * @param condition
   */
  _assetField(fieldName: string, condition: any): void {
    if (!condition) {
      Console.error(`${fieldName} doesnt exist in dataset ${this.id}`);
    }
  }
}

export type Datasets = {
  [key: string]: KeplerTable<Field>;
};

// HELPER FUNCTIONS (MAINLY EXPORTED FOR TEST...)
// have to double excape
const specialCharacterSet = `[#_&@\\.\\-\\ ]`;

function foundMatchingFields(re, suffixPair, allNames, fieldName) {
  const partnerIdx = allNames.findIndex(
    d => d === fieldName.replace(re, match => match.replace(suffixPair[0], suffixPair[1]))
  );
  let altIdx = -1;
  if (partnerIdx > -1) {
    // if found partner, go on and look for altitude
    ALTITUDE_FIELDS.some(alt => {
      altIdx = allNames.findIndex(
        d => d === fieldName.replace(re, match => match.replace(suffixPair[0], alt))
      );
      return altIdx > -1;
    });
  }
  return {partnerIdx, altIdx};
}
/**
 * Find point fields pairs from fields
 *
 * @param fields
 * @returns found point fields
 */
export function findPointFieldPairs(fields: Field[]): FieldPair[] {
  const allNames = fields.map(f => f.name.toLowerCase());

  // get list of all fields with matching suffixes
  const acc: FieldPair[] = [];
  return allNames.reduce((carry, fieldName, idx) => {
    // This search for pairs will early exit if found.
    for (const suffixPair of TRIP_POINT_FIELDS) {
      // match first suffix
      // (^|[#_&@\.\-\ ])lat([#_&@\.\-\ ]|$)
      const re = new RegExp(`(^|${specialCharacterSet})${suffixPair[0]}(${specialCharacterSet}|$)`);

      if (re.test(fieldName)) {
        const {partnerIdx, altIdx} = foundMatchingFields(re, suffixPair, allNames, fieldName);

        if (partnerIdx > -1) {
          const trimName = fieldName.replace(re, '').trim();

          carry.push({
            defaultName: trimName || 'point',
            pair: {
              lat: {
                fieldIdx: idx,
                value: fields[idx].name
              },
              lng: {
                fieldIdx: partnerIdx,
                value: fields[partnerIdx].name
              },
              ...(altIdx > -1
                ? {
                    altitude: {
                      fieldIdx: altIdx,
                      value: fields[altIdx].name
                    }
                  }
                : {})
            },
            suffix: suffixPair
          });
          return carry;
        }
      }
    }
    return carry;
  }, acc);
}

/**
 *
 * @param dataset
 * @param column
 * @param mode
 * @type
 */
export function sortDatasetByColumn(
  dataset: KeplerTable<Field>,
  column: string,
  mode?: string
): KeplerTable<Field> {
  const {allIndexes, fields, dataContainer} = dataset;
  const fieldIndex = fields.findIndex(f => f.name === column);
  if (fieldIndex < 0) {
    return dataset;
  }

  const sortBy = SORT_ORDER[mode || ''] || SORT_ORDER.ASCENDING;

  if (sortBy === SORT_ORDER.UNSORT) {
    dataset.sortColumn = {};
    dataset.sortOrder = null;

    return dataset;
  }

  const sortFunction = sortBy === SORT_ORDER.ASCENDING ? ascending : descending;
  const sortOrder = allIndexes.slice().sort((a, b) => {
    const value1 = dataContainer.valueAt(a, fieldIndex);
    const value2 = dataContainer.valueAt(b, fieldIndex);
    if (!notNullorUndefined(value1) && notNullorUndefined(value2)) {
      return 1;
    } else if (notNullorUndefined(value1) && !notNullorUndefined(value2)) {
      return -1;
    }
    return sortFunction(value1, value2);
  });

  dataset.sortColumn = {
    [column]: sortBy
  };
  dataset.sortOrder = sortOrder;

  return dataset;
}

export function pinTableColumns(dataset: KeplerTable<Field>, column: string): KeplerTable<Field> {
  const field = dataset.getColumnField(column);
  if (!field) {
    return dataset;
  }

  let pinnedColumns;
  if (Array.isArray(dataset.pinnedColumns) && dataset.pinnedColumns.includes(field.name)) {
    // unpin it
    pinnedColumns = dataset.pinnedColumns.filter(co => co !== field.name);
  } else {
    pinnedColumns = (dataset.pinnedColumns || []).concat(field.name);
  }

  // @ts-ignore
  return copyTableAndUpdate(dataset, {pinnedColumns});
}

export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {
  return Object.assign(Object.create(Object.getPrototypeOf(original)), original);
}

/**
 * @type
 * @returns
 */
export function copyTableAndUpdate(
  original: KeplerTable<Field>,
  options: Partial<KeplerTable<Field>> = {}
): KeplerTable<Field> {
  return Object.entries(options).reduce((acc, entry) => {
    acc[entry[0]] = entry[1];
    return acc;
  }, copyTable(original));
}

export function getFieldValueAccessor<
  F extends {
    type?: Field['type'];
    format?: Field['format'];
  }
>(f: F, i: number, dc: DataContainerInterface) {
  return maybeToDate.bind(
    null,
    // is time
    f.type === ALL_FIELD_TYPES.timestamp,
    i,
    f.format || '',
    dc
  );
}

export default KeplerTable;

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import Console from 'global/console';
5	import {ascending, descending} from 'd3-array';
6
7	import {
8	TRIP_POINT_FIELDS,
9	SORT_ORDER,
10	ALL_FIELD_TYPES,
11	ALTITUDE_FIELDS,
12	SCALE_TYPES
13	} from '@kepler.gl/constants';
14	import {
15	RGBColor,
16	Field,
17	FieldPair,
18	FieldDomain,
19	Filter,
20	ProtoDataset,
21	FilterRecord,
22	FilterDatasetOpt,
23	RangeFieldDomain,
24	SelectFieldDomain,
25	MultiSelectFieldDomain,
26	TimeRangeFieldDomain
27	} from '@kepler.gl/types';
28
29	import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils';
30
31	import {
32	getSortingFunction,
33	timeToUnixMilli,
34	createDataContainer,
35	DataForm,
36	diffFilters,
37	filterDataByFilterTypes,
38	FilterResult,
39	getFilterFunction,
40	getFilterProps,
41	getFilterRecord,
42	getNumericFieldDomain,
43	getTimestampFieldDomain,
44	getLinearDomain,
45	getLogDomain,
46	getOrdinalDomain,
47	getQuantileDomain,
48	DataContainerInterface,
49	FilterChanged
50	} from '@kepler.gl/utils';
51	import {generateHashId, notNullorUndefined} from '@kepler.gl/common-utils';
52
53	// TODO isolate layer type, depends on @kepler.gl/layers
54	type Layer = any;
55
56	export type GpuFilter = {
57	filterRange: number[][];
58	filterValueUpdateTriggers: {
59	[id: string]: {name: string; domain0: number} \| null;
60	};
61	filterValueAccessor: (
62	dc: DataContainerInterface
63	) => (
64	getIndex?: (any) => number,
65	getData?: (dc_: DataContainerInterface, d: any, fieldIndex: number) => any
66	) => (d: any, objectInfo?: {index: number}) => (number \| number[])[];
67	};
68
69	export type FilterProps =
70	\| NumericFieldFilterProps
71	\| BooleanFieldFilterProps
72	\| StringFieldFilterProps
73	\| TimeFieldFilterProps;
74
75	export type NumericFieldFilterProps = RangeFieldDomain & {
76	value: [number, number];
77	type: string;
78	typeOptions: string[];
79	gpu: boolean;
80	columnStats?: Record<string, any>;
81	};
82	export type BooleanFieldFilterProps = SelectFieldDomain & {
83	type: string;
84	value: boolean;
85	gpu: boolean;
86	columnStats?: Record<string, any>;
87	};
88	export type StringFieldFilterProps = MultiSelectFieldDomain & {
89	type: string;
90	value: string[];
91	gpu: boolean;
92	columnStats?: Record<string, any>;
93	};
94	export type TimeFieldFilterProps = TimeRangeFieldDomain & {
95	type: string;
96	view: Filter['view'];
97	fixedDomain: boolean;
98	value: number[];
99	gpu: boolean;
100	columnStats?: Record<string, any>;
101	};
102
103	// Unique identifier of each field
104	const FID_KEY = 'name';	13✔
105
106	export function maybeToDate(
107	isTime: boolean,
108	fieldIdx: number,
109	format: string,
110	dc: DataContainerInterface,
111	d: {index: number}
112	) {
113	if (isTime) {	5,244✔
114	return timeToUnixMilli(dc.valueAt(d.index, fieldIdx), format);	768✔
115	}
116
117	return dc.valueAt(d.index, fieldIdx);	4,476✔
118	}
119
120	class KeplerTable<F extends Field = Field> {
121	readonly id: string;
122
123	type?: string;
124	label: string;
125	color: RGBColor;
126
127	// fields and data
128	fields: F[] = [];	149✔
129
130	dataContainer: DataContainerInterface;
131
132	allIndexes: number[] = [];	149✔
133	filteredIndex: number[] = [];	149✔
134	filteredIdxCPU?: number[];
135	filteredIndexForDomain: number[] = [];	149✔
136	fieldPairs: FieldPair[] = [];	149✔
137	gpuFilter: GpuFilter;
138	filterRecord?: FilterRecord;
139	filterRecordCPU?: FilterRecord;
140	changedFilters?: FilterChanged;
141
142	// table-injected metadata
143	sortColumn?: {
144	// column name: sorted idx
145	[key: string]: string; // ASCENDING \| DESCENDING \| UNSORT
146	};
147	sortOrder?: number[] \| null;
148
149	pinnedColumns?: string[];
150	supportedFilterTypes?: string[] \| null;
151	disableDataOperation?: boolean;
152
153	// table-injected metadata
154	metadata: object;
155
156	constructor({
157	info,
158	color,
159	metadata,
160	supportedFilterTypes = null,	148✔
161	disableDataOperation = false	149✔
162	}: {
163	info?: ProtoDataset['info'];
164	color: RGBColor;
165	metadata?: ProtoDataset['metadata'];
166	supportedFilterTypes?: ProtoDataset['supportedFilterTypes'];
167	disableDataOperation?: ProtoDataset['disableDataOperation'];
168	}) {
169	// TODO - what to do if validation fails? Can kepler handle exceptions?
170	// const validatedData = validateInputData(data);
171	// if (!validatedData) {
172	// return this;
173	// }
174
175	const datasetInfo = {	149✔
176	id: generateHashId(4),
177	label: 'new dataset',
178	type: '',
179	...info
180	};
181
182	const defaultMetadata = {	149✔
183	id: datasetInfo.id,
184	// @ts-ignore
185	format: datasetInfo.format \|\| '',	298✔
186	label: datasetInfo.label \|\| ''	149!
187	};
188
189	this.id = datasetInfo.id;	149✔
190	this.type = datasetInfo.type;	149✔
191	this.label = datasetInfo.label;	149✔
192	this.color = color;	149✔
193	this.metadata = {	149✔
194	...defaultMetadata,
195	...metadata
196	};
197
198	this.supportedFilterTypes = supportedFilterTypes;	149✔
199	this.disableDataOperation = disableDataOperation;	149✔
200
201	this.dataContainer = createDataContainer([]);	149✔
202	this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);	149✔
203	}
204
205	importData({data}: {data: ProtoDataset['data']}) {
206	const dataContainerData = data.cols ? data.cols : data.rows;	149!
207	const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;	149!
208
209	const dataContainer = createDataContainer(dataContainerData, {	149✔
210	fields: data.fields,
211	inputDataFormat
212	});
213
214	const fields: Field[] = data.fields.map((f, i) => ({	1,225✔
215	...f,
216	fieldIdx: i,
217	id: f.name,
218	displayName: f.displayName \|\| f.name,	1,652✔
219	analyzerType: f.analyzerType \|\| ALL_FIELD_TYPES.string,	1,238✔
220	format: f.format \|\| '',	2,165✔
221	valueAccessor: getFieldValueAccessor(f, i, dataContainer)
222	}));
223
224	const allIndexes = dataContainer.getPlainIndex();	149✔
225	this.dataContainer = dataContainer;	149✔
226	this.allIndexes = allIndexes;	149✔
227	this.filteredIndex = allIndexes;	149✔
228	this.filteredIndexForDomain = allIndexes;	149✔
229	this.fieldPairs = findPointFieldPairs(fields);	149✔
230	// @ts-expect-error Make sure that fields satisfies F extends Field
231	this.fields = fields;	149✔
232	this.gpuFilter = getGpuFilterProps([], this.id, fields, undefined);	149✔
233	}
234
235	/**
236	* update table with new data
237	* @param data - new data e.g. the arrow data with new batches loaded
238	*/
239	update(data: ProtoDataset['data']) {
240	const dataContainerData = data.cols ? data.cols : data.rows;	×
241	this.dataContainer.update?.(dataContainerData);	×
242	this.allIndexes = this.dataContainer.getPlainIndex();	×
243	this.filteredIndex = this.allIndexes;	×
244	this.filteredIndexForDomain = this.allIndexes;	×
245
246	return this;	×
247	}
248
249	get length() {
250	return this.dataContainer.numRows();	390✔
251	}
252
253	/**
254	* Get field
255	* @param columnName
256	*/
257	getColumnField(columnName: string): F \| undefined {
258	const field = this.fields.find(fd => fd[FID_KEY] === columnName);	3,127✔
259	this._assetField(columnName, field);	514✔
260	return field;	514✔
261	}
262
263	/**
264	* Get fieldIdx
265	* @param columnName
266	*/
267	getColumnFieldIdx(columnName: string): number {
268	const fieldIdx = this.fields.findIndex(fd => fd[FID_KEY] === columnName);	914✔
269	this._assetField(columnName, Boolean(fieldIdx > -1));	215✔
270	return fieldIdx;	215✔
271	}
272
273	/**
274	* Get displayFormat
275	* @param columnName
276	*/
277	getColumnDisplayFormat(columnName) {
278	const field = this.fields.find(fd => fd[FID_KEY] === columnName);	×
279	this._assetField(columnName, field);	×
280	return field?.displayFormat;	×
281	}
282
283	/**
284	* Get the value of a cell
285	*/
286	getValue(columnName: string, rowIdx: number): any {
287	const field = this.getColumnField(columnName);	419✔
288	return field ? field.valueAccessor({index: rowIdx}) : null;	419!
289	}
290
291	/**
292	* Updates existing field with a new object
293	* @param fieldIdx
294	* @param newField
295	*/
296	updateColumnField(fieldIdx: number, newField: F): void {
297	this.fields = Object.assign([...this.fields], {[fieldIdx]: newField});	76✔
298	}
299
300	/**
301	* Update dataset color by custom color
302	* @param newColor
303	*/
304	updateTableColor(newColor: RGBColor): void {
305	this.color = newColor;	×
306	}
307
308	/**
309	* Save filterProps to field and retrieve it
310	* @param columnName
311	*/
312	getColumnFilterProps(columnName: string): F['filterProps'] \| null \| undefined {
313	const fieldIdx = this.getColumnFieldIdx(columnName);	135✔
314	if (fieldIdx < 0) {	135✔
315	return null;	1✔
316	}
317	const field = this.fields[fieldIdx];	134✔
318	if (Object.prototype.hasOwnProperty.call(field, 'filterProps')) {	134✔
319	return field.filterProps;	58✔
320	}
321
322	const fieldDomain = this.getColumnFilterDomain(field);	76✔
323	if (!fieldDomain) {	76!
324	return null;	×
325	}
326
327	const filterProps = getFilterProps(field, fieldDomain);	76✔
328	const newField = {	76✔
329	...field,
330	filterProps
331	};
332
333	this.updateColumnField(fieldIdx, newField);	76✔
334
335	return filterProps;	76✔
336	}
337
338	/**
339	* Apply filters to dataset, return the filtered dataset with updated `gpuFilter`, `filterRecord`, `filteredIndex`, `filteredIndexForDomain`
340	* @param filters
341	* @param layers
342	* @param opt
343	*/
344	filterTable(filters: Filter[], layers: Layer[], opt?: FilterDatasetOpt): KeplerTable<Field> {
345	const {dataContainer, id: dataId, filterRecord: oldFilterRecord, fields} = this;	117✔
346
347	// if there is no filters
348	const filterRecord = getFilterRecord(dataId, filters, opt \|\| {});	117!
349
350	this.filterRecord = filterRecord;	117✔
351	this.gpuFilter = getGpuFilterProps(filters, dataId, fields, this.gpuFilter);	117✔
352
353	this.changedFilters = diffFilters(filterRecord, oldFilterRecord);	117✔
354
355	if (!filters.length) {	117✔
356	this.filteredIndex = this.allIndexes;	4✔
357	this.filteredIndexForDomain = this.allIndexes;	4✔
358	return this;	4✔
359	}
360
361	// generate 2 sets of filter result
362	// filteredIndex used to calculate layer data
363	// filteredIndexForDomain used to calculate layer Domain
364	const shouldCalDomain = Boolean(this.changedFilters.dynamicDomain);	113✔
365	const shouldCalIndex = Boolean(this.changedFilters.cpu);	113✔
366
367	let filterResult: FilterResult = {};	113✔
368	if (shouldCalDomain \|\| shouldCalIndex) {	113✔
369	const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;	55✔
370	const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;	55✔
371
372	const filterFuncs = filters.reduce((acc, filter) => {	55✔
373	const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);	76✔
374	const field = fieldIndex !== -1 ? fields[fieldIndex] : null;	76✔
375
376	return {	76✔
377	...acc,
378	[filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
379	};
380	}, {});
381
382	filterResult = filterDataByFilterTypes(	55✔
383	{dynamicDomainFilters, cpuFilters, filterFuncs},
384	dataContainer
385	);
386	}
387
388	this.filteredIndex = filterResult.filteredIndex \|\| this.filteredIndex;	113✔
389	this.filteredIndexForDomain =	113✔
390	filterResult.filteredIndexForDomain \|\| this.filteredIndexForDomain;	181✔
391
392	return this;	113✔
393	}
394
395	/**
396	* Apply filters to a dataset all on CPU, assign to `filteredIdxCPU`, `filterRecordCPU`
397	* @param filters
398	* @param layers
399	*/
400	filterTableCPU(filters: Filter[], layers: Layer[]): KeplerTable<Field> {
401	const opt = {	7✔
402	cpuOnly: true,
403	ignoreDomain: true
404	};
405
406	// no filter
407	if (!filters.length) {	7✔
408	this.filteredIdxCPU = this.allIndexes;	1✔
409	this.filterRecordCPU = getFilterRecord(this.id, filters, opt);	1✔
410	return this;	1✔
411	}
412
413	// no gpu filter
414	if (!filters.find(f => f.gpu)) {	6✔
415	this.filteredIdxCPU = this.filteredIndex;	3✔
416	this.filterRecordCPU = getFilterRecord(this.id, filters, opt);	3✔
417	return this;	3✔
418	}
419
420	// make a copy for cpu filtering
421	const copied = copyTable(this);	3✔
422
423	copied.filterRecord = this.filterRecordCPU;	3✔
424	copied.filteredIndex = this.filteredIdxCPU \|\| [];	3✔
425
426	const filtered = copied.filterTable(filters, layers, opt);	3✔
427
428	this.filteredIdxCPU = filtered.filteredIndex;	3✔
429	this.filterRecordCPU = filtered.filterRecord;	3✔
430
431	return this;	3✔
432	}
433
434	/**
435	* Calculate field domain based on field type and data
436	* for Filter
437	*/
438	getColumnFilterDomain(field: F): FieldDomain {
439	const {dataContainer} = this;	86✔
440	const {valueAccessor} = field;	86✔
441
442	let domain;
443
444	switch (field.type) {	86!
445	case ALL_FIELD_TYPES.real:
446	case ALL_FIELD_TYPES.integer:
447	// calculate domain and step
448	return getNumericFieldDomain(dataContainer, valueAccessor);	28✔
449
450	case ALL_FIELD_TYPES.boolean:
451	return {domain: [true, false]};	2✔
452
453	case ALL_FIELD_TYPES.string:
454	case ALL_FIELD_TYPES.h3:
455	case ALL_FIELD_TYPES.date:
456	domain = getOrdinalDomain(dataContainer, valueAccessor);	15✔
457	return {domain};	15✔
458
459	case ALL_FIELD_TYPES.timestamp:
460	return getTimestampFieldDomain(dataContainer, valueAccessor);	41✔
461
462	default:
463	return {domain: getOrdinalDomain(dataContainer, valueAccessor)};	×
464	}
465	}
466
467	/**
468	* Get the domain of this column based on scale type
469	*/
470	getColumnLayerDomain(field: F, scaleType: string): number[] \| string[] \| [number, number] \| null {
471	const {dataContainer, filteredIndexForDomain} = this;	131✔
472
473	if (!SCALE_TYPES[scaleType]) {	131!
474	Console.error(`scale type ${scaleType} not supported`);	×
475	return null;	×
476	}
477
478	const {valueAccessor} = field;	131✔
479	const indexValueAccessor = i => valueAccessor({index: i});	1,294✔
480	const sortFunction = getSortingFunction(field.type);	131✔
481
482	switch (scaleType) {	131!
483	case SCALE_TYPES.ordinal:
484	case SCALE_TYPES.customOrdinal:
485	case SCALE_TYPES.point:
486	// do not recalculate ordinal domain based on filtered data
487	// don't need to update ordinal domain every time
488	return getOrdinalDomain(dataContainer, valueAccessor);	34✔
489
490	case SCALE_TYPES.quantile:
491	return getQuantileDomain(filteredIndexForDomain, indexValueAccessor, sortFunction);	47✔
492
493	case SCALE_TYPES.log:
UNCOV 494	return getLogDomain(filteredIndexForDomain, indexValueAccessor);	×
495
496	case SCALE_TYPES.quantize:
497	case SCALE_TYPES.linear:
498	case SCALE_TYPES.sqrt:
499	case SCALE_TYPES.custom:
500	default:
501	return getLinearDomain(filteredIndexForDomain, indexValueAccessor);	50✔
502	}
503	}
504
505	/**
506	* Get a sample of rows to calculate layer boundaries
507	*/
508	// getSampleData(rows)
509
510	/**
511	* Parse cell value based on column type and return a string representation
512	* Value the field value, type the field type
513	*/
514	// parseFieldValue(value, type)
515
516	// sortDatasetByColumn()
517
518	/**
519	* Assert whether field exist
520	* @param fieldName
521	* @param condition
522	*/
523	_assetField(fieldName: string, condition: any): void {
524	if (!condition) {	729✔
525	Console.error(`${fieldName} doesnt exist in dataset ${this.id}`);	2✔
526	}
527	}
528	}
529
530	export type Datasets = {
531	[key: string]: KeplerTable<Field>;
532	};
533
534	// HELPER FUNCTIONS (MAINLY EXPORTED FOR TEST...)
535	// have to double excape
536	const specialCharacterSet = `[#_&@\\.\\-\\ ]`;	13✔
537
538	function foundMatchingFields(re, suffixPair, allNames, fieldName) {
539	const partnerIdx = allNames.findIndex(	142✔
540	d => d === fieldName.replace(re, match => match.replace(suffixPair[0], suffixPair[1]))	559✔
541	);
542	let altIdx = -1;	142✔
543	if (partnerIdx > -1) {	142✔
544	// if found partner, go on and look for altitude
545	ALTITUDE_FIELDS.some(alt => {	136✔
546	altIdx = allNames.findIndex(	268✔
547	d => d === fieldName.replace(re, match => match.replace(suffixPair[0], alt))	2,430✔
548	);
549	return altIdx > -1;	268✔
550	});
551	}
552	return {partnerIdx, altIdx};	142✔
553	}
554	/**
555	* Find point fields pairs from fields
556	*
557	* @param fields
558	* @returns found point fields
559	*/
560	export function findPointFieldPairs(fields: Field[]): FieldPair[] {
561	const allNames = fields.map(f => f.name.toLowerCase());	1,282✔
562
563	// get list of all fields with matching suffixes
564	const acc: FieldPair[] = [];	160✔
565	return allNames.reduce((carry, fieldName, idx) => {	160✔
566	// This search for pairs will early exit if found.
567	for (const suffixPair of TRIP_POINT_FIELDS) {	1,282✔
568	// match first suffix
569	// (^\|[#_&@\.\-\ ])lat([#_&@\.\-\ ]\|$)
570	const re = new RegExp(`(^\|${specialCharacterSet})${suffixPair[0]}(${specialCharacterSet}\|$)`);	4,777✔
571
572	if (re.test(fieldName)) {	4,777✔
573	const {partnerIdx, altIdx} = foundMatchingFields(re, suffixPair, allNames, fieldName);	142✔
574
575	if (partnerIdx > -1) {	142✔
576	const trimName = fieldName.replace(re, '').trim();	136✔
577
578	carry.push({	136✔
579	defaultName: trimName \|\| 'point',	152✔
580	pair: {
581	lat: {
582	fieldIdx: idx,
583	value: fields[idx].name
584	},
585	lng: {
586	fieldIdx: partnerIdx,
587	value: fields[partnerIdx].name
588	},
589	...(altIdx > -1	136✔
590	? {
591	altitude: {
592	fieldIdx: altIdx,
593	value: fields[altIdx].name
594	}
595	}
596	: {})
597	},
598	suffix: suffixPair
599	});
600	return carry;	136✔
601	}
602	}
603	}
604	return carry;	1,146✔
605	}, acc);
606	}
607
608	/**
609	*
610	* @param dataset
611	* @param column
612	* @param mode
613	* @type
614	*/
615	export function sortDatasetByColumn(
616	dataset: KeplerTable<Field>,
617	column: string,
618	mode?: string
619	): KeplerTable<Field> {
620	const {allIndexes, fields, dataContainer} = dataset;	5✔
621	const fieldIndex = fields.findIndex(f => f.name === column);	10✔
622	if (fieldIndex < 0) {	5!
UNCOV 623	return dataset;	×
624	}
625
626	const sortBy = SORT_ORDER[mode \|\| ''] \|\| SORT_ORDER.ASCENDING;	5!
627
628	if (sortBy === SORT_ORDER.UNSORT) {	5✔
629	dataset.sortColumn = {};	1✔
630	dataset.sortOrder = null;	1✔
631
632	return dataset;	1✔
633	}
634
635	const sortFunction = sortBy === SORT_ORDER.ASCENDING ? ascending : descending;	4✔
636	const sortOrder = allIndexes.slice().sort((a, b) => {	4✔
637	const value1 = dataContainer.valueAt(a, fieldIndex);	322✔
638	const value2 = dataContainer.valueAt(b, fieldIndex);	322✔
639	if (!notNullorUndefined(value1) && notNullorUndefined(value2)) {	322!
640	return 1;	×
641	} else if (notNullorUndefined(value1) && !notNullorUndefined(value2)) {	322!
UNCOV 642	return -1;	×
643	}
644	return sortFunction(value1, value2);	322✔
645	});
646
647	dataset.sortColumn = {	4✔
648	[column]: sortBy
649	};
650	dataset.sortOrder = sortOrder;	4✔
651
652	return dataset;	4✔
653	}
654
655	export function pinTableColumns(dataset: KeplerTable<Field>, column: string): KeplerTable<Field> {
656	const field = dataset.getColumnField(column);	3✔
657	if (!field) {	3!
UNCOV 658	return dataset;	×
659	}
660
661	let pinnedColumns;
662	if (Array.isArray(dataset.pinnedColumns) && dataset.pinnedColumns.includes(field.name)) {	3✔
663	// unpin it
664	pinnedColumns = dataset.pinnedColumns.filter(co => co !== field.name);	1✔
665	} else {
666	pinnedColumns = (dataset.pinnedColumns \|\| []).concat(field.name);	2✔
667	}
668
669	// @ts-ignore
670	return copyTableAndUpdate(dataset, {pinnedColumns});	3✔
671	}
672
673	export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {
674	return Object.assign(Object.create(Object.getPrototypeOf(original)), original);	44✔
675	}
676
677	/**
678	* @type
679	* @returns
680	*/
681	export function copyTableAndUpdate(
682	original: KeplerTable<Field>,
683	options: Partial<KeplerTable<Field>> = {}	×
684	): KeplerTable<Field> {
685	return Object.entries(options).reduce((acc, entry) => {	40✔
686	acc[entry[0]] = entry[1];	40✔
687	return acc;	40✔
688	}, copyTable(original));
689	}
690
691	export function getFieldValueAccessor<
692	F extends {
693	type?: Field['type'];
694	format?: Field['format'];
695	}
696	>(f: F, i: number, dc: DataContainerInterface) {
697	return maybeToDate.bind(	1,225✔
698	null,
699	// is time
700	f.type === ALL_FIELD_TYPES.timestamp,
701	i,
702	f.format \|\| '',	2,165✔
703	dc
704	);
705	}
706
707	export default KeplerTable;

keplergl / kepler.gl / 12604159536

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous