22361650031

Committed 24 Feb 2026 05:09PM UTC coverage: 61.612% (-0.2%) from 61.806%

Build # 22361650031

Build Type

Pull #3219

github

Committed by

web-flow

Commit Message

Merge 1d9b34cb5 into cc33b0c8f

Pull Request Pull Request #3219: Update kepler-jupyter to use kepler.gl v3.2.0

Coverage Stats

6382 of 12288 branches covered (51.94%)

Branch coverage included in aggregate %.

13078 of 19297 relevant lines covered (67.77%)

81.44 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.33

/src/table/src/kepler-table.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import Console from 'global/console';
import {ascending, descending} from 'd3-array';

import {
  TRIP_POINT_FIELDS,
  SORT_ORDER,
  ALL_FIELD_TYPES,
  ALTITUDE_FIELDS,
  SCALE_TYPES
} from '@kepler.gl/constants';
import {
  RGBColor,
  Field,
  FieldPair,
  FieldDomain,
  Filter,
  ProtoDataset,
  FilterRecord,
  FilterDatasetOpt,
  RangeFieldDomain,
  SelectFieldDomain,
  MultiSelectFieldDomain,
  TimeRangeFieldDomain
} from '@kepler.gl/types';

import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils';

import {
  getSortingFunction,
  timeToUnixMilli,
  createDataContainer,
  DataForm,
  diffFilters,
  filterDataByFilterTypes,
  FilterResult,
  getFilterFunction,
  getFilterProps,
  getFilterRecord,
  getNumericFieldDomain,
  getTimestampFieldDomain,
  getLinearDomain,
  getLogDomain,
  getOrdinalDomain,
  getQuantileDomain,
  DataContainerInterface,
  FilterChanged
} from '@kepler.gl/utils';
import {generateHashId, notNullorUndefined} from '@kepler.gl/common-utils';

// TODO isolate layer type, depends on @kepler.gl/layers
type Layer = any;

export type GpuFilter = {
  filterRange: number[][];
  filterValueUpdateTriggers: {
    [id: string]: {name: string; domain0: number} | null;
  };
  filterValueAccessor: (
    dc: DataContainerInterface
  ) => (
    getIndex?: (any) => number,
    getData?: (dc_: DataContainerInterface, d: any, fieldIndex: number) => any
  ) => (d: any, objectInfo?: {index: number}) => (number | number[])[];
};

export type FilterProps =
  | NumericFieldFilterProps
  | BooleanFieldFilterProps
  | StringFieldFilterProps
  | TimeFieldFilterProps;

export type NumericFieldFilterProps = RangeFieldDomain & {
  value: [number, number];
  type: string;
  typeOptions: string[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type BooleanFieldFilterProps = SelectFieldDomain & {
  type: string;
  value: boolean;
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type StringFieldFilterProps = MultiSelectFieldDomain & {
  type: string;
  value: string[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};
export type TimeFieldFilterProps = TimeRangeFieldDomain & {
  type: string;
  view: Filter['view'];
  fixedDomain: boolean;
  value: number[];
  gpu: boolean;
  columnStats?: Record<string, any>;
};

// Unique identifier of each field
const FID_KEY = 'name';

export function maybeToDate(
  isTime: boolean,
  fieldIdx: number,
  format: string,
  dc: DataContainerInterface,
  // An object with row index or a materialized row array (for materialized hover info from trip layer)
  d: {index: number} | any[]
) {
  if (isTime) {
    return timeToUnixMilli(Array.isArray(d) ? d[fieldIdx] : dc.valueAt(d.index, fieldIdx), format);
  }

  return Array.isArray(d) ? d[fieldIdx] : dc.valueAt(d.index, fieldIdx);
}

class KeplerTable<F extends Field = Field> {
  readonly id: string;

  type?: string;
  label: string;
  color: RGBColor;

  // fields and data
  fields: F[] = [];

  dataContainer: DataContainerInterface;

  allIndexes: number[] = [];
  filteredIndex: number[] = [];
  filteredIdxCPU?: number[];
  filteredIndexForDomain: number[] = [];
  fieldPairs: FieldPair[] = [];
  gpuFilter: GpuFilter;
  filterRecord?: FilterRecord;
  filterRecordCPU?: FilterRecord;
  changedFilters?: FilterChanged;

  // table-injected metadata
  sortColumn?: {
    // column name: sorted idx
    [key: string]: string; // ASCENDING | DESCENDING | UNSORT
  };
  sortOrder?: number[] | null;

  pinnedColumns?: string[];
  supportedFilterTypes?: string[] | null;
  disableDataOperation?: boolean;

  // table-injected metadata
  metadata: Record<string, any>;

  getFileProcessor?: (data: any, inputFormat?: string) => {data: any; format: string};

  constructor({
    info,
    color,
    metadata,
    supportedFilterTypes = null,
    disableDataOperation = false
  }: {
    info?: ProtoDataset['info'];
    color: RGBColor;
    metadata?: ProtoDataset['metadata'];
    supportedFilterTypes?: ProtoDataset['supportedFilterTypes'];
    disableDataOperation?: ProtoDataset['disableDataOperation'];
  }) {
    // TODO - what to do if validation fails? Can kepler handle exceptions?
    // const validatedData = validateInputData(data);
    // if (!validatedData) {
    //   return this;
    // }

    const datasetInfo = {
      id: generateHashId(4),
      label: 'new dataset',
      type: '',
      ...info
    };

    const defaultMetadata = {
      id: datasetInfo.id,
      // @ts-ignore
      format: datasetInfo.format || '',
      label: datasetInfo.label || ''
    };

    this.id = datasetInfo.id;
    this.type = datasetInfo.type;
    this.label = datasetInfo.label;
    this.color = color;
    this.metadata = {
      ...defaultMetadata,
      ...metadata
    };

    this.supportedFilterTypes = supportedFilterTypes;
    this.disableDataOperation = disableDataOperation;

    this.dataContainer = createDataContainer([]);
    this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);
  }

  async importData({data}: {data: ProtoDataset['data']}) {
    const dataContainerData = data.cols ? data.cols : data.rows;
    const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;

    const dataContainer = createDataContainer(dataContainerData, {
      fields: data.fields,
      arrowTable: data.arrowTable,
      inputDataFormat
    });

    const fields: Field[] = data.fields.map((f, i) => ({
      ...f,
      fieldIdx: i,
      id: f.name,
      displayName: f.displayName || f.name,
      analyzerType: f.analyzerType || ALL_FIELD_TYPES.string,
      format: f.format || '',
      valueAccessor: getFieldValueAccessor(f, i, dataContainer)
    }));

    const allIndexes = dataContainer.getPlainIndex();
    this.dataContainer = dataContainer;
    this.allIndexes = allIndexes;
    this.filteredIndex = allIndexes;
    this.filteredIndexForDomain = allIndexes;
    this.fieldPairs = findPointFieldPairs(fields);
    // @ts-expect-error Make sure that fields satisfies F extends Field
    this.fields = fields;
    this.gpuFilter = getGpuFilterProps([], this.id, fields, undefined);
  }

  /**
   * update table with new data
   * @param data - new data e.g. the arrow data with new batches loaded
   */
  async update(data: ProtoDataset['data']) {
    const dataContainerData = data.arrowTable ?? data.cols ?? data.rows;
    this.dataContainer.update?.(dataContainerData);
    this.allIndexes = this.dataContainer.getPlainIndex();
    this.filteredIndex = this.allIndexes;
    this.filteredIndexForDomain = this.allIndexes;

    return this;
  }

  get length() {
    return this.dataContainer.numRows();
  }

  /**
   * Get field
   * @param columnName
   */
  getColumnField(columnName: string): F | undefined {
    const field = this.fields.find(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, field);
    return field;
  }

  /**
   * Get fieldIdx
   * @param columnName
   */
  getColumnFieldIdx(columnName: string): number {
    const fieldIdx = this.fields.findIndex(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, Boolean(fieldIdx > -1));
    return fieldIdx;
  }

  /**
   * Get displayFormat
   * @param columnName
   */
  getColumnDisplayFormat(columnName) {
    const field = this.fields.find(fd => fd[FID_KEY] === columnName);
    this._assetField(columnName, field);
    return field?.displayFormat;
  }

  /**
   * Get the value of a cell
   */
  getValue(columnName: string, rowIdx: number): any {
    const field = this.getColumnField(columnName);
    return field ? field.valueAccessor({index: rowIdx}) : null;
  }

  /**
   * Updates existing field with a new object
   * @param fieldIdx
   * @param newField
   */
  updateColumnField(fieldIdx: number, newField: F): void {
    this.fields = Object.assign([...this.fields], {[fieldIdx]: newField});
  }

  /**
   * Update dataset color by custom color
   * @param newColor
   */
  updateTableColor(newColor: RGBColor): void {
    this.color = newColor;
  }

  /**
   * Save filterProps to field and retrieve it
   * @param columnName
   */
  getColumnFilterProps(columnName: string): F['filterProps'] | null | undefined {
    const fieldIdx = this.getColumnFieldIdx(columnName);
    if (fieldIdx < 0) {
      return null;
    }
    const field = this.fields[fieldIdx];
    if (Object.prototype.hasOwnProperty.call(field, 'filterProps')) {
      return field.filterProps;
    }

    const fieldDomain = this.getColumnFilterDomain(field);
    if (!fieldDomain) {
      return null;
    }

    const filterProps = getFilterProps(field, fieldDomain);
    const newField = {
      ...field,
      filterProps
    };

    this.updateColumnField(fieldIdx, newField);

    return filterProps;
  }

  /**
   * Apply filters to dataset, return the filtered dataset with updated `gpuFilter`, `filterRecord`, `filteredIndex`, `filteredIndexForDomain`
   * @param filters
   * @param layers
   * @param opt
   */
  filterTable(filters: Filter[], layers: Layer[], opt?: FilterDatasetOpt): KeplerTable<Field> {
    const {dataContainer, id: dataId, filterRecord: oldFilterRecord, fields} = this;

    // if there is no filters
    const filterRecord = getFilterRecord(dataId, filters, opt || {});

    this.filterRecord = filterRecord;
    this.gpuFilter = getGpuFilterProps(filters, dataId, fields, this.gpuFilter);

    this.changedFilters = diffFilters(filterRecord, oldFilterRecord);

    if (!filters.length) {
      this.filteredIndex = this.allIndexes;
      this.filteredIndexForDomain = this.allIndexes;
      return this;
    }

    // generate 2 sets of filter result
    // filteredIndex used to calculate layer data
    // filteredIndexForDomain used to calculate layer Domain
    const shouldCalDomain = Boolean(this.changedFilters.dynamicDomain);
    const shouldCalIndex = Boolean(this.changedFilters.cpu);

    let filterResult: FilterResult = {};
    if (shouldCalDomain || shouldCalIndex) {
      const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;
      const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;

      const filterFuncs = filters.reduce((acc, filter) => {
        const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);
        const field = fieldIndex !== -1 ? fields[fieldIndex] : null;

        return {
          ...acc,
          [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
        };
      }, {});

      filterResult = filterDataByFilterTypes(
        {dynamicDomainFilters, cpuFilters, filterFuncs},
        dataContainer
      );
    }

    this.filteredIndex = filterResult.filteredIndex || this.filteredIndex;
    this.filteredIndexForDomain =
      filterResult.filteredIndexForDomain || this.filteredIndexForDomain;

    return this;
  }

  /**
   * Apply filters to a dataset all on CPU, assign to `filteredIdxCPU`, `filterRecordCPU`
   * @param filters
   * @param layers
   */
  filterTableCPU(filters: Filter[], layers: Layer[]): KeplerTable<Field> {
    const opt = {
      cpuOnly: true,
      ignoreDomain: true
    };

    // no filter
    if (!filters.length) {
      this.filteredIdxCPU = this.allIndexes;
      this.filterRecordCPU = getFilterRecord(this.id, filters, opt);
      return this;
    }

    // no gpu filter
    if (!filters.find(f => f.gpu)) {
      this.filteredIdxCPU = this.filteredIndex;
      this.filterRecordCPU = getFilterRecord(this.id, filters, opt);
      return this;
    }

    // make a copy for cpu filtering
    const copied = copyTable(this);

    copied.filterRecord = this.filterRecordCPU;
    copied.filteredIndex = this.filteredIdxCPU || [];

    const filtered = copied.filterTable(filters, layers, opt);

    this.filteredIdxCPU = filtered.filteredIndex;
    this.filterRecordCPU = filtered.filterRecord;

    return this;
  }

  /**
   * Calculate field domain based on field type and data
   * for Filter
   */
  getColumnFilterDomain(field: F): FieldDomain {
    const {dataContainer} = this;
    const {valueAccessor} = field;

    let domain;

    switch (field.type) {
      case ALL_FIELD_TYPES.real:
      case ALL_FIELD_TYPES.integer:
        // calculate domain and step
        return getNumericFieldDomain(dataContainer, valueAccessor);

      case ALL_FIELD_TYPES.boolean:
        return {domain: [true, false]};

      case ALL_FIELD_TYPES.string:
      case ALL_FIELD_TYPES.h3:
      case ALL_FIELD_TYPES.date:
        domain = getOrdinalDomain(dataContainer, valueAccessor);
        return {domain};

      case ALL_FIELD_TYPES.timestamp:
        return getTimestampFieldDomain(dataContainer, valueAccessor);

      default:
        return {domain: getOrdinalDomain(dataContainer, valueAccessor)};
    }
  }

  /**
   *  Get the domain of this column based on scale type
   */
  getColumnLayerDomain(field: F, scaleType: string): number[] | string[] | [number, number] | null {
    const {dataContainer, filteredIndexForDomain} = this;

    if (!SCALE_TYPES[scaleType]) {
      Console.error(`scale type ${scaleType} not supported`);
      return null;
    }

    const {valueAccessor} = field;
    const indexValueAccessor = i => valueAccessor({index: i});
    const sortFunction = getSortingFunction(field.type);

    switch (scaleType) {
      case SCALE_TYPES.ordinal:
      case SCALE_TYPES.customOrdinal:
      case SCALE_TYPES.point:
        // do not recalculate ordinal domain based on filtered data
        // don't need to update ordinal domain every time
        return getOrdinalDomain(dataContainer, valueAccessor);

      case SCALE_TYPES.quantile:
        return getQuantileDomain(filteredIndexForDomain, indexValueAccessor, sortFunction);

      case SCALE_TYPES.log:
        return getLogDomain(filteredIndexForDomain, indexValueAccessor);

      case SCALE_TYPES.quantize:
      case SCALE_TYPES.linear:
      case SCALE_TYPES.sqrt:
      case SCALE_TYPES.custom:
      default:
        return getLinearDomain(filteredIndexForDomain, indexValueAccessor);
    }
  }

  /**
   * Get a sample of rows to calculate layer boundaries
   */
  // getSampleData(rows)

  /**
   * Parse cell value based on column type and return a string representation
   * Value the field value, type the field type
   */
  // parseFieldValue(value, type)

  // sortDatasetByColumn()

  /**
   * Assert whether field exist
   * @param fieldName
   * @param condition
   */
  _assetField(fieldName: string, condition: any): void {
    if (!condition) {
      Console.error(`${fieldName} doesnt exist in dataset ${this.id}`);
    }
  }
}

export type Datasets = {
  [key: string]: KeplerTable<Field>;
};

// HELPER FUNCTIONS (MAINLY EXPORTED FOR TEST...)
// have to double excape
const specialCharacterSet = `[#_&@\\.\\-\\ ]`;

function foundMatchingFields(re, suffixPair, allNames, fieldName) {
  const partnerIdx = allNames.findIndex(
    d => d === fieldName.replace(re, match => match.replace(suffixPair[0], suffixPair[1]))
  );
  let altIdx = -1;
  if (partnerIdx > -1) {
    // if found partner, go on and look for altitude
    ALTITUDE_FIELDS.some(alt => {
      altIdx = allNames.findIndex(
        d => d === fieldName.replace(re, match => match.replace(suffixPair[0], alt))
      );
      return altIdx > -1;
    });
  }
  return {partnerIdx, altIdx};
}
/**
 * Find point fields pairs from fields
 *
 * @param fields
 * @returns found point fields
 */
export function findPointFieldPairs(fields: Field[]): FieldPair[] {
  const allNames = fields.map(f => f.name.toLowerCase());

  // get list of all fields with matching suffixes
  const acc: FieldPair[] = [];
  return allNames.reduce((carry, fieldName, idx) => {
    // This search for pairs will early exit if found.
    for (const suffixPair of TRIP_POINT_FIELDS) {
      // match first suffix
      // (^|[#_&@\.\-\ ])lat([#_&@\.\-\ ]|$)
      const re = new RegExp(`(^|${specialCharacterSet})${suffixPair[0]}(${specialCharacterSet}|$)`);

      if (re.test(fieldName)) {
        const {partnerIdx, altIdx} = foundMatchingFields(re, suffixPair, allNames, fieldName);

        if (partnerIdx > -1) {
          const trimName = fieldName.replace(re, '').trim();

          carry.push({
            defaultName: trimName || 'point',
            pair: {
              lat: {
                fieldIdx: idx,
                value: fields[idx].name
              },
              lng: {
                fieldIdx: partnerIdx,
                value: fields[partnerIdx].name
              },
              ...(altIdx > -1
                ? {
                    altitude: {
                      fieldIdx: altIdx,
                      value: fields[altIdx].name
                    }
                  }
                : {})
            },
            suffix: suffixPair
          });
          return carry;
        }
      }
    }
    return carry;
  }, acc);
}

/**
 *
 * @param dataset
 * @param column
 * @param mode
 * @type
 */
export function sortDatasetByColumn(
  dataset: KeplerTable<Field>,
  column: string,
  mode?: string
): KeplerTable<Field> {
  const {allIndexes, fields, dataContainer} = dataset;
  const fieldIndex = fields.findIndex(f => f.name === column);
  if (fieldIndex < 0) {
    return dataset;
  }

  const sortBy = SORT_ORDER[mode || ''] || SORT_ORDER.ASCENDING;

  if (sortBy === SORT_ORDER.UNSORT) {
    dataset.sortColumn = {};
    dataset.sortOrder = null;

    return dataset;
  }

  const sortFunction = sortBy === SORT_ORDER.ASCENDING ? ascending : descending;
  const sortOrder = allIndexes.slice().sort((a, b) => {
    const value1 = dataContainer.valueAt(a, fieldIndex);
    const value2 = dataContainer.valueAt(b, fieldIndex);
    if (!notNullorUndefined(value1) && notNullorUndefined(value2)) {
      return 1;
    } else if (notNullorUndefined(value1) && !notNullorUndefined(value2)) {
      return -1;
    }
    return sortFunction(value1, value2);
  });

  dataset.sortColumn = {
    [column]: sortBy
  };
  dataset.sortOrder = sortOrder;

  return dataset;
}

export function pinTableColumns(dataset: KeplerTable<Field>, column: string): KeplerTable<Field> {
  const field = dataset.getColumnField(column);
  if (!field) {
    return dataset;
  }

  let pinnedColumns;
  if (Array.isArray(dataset.pinnedColumns) && dataset.pinnedColumns.includes(field.name)) {
    // unpin it
    pinnedColumns = dataset.pinnedColumns.filter(co => co !== field.name);
  } else {
    pinnedColumns = (dataset.pinnedColumns || []).concat(field.name);
  }

  // @ts-ignore
  return copyTableAndUpdate(dataset, {pinnedColumns});
}

export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {
  return Object.assign(Object.create(Object.getPrototypeOf(original)), original);
}

/**
 * @type
 * @returns
 */
export function copyTableAndUpdate(
  original: KeplerTable<Field>,
  options: Partial<KeplerTable<Field>> = {}
): KeplerTable<Field> {
  return Object.entries(options).reduce((acc, entry) => {
    acc[entry[0]] = entry[1];
    return acc;
  }, copyTable(original));
}

export function getFieldValueAccessor<
  F extends {
    type?: Field['type'];
    format?: Field['format'];
  }
>(f: F, i: number, dc: DataContainerInterface) {
  return maybeToDate.bind(
    null,
    // is time
    f.type === ALL_FIELD_TYPES.timestamp,
    i,
    f.format || '',
    dc
  );
}

export default KeplerTable;

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import Console from 'global/console';
5	import {ascending, descending} from 'd3-array';
6
7	import {
8	TRIP_POINT_FIELDS,
9	SORT_ORDER,
10	ALL_FIELD_TYPES,
11	ALTITUDE_FIELDS,
12	SCALE_TYPES
13	} from '@kepler.gl/constants';
14	import {
15	RGBColor,
16	Field,
17	FieldPair,
18	FieldDomain,
19	Filter,
20	ProtoDataset,
21	FilterRecord,
22	FilterDatasetOpt,
23	RangeFieldDomain,
24	SelectFieldDomain,
25	MultiSelectFieldDomain,
26	TimeRangeFieldDomain
27	} from '@kepler.gl/types';
28
29	import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils';
30
31	import {
32	getSortingFunction,
33	timeToUnixMilli,
34	createDataContainer,
35	DataForm,
36	diffFilters,
37	filterDataByFilterTypes,
38	FilterResult,
39	getFilterFunction,
40	getFilterProps,
41	getFilterRecord,
42	getNumericFieldDomain,
43	getTimestampFieldDomain,
44	getLinearDomain,
45	getLogDomain,
46	getOrdinalDomain,
47	getQuantileDomain,
48	DataContainerInterface,
49	FilterChanged
50	} from '@kepler.gl/utils';
51	import {generateHashId, notNullorUndefined} from '@kepler.gl/common-utils';
52
53	// TODO isolate layer type, depends on @kepler.gl/layers
54	type Layer = any;
55
56	export type GpuFilter = {
57	filterRange: number[][];
58	filterValueUpdateTriggers: {
59	[id: string]: {name: string; domain0: number} \| null;
60	};
61	filterValueAccessor: (
62	dc: DataContainerInterface
63	) => (
64	getIndex?: (any) => number,
65	getData?: (dc_: DataContainerInterface, d: any, fieldIndex: number) => any
66	) => (d: any, objectInfo?: {index: number}) => (number \| number[])[];
67	};
68
69	export type FilterProps =
70	\| NumericFieldFilterProps
71	\| BooleanFieldFilterProps
72	\| StringFieldFilterProps
73	\| TimeFieldFilterProps;
74
75	export type NumericFieldFilterProps = RangeFieldDomain & {
76	value: [number, number];
77	type: string;
78	typeOptions: string[];
79	gpu: boolean;
80	columnStats?: Record<string, any>;
81	};
82	export type BooleanFieldFilterProps = SelectFieldDomain & {
83	type: string;
84	value: boolean;
85	gpu: boolean;
86	columnStats?: Record<string, any>;
87	};
88	export type StringFieldFilterProps = MultiSelectFieldDomain & {
89	type: string;
90	value: string[];
91	gpu: boolean;
92	columnStats?: Record<string, any>;
93	};
94	export type TimeFieldFilterProps = TimeRangeFieldDomain & {
95	type: string;
96	view: Filter['view'];
97	fixedDomain: boolean;
98	value: number[];
99	gpu: boolean;
100	columnStats?: Record<string, any>;
101	};
102
103	// Unique identifier of each field
104	const FID_KEY = 'name';	13✔
105
106	export function maybeToDate(
107	isTime: boolean,
108	fieldIdx: number,
109	format: string,
110	dc: DataContainerInterface,
111	// An object with row index or a materialized row array (for materialized hover info from trip layer)
112	d: {index: number} \| any[]
113	) {
114	if (isTime) {	5,373✔
115	return timeToUnixMilli(Array.isArray(d) ? d[fieldIdx] : dc.valueAt(d.index, fieldIdx), format);	768!
116	}
117
118	return Array.isArray(d) ? d[fieldIdx] : dc.valueAt(d.index, fieldIdx);	4,605!
119	}
120
121	class KeplerTable<F extends Field = Field> {
122	readonly id: string;
123
124	type?: string;
125	label: string;
126	color: RGBColor;
127
128	// fields and data
129	fields: F[] = [];	149✔
130
131	dataContainer: DataContainerInterface;
132
133	allIndexes: number[] = [];	149✔
134	filteredIndex: number[] = [];	149✔
135	filteredIdxCPU?: number[];
136	filteredIndexForDomain: number[] = [];	149✔
137	fieldPairs: FieldPair[] = [];	149✔
138	gpuFilter: GpuFilter;
139	filterRecord?: FilterRecord;
140	filterRecordCPU?: FilterRecord;
141	changedFilters?: FilterChanged;
142
143	// table-injected metadata
144	sortColumn?: {
145	// column name: sorted idx
146	[key: string]: string; // ASCENDING \| DESCENDING \| UNSORT
147	};
148	sortOrder?: number[] \| null;
149
150	pinnedColumns?: string[];
151	supportedFilterTypes?: string[] \| null;
152	disableDataOperation?: boolean;
153
154	// table-injected metadata
155	metadata: Record<string, any>;
156
157	getFileProcessor?: (data: any, inputFormat?: string) => {data: any; format: string};
158
159	constructor({
160	info,
161	color,
162	metadata,
163	supportedFilterTypes = null,	148✔
164	disableDataOperation = false	149✔
165	}: {
166	info?: ProtoDataset['info'];
167	color: RGBColor;
168	metadata?: ProtoDataset['metadata'];
169	supportedFilterTypes?: ProtoDataset['supportedFilterTypes'];
170	disableDataOperation?: ProtoDataset['disableDataOperation'];
171	}) {
172	// TODO - what to do if validation fails? Can kepler handle exceptions?
173	// const validatedData = validateInputData(data);
174	// if (!validatedData) {
175	// return this;
176	// }
177
178	const datasetInfo = {	149✔
179	id: generateHashId(4),
180	label: 'new dataset',
181	type: '',
182	...info
183	};
184
185	const defaultMetadata = {	149✔
186	id: datasetInfo.id,
187	// @ts-ignore
188	format: datasetInfo.format \|\| '',	298✔
189	label: datasetInfo.label \|\| ''	149!
190	};
191
192	this.id = datasetInfo.id;	149✔
193	this.type = datasetInfo.type;	149✔
194	this.label = datasetInfo.label;	149✔
195	this.color = color;	149✔
196	this.metadata = {	149✔
197	...defaultMetadata,
198	...metadata
199	};
200
201	this.supportedFilterTypes = supportedFilterTypes;	149✔
202	this.disableDataOperation = disableDataOperation;	149✔
203
204	this.dataContainer = createDataContainer([]);	149✔
205	this.gpuFilter = getGpuFilterProps([], this.id, [], undefined);	149✔
206	}
207
208	async importData({data}: {data: ProtoDataset['data']}) {
209	const dataContainerData = data.cols ? data.cols : data.rows;	149!
210	const inputDataFormat = data.cols ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY;	149!
211
212	const dataContainer = createDataContainer(dataContainerData, {	149✔
213	fields: data.fields,
214	arrowTable: data.arrowTable,
215	inputDataFormat
216	});
217
218	const fields: Field[] = data.fields.map((f, i) => ({	1,225✔
219	...f,
220	fieldIdx: i,
221	id: f.name,
222	displayName: f.displayName \|\| f.name,	1,652✔
223	analyzerType: f.analyzerType \|\| ALL_FIELD_TYPES.string,	1,238✔
224	format: f.format \|\| '',	2,165✔
225	valueAccessor: getFieldValueAccessor(f, i, dataContainer)
226	}));
227
228	const allIndexes = dataContainer.getPlainIndex();	149✔
229	this.dataContainer = dataContainer;	149✔
230	this.allIndexes = allIndexes;	149✔
231	this.filteredIndex = allIndexes;	149✔
232	this.filteredIndexForDomain = allIndexes;	149✔
233	this.fieldPairs = findPointFieldPairs(fields);	149✔
234	// @ts-expect-error Make sure that fields satisfies F extends Field
235	this.fields = fields;	149✔
236	this.gpuFilter = getGpuFilterProps([], this.id, fields, undefined);	149✔
237	}
238
239	/**
240	* update table with new data
241	* @param data - new data e.g. the arrow data with new batches loaded
242	*/
243	async update(data: ProtoDataset['data']) {
244	const dataContainerData = data.arrowTable ?? data.cols ?? data.rows;	×
245	this.dataContainer.update?.(dataContainerData);	×
246	this.allIndexes = this.dataContainer.getPlainIndex();	×
247	this.filteredIndex = this.allIndexes;	×
248	this.filteredIndexForDomain = this.allIndexes;	×
249
250	return this;	×
251	}
252
253	get length() {
254	return this.dataContainer.numRows();	197✔
255	}
256
257	/**
258	* Get field
259	* @param columnName
260	*/
261	getColumnField(columnName: string): F \| undefined {
262	const field = this.fields.find(fd => fd[FID_KEY] === columnName);	3,967✔
263	this._assetField(columnName, field);	634✔
264	return field;	634✔
265	}
266
267	/**
268	* Get fieldIdx
269	* @param columnName
270	*/
271	getColumnFieldIdx(columnName: string): number {
272	const fieldIdx = this.fields.findIndex(fd => fd[FID_KEY] === columnName);	914✔
273	this._assetField(columnName, Boolean(fieldIdx > -1));	215✔
274	return fieldIdx;	215✔
275	}
276
277	/**
278	* Get displayFormat
279	* @param columnName
280	*/
281	getColumnDisplayFormat(columnName) {
282	const field = this.fields.find(fd => fd[FID_KEY] === columnName);	×
283	this._assetField(columnName, field);	×
284	return field?.displayFormat;	×
285	}
286
287	/**
288	* Get the value of a cell
289	*/
290	getValue(columnName: string, rowIdx: number): any {
291	const field = this.getColumnField(columnName);	539✔
292	return field ? field.valueAccessor({index: rowIdx}) : null;	539!
293	}
294
295	/**
296	* Updates existing field with a new object
297	* @param fieldIdx
298	* @param newField
299	*/
300	updateColumnField(fieldIdx: number, newField: F): void {
301	this.fields = Object.assign([...this.fields], {[fieldIdx]: newField});	76✔
302	}
303
304	/**
305	* Update dataset color by custom color
306	* @param newColor
307	*/
308	updateTableColor(newColor: RGBColor): void {
309	this.color = newColor;	×
310	}
311
312	/**
313	* Save filterProps to field and retrieve it
314	* @param columnName
315	*/
316	getColumnFilterProps(columnName: string): F['filterProps'] \| null \| undefined {
317	const fieldIdx = this.getColumnFieldIdx(columnName);	135✔
318	if (fieldIdx < 0) {	135✔
319	return null;	1✔
320	}
321	const field = this.fields[fieldIdx];	134✔
322	if (Object.prototype.hasOwnProperty.call(field, 'filterProps')) {	134✔
323	return field.filterProps;	58✔
324	}
325
326	const fieldDomain = this.getColumnFilterDomain(field);	76✔
327	if (!fieldDomain) {	76!
328	return null;	×
329	}
330
331	const filterProps = getFilterProps(field, fieldDomain);	76✔
332	const newField = {	76✔
333	...field,
334	filterProps
335	};
336
337	this.updateColumnField(fieldIdx, newField);	76✔
338
339	return filterProps;	76✔
340	}
341
342	/**
343	* Apply filters to dataset, return the filtered dataset with updated `gpuFilter`, `filterRecord`, `filteredIndex`, `filteredIndexForDomain`
344	* @param filters
345	* @param layers
346	* @param opt
347	*/
348	filterTable(filters: Filter[], layers: Layer[], opt?: FilterDatasetOpt): KeplerTable<Field> {
349	const {dataContainer, id: dataId, filterRecord: oldFilterRecord, fields} = this;	117✔
350
351	// if there is no filters
352	const filterRecord = getFilterRecord(dataId, filters, opt \|\| {});	117!
353
354	this.filterRecord = filterRecord;	117✔
355	this.gpuFilter = getGpuFilterProps(filters, dataId, fields, this.gpuFilter);	117✔
356
357	this.changedFilters = diffFilters(filterRecord, oldFilterRecord);	117✔
358
359	if (!filters.length) {	117✔
360	this.filteredIndex = this.allIndexes;	4✔
361	this.filteredIndexForDomain = this.allIndexes;	4✔
362	return this;	4✔
363	}
364
365	// generate 2 sets of filter result
366	// filteredIndex used to calculate layer data
367	// filteredIndexForDomain used to calculate layer Domain
368	const shouldCalDomain = Boolean(this.changedFilters.dynamicDomain);	113✔
369	const shouldCalIndex = Boolean(this.changedFilters.cpu);	113✔
370
371	let filterResult: FilterResult = {};	113✔
372	if (shouldCalDomain \|\| shouldCalIndex) {	113✔
373	const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;	55✔
374	const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;	55✔
375
376	const filterFuncs = filters.reduce((acc, filter) => {	55✔
377	const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);	76✔
378	const field = fieldIndex !== -1 ? fields[fieldIndex] : null;	76✔
379
380	return {	76✔
381	...acc,
382	[filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
383	};
384	}, {});
385
386	filterResult = filterDataByFilterTypes(	55✔
387	{dynamicDomainFilters, cpuFilters, filterFuncs},
388	dataContainer
389	);
390	}
391
392	this.filteredIndex = filterResult.filteredIndex \|\| this.filteredIndex;	113✔
393	this.filteredIndexForDomain =	113✔
394	filterResult.filteredIndexForDomain \|\| this.filteredIndexForDomain;	181✔
395
396	return this;	113✔
397	}
398
399	/**
400	* Apply filters to a dataset all on CPU, assign to `filteredIdxCPU`, `filterRecordCPU`
401	* @param filters
402	* @param layers
403	*/
404	filterTableCPU(filters: Filter[], layers: Layer[]): KeplerTable<Field> {
405	const opt = {	7✔
406	cpuOnly: true,
407	ignoreDomain: true
408	};
409
410	// no filter
411	if (!filters.length) {	7✔
412	this.filteredIdxCPU = this.allIndexes;	1✔
413	this.filterRecordCPU = getFilterRecord(this.id, filters, opt);	1✔
414	return this;	1✔
415	}
416
417	// no gpu filter
418	if (!filters.find(f => f.gpu)) {	6✔
419	this.filteredIdxCPU = this.filteredIndex;	3✔
420	this.filterRecordCPU = getFilterRecord(this.id, filters, opt);	3✔
421	return this;	3✔
422	}
423
424	// make a copy for cpu filtering
425	const copied = copyTable(this);	3✔
426
427	copied.filterRecord = this.filterRecordCPU;	3✔
428	copied.filteredIndex = this.filteredIdxCPU \|\| [];	3✔
429
430	const filtered = copied.filterTable(filters, layers, opt);	3✔
431
432	this.filteredIdxCPU = filtered.filteredIndex;	3✔
433	this.filterRecordCPU = filtered.filterRecord;	3✔
434
435	return this;	3✔
436	}
437
438	/**
439	* Calculate field domain based on field type and data
440	* for Filter
441	*/
442	getColumnFilterDomain(field: F): FieldDomain {
443	const {dataContainer} = this;	86✔
444	const {valueAccessor} = field;	86✔
445
446	let domain;
447
448	switch (field.type) {	86!
449	case ALL_FIELD_TYPES.real:
450	case ALL_FIELD_TYPES.integer:
451	// calculate domain and step
452	return getNumericFieldDomain(dataContainer, valueAccessor);	28✔
453
454	case ALL_FIELD_TYPES.boolean:
455	return {domain: [true, false]};	2✔
456
457	case ALL_FIELD_TYPES.string:
458	case ALL_FIELD_TYPES.h3:
459	case ALL_FIELD_TYPES.date:
460	domain = getOrdinalDomain(dataContainer, valueAccessor);	15✔
461	return {domain};	15✔
462
463	case ALL_FIELD_TYPES.timestamp:
464	return getTimestampFieldDomain(dataContainer, valueAccessor);	41✔
465
466	default:
467	return {domain: getOrdinalDomain(dataContainer, valueAccessor)};	×
468	}
469	}
470
471	/**
472	* Get the domain of this column based on scale type
473	*/
474	getColumnLayerDomain(field: F, scaleType: string): number[] \| string[] \| [number, number] \| null {
475	const {dataContainer, filteredIndexForDomain} = this;	131✔
476
477	if (!SCALE_TYPES[scaleType]) {	131!
478	Console.error(`scale type ${scaleType} not supported`);	×
479	return null;	×
480	}
481
482	const {valueAccessor} = field;	131✔
483	const indexValueAccessor = i => valueAccessor({index: i});	1,294✔
484	const sortFunction = getSortingFunction(field.type);	131✔
485
486	switch (scaleType) {	131!
487	case SCALE_TYPES.ordinal:
488	case SCALE_TYPES.customOrdinal:
489	case SCALE_TYPES.point:
490	// do not recalculate ordinal domain based on filtered data
491	// don't need to update ordinal domain every time
492	return getOrdinalDomain(dataContainer, valueAccessor);	34✔
493
494	case SCALE_TYPES.quantile:
495	return getQuantileDomain(filteredIndexForDomain, indexValueAccessor, sortFunction);	47✔
496
497	case SCALE_TYPES.log:
498	return getLogDomain(filteredIndexForDomain, indexValueAccessor);	×
499
500	case SCALE_TYPES.quantize:
501	case SCALE_TYPES.linear:
502	case SCALE_TYPES.sqrt:
503	case SCALE_TYPES.custom:
504	default:
505	return getLinearDomain(filteredIndexForDomain, indexValueAccessor);	50✔
506	}
507	}
508
509	/**
510	* Get a sample of rows to calculate layer boundaries
511	*/
512	// getSampleData(rows)
513
514	/**
515	* Parse cell value based on column type and return a string representation
516	* Value the field value, type the field type
517	*/
518	// parseFieldValue(value, type)
519
520	// sortDatasetByColumn()
521
522	/**
523	* Assert whether field exist
524	* @param fieldName
525	* @param condition
526	*/
527	_assetField(fieldName: string, condition: any): void {
528	if (!condition) {	849✔
529	Console.error(`${fieldName} doesnt exist in dataset ${this.id}`);	2✔
530	}
531	}
532	}
533
534	export type Datasets = {
535	[key: string]: KeplerTable<Field>;
536	};
537
538	// HELPER FUNCTIONS (MAINLY EXPORTED FOR TEST...)
539	// have to double excape
540	const specialCharacterSet = `[#_&@\\.\\-\\ ]`;	13✔
541
542	function foundMatchingFields(re, suffixPair, allNames, fieldName) {
543	const partnerIdx = allNames.findIndex(	142✔
544	d => d === fieldName.replace(re, match => match.replace(suffixPair[0], suffixPair[1]))	559✔
545	);
546	let altIdx = -1;	142✔
547	if (partnerIdx > -1) {	142✔
548	// if found partner, go on and look for altitude
549	ALTITUDE_FIELDS.some(alt => {	136✔
550	altIdx = allNames.findIndex(	268✔
551	d => d === fieldName.replace(re, match => match.replace(suffixPair[0], alt))	2,430✔
552	);
553	return altIdx > -1;	268✔
554	});
555	}
556	return {partnerIdx, altIdx};	142✔
557	}
558	/**
559	* Find point fields pairs from fields
560	*
561	* @param fields
562	* @returns found point fields
563	*/
564	export function findPointFieldPairs(fields: Field[]): FieldPair[] {
565	const allNames = fields.map(f => f.name.toLowerCase());	1,282✔
566
567	// get list of all fields with matching suffixes
568	const acc: FieldPair[] = [];	160✔
569	return allNames.reduce((carry, fieldName, idx) => {	160✔
570	// This search for pairs will early exit if found.
571	for (const suffixPair of TRIP_POINT_FIELDS) {	1,282✔
572	// match first suffix
573	// (^\|[#_&@\.\-\ ])lat([#_&@\.\-\ ]\|$)
574	const re = new RegExp(`(^\|${specialCharacterSet})${suffixPair[0]}(${specialCharacterSet}\|$)`);	4,777✔
575
576	if (re.test(fieldName)) {	4,777✔
577	const {partnerIdx, altIdx} = foundMatchingFields(re, suffixPair, allNames, fieldName);	142✔
578
579	if (partnerIdx > -1) {	142✔
580	const trimName = fieldName.replace(re, '').trim();	136✔
581
582	carry.push({	136✔
583	defaultName: trimName \|\| 'point',	152✔
584	pair: {
585	lat: {
586	fieldIdx: idx,
587	value: fields[idx].name
588	},
589	lng: {
590	fieldIdx: partnerIdx,
591	value: fields[partnerIdx].name
592	},
593	...(altIdx > -1	136✔
594	? {
595	altitude: {
596	fieldIdx: altIdx,
597	value: fields[altIdx].name
598	}
599	}
600	: {})
601	},
602	suffix: suffixPair
603	});
604	return carry;	136✔
605	}
606	}
607	}
608	return carry;	1,146✔
609	}, acc);
610	}
611
612	/**
613	*
614	* @param dataset
615	* @param column
616	* @param mode
617	* @type
618	*/
619	export function sortDatasetByColumn(
620	dataset: KeplerTable<Field>,
621	column: string,
622	mode?: string
623	): KeplerTable<Field> {
624	const {allIndexes, fields, dataContainer} = dataset;	5✔
625	const fieldIndex = fields.findIndex(f => f.name === column);	10✔
626	if (fieldIndex < 0) {	5!
627	return dataset;	×
628	}
629
630	const sortBy = SORT_ORDER[mode \|\| ''] \|\| SORT_ORDER.ASCENDING;	5!
631
632	if (sortBy === SORT_ORDER.UNSORT) {	5✔
633	dataset.sortColumn = {};	1✔
634	dataset.sortOrder = null;	1✔
635
636	return dataset;	1✔
637	}
638
639	const sortFunction = sortBy === SORT_ORDER.ASCENDING ? ascending : descending;	4✔
640	const sortOrder = allIndexes.slice().sort((a, b) => {	4✔
641	const value1 = dataContainer.valueAt(a, fieldIndex);	322✔
642	const value2 = dataContainer.valueAt(b, fieldIndex);	322✔
643	if (!notNullorUndefined(value1) && notNullorUndefined(value2)) {	322!
644	return 1;	×
645	} else if (notNullorUndefined(value1) && !notNullorUndefined(value2)) {	322!
646	return -1;	×
647	}
648	return sortFunction(value1, value2);	322✔
649	});
650
651	dataset.sortColumn = {	4✔
652	[column]: sortBy
653	};
654	dataset.sortOrder = sortOrder;	4✔
655
656	return dataset;	4✔
657	}
658
659	export function pinTableColumns(dataset: KeplerTable<Field>, column: string): KeplerTable<Field> {
660	const field = dataset.getColumnField(column);	3✔
661	if (!field) {	3!
662	return dataset;	×
663	}
664
665	let pinnedColumns;
666	if (Array.isArray(dataset.pinnedColumns) && dataset.pinnedColumns.includes(field.name)) {	3✔
667	// unpin it
668	pinnedColumns = dataset.pinnedColumns.filter(co => co !== field.name);	1✔
669	} else {
670	pinnedColumns = (dataset.pinnedColumns \|\| []).concat(field.name);	2✔
671	}
672
673	// @ts-ignore
674	return copyTableAndUpdate(dataset, {pinnedColumns});	3✔
675	}
676
677	export function copyTable(original: KeplerTable<Field>): KeplerTable<Field> {
678	return Object.assign(Object.create(Object.getPrototypeOf(original)), original);	44✔
679	}
680
681	/**
682	* @type
683	* @returns
684	*/
685	export function copyTableAndUpdate(
686	original: KeplerTable<Field>,
687	options: Partial<KeplerTable<Field>> = {}	×
688	): KeplerTable<Field> {
689	return Object.entries(options).reduce((acc, entry) => {	40✔
690	acc[entry[0]] = entry[1];	40✔
691	return acc;	40✔
692	}, copyTable(original));
693	}
694
695	export function getFieldValueAccessor<
696	F extends {
697	type?: Field['type'];
698	format?: Field['format'];
699	}
700	>(f: F, i: number, dc: DataContainerInterface) {
701	return maybeToDate.bind(	1,225✔
702	null,
703	// is time
704	f.type === ALL_FIELD_TYPES.timestamp,
705	i,
706	f.format \|\| '',	2,165✔
707	dc
708	);
709	}
710
711	export default KeplerTable;

keplergl / kepler.gl / 22361650031

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous