10302047504

Committed 08 Aug 2024 12:27PM UTC coverage: 89.203% (-0.03%) from 89.229%

Build # 10302047504

Build Type

push

github

Committed by

web-flow

Commit Message

CARTO: Re-aggregate properties on change (#9078)

Run Details

6537 of 7181 branches covered (91.03%)

Branch coverage included in aggregate %.

6 of 31 new or added lines in 2 files covered. (19.35%)

1 existing line in 1 file now uncovered.

54677 of 61442 relevant lines covered (88.99%)

6293.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

23.03

/modules/carto/src/layers/cluster-utils.ts

import {cellToParent} from 'quadbin';
import {_Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers';
import {Accessor, log} from '@deck.gl/core';
import {BinaryFeatureCollection} from '@loaders.gl/schema';

export type Aggregation = 'any' | 'average' | 'count' | 'min' | 'max' | 'sum';
export type AggregationProperties<FeaturePropertiesT> = {
  aggregation: Aggregation;
  name: keyof FeaturePropertiesT;
}[];
export type ClusteredFeaturePropertiesT<FeaturePropertiesT> = FeaturePropertiesT & {
  id: bigint;
  count: number;
  position: [number, number];
  stats: Record<keyof FeaturePropertiesT, {min: number; max: number}>;
};
export type ParsedQuadbinCell<FeaturePropertiesT> = {id: bigint; properties: FeaturePropertiesT};
export type ParsedQuadbinTile<FeaturePropertiesT> = ParsedQuadbinCell<FeaturePropertiesT>[];

/**
 * Aggregates tile by specified properties, caching result in tile.userData
 *
 * @returns true if data was aggregated, false if cache used
 */
export function aggregateTile<FeaturePropertiesT>(
  tile: Tile2DHeader<ParsedQuadbinTile<FeaturePropertiesT>>,
  aggregationLevels: number,
  properties: AggregationProperties<FeaturePropertiesT> = [],
  getPosition: Accessor<ParsedQuadbinCell<FeaturePropertiesT>, [number, number]>,
  getWeight: Accessor<ParsedQuadbinCell<FeaturePropertiesT>, number>
): boolean {
  if (!tile.content) return false;

  // Aggregate on demand and cache result
  if (!tile.userData) tile.userData = {};
  const cell0 = tile.userData[aggregationLevels]?.[0];
  if (cell0) {
    // Have already aggregated this tile
    if (properties.every(property => property.name in cell0)) {
      // Use cached result
      return false;
    }

    // Aggregated properties have changed, re-aggregate
    tile.userData = {};
  }

  const out: Record<number, any> = {};
  for (const cell of tile.content) {
    let id = cell.id;
    const position = typeof getPosition === 'function' ? getPosition(cell, {} as any) : getPosition;

    // Aggregate by parent id
    for (let i = 0; i < aggregationLevels - 1; i++) {
      id = cellToParent(id);
    }

    // Unfortunately TS doesn't support Record<bigint, any>
    // https://github.com/microsoft/TypeScript/issues/46395
    const parentId = Number(id);
    if (!(parentId in out)) {
      out[parentId] = {id, count: 0, position: [0, 0]};
      for (const {name, aggregation} of properties) {
        if (aggregation === 'any') {
          // Just pick first value for ANY
          out[parentId][name] = cell.properties[name];
        } else {
          out[parentId][name] = 0;
        }
      }
    }
    // Layout props
    const prevTotalW = out[parentId].count;
    out[parentId].count += typeof getWeight === 'function' ? getWeight(cell, {} as any) : getWeight;

    const totalW = out[parentId].count;
    const W = totalW - prevTotalW;
    out[parentId].position[0] = (prevTotalW * out[parentId].position[0] + W * position[0]) / totalW;
    out[parentId].position[1] = (prevTotalW * out[parentId].position[1] + W * position[1]) / totalW;

    // Re-aggregate other properties so clusters can be styled
    for (const {name, aggregation} of properties) {
      const prevValue = out[parentId][name];
      const value = cell.properties[name] as number;
      if (aggregation === 'average') {
        out[parentId][name] = (prevTotalW * prevValue + W * value) / totalW;
      } else if (aggregation === 'count' || aggregation === 'sum') {
        out[parentId][name] = prevValue + value;
      } else if (aggregation === 'max') {
        out[parentId][name] = Math.max(prevValue, value);
      } else if (aggregation === 'min') {
        out[parentId][name] = Math.min(prevValue, value);
      }
    }
  }

  tile.userData[aggregationLevels] = Object.values(out);
  return true;
}

export function extractAggregationProperties<FeaturePropertiesT extends {}>(
  tile: Tile2DHeader<ParsedQuadbinTile<FeaturePropertiesT>>
): AggregationProperties<FeaturePropertiesT> {
  const properties: AggregationProperties<FeaturePropertiesT> = [];
  const validAggregations: Aggregation[] = ['any', 'average', 'count', 'min', 'max', 'sum'];
  for (const name of Object.keys(tile.content![0].properties)) {
    let aggregation = name.split('_').pop()!.toLowerCase() as Aggregation;
    if (!validAggregations.includes(aggregation)) {
      log.warn(`No valid aggregation present in ${name} property`)();
      aggregation = 'any';
    }
    properties.push({name: name as keyof FeaturePropertiesT, aggregation});
  }

  return properties;
}

export function computeAggregationStats<FeaturePropertiesT>(
  data: ClusteredFeaturePropertiesT<FeaturePropertiesT>[],
  properties: AggregationProperties<FeaturePropertiesT>
) {
  const stats = {} as Record<keyof FeaturePropertiesT, {min: number; max: number}>;
  for (const {name, aggregation} of properties) {
    stats[name] = {min: Infinity, max: -Infinity};
    if (aggregation !== 'any') {
      for (const d of data) {
        stats[name].min = Math.min(stats[name].min, d[name] as number);
        stats[name].max = Math.max(stats[name].max, d[name] as number);
      }
    }
  }

  return stats;
}

const EMPTY_UINT16ARRAY = new Uint16Array();
const EMPTY_BINARY_PROPS = {
  positions: {value: new Float32Array(), size: 2},
  properties: [],
  numericProps: {},
  featureIds: {value: EMPTY_UINT16ARRAY, size: 1},
  globalFeatureIds: {value: EMPTY_UINT16ARRAY, size: 1}
};

export function clustersToBinary<FeaturePropertiesT>(
  data: ClusteredFeaturePropertiesT<FeaturePropertiesT>[]
): BinaryFeatureCollection {
  const positions = new Float32Array(data.length * 2);
  const featureIds = new Uint16Array(data.length);
  for (let i = 0; i < data.length; i++) {
    positions.set(data[i].position, 2 * i);
    featureIds[i] = i;
  }

  return {
    shape: 'binary-feature-collection',
    points: {
      type: 'Point',
      positions: {value: positions, size: 2},
      properties: data,
      numericProps: {},
      featureIds: {value: featureIds, size: 1},
      globalFeatureIds: {value: featureIds, size: 1}
    },
    lines: {
      type: 'LineString',
      pathIndices: {value: EMPTY_UINT16ARRAY, size: 1},
      ...EMPTY_BINARY_PROPS
    },
    polygons: {
      type: 'Polygon',
      polygonIndices: {value: EMPTY_UINT16ARRAY, size: 1},
      primitivePolygonIndices: {value: EMPTY_UINT16ARRAY, size: 1},
      ...EMPTY_BINARY_PROPS
    }
  };
}

1	import {cellToParent} from 'quadbin';	1✔
2	import {_Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers';	1✔
3	import {Accessor, log} from '@deck.gl/core';	1✔
4	import {BinaryFeatureCollection} from '@loaders.gl/schema';	1✔
5		1✔
6	export type Aggregation = 'any' \| 'average' \| 'count' \| 'min' \| 'max' \| 'sum';	1✔
7	export type AggregationProperties<FeaturePropertiesT> = {	1✔
8	aggregation: Aggregation;	1✔
9	name: keyof FeaturePropertiesT;	1✔
10	}[];	1✔
11	export type ClusteredFeaturePropertiesT<FeaturePropertiesT> = FeaturePropertiesT & {	1✔
12	id: bigint;	1✔
13	count: number;	1✔
14	position: [number, number];	1✔
15	stats: Record<keyof FeaturePropertiesT, {min: number; max: number}>;	1✔
16	};	1✔
17	export type ParsedQuadbinCell<FeaturePropertiesT> = {id: bigint; properties: FeaturePropertiesT};	1✔
18	export type ParsedQuadbinTile<FeaturePropertiesT> = ParsedQuadbinCell<FeaturePropertiesT>[];	1✔
19		1✔
20	/**	1✔
21	* Aggregates tile by specified properties, caching result in tile.userData	1✔
22	*	1✔
23	* @returns true if data was aggregated, false if cache used	1✔
24	*/	1✔
25	export function aggregateTile<FeaturePropertiesT>(	1✔
26	tile: Tile2DHeader<ParsedQuadbinTile<FeaturePropertiesT>>,	×
27	aggregationLevels: number,	×
28	properties: AggregationProperties<FeaturePropertiesT> = [],	×
29	getPosition: Accessor<ParsedQuadbinCell<FeaturePropertiesT>, [number, number]>,	×
30	getWeight: Accessor<ParsedQuadbinCell<FeaturePropertiesT>, number>	×
NEW 31	): boolean {	×
NEW 32	if (!tile.content) return false;	×
33		×
34	// Aggregate on demand and cache result	×
35	if (!tile.userData) tile.userData = {};	×
NEW 36	const cell0 = tile.userData[aggregationLevels]?.[0];	×
NEW 37	if (cell0) {	×
NEW 38	// Have already aggregated this tile	×
NEW 39	if (properties.every(property => property.name in cell0)) {	×
NEW 40	// Use cached result	×
NEW 41	return false;	×
NEW 42	}	×
NEW 43		×
NEW 44	// Aggregated properties have changed, re-aggregate	×
NEW 45	tile.userData = {};	×
NEW 46	}	×
47		×
48	const out: Record<number, any> = {};	×
49	for (const cell of tile.content) {	×
50	let id = cell.id;	×
51	const position = typeof getPosition === 'function' ? getPosition(cell, {} as any) : getPosition;	×
52		×
53	// Aggregate by parent id	×
54	for (let i = 0; i < aggregationLevels - 1; i++) {	×
55	id = cellToParent(id);	×
56	}	×
57		×
58	// Unfortunately TS doesn't support Record<bigint, any>	×
59	// https://github.com/microsoft/TypeScript/issues/46395	×
60	const parentId = Number(id);	×
61	if (!(parentId in out)) {	×
62	out[parentId] = {id, count: 0, position: [0, 0]};	×
63	for (const {name, aggregation} of properties) {	×
64	if (aggregation === 'any') {	×
65	// Just pick first value for ANY	×
66	out[parentId][name] = cell.properties[name];	×
67	} else {	×
68	out[parentId][name] = 0;	×
69	}	×
70	}	×
71	}	×
72	// Layout props	×
73	const prevTotalW = out[parentId].count;	×
74	out[parentId].count += typeof getWeight === 'function' ? getWeight(cell, {} as any) : getWeight;	×
75		×
76	const totalW = out[parentId].count;	×
77	const W = totalW - prevTotalW;	×
78	out[parentId].position[0] = (prevTotalW * out[parentId].position[0] + W * position[0]) / totalW;	×
79	out[parentId].position[1] = (prevTotalW * out[parentId].position[1] + W * position[1]) / totalW;	×
80		×
81	// Re-aggregate other properties so clusters can be styled	×
82	for (const {name, aggregation} of properties) {	×
83	const prevValue = out[parentId][name];	×
84	const value = cell.properties[name] as number;	×
85	if (aggregation === 'average') {	×
86	out[parentId][name] = (prevTotalW * prevValue + W * value) / totalW;	×
87	} else if (aggregation === 'count' \|\| aggregation === 'sum') {	×
88	out[parentId][name] = prevValue + value;	×
89	} else if (aggregation === 'max') {	×
90	out[parentId][name] = Math.max(prevValue, value);	×
91	} else if (aggregation === 'min') {	×
92	out[parentId][name] = Math.min(prevValue, value);	×
93	}	×
94	}	×
95	}	×
96		×
97	tile.userData[aggregationLevels] = Object.values(out);	×
NEW 98	return true;	×
UNCOV 99	}	×
100		1✔
101	export function extractAggregationProperties<FeaturePropertiesT extends {}>(	1✔
102	tile: Tile2DHeader<ParsedQuadbinTile<FeaturePropertiesT>>	×
103	): AggregationProperties<FeaturePropertiesT> {	×
104	const properties: AggregationProperties<FeaturePropertiesT> = [];	×
105	const validAggregations: Aggregation[] = ['any', 'average', 'count', 'min', 'max', 'sum'];	×
106	for (const name of Object.keys(tile.content![0].properties)) {	×
107	let aggregation = name.split('_').pop()!.toLowerCase() as Aggregation;	×
108	if (!validAggregations.includes(aggregation)) {	×
NEW 109	log.warn(`No valid aggregation present in ${name} property`)();	×
110	aggregation = 'any';	×
111	}	×
112	properties.push({name: name as keyof FeaturePropertiesT, aggregation});	×
113	}	×
114		×
115	return properties;	×
116	}	×
117		1✔
118	export function computeAggregationStats<FeaturePropertiesT>(	1✔
119	data: ClusteredFeaturePropertiesT<FeaturePropertiesT>[],	×
120	properties: AggregationProperties<FeaturePropertiesT>	×
121	) {	×
122	const stats = {} as Record<keyof FeaturePropertiesT, {min: number; max: number}>;	×
123	for (const {name, aggregation} of properties) {	×
124	stats[name] = {min: Infinity, max: -Infinity};	×
125	if (aggregation !== 'any') {	×
126	for (const d of data) {	×
127	stats[name].min = Math.min(stats[name].min, d[name] as number);	×
128	stats[name].max = Math.max(stats[name].max, d[name] as number);	×
129	}	×
130	}	×
131	}	×
132		×
133	return stats;	×
134	}	×
135		1✔
136	const EMPTY_UINT16ARRAY = new Uint16Array();	1✔
137	const EMPTY_BINARY_PROPS = {	1✔
138	positions: {value: new Float32Array(), size: 2},	1✔
139	properties: [],	1✔
140	numericProps: {},	1✔
141	featureIds: {value: EMPTY_UINT16ARRAY, size: 1},	1✔
142	globalFeatureIds: {value: EMPTY_UINT16ARRAY, size: 1}	1✔
143	};	1✔
144		1✔
145	export function clustersToBinary<FeaturePropertiesT>(	1✔
146	data: ClusteredFeaturePropertiesT<FeaturePropertiesT>[]	×
147	): BinaryFeatureCollection {	×
148	const positions = new Float32Array(data.length * 2);	×
149	const featureIds = new Uint16Array(data.length);	×
150	for (let i = 0; i < data.length; i++) {	×
151	positions.set(data[i].position, 2 * i);	×
152	featureIds[i] = i;	×
153	}	×
154		×
155	return {	×
156	shape: 'binary-feature-collection',	×
157	points: {	×
158	type: 'Point',	×
159	positions: {value: positions, size: 2},	×
160	properties: data,	×
161	numericProps: {},	×
162	featureIds: {value: featureIds, size: 1},	×
163	globalFeatureIds: {value: featureIds, size: 1}	×
164	},	×
165	lines: {	×
166	type: 'LineString',	×
167	pathIndices: {value: EMPTY_UINT16ARRAY, size: 1},	×
168	...EMPTY_BINARY_PROPS	×
169	},	×
170	polygons: {	×
171	type: 'Polygon',	×
172	polygonIndices: {value: EMPTY_UINT16ARRAY, size: 1},	×
173	primitivePolygonIndices: {value: EMPTY_UINT16ARRAY, size: 1},	×
174	...EMPTY_BINARY_PROPS	×
175	}	×
176	};	×
177	}	×

visgl / deck.gl / 10302047504

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous