9003776237

Committed 08 May 2024 02:40PM UTC coverage: 89.82%. Remained the same

Build # 9003776237

Build Type

Pull #8856

github

Committed by

web-flow

Commit Message

Merge b1eec586d into 86f3f84f4

Pull Request Pull Request #8856: Carto: fetchMap to support custom basemaps

Run Details

6335 of 6975 branches covered (90.82%)

Branch coverage included in aggregate %.

154 of 172 new or added lines in 5 files covered. (89.53%)

21 existing lines in 1 file now uncovered.

52214 of 58210 relevant lines covered (89.7%)

6617.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

18.69

/modules/carto/src/api/fetch-map.ts

/* eslint-disable camelcase */
/**
 * Maps API Client for Carto 3
 */
import {CartoAPIError} from './carto-api-error';
import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT} from './common';
import {buildPublicMapUrl, buildStatsUrl} from './endpoints';
import {
  GeojsonResult,
  JsonResult,
  TilejsonResult,
  h3QuerySource,
  h3TableSource,
  quadbinQuerySource,
  quadbinTableSource,
  vectorQuerySource,
  vectorTableSource,
  vectorTilesetSource
} from '../sources/index';
import {parseMap} from './parse-map';
import {requestWithParameters} from './request-with-parameters';
import {assert} from '../utils';
import type {APIErrorContext, Format, MapType, QueryParameters} from './types';
import {fetchBasemapProps} from './basemap';

type Dataset = {
  id: string;
  type: MapType;
  source: string;
  cache?: number;
  connectionName: string;
  geoColumn: string;
  data: TilejsonResult | GeojsonResult | JsonResult;
  columns: string[];
  format: Format;
  aggregationExp: string;
  aggregationResLevel: number;
  queryParameters: QueryParameters;
};

/* global clearInterval, setInterval, URL */
/* eslint-disable complexity, max-statements */
async function _fetchMapDataset(
  dataset: Dataset,
  accessToken: string,
  apiBaseUrl: string,
  clientId?: string,
  headers?: Record<string, string>
) {
  const {
    aggregationExp,
    aggregationResLevel,
    connectionName,
    columns,
    format,
    geoColumn,
    source,
    type,
    queryParameters
  } = dataset;

  const cache: {value?: number} = {};
  const globalOptions: any = {
    accessToken,
    apiBaseUrl,
    cache,
    clientId,
    connectionName,
    format,
    headers
  };

  if (type === 'tileset') {
    // TODO do we want a generic tilesetSource?
    // @ts-ignore
    dataset.data = await vectorTilesetSource({...globalOptions, tableName: source});
  } else {
    const [spatialDataType, spatialDataColumn] = geoColumn ? geoColumn.split(':') : ['geom'];
    if (spatialDataType === 'geom') {
      const options = {...globalOptions, spatialDataColumn};
      if (type === 'table') {
        dataset.data = await vectorTableSource({...options, columns, tableName: source});
      } else if (type === 'query') {
        dataset.data = await vectorQuerySource({
          ...options,
          columns,
          sqlQuery: source,
          queryParameters
        });
      }
    } else if (spatialDataType === 'h3') {
      const options = {...globalOptions, aggregationExp, aggregationResLevel, spatialDataColumn};
      if (type === 'table') {
        dataset.data = await h3TableSource({...options, tableName: source});
      } else if (type === 'query') {
        dataset.data = await h3QuerySource({...options, sqlQuery: source, queryParameters});
      }
    } else if (spatialDataType === 'quadbin') {
      const options = {...globalOptions, aggregationExp, aggregationResLevel, spatialDataColumn};
      if (type === 'table') {
        dataset.data = await quadbinTableSource({...options, tableName: source});
      } else if (type === 'query') {
        dataset.data = await quadbinQuerySource({...options, sqlQuery: source, queryParameters});
      }
    }
  }
  let cacheChanged = true;
  if (cache.value) {
    cacheChanged = dataset.cache !== cache.value;
    dataset.cache = cache.value;
  }

  return cacheChanged;
}

async function _fetchTilestats(
  attribute: string,
  dataset: Dataset,
  accessToken: string,
  apiBaseUrl: string
) {
  const {connectionName, data, id, source, type, queryParameters} = dataset;
  const errorContext: APIErrorContext = {
    requestType: 'Tile stats',
    connection: connectionName,
    type,
    source
  };
  if (!('tilestats' in data)) {
    throw new CartoAPIError(new Error(`Invalid dataset for tilestats: ${id}`), errorContext);
  }

  const baseUrl = buildStatsUrl({attribute, apiBaseUrl, ...dataset});
  const client = new URLSearchParams(data.tiles[0]).get('client');
  const headers = {Authorization: `Bearer ${accessToken}`};
  const parameters: Record<string, string> = {};
  if (client) {
    parameters.client = client;
  }
  if (type === 'query') {
    parameters.q = source;
    if (queryParameters) {
      parameters.queryParameters = JSON.stringify(queryParameters);
    }
  }
  const stats = await requestWithParameters({
    baseUrl,
    headers,
    parameters,
    errorContext
  });

  // Replace tilestats for attribute with value from API
  const {attributes} = data.tilestats.layers[0];
  const index = attributes.findIndex(d => d.attribute === attribute);
  attributes[index] = stats;
  return true;
}

async function fillInMapDatasets(
  {datasets, token}: {datasets: Dataset[]; token: string},
  clientId: string,
  apiBaseUrl: string,
  headers?: Record<string, string>
) {
  const promises = datasets.map(dataset =>
    _fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers)
  );
  return await Promise.all(promises);
}

async function fillInTileStats(
  {datasets, keplerMapConfig, token}: {datasets: Dataset[]; keplerMapConfig: any; token: string},
  apiBaseUrl: string
) {
  const attributes: {attribute: string; dataset: any}[] = [];
  const {layers} = keplerMapConfig.config.visState;
  for (const layer of layers) {
    for (const channel of Object.keys(layer.visualChannels)) {
      const attribute = layer.visualChannels[channel]?.name;
      if (attribute) {
        const dataset = datasets.find(d => d.id === layer.config.dataId);
        if (dataset && dataset.type !== 'tileset' && (dataset.data as TilejsonResult).tilestats) {
          // Only fetch stats for QUERY & TABLE map types
          attributes.push({attribute, dataset});
        }
      }
    }
  }
  // Remove duplicates to avoid repeated requests
  const filteredAttributes: {attribute: string; dataset: any}[] = [];
  for (const a of attributes) {
    if (
      !filteredAttributes.find(
        ({attribute, dataset}) => attribute === a.attribute && dataset === a.dataset
      )
    ) {
      filteredAttributes.push(a);
    }
  }

  const promises = filteredAttributes.map(({attribute, dataset}) =>
    _fetchTilestats(attribute, dataset, token, apiBaseUrl)
  );
  return await Promise.all(promises);
}

export type FetchMapOptions = {
  apiBaseUrl?: string;
  cartoMapId: string;
  clientId?: string;
  headers?: Record<string, string>;
  autoRefresh?: number;
  onNewData?: (map: any) => void;
};

/* eslint-disable max-statements */
export async function fetchMap({
  apiBaseUrl = DEFAULT_API_BASE_URL,
  cartoMapId,
  clientId = DEFAULT_CLIENT,
  headers = {},
  autoRefresh,
  onNewData
}: FetchMapOptions) {
  assert(cartoMapId, 'Must define CARTO map id: fetchMap({cartoMapId: "XXXX-XXXX-XXXX"})');
  assert(apiBaseUrl, 'Must define apiBaseUrl');

  if (autoRefresh || onNewData) {
    assert(onNewData, 'Must define `onNewData` when using autoRefresh');
    assert(typeof onNewData === 'function', '`onNewData` must be a function');
    assert(
      typeof autoRefresh === 'number' && autoRefresh > 0,
      '`autoRefresh` must be a positive number'
    );
  }

  const baseUrl = buildPublicMapUrl({apiBaseUrl, cartoMapId});
  const errorContext: APIErrorContext = {requestType: 'Public map', mapId: cartoMapId};
  const map = await requestWithParameters({baseUrl, headers, errorContext});

  // Periodically check if the data has changed. Note that this
  // will not update when a map is published.
  let stopAutoRefresh: (() => void) | undefined;
  if (autoRefresh) {
    // eslint-disable-next-line @typescript-eslint/no-misused-promises
    const intervalId = setInterval(async () => {
      const changed = await fillInMapDatasets(map, clientId, apiBaseUrl, {
        ...headers,
        'If-Modified-Since': new Date().toUTCString()
      });
      if (onNewData && changed.some(v => v === true)) {
        onNewData(parseMap(map));
      }
    }, autoRefresh * 1000);
    stopAutoRefresh = () => {
      clearInterval(intervalId);
    };
  }

  const geojsonLayers = map.keplerMapConfig.config.visState.layers.filter(
    ({type}) => type === 'geojson' || type === 'point'
  );
  const geojsonDatasetIds = geojsonLayers.map(({config}) => config.dataId);
  map.datasets.forEach(dataset => {
    if (geojsonDatasetIds.includes(dataset.id)) {
      const {config} = geojsonLayers.find(({config}) => config.dataId === dataset.id);
      dataset.format = 'geojson';
      // Support for very old maps. geoColumn was not stored in the past
      if (!dataset.geoColumn && config.columns.geojson) {
        dataset.geoColumn = config.columns.geojson;
      }
    }
  });

  const [basemap] = await Promise.all([
    fetchBasemapProps({config: map.keplerMapConfig.config, errorContext}),

    // Mutates map.datasets so that dataset.data contains data
    fillInMapDatasets(map, clientId, apiBaseUrl, headers)
  ]);

  // Mutates attributes in visualChannels to contain tile stats
  await fillInTileStats(map, apiBaseUrl);

  const out = {...parseMap(map), basemap, ...{stopAutoRefresh}};

  const textLayers = out.layers.filter(layer => {
    const pointType = layer.props.pointType || '';
    return pointType.includes('text');
  });

  /* global FontFace, window, document */
  if (textLayers.length && window.FontFace && !document.fonts.check('12px Inter')) {
    // Fetch font needed for labels
    const font = new FontFace(
      'Inter',
      'url(https://fonts.gstatic.com/s/inter/v12/UcC73FwrK3iLTeHuS_fvQtMwCp50KnMa1ZL7W0Q5nw.woff2)'
    );
    await font.load().then(f => document.fonts.add(f));
  }

  return out;
}

1	/* eslint-disable camelcase */	1✔
2	/**	1✔
3	* Maps API Client for Carto 3	1✔
4	*/	1✔
5	import {CartoAPIError} from './carto-api-error';	1✔
6	import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT} from './common';	1✔
7	import {buildPublicMapUrl, buildStatsUrl} from './endpoints';	1✔
8	import {	1✔
9	GeojsonResult,	1✔
10	JsonResult,	1✔
11	TilejsonResult,	1✔
12	h3QuerySource,	1✔
13	h3TableSource,	1✔
14	quadbinQuerySource,	1✔
15	quadbinTableSource,	1✔
16	vectorQuerySource,	1✔
17	vectorTableSource,	1✔
18	vectorTilesetSource	1✔
19	} from '../sources/index';	1✔
20	import {parseMap} from './parse-map';	1✔
21	import {requestWithParameters} from './request-with-parameters';	1✔
22	import {assert} from '../utils';	1✔
23	import type {APIErrorContext, Format, MapType, QueryParameters} from './types';	1✔
24	import {fetchBasemapProps} from './basemap';	1✔
25		1✔
26	type Dataset = {	1✔
27	id: string;	1✔
28	type: MapType;	1✔
29	source: string;	1✔
30	cache?: number;	1✔
31	connectionName: string;	1✔
32	geoColumn: string;	1✔
33	data: TilejsonResult \| GeojsonResult \| JsonResult;	1✔
34	columns: string[];	1✔
35	format: Format;	1✔
36	aggregationExp: string;	1✔
37	aggregationResLevel: number;	1✔
38	queryParameters: QueryParameters;	1✔
39	};	1✔
40		1✔
41	/* global clearInterval, setInterval, URL */	1✔
42	/* eslint-disable complexity, max-statements */	1✔
43	async function _fetchMapDataset(	×
44	dataset: Dataset,	×
45	accessToken: string,	×
46	apiBaseUrl: string,	×
47	clientId?: string,	×
48	headers?: Record<string, string>	×
49	) {	×
50	const {	×
51	aggregationExp,	×
52	aggregationResLevel,	×
53	connectionName,	×
54	columns,	×
55	format,	×
56	geoColumn,	×
57	source,	×
58	type,	×
59	queryParameters	×
60	} = dataset;	×
61		×
62	const cache: {value?: number} = {};	×
63	const globalOptions: any = {	×
64	accessToken,	×
65	apiBaseUrl,	×
66	cache,	×
67	clientId,	×
68	connectionName,	×
69	format,	×
70	headers	×
71	};	×
72		×
73	if (type === 'tileset') {	×
74	// TODO do we want a generic tilesetSource?	×
75	// @ts-ignore	×
76	dataset.data = await vectorTilesetSource({...globalOptions, tableName: source});	×
77	} else {	×
78	const [spatialDataType, spatialDataColumn] = geoColumn ? geoColumn.split(':') : ['geom'];	×
79	if (spatialDataType === 'geom') {	×
80	const options = {...globalOptions, spatialDataColumn};	×
81	if (type === 'table') {	×
82	dataset.data = await vectorTableSource({...options, columns, tableName: source});	×
83	} else if (type === 'query') {	×
84	dataset.data = await vectorQuerySource({	×
85	...options,	×
86	columns,	×
87	sqlQuery: source,	×
88	queryParameters	×
89	});	×
90	}	×
91	} else if (spatialDataType === 'h3') {	×
92	const options = {...globalOptions, aggregationExp, aggregationResLevel, spatialDataColumn};	×
93	if (type === 'table') {	×
94	dataset.data = await h3TableSource({...options, tableName: source});	×
95	} else if (type === 'query') {	×
96	dataset.data = await h3QuerySource({...options, sqlQuery: source, queryParameters});	×
97	}	×
98	} else if (spatialDataType === 'quadbin') {	×
99	const options = {...globalOptions, aggregationExp, aggregationResLevel, spatialDataColumn};	×
100	if (type === 'table') {	×
101	dataset.data = await quadbinTableSource({...options, tableName: source});	×
102	} else if (type === 'query') {	×
103	dataset.data = await quadbinQuerySource({...options, sqlQuery: source, queryParameters});	×
104	}	×
105	}	×
106	}	×
107	let cacheChanged = true;	×
108	if (cache.value) {	×
109	cacheChanged = dataset.cache !== cache.value;	×
110	dataset.cache = cache.value;	×
111	}	×
112		×
113	return cacheChanged;	×
114	}	×
115		1✔
116	async function _fetchTilestats(	×
117	attribute: string,	×
118	dataset: Dataset,	×
119	accessToken: string,	×
120	apiBaseUrl: string	×
121	) {	×
122	const {connectionName, data, id, source, type, queryParameters} = dataset;	×
123	const errorContext: APIErrorContext = {	×
124	requestType: 'Tile stats',	×
125	connection: connectionName,	×
126	type,	×
127	source	×
128	};	×
129	if (!('tilestats' in data)) {	×
130	throw new CartoAPIError(new Error(`Invalid dataset for tilestats: ${id}`), errorContext);	×
131	}	×
132		×
133	const baseUrl = buildStatsUrl({attribute, apiBaseUrl, ...dataset});	×
134	const client = new URLSearchParams(data.tiles[0]).get('client');	×
135	const headers = {Authorization: `Bearer ${accessToken}`};	×
136	const parameters: Record<string, string> = {};	×
137	if (client) {	×
138	parameters.client = client;	×
139	}	×
140	if (type === 'query') {	×
141	parameters.q = source;	×
142	if (queryParameters) {	×
143	parameters.queryParameters = JSON.stringify(queryParameters);	×
144	}	×
145	}	×
146	const stats = await requestWithParameters({	×
147	baseUrl,	×
148	headers,	×
149	parameters,	×
150	errorContext	×
151	});	×
152		×
153	// Replace tilestats for attribute with value from API	×
154	const {attributes} = data.tilestats.layers[0];	×
155	const index = attributes.findIndex(d => d.attribute === attribute);	×
156	attributes[index] = stats;	×
157	return true;	×
158	}	×
159		1✔
160	async function fillInMapDatasets(	×
161	{datasets, token}: {datasets: Dataset[]; token: string},	×
162	clientId: string,	×
163	apiBaseUrl: string,	×
164	headers?: Record<string, string>	×
165	) {	×
166	const promises = datasets.map(dataset =>	×
167	_fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers)	×
168	);	×
169	return await Promise.all(promises);	×
170	}	×
171		1✔
172	async function fillInTileStats(	×
173	{datasets, keplerMapConfig, token}: {datasets: Dataset[]; keplerMapConfig: any; token: string},	×
174	apiBaseUrl: string	×
175	) {	×
176	const attributes: {attribute: string; dataset: any}[] = [];	×
177	const {layers} = keplerMapConfig.config.visState;	×
178	for (const layer of layers) {	×
179	for (const channel of Object.keys(layer.visualChannels)) {	×
180	const attribute = layer.visualChannels[channel]?.name;	×
181	if (attribute) {	×
182	const dataset = datasets.find(d => d.id === layer.config.dataId);	×
183	if (dataset && dataset.type !== 'tileset' && (dataset.data as TilejsonResult).tilestats) {	×
184	// Only fetch stats for QUERY & TABLE map types	×
185	attributes.push({attribute, dataset});	×
186	}	×
187	}	×
188	}	×
189	}	×
190	// Remove duplicates to avoid repeated requests	×
191	const filteredAttributes: {attribute: string; dataset: any}[] = [];	×
192	for (const a of attributes) {	×
193	if (	×
194	!filteredAttributes.find(	×
195	({attribute, dataset}) => attribute === a.attribute && dataset === a.dataset	×
196	)	×
197	) {	×
198	filteredAttributes.push(a);	×
199	}	×
200	}	×
201		×
202	const promises = filteredAttributes.map(({attribute, dataset}) =>	×
203	_fetchTilestats(attribute, dataset, token, apiBaseUrl)	×
204	);	×
205	return await Promise.all(promises);	×
206	}	×
207		1✔
208	export type FetchMapOptions = {	1✔
209	apiBaseUrl?: string;	1✔
210	cartoMapId: string;	1✔
211	clientId?: string;	1✔
212	headers?: Record<string, string>;	1✔
213	autoRefresh?: number;	1✔
214	onNewData?: (map: any) => void;	1✔
215	};	1✔
216		1✔
217	/* eslint-disable max-statements */	1✔
218	export async function fetchMap({	×
219	apiBaseUrl = DEFAULT_API_BASE_URL,	×
220	cartoMapId,	×
221	clientId = DEFAULT_CLIENT,	×
222	headers = {},	×
223	autoRefresh,	×
224	onNewData	×
225	}: FetchMapOptions) {	×
226	assert(cartoMapId, 'Must define CARTO map id: fetchMap({cartoMapId: "XXXX-XXXX-XXXX"})');	×
227	assert(apiBaseUrl, 'Must define apiBaseUrl');	×
228		×
229	if (autoRefresh \|\| onNewData) {	×
230	assert(onNewData, 'Must define `onNewData` when using autoRefresh');	×
231	assert(typeof onNewData === 'function', '`onNewData` must be a function');	×
232	assert(	×
233	typeof autoRefresh === 'number' && autoRefresh > 0,	×
234	'`autoRefresh` must be a positive number'	×
235	);	×
236	}	×
237		×
238	const baseUrl = buildPublicMapUrl({apiBaseUrl, cartoMapId});	×
239	const errorContext: APIErrorContext = {requestType: 'Public map', mapId: cartoMapId};	×
240	const map = await requestWithParameters({baseUrl, headers, errorContext});	×
241		×
242	// Periodically check if the data has changed. Note that this	×
243	// will not update when a map is published.	×
244	let stopAutoRefresh: (() => void) \| undefined;	×
245	if (autoRefresh) {	×
246	// eslint-disable-next-line @typescript-eslint/no-misused-promises	×
247	const intervalId = setInterval(async () => {	×
248	const changed = await fillInMapDatasets(map, clientId, apiBaseUrl, {	×
249	...headers,	×
250	'If-Modified-Since': new Date().toUTCString()	×
251	});	×
252	if (onNewData && changed.some(v => v === true)) {	×
253	onNewData(parseMap(map));	×
254	}	×
255	}, autoRefresh * 1000);	×
256	stopAutoRefresh = () => {	×
257	clearInterval(intervalId);	×
258	};	×
259	}	×
260		×
261	const geojsonLayers = map.keplerMapConfig.config.visState.layers.filter(	×
262	({type}) => type === 'geojson' \|\| type === 'point'	×
263	);	×
264	const geojsonDatasetIds = geojsonLayers.map(({config}) => config.dataId);	×
265	map.datasets.forEach(dataset => {	×
266	if (geojsonDatasetIds.includes(dataset.id)) {	×
267	const {config} = geojsonLayers.find(({config}) => config.dataId === dataset.id);	×
268	dataset.format = 'geojson';	×
269	// Support for very old maps. geoColumn was not stored in the past	×
270	if (!dataset.geoColumn && config.columns.geojson) {	×
271	dataset.geoColumn = config.columns.geojson;	×
272	}	×
273	}	×
274	});	×
275		×
NEW 276	const [basemap] = await Promise.all([	×
NEW 277	fetchBasemapProps({config: map.keplerMapConfig.config, errorContext}),	×
NEW 278		×
NEW 279	// Mutates map.datasets so that dataset.data contains data	×
NEW 280	fillInMapDatasets(map, clientId, apiBaseUrl, headers)	×
NEW 281	]);	×
282		×
283	// Mutates attributes in visualChannels to contain tile stats	×
284	await fillInTileStats(map, apiBaseUrl);	×
NEW 285		×
NEW 286	const out = {...parseMap(map), basemap, ...{stopAutoRefresh}};	×
287		×
288	const textLayers = out.layers.filter(layer => {	×
289	const pointType = layer.props.pointType \|\| '';	×
290	return pointType.includes('text');	×
291	});	×
292		×
293	/* global FontFace, window, document */	×
294	if (textLayers.length && window.FontFace && !document.fonts.check('12px Inter')) {	×
295	// Fetch font needed for labels	×
296	const font = new FontFace(	×
297	'Inter',	×
298	'url(https://fonts.gstatic.com/s/inter/v12/UcC73FwrK3iLTeHuS_fvQtMwCp50KnMa1ZL7W0Q5nw.woff2)'	×
299	);	×
300	await font.load().then(f => document.fonts.add(f));	×
301	}	×
302		×
303	return out;	×
304	}	×

visgl / deck.gl / 9003776237

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous