22361650031

Committed 24 Feb 2026 05:09PM UTC coverage: 61.612% (-0.2%) from 61.806%

Build # 22361650031

Build Type

Pull #3219

github

Committed by

web-flow

Commit Message

Merge 1d9b34cb5 into cc33b0c8f

Pull Request Pull Request #3219: Update kepler-jupyter to use kepler.gl v3.2.0

Coverage Stats

6382 of 12288 branches covered (51.94%)

Branch coverage included in aggregate %.

13078 of 19297 relevant lines covered (67.77%)

81.44 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

51.57

/src/processors/src/data-processor.ts

// SPDX-License-Identifier: MIT
// Copyright contributors to the kepler.gl project

import * as arrow from 'apache-arrow';
import {csvParseRows} from 'd3-dsv';
import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
import normalize from '@mapbox/geojson-normalize';
import {parseSync} from '@loaders.gl/core';
import {ArrowTable} from '@loaders.gl/schema';
import {WKBLoader} from '@loaders.gl/wkt';

import {
  ALL_FIELD_TYPES,
  DATASET_FORMATS,
  GEOARROW_EXTENSIONS,
  GEOARROW_METADATA_KEY,
  GUIDES_FILE_FORMAT_DOC
} from '@kepler.gl/constants';
import {ProcessorResult, Field} from '@kepler.gl/types';
import {
  arrowDataTypeToAnalyzerDataType,
  arrowDataTypeToFieldType,
  hasOwnProperty,
  isPlainObject
} from '@kepler.gl/utils';
import {
  analyzerTypeToFieldType,
  getSampleForTypeAnalyze,
  getSampleForTypeAnalyzeArrow,
  getFieldsFromData,
  h3IsValid,
  notNullorUndefined,
  toArray
} from '@kepler.gl/common-utils';
import {KeplerGlSchema, ParsedDataset, SavedMap, LoadedMap} from '@kepler.gl/schemas';
import {Feature} from '@nebula.gl/edit-modes';

// if any of these value occurs in csv, parse it to null;
// const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
// matches empty string
export const CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;

function tryParseJsonString(str) {
  try {
    return JSON.parse(str);
  } catch (e) {
    return null;
  }
}

export const PARSE_FIELD_VALUE_FROM_STRING = {
  [ALL_FIELD_TYPES.boolean]: {
    valid: (d: unknown): boolean => typeof d === 'boolean',
    parse: (d: unknown): boolean => d === 'true' || d === 'True' || d === 'TRUE' || d === '1'
  },
  [ALL_FIELD_TYPES.integer]: {
    // @ts-ignore
    valid: (d: unknown): boolean => parseInt(d, 10) === d,
    // @ts-ignore
    parse: (d: unknown): number => parseInt(d, 10)
  },
  [ALL_FIELD_TYPES.timestamp]: {
    valid: (d: unknown, field: Field): boolean =>
      ['x', 'X'].includes(field.format) ? typeof d === 'number' : typeof d === 'string',
    parse: (d: any, field: Field) => (['x', 'X'].includes(field.format) ? Number(d) : d)
  },
  [ALL_FIELD_TYPES.real]: {
    // @ts-ignore
    valid: (d: unknown): boolean => parseFloat(d) === d,
    // Note this will result in NaN for some string
    parse: parseFloat
  },
  [ALL_FIELD_TYPES.object]: {
    valid: isPlainObject,
    parse: tryParseJsonString
  },

  [ALL_FIELD_TYPES.array]: {
    valid: Array.isArray,
    parse: tryParseJsonString
  },

  [ALL_FIELD_TYPES.h3]: {
    valid: d => h3IsValid(d),
    parse: d => d
  }
};

/**
 * Process csv data, output a data object with `{fields: [], rows: []}`.
 * The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
 * @param rawData raw csv string
 * @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
 * @public
 * @example
 * import {processCsvData} from '@kepler.gl/processors';
 *
 * const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
 * 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
 * 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
 * 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
 * 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
 *
 * const dataset = {
 *  info: {id: 'test_data', label: 'My Csv'},
 *  data: processCsvData(testData)
 * };
 *
 * dispatch(addDataToMap({
 *  datasets: [dataset],
 *  options: {centerMap: true, readOnly: true}
 * }));
 */
export function processCsvData(rawData: unknown[][] | string, header?: string[]): ProcessorResult {
  let rows: unknown[][] | undefined;
  let headerRow: string[] | undefined;

  if (typeof rawData === 'string') {
    const parsedRows: string[][] = csvParseRows(rawData);

    if (!Array.isArray(parsedRows) || parsedRows.length < 2) {
      // looks like an empty file, throw error to be catch
      throw new Error('process Csv Data Failed: CSV is empty');
    }
    headerRow = parsedRows[0];
    rows = parsedRows.slice(1);
  } else if (Array.isArray(rawData) && rawData.length) {
    rows = rawData;
    headerRow = header;

    if (!Array.isArray(headerRow)) {
      // if data is passed in as array of rows and missing header
      // assume first row is header
      // @ts-ignore
      headerRow = rawData[0];
      rows = rawData.slice(1);
    }
  }

  if (!rows || !headerRow) {
    throw new Error('invalid input passed to processCsvData');
  }

  // here we assume the csv file that people uploaded will have first row
  // as name of the column

  cleanUpFalsyCsvValue(rows);
  // No need to run type detection on every data point
  // here we get a list of none null values to run analyze on
  const sample = getSampleForTypeAnalyze({fields: headerRow, rows});
  const fields = getFieldsFromData(sample, headerRow);
  const parsedRows = parseRowsByFields(rows, fields);

  return {fields, rows: parsedRows};
}

/**
 * Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
 * @param rows
 * @param fields
 */
export function parseRowsByFields(rows: any[][], fields: Field[]) {
  // Edit rows in place
  const geojsonFieldIdx = fields.findIndex(f => f.name === '_geojson');
  fields.forEach(parseCsvRowsByFieldType.bind(null, rows, geojsonFieldIdx));

  return rows;
}

/**
 * Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,
 * so that type-analyzer won't detect it as string
 *
 * @param rows
 */
function cleanUpFalsyCsvValue(rows: unknown[][]): void {
  const re = new RegExp(CSV_NULLS, 'g');
  for (let i = 0; i < rows.length; i++) {
    for (let j = 0; j < rows[i].length; j++) {
      // analyzer will set any fields to 'string' if there are empty values
      // which will be parsed as '' by d3.csv
      // here we parse empty data as null
      // TODO: create warning when deltect `CSV_NULLS` in the data
      if (typeof rows[i][j] === 'string' && (rows[i][j] as string).match(re)) {
        rows[i][j] = null;
      }
    }
  }
}

/**
 * Process uploaded csv file to parse value by field type
 *
 * @param rows
 * @param geoFieldIdx field index
 * @param field
 * @param i
 */
export function parseCsvRowsByFieldType(
  rows: unknown[][],
  geoFieldIdx: number,
  field: Field,
  i: number
): void {
  const parser = PARSE_FIELD_VALUE_FROM_STRING[field.type];
  if (parser) {
    // check first not null value of it's already parsed
    const first = rows.find(r => notNullorUndefined(r[i]));
    if (!first || parser.valid(first[i], field)) {
      return;
    }
    rows.forEach(row => {
      // parse string value based on field type
      if (row[i] !== null) {
        row[i] = parser.parse(row[i], field);
        if (
          geoFieldIdx > -1 &&
          isPlainObject(row[geoFieldIdx]) &&
          // @ts-ignore
          hasOwnProperty(row[geoFieldIdx], 'properties')
        ) {
          // @ts-ignore
          row[geoFieldIdx].properties[field.name] = row[i];
        }
      }
    });
  }
}

/* eslint-enable complexity */

/**
 * Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
 * NOTE: This function may mutate input.
 * @param rawData an array of row object, each object should have the same number of keys
 * @returns dataset containing `fields` and `rows`
 * @public
 * @example
 * import {addDataToMap} from '@kepler.gl/actions';
 * import {processRowObject} from '@kepler.gl/processors';
 *
 * const data = [
 *  {lat: 31.27, lng: 127.56, value: 3},
 *  {lat: 31.22, lng: 126.26, value: 1}
 * ];
 *
 * dispatch(addDataToMap({
 *  datasets: {
 *    info: {label: 'My Data', id: 'my_data'},
 *    data: processRowObject(data)
 *  }
 * }));
 */
export function processRowObject(rawData: unknown[]): ProcessorResult {
  if (!Array.isArray(rawData)) {
    return null;
  } else if (!rawData.length) {
    // data is empty
    return {
      fields: [],
      rows: []
    };
  }

  const firstRow = rawData[0] as Record<string, unknown>;
  const keys = Object.keys(firstRow); // [lat, lng, value]
  const rows = rawData.map(d => keys.map(key => (d as Record<string, unknown>)[key])); // [[31.27, 127.56, 3]]

  // row object can still contain values like `Null` or `N/A`
  cleanUpFalsyCsvValue(rows);

  return processCsvData(rows, keys);
}

/**
 * Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
 * output a data object with `{fields: [], rows: []}`.
 * The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
 * NOTE: This function may mutate input.
 *
 * @param rawData raw geojson feature collection
 * @returns dataset containing `fields` and `rows`
 * @public
 * @example
 * import {addDataToMap} from '@kepler.gl/actions';
 * import {processGeojson} from '@kepler.gl/processors';
 *
 * const geojson = {
 *         "type" : "FeatureCollection",
 *         "features" : [{
 *                 "type" : "Feature",
 *                 "properties" : {
 *                         "capacity" : "10",
 *                         "type" : "U-Rack"
 *                 },
 *                 "geometry" : {
 *                         "type" : "Point",
 *                         "coordinates" : [ -71.073283, 42.417500 ]
 *                 }
 *         }]
 * };
 *
 * dispatch(addDataToMap({
 *  datasets: {
 *    info: {
 *      label: 'Sample Taxi Trips in New York City',
 *      id: 'test_trip_data'
 *    },
 *    data: processGeojson(geojson)
 *  }
 * }));
 */
export function processGeojson(rawData: unknown): ProcessorResult {
  const normalizedGeojson = normalize(rawData);

  if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {
    throw new Error(
      `Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](${GUIDES_FILE_FORMAT_DOC})`
    );
  }

  // getting all feature fields
  const allDataRows: Array<{_geojson: Feature} & keyof Feature> = [];
  for (let i = 0; i < normalizedGeojson.features.length; i++) {
    const f = normalizedGeojson.features[i];
    if (f.geometry) {
      allDataRows.push({
        // add feature to _geojson field
        _geojson: f,
        ...(f.properties || {})
      });
    }
  }
  // get all the field
  const fields = allDataRows.reduce<string[]>((accu, curr) => {
    Object.keys(curr).forEach(key => {
      if (!accu.includes(key)) {
        accu.push(key);
      }
    });
    return accu;
  }, []);

  // make sure each feature has exact same fields
  allDataRows.forEach(d => {
    fields.forEach(f => {
      if (!(f in d)) {
        d[f] = null;
        if (d._geojson.properties) {
          d._geojson.properties[f] = null;
        }
      }
    });
  });

  return processRowObject(allDataRows);
}

/**
 * Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
 * The json object should contain `datasets` and `config`.
 * @param rawData
 * @param schema
 * @returns datasets and config `{datasets: {}, config: {}}`
 * @public
 * @example
 * import {addDataToMap} from '@kepler.gl/actions';
 * import {processKeplerglJSON} from '@kepler.gl/processors';
 *
 * dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
 */
export function processKeplerglJSON(rawData: SavedMap, schema = KeplerGlSchema): LoadedMap | null {
  return rawData ? schema.load(rawData.datasets, rawData.config) : null;
}

/**
 * Parse a single or an array of datasets saved using kepler.gl schema
 * @param rawData
 * @param schema
 */
export function processKeplerglDataset(
  rawData: object | object[],
  schema = KeplerGlSchema
): ParsedDataset | ParsedDataset[] | null {
  if (!rawData) {
    return null;
  }

  const results = schema.parseSavedData(toArray(rawData));
  if (!results) {
    return null;
  }
  return Array.isArray(rawData) ? results : results[0];
}

/**
 * Parse arrow table and return a dataset
 *
 * @param arrowTable ArrowTable to parse, see loaders.gl/schema
 * @returns dataset containing `fields` and `rows` or null
 */
export function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null {
  // @ts-ignore - Unknown data type causing build failures
  return processArrowBatches(arrowTable.data.batches);
}

/**
 * Extracts GeoArrow metadata from an Apache Arrow table schema.
 * For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
 * @param table The Apache Arrow table to extract metadata from.
 * @returns An object mapping column names to their GeoArrow encoding type.
 * @throws Logs an error message if parsing of metadata fails.
 */
export function getGeoArrowMetadataFromSchema(table: arrow.Table): Record<string, string> {
  const geoArrowMetadata: Record<string, string> = {};
  try {
    const geoString = table.schema.metadata?.get('geo');
    if (geoString) {
      const parsedGeoString = JSON.parse(geoString);
      if (parsedGeoString.columns) {
        Object.keys(parsedGeoString.columns).forEach(columnName => {
          const columnData = parsedGeoString.columns[columnName];
          if (columnData?.encoding === 'WKB') {
            geoArrowMetadata[columnName] = GEOARROW_EXTENSIONS.WKB;
          }
          // TODO potentially there are other types but no datasets to test
        });
      }
    }
  } catch (error) {
    console.error('An error during arrow table schema metadata parsing');
  }
  return geoArrowMetadata;
}

/**
 * Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
 * @param table The Apache Arrow table whose schema needs to be converted.
 * @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
 * @returns An array of field objects suitable for Kepler.gl.
 */
export function arrowSchemaToFields(
  table: arrow.Table,
  fieldTypeSuggestions: Record<string, string> = {}
): Field[] {
  const headerRow = table.schema.fields.map(f => f.name);
  const sample = getSampleForTypeAnalyzeArrow(table, headerRow);
  const keplerFields = getFieldsFromData(sample, headerRow);
  const geoArrowMetadata = getGeoArrowMetadataFromSchema(table);

  return table.schema.fields.map((field: arrow.Field, fieldIndex: number) => {
    let type = arrowDataTypeToFieldType(field.type);
    let analyzerType = arrowDataTypeToAnalyzerDataType(field.type);
    let format = '';

    const fieldTypeSuggestion = fieldTypeSuggestions[field.name];
    const keplerField = keplerFields[fieldIndex];

    // geometry fields produced by DuckDB's st_asgeojson()
    if (fieldTypeSuggestion === 'JSON') {
      type = ALL_FIELD_TYPES.geojson;
      analyzerType = AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING;
    } else if (
      fieldTypeSuggestion === 'GEOMETRY' ||
      field.metadata.get(GEOARROW_METADATA_KEY)?.startsWith('geoarrow')
    ) {
      type = ALL_FIELD_TYPES.geoarrow;
      analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
    } else if (geoArrowMetadata[field.name]) {
      type = ALL_FIELD_TYPES.geoarrow;
      analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
      field.metadata?.set(GEOARROW_METADATA_KEY, geoArrowMetadata[field.name]);
    } else if (fieldTypeSuggestion === 'BLOB') {
      // When arrow wkb column saved to DuckDB as BLOB without any metadata, then queried back
      try {
        const data = table.getChildAt(fieldIndex)?.get(0);
        if (data) {
          const binaryGeo = parseSync(data, WKBLoader);
          if (binaryGeo) {
            type = ALL_FIELD_TYPES.geoarrow;
            analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
            field.metadata?.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB);
          }
        }
      } catch (error) {
        // ignore, not WKB
      }
    } else if (
      fieldTypeSuggestion === 'VARCHAR' &&
      (keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY ||
        keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING)
    ) {
      // When wkb/wkt was saved as varchar in DuckDB
      type = keplerField.type;
      analyzerType = keplerField.analyzerType;
      format = keplerField.format;
    } else if (fieldTypeSuggestion === 'VARCHAR' && keplerField.type === ALL_FIELD_TYPES.h3) {
      // when kepler detected h3 column using getFieldsFromData(), set type to h3 and analyzerType to H3
      type = ALL_FIELD_TYPES.h3;
      analyzerType = keplerField.analyzerType;
    } else {
      // TODO should we use Kepler getFieldsFromData instead
      // of arrowDataTypeToFieldType for all fields?
      if (keplerField.type === ALL_FIELD_TYPES.timestamp) {
        type = keplerField.type;
        analyzerType = keplerField.analyzerType;
        format = keplerField.format;
      }
    }

    return {
      ...field,
      name: field.name,
      id: field.name,
      displayName: field.name,
      format: format,
      fieldIdx: fieldIndex,
      type,
      analyzerType,
      valueAccessor: (dc: any) => d => {
        return dc.valueAt(d.index, fieldIndex);
      },
      metadata: field.metadata
    };
  });
}

/**
 * Parse arrow batches returned from parseInBatches()
 *
 * @param arrowTable the arrow table to parse
 * @returns dataset containing `fields` and `rows` or null
 */
export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null {
  if (arrowBatches.length === 0) {
    return null;
  }
  const arrowTable = new arrow.Table(arrowBatches);
  const fields = arrowSchemaToFields(arrowTable);

  const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i));

  // return empty rows and use raw arrow table to construct column-wise data container
  return {
    fields,
    rows: [],
    cols,
    metadata: arrowTable.schema.metadata,
    // Save original arrow schema, for better ingestion into DuckDB.
    // TODO consider returning arrowTable in cols, not an array of Vectors from arrowTable.
    arrowSchema: arrowTable.schema
  };
}

export const DATASET_HANDLERS = {
  [DATASET_FORMATS.row]: processRowObject,
  [DATASET_FORMATS.geojson]: processGeojson,
  [DATASET_FORMATS.csv]: processCsvData,
  [DATASET_FORMATS.arrow]: processArrowTable,
  [DATASET_FORMATS.keplergl]: processKeplerglDataset
};

export const Processors: {
  processGeojson: typeof processGeojson;
  processCsvData: typeof processCsvData;
  processArrowTable: typeof processArrowTable;
  processArrowBatches: typeof processArrowBatches;
  processRowObject: typeof processRowObject;
  processKeplerglJSON: typeof processKeplerglJSON;
  processKeplerglDataset: typeof processKeplerglDataset;
  analyzerTypeToFieldType: typeof analyzerTypeToFieldType;
  getFieldsFromData: typeof getFieldsFromData;
  parseCsvRowsByFieldType: typeof parseCsvRowsByFieldType;
} = {
  processGeojson,
  processCsvData,
  processArrowTable,
  processArrowBatches,
  processRowObject,
  processKeplerglJSON,
  processKeplerglDataset,
  analyzerTypeToFieldType,
  getFieldsFromData,
  parseCsvRowsByFieldType
};

1	// SPDX-License-Identifier: MIT
2	// Copyright contributors to the kepler.gl project
3
4	import * as arrow from 'apache-arrow';
5	import {csvParseRows} from 'd3-dsv';
6	import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
7	import normalize from '@mapbox/geojson-normalize';
8	import {parseSync} from '@loaders.gl/core';
9	import {ArrowTable} from '@loaders.gl/schema';
10	import {WKBLoader} from '@loaders.gl/wkt';
11
12	import {
13	ALL_FIELD_TYPES,
14	DATASET_FORMATS,
15	GEOARROW_EXTENSIONS,
16	GEOARROW_METADATA_KEY,
17	GUIDES_FILE_FORMAT_DOC
18	} from '@kepler.gl/constants';
19	import {ProcessorResult, Field} from '@kepler.gl/types';
20	import {
21	arrowDataTypeToAnalyzerDataType,
22	arrowDataTypeToFieldType,
23	hasOwnProperty,
24	isPlainObject
25	} from '@kepler.gl/utils';
26	import {
27	analyzerTypeToFieldType,
28	getSampleForTypeAnalyze,
29	getSampleForTypeAnalyzeArrow,
30	getFieldsFromData,
31	h3IsValid,
32	notNullorUndefined,
33	toArray
34	} from '@kepler.gl/common-utils';
35	import {KeplerGlSchema, ParsedDataset, SavedMap, LoadedMap} from '@kepler.gl/schemas';
36	import {Feature} from '@nebula.gl/edit-modes';
37
38	// if any of these value occurs in csv, parse it to null;
39	// const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
40	// matches empty string
41	export const CSV_NULLS = /^(null\|NULL\|Null\|NaN\|\/N\|\|)$/;	13✔
42
43	function tryParseJsonString(str) {
44	try {	31✔
45	return JSON.parse(str);	31✔
46	} catch (e) {
47	return null;	×
48	}
49	}
50
51	export const PARSE_FIELD_VALUE_FROM_STRING = {	13✔
52	[ALL_FIELD_TYPES.boolean]: {
53	valid: (d: unknown): boolean => typeof d === 'boolean',	29✔
54	parse: (d: unknown): boolean => d === 'true' \|\| d === 'True' \|\| d === 'TRUE' \|\| d === '1'	359✔
55	},
56	[ALL_FIELD_TYPES.integer]: {
57	// @ts-ignore
58	valid: (d: unknown): boolean => parseInt(d, 10) === d,	121✔
59	// @ts-ignore
60	parse: (d: unknown): number => parseInt(d, 10)	494✔
61	},
62	[ALL_FIELD_TYPES.timestamp]: {
63	valid: (d: unknown, field: Field): boolean =>
64	['x', 'X'].includes(field.format) ? typeof d === 'number' : typeof d === 'string',	100✔
65	parse: (d: any, field: Field) => (['x', 'X'].includes(field.format) ? Number(d) : d)	362!
66	},
67	[ALL_FIELD_TYPES.real]: {
68	// @ts-ignore
69	valid: (d: unknown): boolean => parseFloat(d) === d,	124✔
70	// Note this will result in NaN for some string
71	parse: parseFloat
72	},
73	[ALL_FIELD_TYPES.object]: {
74	valid: isPlainObject,
75	parse: tryParseJsonString
76	},
77
78	[ALL_FIELD_TYPES.array]: {
79	valid: Array.isArray,
80	parse: tryParseJsonString
81	},
82
83	[ALL_FIELD_TYPES.h3]: {
84	valid: d => h3IsValid(d),	15✔
85	parse: d => d	×
86	}
87	};
88
89	/**
90	* Process csv data, output a data object with `{fields: [], rows: []}`.
91	* The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
92	* @param rawData raw csv string
93	* @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
94	* @public
95	* @example
96	* import {processCsvData} from '@kepler.gl/processors';
97	*
98	* const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
99	* 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
100	* 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
101	* 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
102	* 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
103	*
104	* const dataset = {
105	* info: {id: 'test_data', label: 'My Csv'},
106	* data: processCsvData(testData)
107	* };
108	*
109	* dispatch(addDataToMap({
110	* datasets: [dataset],
111	* options: {centerMap: true, readOnly: true}
112	* }));
113	*/
114	export function processCsvData(rawData: unknown[][] \| string, header?: string[]): ProcessorResult {
115	let rows: unknown[][] \| undefined;
116	let headerRow: string[] \| undefined;
117
118	if (typeof rawData === 'string') {	75✔
119	const parsedRows: string[][] = csvParseRows(rawData);	39✔
120
121	if (!Array.isArray(parsedRows) \|\| parsedRows.length < 2) {	39✔
122	// looks like an empty file, throw error to be catch
123	throw new Error('process Csv Data Failed: CSV is empty');	1✔
124	}
125	headerRow = parsedRows[0];	38✔
126	rows = parsedRows.slice(1);	38✔
127	} else if (Array.isArray(rawData) && rawData.length) {	36!
128	rows = rawData;	36✔
129	headerRow = header;	36✔
130
131	if (!Array.isArray(headerRow)) {	36!
132	// if data is passed in as array of rows and missing header
133	// assume first row is header
134	// @ts-ignore
135	headerRow = rawData[0];	×
136	rows = rawData.slice(1);	×
137	}
138	}
139
140	if (!rows \|\| !headerRow) {	74!
141	throw new Error('invalid input passed to processCsvData');	×
142	}
143
144	// here we assume the csv file that people uploaded will have first row
145	// as name of the column
146
147	cleanUpFalsyCsvValue(rows);	74✔
148	// No need to run type detection on every data point
149	// here we get a list of none null values to run analyze on
150	const sample = getSampleForTypeAnalyze({fields: headerRow, rows});	74✔
151	const fields = getFieldsFromData(sample, headerRow);	74✔
152	const parsedRows = parseRowsByFields(rows, fields);	74✔
153
154	return {fields, rows: parsedRows};	74✔
155	}
156
157	/**
158	* Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
159	* @param rows
160	* @param fields
161	*/
162	export function parseRowsByFields(rows: any[][], fields: Field[]) {
163	// Edit rows in place
164	const geojsonFieldIdx = fields.findIndex(f => f.name === '_geojson');	439✔
165	fields.forEach(parseCsvRowsByFieldType.bind(null, rows, geojsonFieldIdx));	74✔
166
167	return rows;	74✔
168	}
169
170	/**
171	* Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,
172	* so that type-analyzer won't detect it as string
173	*
174	* @param rows
175	*/
176	function cleanUpFalsyCsvValue(rows: unknown[][]): void {
177	const re = new RegExp(CSV_NULLS, 'g');	110✔
178	for (let i = 0; i < rows.length; i++) {	110✔
179	for (let j = 0; j < rows[i].length; j++) {	1,036✔
180	// analyzer will set any fields to 'string' if there are empty values
181	// which will be parsed as '' by d3.csv
182	// here we parse empty data as null
183	// TODO: create warning when deltect `CSV_NULLS` in the data
184	if (typeof rows[i][j] === 'string' && (rows[i][j] as string).match(re)) {	8,486✔
185	rows[i][j] = null;	905✔
186	}
187	}
188	}
189	}
190
191	/**
192	* Process uploaded csv file to parse value by field type
193	*
194	* @param rows
195	* @param geoFieldIdx field index
196	* @param field
197	* @param i
198	*/
199	export function parseCsvRowsByFieldType(
200	rows: unknown[][],
201	geoFieldIdx: number,
202	field: Field,
203	i: number
204	): void {
205	const parser = PARSE_FIELD_VALUE_FROM_STRING[field.type];	563✔
206	if (parser) {	563✔
207	// check first not null value of it's already parsed
208	const first = rows.find(r => notNullorUndefined(r[i]));	441✔
209	if (!first \|\| parser.valid(first[i], field)) {	418✔
210	return;	210✔
211	}
212	rows.forEach(row => {	208✔
213	// parse string value based on field type
214	if (row[i] !== null) {	2,938✔
215	row[i] = parser.parse(row[i], field);	2,635✔
216	if (	2,635✔
217	geoFieldIdx > -1 &&	2,653✔
218	isPlainObject(row[geoFieldIdx]) &&
219	// @ts-ignore
220	hasOwnProperty(row[geoFieldIdx], 'properties')
221	) {
222	// @ts-ignore
223	row[geoFieldIdx].properties[field.name] = row[i];	9✔
224	}
225	}
226	});
227	}
228	}
229
230	/* eslint-enable complexity */
231
232	/**
233	* Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
234	* NOTE: This function may mutate input.
235	* @param rawData an array of row object, each object should have the same number of keys
236	* @returns dataset containing `fields` and `rows`
237	* @public
238	* @example
239	* import {addDataToMap} from '@kepler.gl/actions';
240	* import {processRowObject} from '@kepler.gl/processors';
241	*
242	* const data = [
243	* {lat: 31.27, lng: 127.56, value: 3},
244	* {lat: 31.22, lng: 126.26, value: 1}
245	* ];
246	*
247	* dispatch(addDataToMap({
248	* datasets: {
249	* info: {label: 'My Data', id: 'my_data'},
250	* data: processRowObject(data)
251	* }
252	* }));
253	*/
254	export function processRowObject(rawData: unknown[]): ProcessorResult {
255	if (!Array.isArray(rawData)) {	37✔
256	return null;	1✔
257	} else if (!rawData.length) {	36!
258	// data is empty
259	return {	×
260	fields: [],
261	rows: []
262	};
263	}
264
265	const firstRow = rawData[0] as Record<string, unknown>;	36✔
266	const keys = Object.keys(firstRow); // [lat, lng, value]	36✔
267	const rows = rawData.map(d => keys.map(key => (d as Record<string, unknown>)[key])); // [[31.27, 127.56, 3]]	1,551✔
268
269	// row object can still contain values like `Null` or `N/A`
270	cleanUpFalsyCsvValue(rows);	36✔
271
272	return processCsvData(rows, keys);	36✔
273	}
274
275	/**
276	* Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
277	* output a data object with `{fields: [], rows: []}`.
278	* The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
279	* NOTE: This function may mutate input.
280	*
281	* @param rawData raw geojson feature collection
282	* @returns dataset containing `fields` and `rows`
283	* @public
284	* @example
285	* import {addDataToMap} from '@kepler.gl/actions';
286	* import {processGeojson} from '@kepler.gl/processors';
287	*
288	* const geojson = {
289	* "type" : "FeatureCollection",
290	* "features" : [{
291	* "type" : "Feature",
292	* "properties" : {
293	* "capacity" : "10",
294	* "type" : "U-Rack"
295	* },
296	* "geometry" : {
297	* "type" : "Point",
298	* "coordinates" : [ -71.073283, 42.417500 ]
299	* }
300	* }]
301	* };
302	*
303	* dispatch(addDataToMap({
304	* datasets: {
305	* info: {
306	* label: 'Sample Taxi Trips in New York City',
307	* id: 'test_trip_data'
308	* },
309	* data: processGeojson(geojson)
310	* }
311	* }));
312	*/
313	export function processGeojson(rawData: unknown): ProcessorResult {
314	const normalizedGeojson = normalize(rawData);	28✔
315
316	if (!normalizedGeojson \|\| !Array.isArray(normalizedGeojson.features)) {	28✔
317	throw new Error(	1✔
318	`Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](${GUIDES_FILE_FORMAT_DOC})`
319	);
320	}
321
322	// getting all feature fields
323	const allDataRows: Array<{_geojson: Feature} & keyof Feature> = [];	27✔
324	for (let i = 0; i < normalizedGeojson.features.length; i++) {	27✔
325	const f = normalizedGeojson.features[i];	160✔
326	if (f.geometry) {	160!
327	allDataRows.push({	160✔
328	// add feature to _geojson field
329	_geojson: f,
330	...(f.properties \|\| {})	161✔
331	});
332	}
333	}
334	// get all the field
335	const fields = allDataRows.reduce<string[]>((accu, curr) => {	27✔
336	Object.keys(curr).forEach(key => {	160✔
337	if (!accu.includes(key)) {	807✔
338	accu.push(key);	148✔
339	}
340	});
341	return accu;	160✔
342	}, []);
343
344	// make sure each feature has exact same fields
345	allDataRows.forEach(d => {	27✔
346	fields.forEach(f => {	160✔
347	if (!(f in d)) {	860✔
348	d[f] = null;	53✔
349	if (d._geojson.properties) {	53!
350	d._geojson.properties[f] = null;	53✔
351	}
352	}
353	});
354	});
355
356	return processRowObject(allDataRows);	27✔
357	}
358
359	/**
360	* Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
361	* The json object should contain `datasets` and `config`.
362	* @param rawData
363	* @param schema
364	* @returns datasets and config `{datasets: {}, config: {}}`
365	* @public
366	* @example
367	* import {addDataToMap} from '@kepler.gl/actions';
368	* import {processKeplerglJSON} from '@kepler.gl/processors';
369	*
370	* dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
371	*/
372	export function processKeplerglJSON(rawData: SavedMap, schema = KeplerGlSchema): LoadedMap \| null {	5✔
373	return rawData ? schema.load(rawData.datasets, rawData.config) : null;	5!
374	}
375
376	/**
377	* Parse a single or an array of datasets saved using kepler.gl schema
378	* @param rawData
379	* @param schema
380	*/
381	export function processKeplerglDataset(
382	rawData: object \| object[],
383	schema = KeplerGlSchema	×
384	): ParsedDataset \| ParsedDataset[] \| null {
385	if (!rawData) {	×
386	return null;	×
387	}
388
389	const results = schema.parseSavedData(toArray(rawData));	×
390	if (!results) {	×
391	return null;	×
392	}
393	return Array.isArray(rawData) ? results : results[0];	×
394	}
395
396	/**
397	* Parse arrow table and return a dataset
398	*
399	* @param arrowTable ArrowTable to parse, see loaders.gl/schema
400	* @returns dataset containing `fields` and `rows` or null
401	*/
402	export function processArrowTable(arrowTable: ArrowTable): ProcessorResult \| null {
403	// @ts-ignore - Unknown data type causing build failures
404	return processArrowBatches(arrowTable.data.batches);	×
405	}
406
407	/**
408	* Extracts GeoArrow metadata from an Apache Arrow table schema.
409	* For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
410	* @param table The Apache Arrow table to extract metadata from.
411	* @returns An object mapping column names to their GeoArrow encoding type.
412	* @throws Logs an error message if parsing of metadata fails.
413	*/
414	export function getGeoArrowMetadataFromSchema(table: arrow.Table): Record<string, string> {
415	const geoArrowMetadata: Record<string, string> = {};	×
416	try {	×
417	const geoString = table.schema.metadata?.get('geo');	×
418	if (geoString) {	×
419	const parsedGeoString = JSON.parse(geoString);	×
420	if (parsedGeoString.columns) {	×
421	Object.keys(parsedGeoString.columns).forEach(columnName => {	×
422	const columnData = parsedGeoString.columns[columnName];	×
423	if (columnData?.encoding === 'WKB') {	×
424	geoArrowMetadata[columnName] = GEOARROW_EXTENSIONS.WKB;	×
425	}
426	// TODO potentially there are other types but no datasets to test
427	});
428	}
429	}
430	} catch (error) {
431	console.error('An error during arrow table schema metadata parsing');	×
432	}
433	return geoArrowMetadata;	×
434	}
435
436	/**
437	* Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
438	* @param table The Apache Arrow table whose schema needs to be converted.
439	* @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
440	* @returns An array of field objects suitable for Kepler.gl.
441	*/
442	export function arrowSchemaToFields(
443	table: arrow.Table,
444	fieldTypeSuggestions: Record<string, string> = {}	×
445	): Field[] {
446	const headerRow = table.schema.fields.map(f => f.name);	×
447	const sample = getSampleForTypeAnalyzeArrow(table, headerRow);	×
448	const keplerFields = getFieldsFromData(sample, headerRow);	×
449	const geoArrowMetadata = getGeoArrowMetadataFromSchema(table);	×
450
451	return table.schema.fields.map((field: arrow.Field, fieldIndex: number) => {	×
452	let type = arrowDataTypeToFieldType(field.type);	×
453	let analyzerType = arrowDataTypeToAnalyzerDataType(field.type);	×
454	let format = '';	×
455
456	const fieldTypeSuggestion = fieldTypeSuggestions[field.name];	×
457	const keplerField = keplerFields[fieldIndex];	×
458
459	// geometry fields produced by DuckDB's st_asgeojson()
460	if (fieldTypeSuggestion === 'JSON') {	×
461	type = ALL_FIELD_TYPES.geojson;	×
462	analyzerType = AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING;	×
463	} else if (	×
464	fieldTypeSuggestion === 'GEOMETRY' \|\|	×
465	field.metadata.get(GEOARROW_METADATA_KEY)?.startsWith('geoarrow')
466	) {
467	type = ALL_FIELD_TYPES.geoarrow;	×
468	analyzerType = AnalyzerDATA_TYPES.GEOMETRY;	×
469	} else if (geoArrowMetadata[field.name]) {	×
470	type = ALL_FIELD_TYPES.geoarrow;	×
471	analyzerType = AnalyzerDATA_TYPES.GEOMETRY;	×
472	field.metadata?.set(GEOARROW_METADATA_KEY, geoArrowMetadata[field.name]);	×
473	} else if (fieldTypeSuggestion === 'BLOB') {	×
474	// When arrow wkb column saved to DuckDB as BLOB without any metadata, then queried back
475	try {	×
476	const data = table.getChildAt(fieldIndex)?.get(0);	×
477	if (data) {	×
478	const binaryGeo = parseSync(data, WKBLoader);	×
479	if (binaryGeo) {	×
480	type = ALL_FIELD_TYPES.geoarrow;	×
481	analyzerType = AnalyzerDATA_TYPES.GEOMETRY;	×
482	field.metadata?.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB);	×
483	}
484	}
485	} catch (error) {
486	// ignore, not WKB
487	}
488	} else if (	×
489	fieldTypeSuggestion === 'VARCHAR' &&	×
490	(keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY \|\|
491	keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING)
492	) {
493	// When wkb/wkt was saved as varchar in DuckDB
494	type = keplerField.type;	×
495	analyzerType = keplerField.analyzerType;	×
496	format = keplerField.format;	×
497	} else if (fieldTypeSuggestion === 'VARCHAR' && keplerField.type === ALL_FIELD_TYPES.h3) {	×
498	// when kepler detected h3 column using getFieldsFromData(), set type to h3 and analyzerType to H3
499	type = ALL_FIELD_TYPES.h3;	×
500	analyzerType = keplerField.analyzerType;	×
501	} else {
502	// TODO should we use Kepler getFieldsFromData instead
503	// of arrowDataTypeToFieldType for all fields?
504	if (keplerField.type === ALL_FIELD_TYPES.timestamp) {	×
505	type = keplerField.type;	×
506	analyzerType = keplerField.analyzerType;	×
507	format = keplerField.format;	×
508	}
509	}
510
511	return {	×
512	...field,
513	name: field.name,
514	id: field.name,
515	displayName: field.name,
516	format: format,
517	fieldIdx: fieldIndex,
518	type,
519	analyzerType,
520	valueAccessor: (dc: any) => d => {	×
521	return dc.valueAt(d.index, fieldIndex);	×
522	},
523	metadata: field.metadata
524	};
525	});
526	}
527
528	/**
529	* Parse arrow batches returned from parseInBatches()
530	*
531	* @param arrowTable the arrow table to parse
532	* @returns dataset containing `fields` and `rows` or null
533	*/
534	export function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult \| null {
535	if (arrowBatches.length === 0) {	×
536	return null;	×
537	}
538	const arrowTable = new arrow.Table(arrowBatches);	×
539	const fields = arrowSchemaToFields(arrowTable);	×
540
541	const cols = [...Array(arrowTable.numCols).keys()].map(i => arrowTable.getChildAt(i));	×
542
543	// return empty rows and use raw arrow table to construct column-wise data container
544	return {	×
545	fields,
546	rows: [],
547	cols,
548	metadata: arrowTable.schema.metadata,
549	// Save original arrow schema, for better ingestion into DuckDB.
550	// TODO consider returning arrowTable in cols, not an array of Vectors from arrowTable.
551	arrowSchema: arrowTable.schema
552	};
553	}
554
555	export const DATASET_HANDLERS = {	13✔
556	[DATASET_FORMATS.row]: processRowObject,
557	[DATASET_FORMATS.geojson]: processGeojson,
558	[DATASET_FORMATS.csv]: processCsvData,
559	[DATASET_FORMATS.arrow]: processArrowTable,
560	[DATASET_FORMATS.keplergl]: processKeplerglDataset
561	};
562
563	export const Processors: {
564	processGeojson: typeof processGeojson;
565	processCsvData: typeof processCsvData;
566	processArrowTable: typeof processArrowTable;
567	processArrowBatches: typeof processArrowBatches;
568	processRowObject: typeof processRowObject;
569	processKeplerglJSON: typeof processKeplerglJSON;
570	processKeplerglDataset: typeof processKeplerglDataset;
571	analyzerTypeToFieldType: typeof analyzerTypeToFieldType;
572	getFieldsFromData: typeof getFieldsFromData;
573	parseCsvRowsByFieldType: typeof parseCsvRowsByFieldType;
574	} = {	13✔
575	processGeojson,
576	processCsvData,
577	processArrowTable,
578	processArrowBatches,
579	processRowObject,
580	processKeplerglJSON,
581	processKeplerglDataset,
582	analyzerTypeToFieldType,
583	getFieldsFromData,
584	parseCsvRowsByFieldType
585	};

keplergl / kepler.gl / 22361650031

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous