26033117099

Committed 18 May 2026 12:20PM UTC coverage: 74.769% (-0.1%) from 74.887%

Build # 26033117099

Build Type

push

github

Committed by

web-flow

Commit Message

feat(arrow) Streaming ArrowTextLayer (#2620)

Coverage Stats

6882 of 10396 branches covered (66.2%)

Branch coverage included in aggregate %.

194 of 250 new or added lines in 9 files covered. (77.6%)

13 existing lines in 7 files now uncovered.

15038 of 18921 relevant lines covered (79.48%)

914.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.07

/modules/arrow/src/arrow/arrow-paths.ts

// luma.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import * as arrow from 'apache-arrow';

/** Returns all leaf column paths in an Arrow object, using dot notation for nested structs. */
export function getArrowPaths(
  arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector
): string[] {
  const data = getArrowDataArray(arrowObject)[0];
  return getArrowPathsRecursive(data, []);
}

/** Returns all leaf field paths in an Arrow schema, using dot notation for nested structs. */
export function getArrowSchemaPaths(schema: arrow.Schema): string[] {
  return getArrowSchemaPathsRecursive(schema.fields, []);
}

/** Recursively returns all leaf paths below an Arrow data node. */
export function getArrowPathsRecursive(arrowData: arrow.Data, currentPath: string[]): string[] {
  if (!arrow.DataType.isStruct(arrowData.type)) {
    return [currentPath.join('.')];
  }

  const fields = arrowData.type.children;
  const nestedPaths: any[] = [];
  for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) {
    const field = fields[fieldIndex];
    const fieldData = arrowData.children[fieldIndex];
    const fieldPath = [...currentPath, field.name];
    const paths = getArrowPathsRecursive(fieldData, fieldPath);
    nestedPaths.push(...paths);
  }

  return nestedPaths;
}

/** Returns the schema leaf field at a dot-separated path, or `null` when it cannot resolve. */
export function findArrowFieldByPath(
  schemaOrTable: arrow.Schema | arrow.Table,
  columnPath: string
): arrow.Field | null {
  const schema = schemaOrTable instanceof arrow.Table ? schemaOrTable.schema : schemaOrTable;
  const path = decomposePath(columnPath);
  let fields = schema.fields;
  let resolvedField: arrow.Field | null = null;

  for (let pathIndex = 0; pathIndex < path.length; pathIndex++) {
    const key = path[pathIndex];
    const isLeafField = pathIndex === path.length - 1;
    resolvedField = fields.find(field => field.name === key) ?? null;
    if (!resolvedField) {
      return null;
    }
    if (!isLeafField) {
      if (!arrow.DataType.isStruct(resolvedField.type)) {
        return null;
      }
      fields = resolvedField.type.children;
    }
  }

  return resolvedField && !arrow.DataType.isStruct(resolvedField.type) ? resolvedField : null;
}

/** Returns the Arrow data node at a dot-separated column path. */
export function getArrowDataByPath(
  arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector,
  columnPath: string
): arrow.Data {
  const data = getArrowDataArray(arrowObject)[0];

  const path = decomposePath(columnPath);
  let nestedData = data;
  for (const key of path) {
    if (!arrow.DataType.isStruct(nestedData.type)) {
      throw new Error(
        `Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
      );
    }
    const fields = nestedData.type.children;
    const indexByField = fields.findIndex(field => field.name === key);
    if (indexByField === -1) {
      throw new Error(
        `Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
      );
    }

    nestedData = nestedData.children[indexByField];
  }

  // Check that we resolved all the intermediate structs
  if (arrow.DataType.isStruct(nestedData.type)) {
    throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);
  }

  return nestedData;
}

/** Returns the Arrow vector at a dot-separated table column path. */
export function getArrowVectorByPath(arrowTable: arrow.Table, columnPath: string): arrow.Vector {
  // Make a temporary vector from the top level struct data.
  const vector = arrow.makeVector(arrowTable.data);

  const path = decomposePath(columnPath);
  let nestedVector = vector;
  for (const key of path) {
    if (!arrow.DataType.isStruct(nestedVector.type)) {
      throw new Error(
        `Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
      );
    }
    const fields = nestedVector.type.children;
    const indexByField = fields.findIndex(field => field.name === key);
    if (indexByField === -1) {
      throw new Error(
        `Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
      );
    }

    nestedVector = nestedVector.getChildAt(indexByField)!;
  }

  // Check that we resolved all the intermediate structs
  if (arrow.DataType.isStruct(nestedVector.type)) {
    throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);
  }

  return nestedVector;
}

/** Returns the Arrow schema field at a dot-separated table column path. */
export function getArrowFieldByPath(arrowTable: arrow.Table, columnPath: string): arrow.Field {
  const path = decomposePath(columnPath);
  let fields = arrowTable.schema.fields;
  let resolvedField: arrow.Field | null = null;

  for (let pathIndex = 0; pathIndex < path.length; pathIndex++) {
    const key = path[pathIndex];
    const isLeafField = pathIndex === path.length - 1;
    const indexByField = fields.findIndex(field => field.name === key);
    if (indexByField === -1) {
      throw new Error(
        `Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
      );
    }

    resolvedField = fields[indexByField];
    if (!isLeafField) {
      if (!arrow.DataType.isStruct(resolvedField.type)) {
        throw new Error(
          `Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
        );
      }
      fields = resolvedField.type.children;
    }
  }

  if (!resolvedField || arrow.DataType.isStruct(resolvedField.type)) {
    throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);
  }

  return resolvedField;
}

/** Returns the data chunks contained by an Arrow object. */
export function getArrowDataArray(
  arrowObject: arrow.Data | arrow.Table | arrow.RecordBatch | arrow.Vector
): arrow.Data[] {
  if (arrowObject instanceof arrow.Table) {
    return arrowObject.data;
  } else if (arrowObject instanceof arrow.RecordBatch) {
    return [arrowObject.data];
  } else if (arrowObject instanceof arrow.Vector) {
    // @ts-expect-error for some reason read-only in this context
    return arrowObject.data;
  }
  return [arrowObject];
}

// HELPER FUNCTIONS

function decomposePath(path: string): string[] {
  return path.split('.');
}

function getArrowSchemaPathsRecursive(fields: arrow.Field[], currentPath: string[]): string[] {
  const paths: string[] = [];
  for (const field of fields) {
    const fieldPath = [...currentPath, field.name];
    if (arrow.DataType.isStruct(field.type)) {
      paths.push(...getArrowSchemaPathsRecursive(field.type.children, fieldPath));
    } else {
      paths.push(fieldPath.join('.'));
    }
  }
  return paths;
}

1	// luma.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import * as arrow from 'apache-arrow';
6
7	/** Returns all leaf column paths in an Arrow object, using dot notation for nested structs. */
8	export function getArrowPaths(
9	arrowObject: arrow.Data \| arrow.Table \| arrow.RecordBatch \| arrow.Vector
10	): string[] {
11	const data = getArrowDataArray(arrowObject)[0];	86✔
12	return getArrowPathsRecursive(data, []);	86✔
13	}
14
15	/** Returns all leaf field paths in an Arrow schema, using dot notation for nested structs. */
16	export function getArrowSchemaPaths(schema: arrow.Schema): string[] {
17	return getArrowSchemaPathsRecursive(schema.fields, []);	3✔
18	}
19
20	/** Recursively returns all leaf paths below an Arrow data node. */
21	export function getArrowPathsRecursive(arrowData: arrow.Data, currentPath: string[]): string[] {
22	if (!arrow.DataType.isStruct(arrowData.type)) {	248✔
23	return [currentPath.join('.')];	158✔
24	}
25
26	const fields = arrowData.type.children;	90✔
27	const nestedPaths: any[] = [];	90✔
28	for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) {	90✔
29	const field = fields[fieldIndex];	162✔
30	const fieldData = arrowData.children[fieldIndex];	162✔
31	const fieldPath = [...currentPath, field.name];	162✔
32	const paths = getArrowPathsRecursive(fieldData, fieldPath);	162✔
33	nestedPaths.push(...paths);	162✔
34	}
35
36	return nestedPaths;	90✔
37	}
38
39	/** Returns the schema leaf field at a dot-separated path, or `null` when it cannot resolve. */
40	export function findArrowFieldByPath(
41	schemaOrTable: arrow.Schema \| arrow.Table,
42	columnPath: string
43	): arrow.Field \| null {
44	const schema = schemaOrTable instanceof arrow.Table ? schemaOrTable.schema : schemaOrTable;	16!
45	const path = decomposePath(columnPath);	16✔
46	let fields = schema.fields;	16✔
47	let resolvedField: arrow.Field \| null = null;	16✔
48
49	for (let pathIndex = 0; pathIndex < path.length; pathIndex++) {	16✔
50	const key = path[pathIndex];	16✔
51	const isLeafField = pathIndex === path.length - 1;	16✔
52	resolvedField = fields.find(field => field.name === key) ?? null;	24!
53	if (!resolvedField) {	16!
UNCOV 54	return null;	×
55	}
56	if (!isLeafField) {	16!
57	if (!arrow.DataType.isStruct(resolvedField.type)) {	×
58	return null;	×
59	}
60	fields = resolvedField.type.children;	×
61	}
62	}
63
64	return resolvedField && !arrow.DataType.isStruct(resolvedField.type) ? resolvedField : null;	16!
65	}
66
67	/** Returns the Arrow data node at a dot-separated column path. */
68	export function getArrowDataByPath(
69	arrowObject: arrow.Data \| arrow.Table \| arrow.RecordBatch \| arrow.Vector,
70	columnPath: string
71	): arrow.Data {
72	const data = getArrowDataArray(arrowObject)[0];	12✔
73
74	const path = decomposePath(columnPath);	12✔
75	let nestedData = data;	12✔
76	for (const key of path) {	12✔
77	if (!arrow.DataType.isStruct(nestedData.type)) {	13!
78	throw new Error(	×
79	`Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
80	);
81	}
82	const fields = nestedData.type.children;	13✔
83	const indexByField = fields.findIndex(field => field.name === key);	19✔
84	if (indexByField === -1) {	13!
85	throw new Error(	×
86	`Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
87	);
88	}
89
90	nestedData = nestedData.children[indexByField];	13✔
91	}
92
93	// Check that we resolved all the intermediate structs
94	if (arrow.DataType.isStruct(nestedData.type)) {	12!
95	throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);	×
96	}
97
98	return nestedData;	12✔
99	}
100
101	/** Returns the Arrow vector at a dot-separated table column path. */
102	export function getArrowVectorByPath(arrowTable: arrow.Table, columnPath: string): arrow.Vector {
103	// Make a temporary vector from the top level struct data.
104	const vector = arrow.makeVector(arrowTable.data);	250✔
105
106	const path = decomposePath(columnPath);	250✔
107	let nestedVector = vector;	250✔
108	for (const key of path) {	250✔
109	if (!arrow.DataType.isStruct(nestedVector.type)) {	256!
110	throw new Error(	×
111	`Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
112	);
113	}
114	const fields = nestedVector.type.children;	256✔
115	const indexByField = fields.findIndex(field => field.name === key);	359✔
116	if (indexByField === -1) {	256✔
117	throw new Error(	1✔
118	`Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
119	);
120	}
121
122	nestedVector = nestedVector.getChildAt(indexByField)!;	255✔
123	}
124
125	// Check that we resolved all the intermediate structs
126	if (arrow.DataType.isStruct(nestedVector.type)) {	249!
127	throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);	×
128	}
129
130	return nestedVector;	249✔
131	}
132
133	/** Returns the Arrow schema field at a dot-separated table column path. */
134	export function getArrowFieldByPath(arrowTable: arrow.Table, columnPath: string): arrow.Field {
135	const path = decomposePath(columnPath);	123✔
136	let fields = arrowTable.schema.fields;	123✔
137	let resolvedField: arrow.Field \| null = null;	123✔
138
139	for (let pathIndex = 0; pathIndex < path.length; pathIndex++) {	123✔
140	const key = path[pathIndex];	125✔
141	const isLeafField = pathIndex === path.length - 1;	125✔
142	const indexByField = fields.findIndex(field => field.name === key);	175✔
143	if (indexByField === -1) {	125!
144	throw new Error(	×
145	`Arrow table schema does not contain nested column '${key} in '${path.join('.')}'`
146	);
147	}
148
149	resolvedField = fields[indexByField];	125✔
150	if (!isLeafField) {	125✔
151	if (!arrow.DataType.isStruct(resolvedField.type)) {	2!
152	throw new Error(	×
153	`Arrow table nested column is a not a struct: '${key} in '${path.join('.')}'`
154	);
155	}
156	fields = resolvedField.type.children;	2✔
157	}
158	}
159
160	if (!resolvedField \|\| arrow.DataType.isStruct(resolvedField.type)) {	125!
161	throw new Error(`Arrow table nested column '${path.join('.')}' is a struct`);	×
162	}
163
164	return resolvedField;	123✔
165	}
166
167	/** Returns the data chunks contained by an Arrow object. */
168	export function getArrowDataArray(
169	arrowObject: arrow.Data \| arrow.Table \| arrow.RecordBatch \| arrow.Vector
170	): arrow.Data[] {
171	if (arrowObject instanceof arrow.Table) {	98✔
172	return arrowObject.data;	88✔
173	} else if (arrowObject instanceof arrow.RecordBatch) {	10!
174	return [arrowObject.data];	10✔
175	} else if (arrowObject instanceof arrow.Vector) {	×
176	// @ts-expect-error for some reason read-only in this context
177	return arrowObject.data;	×
178	}
179	return [arrowObject];	×
180	}
181
182	// HELPER FUNCTIONS
183
184	function decomposePath(path: string): string[] {
185	return path.split('.');	401✔
186	}
187
188	function getArrowSchemaPathsRecursive(fields: arrow.Field[], currentPath: string[]): string[] {
189	const paths: string[] = [];	3✔
190	for (const field of fields) {	3✔
191	const fieldPath = [...currentPath, field.name];	6✔
192	if (arrow.DataType.isStruct(field.type)) {	6!
193	paths.push(...getArrowSchemaPathsRecursive(field.type.children, fieldPath));	×
194	} else {
195	paths.push(fieldPath.join('.'));	6✔
196	}
197	}
198	return paths;	3✔
199	}

visgl / luma.gl / 26033117099

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous