20352515932

Committed 18 Dec 2025 09:56PM UTC coverage: 35.115% (-28.4%) from 63.485%

Build # 20352515932

Build Type

push

github

Committed by

web-flow

Commit Message

feat(loader-utils): Export is-type helpers (#3258)

Coverage Stats

1188 of 1998 branches covered (59.46%)

Branch coverage included in aggregate %.

147 of 211 new or added lines in 13 files covered. (69.67%)

30011 existing lines in 424 files now uncovered.

37457 of 108056 relevant lines covered (34.66%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.82

/modules/schema-utils/src/lib/table/batch-builder/arrow-table-builder.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Schema, ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
import * as arrow from 'apache-arrow';
import {convertSchemaToArrow} from '@loaders.gl/schema-utils';

/** Builds an arrow table or batches */
export class ArrowTableBuilder {
  schema: Schema;
  arrowSchema: arrow.Schema;
  arrowBuilders: arrow.Builder[];
  length: number;

  constructor(schema: Schema) {
    this.schema = schema;
    this.arrowSchema = convertSchemaToArrow(schema);
    this.arrowBuilders = this.arrowSchema.fields.map((field) =>
      arrow.makeBuilder({type: field.type, nullValues: [null]})
    );
    this.length = 0;
  }

  addObjectRow(row: {[key: string]: any}) {
    for (let i = 0; i < this.arrowBuilders.length; i++) {
      const columnName = this.schema.fields[i].name;
      const value = row[columnName];
      // if (this.schema.fields[i].type.toString() === 'bool') {
      //   debugger;
      // }
      this.arrowBuilders[i].append(value);
    }
    this.length++;
  }

  addArrayRow(row: any[]) {
    for (let i = 0; i < this.arrowBuilders.length; i++) {
      this.arrowBuilders[i].append(row[i]);
    }
    this.length++;
  }

  /** Makes sure that a first batch with schema is sent even if no rows */
  firstBatch(): ArrowTableBatch | null {
    const arrowRecordBatch = this._getArrowRecordBatch();
    // If there is data, a batch will be sent later
    if (arrowRecordBatch.numCols !== 0) {
      return null;
    }
    return {
      shape: 'arrow-table',
      batchType: 'data',
      length: arrowRecordBatch.numRows,
      schema: this.schema,
      data: new arrow.Table(arrowRecordBatch)
    };
  }

  /** Flush the current batch if conditions are right */
  flushBatch(): ArrowTableBatch | null {
    const arrowRecordBatch = this._getArrowRecordBatch();
    if (arrowRecordBatch.numCols === 0) {
      return null;
    }
    return {
      shape: 'arrow-table',
      batchType: 'data',
      length: arrowRecordBatch.numRows,
      schema: this.schema,
      data: new arrow.Table(arrowRecordBatch)
    };
  }

  /** Get a last batch if any data is left */
  finishBatch(): ArrowTableBatch | null {
    const arrowRecordBatch = this._getArrowRecordBatch();
    this.arrowBuilders.forEach((builder) => builder.finish());
    if (arrowRecordBatch.numCols === 0) {
      return null;
    }
    return {
      shape: 'arrow-table',
      batchType: 'data',
      length: arrowRecordBatch.numRows,
      schema: this.schema,
      data: new arrow.Table(arrowRecordBatch)
    };
  }

  /** Return a table with all the accumulated data */
  finishTable(): ArrowTable {
    const arrowRecordBatch = this._getArrowRecordBatch();
    this.arrowBuilders.forEach((builder) => builder.finish());
    return {
      shape: 'arrow-table',
      schema: this.schema,
      data: new arrow.Table(arrowRecordBatch)
    };
  }

  /** Extract a record batch flushing the currently accumulated data in the builders */
  _getArrowRecordBatch(): arrow.RecordBatch {
    const {arrowBuilders, arrowSchema} = this;
    const arrowDatas = arrowBuilders.map((builder) => builder.flush());
    const length = arrowDatas[0].length;
    const structField = new arrow.Struct(arrowSchema.fields);
    const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);
    const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);
    return arrowRecordBatch;
  }
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {Schema, ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';	1✔
6	import * as arrow from 'apache-arrow';	1✔
7	import {convertSchemaToArrow} from '@loaders.gl/schema-utils';	1✔
8		1✔
9	/** Builds an arrow table or batches */	1✔
10	export class ArrowTableBuilder {	1✔
11	schema: Schema;	1!
UNCOV 12	arrowSchema: arrow.Schema;	×
UNCOV 13	arrowBuilders: arrow.Builder[];	×
UNCOV 14	length: number;	×
15		1✔
16	constructor(schema: Schema) {	1✔
UNCOV 17	this.schema = schema;	×
UNCOV 18	this.arrowSchema = convertSchemaToArrow(schema);	×
UNCOV 19	this.arrowBuilders = this.arrowSchema.fields.map((field) =>	×
UNCOV 20	arrow.makeBuilder({type: field.type, nullValues: [null]})	×
UNCOV 21	);	×
UNCOV 22	this.length = 0;	×
UNCOV 23	}	×
24		1✔
25	addObjectRow(row: {[key: string]: any}) {	1✔
UNCOV 26	for (let i = 0; i < this.arrowBuilders.length; i++) {	×
UNCOV 27	const columnName = this.schema.fields[i].name;	×
UNCOV 28	const value = row[columnName];	×
UNCOV 29	// if (this.schema.fields[i].type.toString() === 'bool') {	×
UNCOV 30	// debugger;	×
UNCOV 31	// }	×
UNCOV 32	this.arrowBuilders[i].append(value);	×
UNCOV 33	}	×
UNCOV 34	this.length++;	×
UNCOV 35	}	×
36		1✔
37	addArrayRow(row: any[]) {	1✔
38	for (let i = 0; i < this.arrowBuilders.length; i++) {	×
39	this.arrowBuilders[i].append(row[i]);	×
40	}	×
41	this.length++;	×
42	}	×
43		1✔
44	/** Makes sure that a first batch with schema is sent even if no rows */	1✔
45	firstBatch(): ArrowTableBatch \| null {	1✔
46	const arrowRecordBatch = this._getArrowRecordBatch();	×
47	// If there is data, a batch will be sent later	×
48	if (arrowRecordBatch.numCols !== 0) {	×
49	return null;	×
50	}	×
51	return {	×
52	shape: 'arrow-table',	×
53	batchType: 'data',	×
54	length: arrowRecordBatch.numRows,	×
55	schema: this.schema,	×
56	data: new arrow.Table(arrowRecordBatch)	×
57	};	×
58	}	×
59		1✔
60	/** Flush the current batch if conditions are right */	1✔
61	flushBatch(): ArrowTableBatch \| null {	1✔
62	const arrowRecordBatch = this._getArrowRecordBatch();	×
63	if (arrowRecordBatch.numCols === 0) {	×
64	return null;	×
65	}	×
66	return {	×
67	shape: 'arrow-table',	×
68	batchType: 'data',	×
69	length: arrowRecordBatch.numRows,	×
70	schema: this.schema,	×
71	data: new arrow.Table(arrowRecordBatch)	×
72	};	×
73	}	×
74		1✔
75	/** Get a last batch if any data is left */	1✔
76	finishBatch(): ArrowTableBatch \| null {	1✔
77	const arrowRecordBatch = this._getArrowRecordBatch();	×
78	this.arrowBuilders.forEach((builder) => builder.finish());	×
79	if (arrowRecordBatch.numCols === 0) {	×
80	return null;	×
81	}	×
82	return {	×
83	shape: 'arrow-table',	×
84	batchType: 'data',	×
85	length: arrowRecordBatch.numRows,	×
86	schema: this.schema,	×
87	data: new arrow.Table(arrowRecordBatch)	×
88	};	×
89	}	×
90		1✔
91	/** Return a table with all the accumulated data */	1✔
92	finishTable(): ArrowTable {	1✔
UNCOV 93	const arrowRecordBatch = this._getArrowRecordBatch();	×
UNCOV 94	this.arrowBuilders.forEach((builder) => builder.finish());	×
UNCOV 95	return {	×
UNCOV 96	shape: 'arrow-table',	×
UNCOV 97	schema: this.schema,	×
UNCOV 98	data: new arrow.Table(arrowRecordBatch)	×
UNCOV 99	};	×
UNCOV 100	}	×
101		1✔
102	/** Extract a record batch flushing the currently accumulated data in the builders */	1✔
103	_getArrowRecordBatch(): arrow.RecordBatch {	1✔
UNCOV 104	const {arrowBuilders, arrowSchema} = this;	×
UNCOV 105	const arrowDatas = arrowBuilders.map((builder) => builder.flush());	×
UNCOV 106	const length = arrowDatas[0].length;	×
UNCOV 107	const structField = new arrow.Struct(arrowSchema.fields);	×
UNCOV 108	const arrowStructData = new arrow.Data(structField, 0, length, 0, undefined, arrowDatas);	×
UNCOV 109	const arrowRecordBatch = new arrow.RecordBatch(arrowSchema, arrowStructData);	×
UNCOV 110	return arrowRecordBatch;	×
UNCOV 111	}	×
112	}	1✔

visgl / loaders.gl / 20352515932

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous