20352515932

Committed 18 Dec 2025 09:56PM UTC coverage: 35.115% (-28.4%) from 63.485%

Build # 20352515932

Build Type

push

github

Committed by

web-flow

Commit Message

feat(loader-utils): Export is-type helpers (#3258)

Coverage Stats

1188 of 1998 branches covered (59.46%)

Branch coverage included in aggregate %.

147 of 211 new or added lines in 13 files covered. (69.67%)

30011 existing lines in 424 files now uncovered.

37457 of 108056 relevant lines covered (34.66%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.68

/modules/schema-utils/src/lib/table/batch-builder/columnar-table-batch-aggregator.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Schema, ColumnarTableBatch, ArrowTableBatch, TypedArray} from '@loaders.gl/schema';
import {isTypedArray} from '@math.gl/types';
import {getArrayTypeFromDataType} from '../../schema/data-type';
import {TableBatchAggregator} from './table-batch-aggregator';
type ColumnarTableBatchOptions = {};

const DEFAULT_ROW_COUNT = 100;

export class ColumnarTableBatchAggregator implements TableBatchAggregator {
  schema: Schema;
  length: number = 0;
  allocated: number = 0;
  columns: Record<string, TypedArray | Array<any>> = {};

  constructor(schema: Schema, options: ColumnarTableBatchOptions) {
    this.schema = schema;
    this._reallocateColumns();
  }

  rowCount(): number {
    return this.length;
  }

  addArrayRow(row: any[]) {
    // If user keeps pushing rows beyond batch size, reallocate
    this._reallocateColumns();
    let i = 0;
    // TODO what if no csv header, columns not populated?
    for (const fieldName in this.columns) {
      this.columns[fieldName][this.length] = row[i++];
    }
    this.length++;
  }

  addObjectRow(row: {[columnName: string]: any}): void {
    // If user keeps pushing rows beyond batch size, reallocate
    this._reallocateColumns();
    for (const fieldName in row) {
      this.columns[fieldName][this.length] = row[fieldName];
    }
    this.length++;
  }

  getBatch(): ColumnarTableBatch | ArrowTableBatch | null {
    this._pruneColumns();

    const batch: ColumnarTableBatch = {
      shape: 'columnar-table',
      batchType: 'data',
      data: this.columns,
      schema: this.schema,
      length: this.length
    };

    return batch;
  }

  // HELPERS

  _reallocateColumns() {
    if (this.length < this.allocated) {
      return;
    }

    // @ts-ignore TODO
    this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;
    this.columns = {};

    for (const field of this.schema.fields) {
      const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
      const oldColumn = this.columns[field.name];

      if (!oldColumn) {
        // Create new
        this.columns[field.name] = new ArrayType(this.allocated);
      } else if (Array.isArray(oldColumn)) {
        // Plain array, just increase its size
        oldColumn.length = this.allocated;
      } else if (isTypedArray(oldColumn)) {
        const typedArray = new ArrayType(this.allocated) as TypedArray;
        // Copy the old data to the new array
        typedArray.set(oldColumn);
        this.columns[field.name] = typedArray;
      } else {
        throw new Error('Invalid column type');
      }
    }
  }

  _pruneColumns() {
    for (const [columnName, column] of Object.entries(this.columns)) {
      this.columns[columnName] = column.slice(0, this.length);
    }
  }
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {Schema, ColumnarTableBatch, ArrowTableBatch, TypedArray} from '@loaders.gl/schema';	1✔
6	import {isTypedArray} from '@math.gl/types';	1✔
7	import {getArrayTypeFromDataType} from '../../schema/data-type';	1✔
8	import {TableBatchAggregator} from './table-batch-aggregator';	1✔
9	type ColumnarTableBatchOptions = {};	1✔
10		1✔
11	const DEFAULT_ROW_COUNT = 100;	1✔
12		1✔
13	export class ColumnarTableBatchAggregator implements TableBatchAggregator {	1✔
14	schema: Schema;	1!
UNCOV 15	length: number = 0;	×
UNCOV 16	allocated: number = 0;	×
UNCOV 17	columns: Record<string, TypedArray \| Array<any>> = {};	×
18		1✔
19	constructor(schema: Schema, options: ColumnarTableBatchOptions) {	1✔
UNCOV 20	this.schema = schema;	×
UNCOV 21	this._reallocateColumns();	×
UNCOV 22	}	×
23		1✔
24	rowCount(): number {	1✔
UNCOV 25	return this.length;	×
UNCOV 26	}	×
27		1✔
28	addArrayRow(row: any[]) {	1✔
UNCOV 29	// If user keeps pushing rows beyond batch size, reallocate	×
UNCOV 30	this._reallocateColumns();	×
UNCOV 31	let i = 0;	×
UNCOV 32	// TODO what if no csv header, columns not populated?	×
UNCOV 33	for (const fieldName in this.columns) {	×
UNCOV 34	this.columns[fieldName][this.length] = row[i++];	×
UNCOV 35	}	×
UNCOV 36	this.length++;	×
UNCOV 37	}	×
38		1✔
39	addObjectRow(row: {[columnName: string]: any}): void {	1✔
40	// If user keeps pushing rows beyond batch size, reallocate	×
41	this._reallocateColumns();	×
42	for (const fieldName in row) {	×
43	this.columns[fieldName][this.length] = row[fieldName];	×
44	}	×
45	this.length++;	×
46	}	×
47		1✔
48	getBatch(): ColumnarTableBatch \| ArrowTableBatch \| null {	1✔
UNCOV 49	this._pruneColumns();	×
UNCOV 50		×
UNCOV 51	const batch: ColumnarTableBatch = {	×
UNCOV 52	shape: 'columnar-table',	×
UNCOV 53	batchType: 'data',	×
UNCOV 54	data: this.columns,	×
UNCOV 55	schema: this.schema,	×
UNCOV 56	length: this.length	×
UNCOV 57	};	×
UNCOV 58		×
UNCOV 59	return batch;	×
UNCOV 60	}	×
61		1✔
62	// HELPERS	1✔
63		1✔
64	_reallocateColumns() {	1✔
UNCOV 65	if (this.length < this.allocated) {	×
UNCOV 66	return;	×
UNCOV 67	}	×
UNCOV 68		×
UNCOV 69	// @ts-ignore TODO	×
UNCOV 70	this.allocated = this.allocated > 0 ? (this.allocated *= 2) : DEFAULT_ROW_COUNT;	×
UNCOV 71	this.columns = {};	×
UNCOV 72		×
UNCOV 73	for (const field of this.schema.fields) {	×
UNCOV 74	const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);	×
UNCOV 75	const oldColumn = this.columns[field.name];	×
UNCOV 76		×
UNCOV 77	if (!oldColumn) {	×
UNCOV 78	// Create new	×
UNCOV 79	this.columns[field.name] = new ArrayType(this.allocated);	×
UNCOV 80	} else if (Array.isArray(oldColumn)) {	×
81	// Plain array, just increase its size	×
82	oldColumn.length = this.allocated;	×
83	} else if (isTypedArray(oldColumn)) {	×
84	const typedArray = new ArrayType(this.allocated) as TypedArray;	×
85	// Copy the old data to the new array	×
86	typedArray.set(oldColumn);	×
87	this.columns[field.name] = typedArray;	×
88	} else {	×
89	throw new Error('Invalid column type');	×
90	}	×
UNCOV 91	}	×
UNCOV 92	}	×
93		1✔
94	_pruneColumns() {	1✔
UNCOV 95	for (const [columnName, column] of Object.entries(this.columns)) {	×
UNCOV 96	this.columns[columnName] = column.slice(0, this.length);	×
UNCOV 97	}	×
UNCOV 98	}	×
99	}	1✔

visgl / loaders.gl / 20352515932

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous