20352515932

Committed 18 Dec 2025 09:56PM UTC coverage: 35.115% (-28.4%) from 63.485%

Build # 20352515932

Build Type

push

github

Committed by

web-flow

Commit Message

feat(loader-utils): Export is-type helpers (#3258)

Coverage Stats

1188 of 1998 branches covered (59.46%)

Branch coverage included in aggregate %.

147 of 211 new or added lines in 13 files covered. (69.67%)

30011 existing lines in 424 files now uncovered.

37457 of 108056 relevant lines covered (34.66%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.83

/modules/schema-utils/src/lib/table/batch-builder/table-batch-builder.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Schema, TableBatch} from '@loaders.gl/schema';
import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
import {RowTableBatchAggregator} from './row-table-batch-aggregator';
import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';

// TODO define interface instead
type TableBatchBuilderOptions = {
  shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
  batchSize?: number | 'auto';
  batchDebounceMs?: number;
  limit?: number;
  _limitMB?: number;
};

type GetBatchOptions = {
  bytesUsed?: number;
  [key: string]: any;
};

const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
  shape: undefined!,
  batchSize: 'auto',
  batchDebounceMs: 0,
  limit: 0,
  _limitMB: 0
};

/** Incrementally builds batches from a stream of rows */
export class TableBatchBuilder {
  schema: Schema;
  options: Required<TableBatchBuilderOptions>;

  private aggregator: TableBatchAggregator | null = null;
  private batchCount: number = 0;
  private bytesUsed: number = 0;
  private isChunkComplete: boolean = false;
  private lastBatchEmittedMs: number = Date.now();
  private totalLength: number = 0;
  private totalBytes: number = 0;
  private rowBytes: number = 0;

  static ArrowBatch?: TableBatchConstructor;

  constructor(schema: Schema, options?: TableBatchBuilderOptions) {
    this.schema = schema;
    this.options = {...DEFAULT_OPTIONS, ...options};
  }

  limitReached(): boolean {
    if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
      return true;
    }
    if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
      return true;
    }
    return false;
  }

  /** @deprecated Use addArrayRow or addObjectRow */
  addRow(row: any[] | {[columnName: string]: any}): void {
    if (this.limitReached()) {
      return;
    }
    this.totalLength++;
    this.rowBytes = this.rowBytes || this._estimateRowMB(row);
    this.totalBytes += this.rowBytes;
    if (Array.isArray(row)) {
      this.addArrayRow(row);
    } else {
      this.addObjectRow(row);
    }
  }

  /** Add one row to the batch */
  protected addArrayRow(row: any[]) {
    if (!this.aggregator) {
      const TableBatchType = this._getTableBatchType();
      this.aggregator = new TableBatchType(this.schema, this.options);
    }
    this.aggregator.addArrayRow(row);
  }

  /** Add one row to the batch */
  protected addObjectRow(row: {[columnName: string]: any}): void {
    if (!this.aggregator) {
      const TableBatchType = this._getTableBatchType();
      this.aggregator = new TableBatchType(this.schema, this.options);
    }
    this.aggregator.addObjectRow(row);
  }

  /** Mark an incoming raw memory chunk has completed */
  chunkComplete(chunk: ArrayBuffer | string): void {
    if (chunk instanceof ArrayBuffer) {
      this.bytesUsed += chunk.byteLength;
    }
    if (typeof chunk === 'string') {
      this.bytesUsed += chunk.length;
    }
    this.isChunkComplete = true;
  }

  getFullBatch(options?: GetBatchOptions): TableBatch | null {
    return this._isFull() ? this._getBatch(options) : null;
  }

  getFinalBatch(options?: GetBatchOptions): TableBatch | null {
    return this._getBatch(options);
  }

  // INTERNAL

  _estimateRowMB(row: any[] | object): number {
    return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
  }

  private _isFull(): boolean {
    // No batch, not ready
    if (!this.aggregator || this.aggregator.rowCount() === 0) {
      return false;
    }

    // if batchSize === 'auto' we wait for chunk to complete
    // if batchSize === number, ensure we have enough rows
    if (this.options.batchSize === 'auto') {
      if (!this.isChunkComplete) {
        return false;
      }
    } else if (this.options.batchSize > this.aggregator.rowCount()) {
      return false;
    }

    // Debounce batches
    if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
      return false;
    }

    // Emit batch
    this.isChunkComplete = false;
    this.lastBatchEmittedMs = Date.now();
    return true;
  }

  /**
   * bytesUsed can be set via chunkComplete or via getBatch*
   */
  private _getBatch(options?: GetBatchOptions): TableBatch | null {
    if (!this.aggregator) {
      return null;
    }

    // TODO - this can overly increment bytes used?
    if (options?.bytesUsed) {
      this.bytesUsed = options.bytesUsed;
    }
    const normalizedBatch = this.aggregator.getBatch() as TableBatch;
    normalizedBatch.count = this.batchCount;
    normalizedBatch.bytesUsed = this.bytesUsed;
    Object.assign(normalizedBatch, options);

    this.batchCount++;
    this.aggregator = null;
    return normalizedBatch;
  }

  private _getTableBatchType(): TableBatchConstructor {
    switch (this.options.shape) {
      case 'array-row-table':
      case 'object-row-table':
        return RowTableBatchAggregator;
      case 'columnar-table':
        return ColumnarTableBatchAggregator;
      case 'arrow-table':
        return ArrowTableBatchAggregator;
      default:
        return BaseTableBatchAggregator;
    }
  }
}

1	// loaders.gl	1✔
2	// SPDX-License-Identifier: MIT	1✔
3	// Copyright (c) vis.gl contributors	1✔
4		1✔
5	import type {Schema, TableBatch} from '@loaders.gl/schema';	1✔
6	import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';	1✔
7	import {BaseTableBatchAggregator} from './base-table-batch-aggregator';	1✔
8	import {RowTableBatchAggregator} from './row-table-batch-aggregator';	1✔
9	import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';	1✔
10	import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';	1✔
11		1✔
12	// TODO define interface instead	1✔
13	type TableBatchBuilderOptions = {	1✔
14	shape?: 'array-row-table' \| 'object-row-table' \| 'columnar-table' \| 'arrow-table';	1✔
15	batchSize?: number \| 'auto';	1✔
16	batchDebounceMs?: number;	1✔
17	limit?: number;	1✔
18	_limitMB?: number;	1✔
19	};	1✔
20		1✔
21	type GetBatchOptions = {	1✔
22	bytesUsed?: number;	1✔
23	[key: string]: any;	1✔
24	};	1✔
25		1✔
26	const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {	1✔
27	shape: undefined!,	1✔
28	batchSize: 'auto',	1✔
29	batchDebounceMs: 0,	1✔
30	limit: 0,	1✔
31	_limitMB: 0	1✔
32	};	1✔
33		1✔
34	/** Incrementally builds batches from a stream of rows */	1✔
35	export class TableBatchBuilder {	1✔
36	schema: Schema;	1✔
37	options: Required<TableBatchBuilderOptions>;	1✔
38		1✔
39	private aggregator: TableBatchAggregator \| null = null;	1✔
40	private batchCount: number = 0;	1✔
41	private bytesUsed: number = 0;	1✔
42	private isChunkComplete: boolean = false;	1✔
43	private lastBatchEmittedMs: number = Date.now();	1✔
44	private totalLength: number = 0;	1✔
45	private totalBytes: number = 0;	1✔
46	private rowBytes: number = 0;	1✔
47		1✔
48	static ArrowBatch?: TableBatchConstructor;	1✔
49		1✔
50	constructor(schema: Schema, options?: TableBatchBuilderOptions) {	1✔
51	this.schema = schema;	1✔
52	this.options = {...DEFAULT_OPTIONS, ...options};	1✔
53	}	1✔
54		1✔
55	limitReached(): boolean {	1✔
56	if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {	4!
UNCOV 57	return true;	×
UNCOV 58	}	×
59	if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {	4!
60	return true;	×
61	}	×
62	return false;	4✔
63	}	4✔
64		1✔
65	/** @deprecated Use addArrayRow or addObjectRow */	1✔
66	addRow(row: any[] \| {[columnName: string]: any}): void {	1✔
67	if (this.limitReached()) {	4!
UNCOV 68	return;	×
UNCOV 69	}	×
70	this.totalLength++;	4✔
71	this.rowBytes = this.rowBytes \|\| this._estimateRowMB(row);	4✔
72	this.totalBytes += this.rowBytes;	4✔
73	if (Array.isArray(row)) {	4!
UNCOV 74	this.addArrayRow(row);	×
75	} else {	4✔
76	this.addObjectRow(row);	4✔
77	}	4✔
78	}	4✔
79		1✔
80	/** Add one row to the batch */	1✔
81	protected addArrayRow(row: any[]) {	1✔
UNCOV 82	if (!this.aggregator) {	×
UNCOV 83	const TableBatchType = this._getTableBatchType();	×
UNCOV 84	this.aggregator = new TableBatchType(this.schema, this.options);	×
UNCOV 85	}	×
UNCOV 86	this.aggregator.addArrayRow(row);	×
UNCOV 87	}	×
88		1✔
89	/** Add one row to the batch */	1✔
90	protected addObjectRow(row: {[columnName: string]: any}): void {	1✔
91	if (!this.aggregator) {	4✔
92	const TableBatchType = this._getTableBatchType();	4✔
93	this.aggregator = new TableBatchType(this.schema, this.options);	4✔
94	}	4✔
95	this.aggregator.addObjectRow(row);	4✔
96	}	4✔
97		1✔
98	/** Mark an incoming raw memory chunk has completed */	1✔
99	chunkComplete(chunk: ArrayBuffer \| string): void {	1✔
100	if (chunk instanceof ArrayBuffer) {	4!
101	this.bytesUsed += chunk.byteLength;	×
102	}	×
103	if (typeof chunk === 'string') {	4✔
104	this.bytesUsed += chunk.length;	4✔
105	}	4✔
106	this.isChunkComplete = true;	4✔
107	}	4✔
108		1✔
109	getFullBatch(options?: GetBatchOptions): TableBatch \| null {	1✔
110	return this._isFull() ? this._getBatch(options) : null;	4!
111	}	4✔
112		1✔
113	getFinalBatch(options?: GetBatchOptions): TableBatch \| null {	1✔
114	return this._getBatch(options);	1✔
115	}	1✔
116		1✔
117	// INTERNAL	1✔
118		1✔
119	_estimateRowMB(row: any[] \| object): number {	1✔
120	return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;	1!
121	}	1✔
122		1✔
123	private _isFull(): boolean {	1✔
124	// No batch, not ready	4✔
125	if (!this.aggregator \|\| this.aggregator.rowCount() === 0) {	4!
UNCOV 126	return false;	×
UNCOV 127	}	×
128		4✔
129	// if batchSize === 'auto' we wait for chunk to complete	4✔
130	// if batchSize === number, ensure we have enough rows	4✔
131	if (this.options.batchSize === 'auto') {	4✔
132	if (!this.isChunkComplete) {	4!
UNCOV 133	return false;	×
UNCOV 134	}	×
135	} else if (this.options.batchSize > this.aggregator.rowCount()) {	4!
UNCOV 136	return false;	×
UNCOV 137	}	×
138		4✔
139	// Debounce batches	4✔
140	if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {	4!
141	return false;	×
142	}	×
143		4✔
144	// Emit batch	4✔
145	this.isChunkComplete = false;	4✔
146	this.lastBatchEmittedMs = Date.now();	4✔
147	return true;	4✔
148	}	4✔
149		1✔
150	/**	1✔
151	* bytesUsed can be set via chunkComplete or via getBatch*	1✔
152	*/	1✔
153	private _getBatch(options?: GetBatchOptions): TableBatch \| null {	1✔
154	if (!this.aggregator) {	5✔
155	return null;	1✔
156	}	1✔
157		4✔
158	// TODO - this can overly increment bytes used?	4✔
159	if (options?.bytesUsed) {	5!
UNCOV 160	this.bytesUsed = options.bytesUsed;	×
UNCOV 161	}	✔
162	const normalizedBatch = this.aggregator.getBatch() as TableBatch;	4✔
163	normalizedBatch.count = this.batchCount;	4✔
164	normalizedBatch.bytesUsed = this.bytesUsed;	4✔
165	Object.assign(normalizedBatch, options);	4✔
166		4✔
167	this.batchCount++;	4✔
168	this.aggregator = null;	4✔
169	return normalizedBatch;	4✔
170	}	5✔
171		1✔
172	private _getTableBatchType(): TableBatchConstructor {	1✔
173	switch (this.options.shape) {	4✔
174	case 'array-row-table':	4!
175	case 'object-row-table':	4!
UNCOV 176	return RowTableBatchAggregator;	×
177	case 'columnar-table':	4!
UNCOV 178	return ColumnarTableBatchAggregator;	×
179	case 'arrow-table':	4!
180	return ArrowTableBatchAggregator;	×
181	default:	4✔
182	return BaseTableBatchAggregator;	4✔
183	}	4✔
184	}	4✔
185	}	1✔

visgl / loaders.gl / 20352515932

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous