24153816851

Committed 08 Apr 2026 07:17PM UTC coverage: 53.247% (-12.1%) from 65.319%

Build # 24153816851

Build Type

push

github

Committed by

web-flow

Commit Message

chore: Move from tape to vitest (#3351)

Coverage Stats

8651 of 17291 branches covered (50.03%)

Branch coverage included in aggregate %.

7 of 7 new or added lines in 1 file covered. (100.0%)

2031 existing lines in 296 files now uncovered.

17563 of 31940 relevant lines covered (54.99%)

5279.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.3

/modules/schema-utils/src/lib/table/batch-builder/table-batch-builder.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {Schema, TableBatch} from '@loaders.gl/schema';
import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
import {RowTableBatchAggregator} from './row-table-batch-aggregator';
import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';

// TODO define interface instead
type TableBatchBuilderOptions = {
  shape?: 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';
  batchSize?: number | 'auto';
  batchDebounceMs?: number;
  limit?: number;
  _limitMB?: number;
};

type GetBatchOptions = {
  bytesUsed?: number;
  [key: string]: any;
};

const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {
  shape: undefined!,
  batchSize: 'auto',
  batchDebounceMs: 0,
  limit: 0,
  _limitMB: 0
};

/** Incrementally builds batches from a stream of rows */
export class TableBatchBuilder {
  schema: Schema;
  options: Required<TableBatchBuilderOptions>;

  private aggregator: TableBatchAggregator | null = null;
  private batchCount: number = 0;
  private bytesUsed: number = 0;
  private isChunkComplete: boolean = false;
  private lastBatchEmittedMs: number = Date.now();
  private totalLength: number = 0;
  private totalBytes: number = 0;
  private rowBytes: number = 0;

  static ArrowBatch?: TableBatchConstructor;

  constructor(schema: Schema, options?: TableBatchBuilderOptions) {
    this.schema = schema;
    this.options = {...DEFAULT_OPTIONS, ...options};
  }

  limitReached(): boolean {
    if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
      return true;
    }
    if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
      return true;
    }
    return false;
  }

  /** @deprecated Use addArrayRow or addObjectRow */
  addRow(row: any[] | {[columnName: string]: any}): void {
    if (this.limitReached()) {
      return;
    }
    this.totalLength++;
    this.rowBytes = this.rowBytes || this._estimateRowMB(row);
    this.totalBytes += this.rowBytes;
    if (Array.isArray(row)) {
      this.addArrayRow(row);
    } else {
      this.addObjectRow(row);
    }
  }

  /** Add one row to the batch */
  protected addArrayRow(row: any[]) {
    if (!this.aggregator) {
      const TableBatchType = this._getTableBatchType();
      this.aggregator = new TableBatchType(this.schema, this.options);
    }
    this.aggregator.addArrayRow(row);
  }

  /** Add one row to the batch */
  protected addObjectRow(row: {[columnName: string]: any}): void {
    if (!this.aggregator) {
      const TableBatchType = this._getTableBatchType();
      this.aggregator = new TableBatchType(this.schema, this.options);
    }
    this.aggregator.addObjectRow(row);
  }

  /** Mark an incoming raw memory chunk has completed */
  chunkComplete(chunk: ArrayBuffer | string): void {
    if (chunk instanceof ArrayBuffer) {
      this.bytesUsed += chunk.byteLength;
    }
    if (typeof chunk === 'string') {
      this.bytesUsed += chunk.length;
    }
    this.isChunkComplete = true;
  }

  getFullBatch(options?: GetBatchOptions): TableBatch | null {
    return this._isFull() ? this._getBatch(options) : null;
  }

  getFinalBatch(options?: GetBatchOptions): TableBatch | null {
    return this._getBatch(options);
  }

  // INTERNAL

  _estimateRowMB(row: any[] | object): number {
    return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
  }

  private _isFull(): boolean {
    // No batch, not ready
    if (!this.aggregator || this.aggregator.rowCount() === 0) {
      return false;
    }

    // if batchSize === 'auto' we wait for chunk to complete
    // if batchSize === number, ensure we have enough rows
    if (this.options.batchSize === 'auto') {
      if (!this.isChunkComplete) {
        return false;
      }
    } else if (this.options.batchSize > this.aggregator.rowCount()) {
      return false;
    }

    // Debounce batches
    if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
      return false;
    }

    // Emit batch
    this.isChunkComplete = false;
    this.lastBatchEmittedMs = Date.now();
    return true;
  }

  /**
   * bytesUsed can be set via chunkComplete or via getBatch*
   */
  private _getBatch(options?: GetBatchOptions): TableBatch | null {
    if (!this.aggregator) {
      return null;
    }

    // TODO - this can overly increment bytes used?
    if (options?.bytesUsed) {
      this.bytesUsed = options.bytesUsed;
    }
    const normalizedBatch = this.aggregator.getBatch() as TableBatch;
    normalizedBatch.count = this.batchCount;
    normalizedBatch.bytesUsed = this.bytesUsed;
    Object.assign(normalizedBatch, options);

    this.batchCount++;
    this.aggregator = null;
    return normalizedBatch;
  }

  private _getTableBatchType(): TableBatchConstructor {
    switch (this.options.shape) {
      case 'array-row-table':
      case 'object-row-table':
        return RowTableBatchAggregator;
      case 'columnar-table':
        return ColumnarTableBatchAggregator;
      case 'arrow-table':
        return ArrowTableBatchAggregator;
      default:
        return BaseTableBatchAggregator;
    }
  }
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import type {Schema, TableBatch} from '@loaders.gl/schema';
6	import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
7	import {BaseTableBatchAggregator} from './base-table-batch-aggregator';
8	import {RowTableBatchAggregator} from './row-table-batch-aggregator';
9	import {ColumnarTableBatchAggregator} from './columnar-table-batch-aggregator';
10	import {ArrowTableBatchAggregator} from './arrow-table-batch-aggregator';
11
12	// TODO define interface instead
13	type TableBatchBuilderOptions = {
14	shape?: 'array-row-table' \| 'object-row-table' \| 'columnar-table' \| 'arrow-table';
15	batchSize?: number \| 'auto';
16	batchDebounceMs?: number;
17	limit?: number;
18	_limitMB?: number;
19	};
20
21	type GetBatchOptions = {
22	bytesUsed?: number;
23	[key: string]: any;
24	};
25
26	const DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {	218✔
27	shape: undefined!,
28	batchSize: 'auto',
29	batchDebounceMs: 0,
30	limit: 0,
31	_limitMB: 0
32	};
33
34	/** Incrementally builds batches from a stream of rows */
35	export class TableBatchBuilder {
36	schema: Schema;
37	options: Required<TableBatchBuilderOptions>;
38
39	private aggregator: TableBatchAggregator \| null = null;	42✔
40	private batchCount: number = 0;	42✔
41	private bytesUsed: number = 0;	42✔
42	private isChunkComplete: boolean = false;	42✔
43	private lastBatchEmittedMs: number = Date.now();	42✔
44	private totalLength: number = 0;	42✔
45	private totalBytes: number = 0;	42✔
46	private rowBytes: number = 0;	42✔
47
48	static ArrowBatch?: TableBatchConstructor;
49
50	constructor(schema: Schema, options?: TableBatchBuilderOptions) {
51	this.schema = schema;	42✔
52	this.options = {...DEFAULT_OPTIONS, ...options};	42✔
53	}
54
55	limitReached(): boolean {
56	if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {	9,990✔
57	return true;	1,900✔
58	}
59	if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {	8,090!
UNCOV 60	return true;	×
61	}
62	return false;	8,090✔
63	}
64
65	/** @deprecated Use addArrayRow or addObjectRow */
66	addRow(row: any[] \| {[columnName: string]: any}): void {
67	if (this.limitReached()) {	9,990✔
68	return;	1,900✔
69	}
70	this.totalLength++;	8,090✔
71	this.rowBytes = this.rowBytes \|\| this._estimateRowMB(row);	8,090✔
72	this.totalBytes += this.rowBytes;	9,990✔
73	if (Array.isArray(row)) {	9,990✔
74	this.addArrayRow(row);	6,184✔
75	} else {
76	this.addObjectRow(row);	1,906✔
77	}
78	}
79
80	/** Add one row to the batch */
81	protected addArrayRow(row: any[]) {
82	if (!this.aggregator) {	6,184✔
83	const TableBatchType = this._getTableBatchType();	112✔
84	this.aggregator = new TableBatchType(this.schema, this.options);	112✔
85	}
86	this.aggregator.addArrayRow(row);	6,184✔
87	}
88
89	/** Add one row to the batch */
90	protected addObjectRow(row: {[columnName: string]: any}): void {
91	if (!this.aggregator) {	1,906✔
92	const TableBatchType = this._getTableBatchType();	70✔
93	this.aggregator = new TableBatchType(this.schema, this.options);	70✔
94	}
95	this.aggregator.addObjectRow(row);	1,906✔
96	}
97
98	/** Mark an incoming raw memory chunk has completed */
99	chunkComplete(chunk: ArrayBuffer \| string): void {
100	if (chunk instanceof ArrayBuffer) {	72!
101	this.bytesUsed += chunk.byteLength;	×
102	}
103	if (typeof chunk === 'string') {	72!
104	this.bytesUsed += chunk.length;	72✔
105	}
106	this.isChunkComplete = true;	72✔
107	}
108
109	getFullBatch(options?: GetBatchOptions): TableBatch \| null {
110	return this._isFull() ? this._getBatch(options) : null;	10,008✔
111	}
112
113	getFinalBatch(options?: GetBatchOptions): TableBatch \| null {
114	return this._getBatch(options);	42✔
115	}
116
117	// INTERNAL
118
119	_estimateRowMB(row: any[] \| object): number {
120	return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;	45✔
121	}
122
123	private _isFull(): boolean {
124	// No batch, not ready
125	if (!this.aggregator \|\| this.aggregator.rowCount() === 0) {	10,008✔
126	return false;	3✔
127	}
128
129	// if batchSize === 'auto' we wait for chunk to complete
130	// if batchSize === number, ensure we have enough rows
131	if (this.options.batchSize === 'auto') {	10,005✔
132	if (!this.isChunkComplete) {	1,818✔
133	return false;	1,789✔
134	}
135	} else if (this.options.batchSize > this.aggregator.rowCount()) {	8,187✔
136	return false;	8,065✔
137	}
138
139	// Debounce batches
140	if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {	151!
UNCOV 141	return false;	×
142	}
143
144	// Emit batch
145	this.isChunkComplete = false;	151✔
146	this.lastBatchEmittedMs = Date.now();	151✔
147	return true;	151✔
148	}
149
150	/**
151	* bytesUsed can be set via chunkComplete or via getBatch*
152	*/
153	private _getBatch(options?: GetBatchOptions): TableBatch \| null {
154	if (!this.aggregator) {	193✔
155	return null;	11✔
156	}
157
158	// TODO - this can overly increment bytes used?
159	if (options?.bytesUsed) {	182✔
160	this.bytesUsed = options.bytesUsed;	84✔
161	}
162	const normalizedBatch = this.aggregator.getBatch() as TableBatch;	182✔
163	normalizedBatch.count = this.batchCount;	182✔
164	normalizedBatch.bytesUsed = this.bytesUsed;	182✔
165	Object.assign(normalizedBatch, options);	182✔
166
167	this.batchCount++;	182✔
168	this.aggregator = null;	182✔
169	return normalizedBatch;	182✔
170	}
171
172	private _getTableBatchType(): TableBatchConstructor {
173	switch (this.options.shape) {	182!
174	case 'array-row-table':
175	case 'object-row-table':
176	return RowTableBatchAggregator;	55✔
177	case 'columnar-table':
178	return ColumnarTableBatchAggregator;	82✔
179	case 'arrow-table':
180	return ArrowTableBatchAggregator;	×
181	default:
182	return BaseTableBatchAggregator;	45✔
183	}
184	}
185	}

visgl / loaders.gl / 24153816851

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous