24839896359

Committed 23 Apr 2026 02:06PM UTC coverage: 59.334% (-0.3%) from 59.627%

Build # 24839896359

Build Type

push

github

Committed by

web-flow

Commit Message

fix(json) Only emit batches when we have complete elements (#3400)

Coverage Stats

11234 of 20699 branches covered (54.27%)

Branch coverage included in aggregate %.

24 of 25 new or added lines in 1 file covered. (96.0%)

123 existing lines in 8 files now uncovered.

23043 of 37071 relevant lines covered (62.16%)

16510.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.51

/modules/json/src/lib/json-parser/streaming-json-parser.ts

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {default as JSONParser} from './json-parser';
import JSONPath from '../jsonpath/jsonpath';

/**
 * The `StreamingJSONParser` looks for the first array in the JSON structure.
 * and emits an array of chunks
 */
export default class StreamingJSONParser extends JSONParser {
  /** JSONPaths that identify arrays eligible for streaming. */
  private jsonPaths: JSONPath[];
  /** JSONPath of the selected streaming array. */
  private streamingJsonPath: JSONPath | null = null;
  /** Parser-owned array for the selected streaming rows. */
  private streamingArray: any[] | null = null;
  /** Number of direct streaming-array children that are fully parsed and ready to emit. */
  private completedStreamingRowCount: number = 0;
  /** Root object used by metadata batches when streaming an embedded array. */
  private topLevelObject: object | null = null;

  constructor(options: {[key: string]: any} = {}) {
    super({
      onopenarray: () => {
        if (!this.streamingArray) {
          if (this._matchJSONPath()) {
            // @ts-ignore
            this.streamingJsonPath = this.getJsonPath().clone();
            this.streamingArray = [];
            this._openArray(this.streamingArray as []);
            return;
          }
        }

        this._openArray();
      },

      // Redefine onopenarray to inject value for top-level object
      onopenobject: name => {
        if (!this.topLevelObject) {
          this.topLevelObject = {};
          this._openObject(this.topLevelObject);
        } else {
          this._openObject({});
        }
        if (typeof name !== 'undefined') {
          this.parser.emit('onkey', name);
        }
      },

      oncloseobject: () => {
        const directStreamingChild = this._isClosingDirectStreamingChild();
        this._closeObject();
        if (directStreamingChild) {
          this.completedStreamingRowCount++;
        }
      },

      onclosearray: () => {
        const directStreamingChild = this._isClosingDirectStreamingChild();
        this._closeArray();
        if (directStreamingChild) {
          this.completedStreamingRowCount++;
        }
      },

      onvalue: value => {
        const directStreamingValue = this._isInStreamingArray();
        this._pushOrSet(value);
        if (directStreamingValue) {
          this.completedStreamingRowCount++;
        }
      }
    });
    const jsonpaths = options.jsonpaths || [];
    this.jsonPaths = jsonpaths.map(jsonpath => new JSONPath(jsonpath));
  }

  /**
   * write REDEFINITION
   * - super.write() chunk to parser
   * - get the contents (so far) of "topmost-level" array as batch of rows
   * - clear top-level array
   * - return the batch of rows\
   */
  write(chunk) {
    super.write(chunk);
    return this._drainCompletedRows();
  }

  /**
   * Returns a partially formed result object
   * Useful for returning the "wrapper" object when array is not top level
   * e.g. GeoJSON
   */
  getPartialResult() {
    return this.topLevelObject;
  }

  getStreamingJsonPath() {
    return this.streamingJsonPath;
  }

  getStreamingJsonPathAsString() {
    return this.streamingJsonPath && this.streamingJsonPath.toString();
  }

  getJsonPath() {
    return this.jsonpath;
  }

  // PRIVATE METHODS

  /**
   * Returns completed rows and removes them from the parser-owned streaming array.
   */
  _drainCompletedRows(): any[] {
    if (!this.streamingArray || this.completedStreamingRowCount === 0) {
      return [];
    }

    const rows = this.streamingArray.slice(0, this.completedStreamingRowCount);
    this.streamingArray.splice(0, this.completedStreamingRowCount);
    this.completedStreamingRowCount = 0;
    return rows;
  }

  /**
   * Checks whether the parser is currently writing a direct value into the streaming array.
   */
  _isInStreamingArray(): boolean {
    return Boolean(this.streamingArray && this.currentState.container === this.streamingArray);
  }

  /**
   * Checks whether the current close event completes a direct streaming-array child.
   */
  _isClosingDirectStreamingChild(): boolean {
    if (!this.streamingArray || this.currentState.container === this.streamingArray) {
      return false;
    }

    const parentState = (this.previousStates as Array<{container: unknown}>)[
      this.previousStates.length - 1
    ];
    return parentState?.container === this.streamingArray;
  }

  /**
   * Checks is this.getJsonPath matches the jsonpaths provided in options
   */
  _matchJSONPath() {
    const currentPath = this.getJsonPath();
    // console.debug(`Testing JSONPath`, currentPath);

    // Backwards compatibility, match any array
    // TODO implement using wildcard once that is supported
    if (this.jsonPaths.length === 0) {
      return true;
    }

    for (const jsonPath of this.jsonPaths) {
      if (jsonPath.equals(currentPath)) {
        return true;
      }
    }

    return false;
  }
}

1	// loaders.gl
2	// SPDX-License-Identifier: MIT
3	// Copyright (c) vis.gl contributors
4
5	import {default as JSONParser} from './json-parser';
6	import JSONPath from '../jsonpath/jsonpath';
7
8	/**
9	* The `StreamingJSONParser` looks for the first array in the JSON structure.
10	* and emits an array of chunks
11	*/
12	export default class StreamingJSONParser extends JSONParser {
13	/** JSONPaths that identify arrays eligible for streaming. */
14	private jsonPaths: JSONPath[];
15	/** JSONPath of the selected streaming array. */
16	private streamingJsonPath: JSONPath \| null = null;	22✔
17	/** Parser-owned array for the selected streaming rows. */
18	private streamingArray: any[] \| null = null;	22✔
19	/** Number of direct streaming-array children that are fully parsed and ready to emit. */
20	private completedStreamingRowCount: number = 0;	22✔
21	/** Root object used by metadata batches when streaming an embedded array. */
22	private topLevelObject: object \| null = null;	22✔
23
24	constructor(options: {[key: string]: any} = {}) {	22✔
25	super({	22✔
26	onopenarray: () => {
27	if (!this.streamingArray) {	6,070✔
28	if (this._matchJSONPath()) {	640✔
29	// @ts-ignore
30	this.streamingJsonPath = this.getJsonPath().clone();	20✔
31	this.streamingArray = [];	20✔
32	this._openArray(this.streamingArray as []);	20✔
33	return;	20✔
34	}
35	}
36
37	this._openArray();	6,050✔
38	},
39
40	// Redefine onopenarray to inject value for top-level object
41	onopenobject: name => {
42	if (!this.topLevelObject) {	16,680✔
43	this.topLevelObject = {};	22✔
44	this._openObject(this.topLevelObject);	22✔
45	} else {
46	this._openObject({});	16,658✔
47	}
48	if (typeof name !== 'undefined') {	16,680!
49	this.parser.emit('onkey', name);	16,680✔
50	}
51	},
52
53	oncloseobject: () => {
54	const directStreamingChild = this._isClosingDirectStreamingChild();	16,680✔
55	this._closeObject();	16,680✔
56	if (directStreamingChild) {	16,680✔
57	this.completedStreamingRowCount++;	4,934✔
58	}
59	},
60
61	onclosearray: () => {
62	const directStreamingChild = this._isClosingDirectStreamingChild();	6,070✔
63	this._closeArray();	6,070✔
64	if (directStreamingChild) {	6,070✔
65	this.completedStreamingRowCount++;	494✔
66	}
67	},
68
69	onvalue: value => {
70	const directStreamingValue = this._isInStreamingArray();	78,328✔
71	this._pushOrSet(value);	78,328✔
72	if (directStreamingValue) {	78,328!
NEW 73	this.completedStreamingRowCount++;	×
74	}
75	}
76	});
77	const jsonpaths = options.jsonpaths \|\| [];	22✔
78	this.jsonPaths = jsonpaths.map(jsonpath => new JSONPath(jsonpath));	22✔
79	}
80
81	/**
82	* write REDEFINITION
83	* - super.write() chunk to parser
84	* - get the contents (so far) of "topmost-level" array as batch of rows
85	* - clear top-level array
86	* - return the batch of rows\
87	*/
88	write(chunk) {
89	super.write(chunk);	512✔
90	return this._drainCompletedRows();	512✔
91	}
92
93	/**
94	* Returns a partially formed result object
95	* Useful for returning the "wrapper" object when array is not top level
96	* e.g. GeoJSON
97	*/
98	getPartialResult() {
99	return this.topLevelObject;	8✔
100	}
101
102	getStreamingJsonPath() {
103	return this.streamingJsonPath;	×
104	}
105
106	getStreamingJsonPathAsString() {
107	return this.streamingJsonPath && this.streamingJsonPath.toString();	69✔
108	}
109
110	getJsonPath() {
111	return this.jsonpath;	660✔
112	}
113
114	// PRIVATE METHODS
115
116	/**
117	* Returns completed rows and removes them from the parser-owned streaming array.
118	*/
119	_drainCompletedRows(): any[] {
120	if (!this.streamingArray \|\| this.completedStreamingRowCount === 0) {	512✔
121	return [];	462✔
122	}
123
124	const rows = this.streamingArray.slice(0, this.completedStreamingRowCount);	50✔
125	this.streamingArray.splice(0, this.completedStreamingRowCount);	50✔
126	this.completedStreamingRowCount = 0;	50✔
127	return rows;	50✔
128	}
129
130	/**
131	* Checks whether the parser is currently writing a direct value into the streaming array.
132	*/
133	_isInStreamingArray(): boolean {
134	return Boolean(this.streamingArray && this.currentState.container === this.streamingArray);	78,328✔
135	}
136
137	/**
138	* Checks whether the current close event completes a direct streaming-array child.
139	*/
140	_isClosingDirectStreamingChild(): boolean {
141	if (!this.streamingArray \|\| this.currentState.container === this.streamingArray) {	22,750✔
142	return false;	2,490✔
143	}
144
145	const parentState = (this.previousStates as Array<{container: unknown}>)[	20,260✔
146	this.previousStates.length - 1
147	];
148	return parentState?.container === this.streamingArray;	20,260✔
149	}
150
151	/**
152	* Checks is this.getJsonPath matches the jsonpaths provided in options
153	*/
154	_matchJSONPath() {
155	const currentPath = this.getJsonPath();	640✔
156	// console.debug(`Testing JSONPath`, currentPath);
157
158	// Backwards compatibility, match any array
159	// TODO implement using wildcard once that is supported
160	if (this.jsonPaths.length === 0) {	640✔
161	return true;	12✔
162	}
163
164	for (const jsonPath of this.jsonPaths) {	628✔
165	if (jsonPath.equals(currentPath)) {	628✔
166	return true;	8✔
167	}
168	}
169
170	return false;	620✔
171	}
172	}

visgl / loaders.gl / 24839896359

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous