• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

graphty-org / graphty-monorepo / 20661584252

02 Jan 2026 03:50PM UTC coverage: 77.924% (+7.3%) from 70.62%
20661584252

push

github

apowers313
ci: fix flakey performance test

13438 of 17822 branches covered (75.4%)

Branch coverage included in aggregate %.

41247 of 52355 relevant lines covered (78.78%)

145534.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.33
/graphty-element/src/data/JsonDataSource.ts
1
import jmespath from "jmespath";
15!
2
import { z } from "zod/v4";
15✔
3
import * as z4 from "zod/v4/core";
4

5
// import {JSONParser} from "@streamparser/json";
6
import type { AdHocData, PartiallyOptional } from "../config/common";
7
import { BaseDataSourceConfig, DataSource, DataSourceChunk } from "./DataSource";
15✔
8

9
const JsonNodeConfig = z
15✔
10
    .strictObject({
15✔
11
        path: z.string().default("nodes"),
15✔
12
        schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
13
    })
15✔
14
    .prefault({});
15✔
15

16
const JsonEdgeConfig = z
15✔
17
    .strictObject({
15✔
18
        path: z.string().default("edges"),
15✔
19
        schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
20
    })
15✔
21
    .prefault({});
15✔
22

23
export const JsonDataSourceConfig = z.object({
15✔
24
    data: z.string().optional(),
15✔
25
    file: z.instanceof(File).optional(),
15✔
26
    url: z.string().optional(),
15✔
27
    chunkSize: z.number().optional(),
15✔
28
    errorLimit: z.number().optional(),
15✔
29
    nodeIdPath: z.string().optional(),
15✔
30
    edgeSrcIdPath: z.string().optional(),
15✔
31
    edgeDstIdPath: z.string().optional(),
15✔
32
    node: JsonNodeConfig,
15✔
33
    edge: JsonEdgeConfig,
15✔
34
});
15✔
35

36
export type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
37
export type JsonDataSourceConfigOpts = PartiallyOptional<JsonDataSourceConfigType, "node" | "edge">;
38

39
/**
40
 * Data source for loading graph data from JSON files.
41
 * Supports JMESPath queries for extracting nodes and edges from complex JSON structures.
42
 */
43
export class JsonDataSource extends DataSource {
15✔
44
    static type = "json";
21✔
45
    opts: JsonDataSourceConfigType;
46

47
    /**
48
     * Creates a new JsonDataSource instance.
49
     * @param anyOpts - Configuration options for JSON parsing and data extraction
50
     */
51
    constructor(anyOpts: object) {
21✔
52
        const opts = JsonDataSourceConfig.parse(anyOpts);
121✔
53

54
        // Pass errorLimit and chunkSize to base class
55
        super(opts.errorLimit ?? 100, opts.chunkSize ?? DataSource.DEFAULT_CHUNK_SIZE);
121✔
56

57
        this.opts = opts;
121✔
58
        if (opts.node.schema) {
121!
59
            this.nodeSchema = opts.node.schema;
4✔
60
        }
4✔
61

62
        if (opts.edge.schema) {
121!
63
            this.edgeSchema = opts.edge.schema;
1✔
64
        }
1✔
65
    }
121✔
66

67
    protected getConfig(): BaseDataSourceConfig {
21✔
68
        // JsonDataSource has special handling for 'data' field:
69
        // If data starts with http/https/data:, treat it as URL
70
        // Otherwise treat it as inline JSON
71
        const isUrl =
120✔
72
            (this.opts.data?.startsWith("http://") ?? false) ||
120✔
73
            (this.opts.data?.startsWith("https://") ?? false) ||
120!
74
            (this.opts.data?.startsWith("data:") ?? false);
34!
75

76
        return {
120✔
77
            data: isUrl ? undefined : this.opts.data,
120!
78
            file: this.opts.file,
120✔
79
            url: isUrl ? this.opts.data : this.opts.url,
120!
80
            chunkSize: this.opts.chunkSize,
120✔
81
            errorLimit: this.opts.errorLimit,
120✔
82
        };
120✔
83
    }
120✔
84

85
    /**
86
     * Fetches and parses JSON data into graph chunks.
87
     * Uses JMESPath to extract nodes and edges from the JSON structure.
88
     * @yields DataSourceChunk objects containing parsed nodes and edges
89
     */
90
    async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {
21✔
91
        let data: unknown;
120✔
92

93
        // Get JSON content (could be from data, file, or url)
94
        const jsonString = await this.getContent();
120✔
95

96
        // Handle empty content gracefully
97
        if (jsonString.trim() === "") {
119!
98
            yield* this.chunkData([], []);
1✔
99
            return;
1✔
100
        }
1✔
101

102
        // Parse JSON
103
        try {
118✔
104
            data = JSON.parse(jsonString);
118✔
105
        } catch (error) {
119!
106
            const canContinue = this.errorAggregator.addError({
2✔
107
                message: `Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`,
2!
108
                category: "parse-error",
2✔
109
            });
2✔
110

111
            if (!canContinue) {
2!
112
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
113
            }
×
114

115
            yield* this.chunkData([], []);
2✔
116
            return;
2✔
117
        }
2✔
118

119
        // Extract nodes using JMESPath
120
        let rawNodes: unknown[] = [];
116✔
121
        try {
116✔
122
            const nodes = jmespath.search(data, this.opts.node.path);
116✔
123
            if (Array.isArray(nodes)) {
119✔
124
                rawNodes = nodes;
113✔
125
            } else if (nodes !== null && nodes !== undefined) {
119!
126
                // Log error but continue with empty nodes
127
                const canContinue = this.errorAggregator.addError({
1✔
128
                    message: `Expected 'nodes' at path '${this.opts.node.path}' to be an array, got ${typeof nodes}`,
1✔
129
                    category: "validation-error",
1✔
130
                    field: "nodes",
1✔
131
                });
1✔
132

133
                if (!canContinue) {
1!
134
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
135
                }
×
136
            }
1✔
137
            // If nodes is null/undefined, just use empty array
138
        } catch (error) {
120!
139
            const canContinue = this.errorAggregator.addError({
×
140
                message: `Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`,
×
141
                category: "parse-error",
×
142
                field: "nodes",
×
143
            });
×
144

145
            if (!canContinue) {
×
146
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
147
            }
×
148
        }
✔
149

150
        // Extract edges using JMESPath
151
        let rawEdges: unknown[] = [];
116✔
152
        try {
116✔
153
            const edges = jmespath.search(data, this.opts.edge.path);
116✔
154
            if (Array.isArray(edges)) {
119✔
155
                rawEdges = edges;
113✔
156
            } else if (edges !== null && edges !== undefined) {
119!
157
                // Log error but continue with empty edges
158
                const canContinue = this.errorAggregator.addError({
1✔
159
                    message: `Expected 'edges' at path '${this.opts.edge.path}' to be an array, got ${typeof edges}`,
1✔
160
                    category: "validation-error",
1✔
161
                    field: "edges",
1✔
162
                });
1✔
163

164
                if (!canContinue) {
1!
165
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
166
                }
×
167
            }
1✔
168
            // If edges is null/undefined, just use empty array
169
        } catch (error) {
120!
170
            const canContinue = this.errorAggregator.addError({
×
171
                message: `Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`,
×
172
                category: "parse-error",
×
173
                field: "edges",
×
174
            });
×
175

176
            if (!canContinue) {
×
177
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
178
            }
×
179
        }
✔
180

181
        // Validate individual nodes and filter out invalid ones
182
        const validNodes: AdHocData[] = [];
116✔
183
        for (let i = 0; i < rawNodes.length; i++) {
119✔
184
            const node = rawNodes[i];
5,948✔
185
            if (this.isValidNode(node, i)) {
5,948✔
186
                validNodes.push(node as AdHocData);
5,947✔
187
            }
5,947✔
188
        }
5,948✔
189

190
        // Validate individual edges and filter out invalid ones
191
        const validEdges: AdHocData[] = [];
116✔
192
        for (let i = 0; i < rawEdges.length; i++) {
119✔
193
            const edge = rawEdges[i];
7,518✔
194
            if (this.isValidEdge(edge, i)) {
7,518✔
195
                validEdges.push(edge as AdHocData);
7,517✔
196
            }
7,517✔
197
        }
7,518✔
198

199
        // Yield data in chunks using inherited helper
200
        yield* this.chunkData(validNodes, validEdges);
116✔
201
    }
120✔
202

203
    /**
204
     * Validates a node object and logs errors if invalid.
205
     * Returns true if the node is valid and should be included.
206
     * @param node - The node object to validate
207
     * @param index - Index of the node in the array
208
     * @returns True if valid, false otherwise
209
     */
210
    private isValidNode(node: unknown, index: number): boolean {
21✔
211
        if (node === null || node === undefined) {
5,948!
212
            const canContinue = this.errorAggregator.addError({
×
213
                message: `Node at index ${index} is null or undefined`,
×
214
                category: "validation-error",
×
215
                field: "nodes",
×
216
                line: index,
×
217
            });
×
218

219
            if (!canContinue) {
×
220
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
221
            }
×
222

223
            return false;
×
224
        }
×
225

226
        if (typeof node !== "object") {
5,948!
227
            const canContinue = this.errorAggregator.addError({
×
228
                message: `Node at index ${index} is not an object (got ${typeof node})`,
×
229
                category: "validation-error",
×
230
                field: "nodes",
×
231
                line: index,
×
232
            });
×
233

234
            if (!canContinue) {
×
235
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
236
            }
×
237

238
            return false;
×
239
        }
×
240

241
        // Check for id field (common requirement - accept common identifier field names)
242
        // If a custom nodeIdPath is specified, also accept that field
243
        const nodeObj = node as Record<string, unknown>;
5,948✔
244
        const customIdPath = this.opts.nodeIdPath;
5,948✔
245
        const hasId =
5,948✔
246
            "id" in nodeObj ||
5,948!
247
            "name" in nodeObj ||
90!
248
            "key" in nodeObj ||
13!
249
            "label" in nodeObj ||
3!
250
            (customIdPath !== undefined && customIdPath in nodeObj);
3!
251
        if (!hasId) {
5,948!
252
            const expectedFields = customIdPath
1!
253
                ? `'id', 'name', 'key', 'label', or '${customIdPath}'`
×
254
                : "'id', 'name', 'key', or 'label'";
1✔
255
            const canContinue = this.errorAggregator.addError({
1✔
256
                message: `Node at index ${index} is missing identifier field (expected ${expectedFields})`,
1✔
257
                category: "missing-value",
1✔
258
                field: "nodes.id",
1✔
259
                line: index,
1✔
260
            });
1✔
261

262
            if (!canContinue) {
1!
263
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
264
            }
×
265

266
            return false;
1✔
267
        }
1✔
268

269
        return true;
5,947✔
270
    }
5,948✔
271

272
    /**
273
     * Validates an edge object and logs errors if invalid.
274
     * Returns true if the edge is valid and should be included.
275
     * @param edge - The edge object to validate
276
     * @param index - Index of the edge in the array
277
     * @returns True if valid, false otherwise
278
     */
279
    private isValidEdge(edge: unknown, index: number): boolean {
21✔
280
        if (edge === null || edge === undefined) {
7,518!
281
            const canContinue = this.errorAggregator.addError({
×
282
                message: `Edge at index ${index} is null or undefined`,
×
283
                category: "validation-error",
×
284
                field: "edges",
×
285
                line: index,
×
286
            });
×
287

288
            if (!canContinue) {
×
289
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
290
            }
×
291

292
            return false;
×
293
        }
×
294

295
        if (typeof edge !== "object") {
7,518!
296
            const canContinue = this.errorAggregator.addError({
×
297
                message: `Edge at index ${index} is not an object (got ${typeof edge})`,
×
298
                category: "validation-error",
×
299
                field: "edges",
×
300
                line: index,
×
301
            });
×
302

303
            if (!canContinue) {
×
304
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
305
            }
×
306

307
            return false;
×
308
        }
×
309

310
        // Check for source/target fields (common requirement)
311
        const edgeObj = edge as Record<string, unknown>;
7,518✔
312
        const hasSource = "source" in edgeObj || "src" in edgeObj || "from" in edgeObj;
7,518!
313
        const hasTarget = "target" in edgeObj || "dst" in edgeObj || "to" in edgeObj;
7,518!
314

315
        if (!hasSource) {
7,518!
316
            const canContinue = this.errorAggregator.addError({
1✔
317
                message: `Edge at index ${index} is missing source field (expected 'source', 'src', or 'from')`,
1✔
318
                category: "missing-value",
1✔
319
                field: "edges.source",
1✔
320
                line: index,
1✔
321
            });
1✔
322

323
            if (!canContinue) {
1!
324
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
325
            }
×
326

327
            return false;
1✔
328
        }
1✔
329

330
        if (!hasTarget) {
7,518!
331
            const canContinue = this.errorAggregator.addError({
×
332
                message: `Edge at index ${index} is missing target field (expected 'target', 'dst', or 'to')`,
×
333
                category: "missing-value",
×
334
                field: "edges.target",
×
335
                line: index,
×
336
            });
×
337

338
            if (!canContinue) {
×
339
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
340
            }
×
341

342
            return false;
×
343
        }
✔
344

345
        return true;
7,517✔
346
    }
7,518✔
347
}
21✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc