• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

graphty-org / graphty-monorepo / 20912147990

12 Jan 2026 07:48AM UTC coverage: 82.139% (+0.4%) from 81.707%
20912147990

push

github

apowers313
ci: fix flaky socket

13680 of 18076 branches covered (75.68%)

Branch coverage included in aggregate %.

42217 of 49976 relevant lines covered (84.47%)

152436.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.33
/graphty-element/src/data/JsonDataSource.ts
1
import jmespath from "jmespath";
15!
2
import { z } from "zod/v4";
15✔
3
import * as z4 from "zod/v4/core";
4

5
// import {JSONParser} from "@streamparser/json";
6
import type { AdHocData } from "../config/common";
7
import { BaseDataSourceConfig, DataSource, DataSourceChunk } from "./DataSource";
15✔
8

9
const JsonNodeConfig = z
15✔
10
    .strictObject({
15✔
11
        path: z.string().default("nodes"),
15✔
12
        schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
13
    })
15✔
14
    .prefault({});
15✔
15

16
const JsonEdgeConfig = z
15✔
17
    .strictObject({
15✔
18
        path: z.string().default("edges"),
15✔
19
        schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
20
    })
15✔
21
    .prefault({});
15✔
22

23
export const JsonDataSourceConfig = z.object({
15✔
24
    data: z.string().optional(),
15✔
25
    file: z.instanceof(File).optional(),
15✔
26
    url: z.string().optional(),
15✔
27
    chunkSize: z.number().optional(),
15✔
28
    errorLimit: z.number().optional(),
15✔
29
    nodeIdPath: z.string().optional(),
15✔
30
    edgeSrcIdPath: z.string().optional(),
15✔
31
    edgeDstIdPath: z.string().optional(),
15✔
32
    node: JsonNodeConfig,
15✔
33
    edge: JsonEdgeConfig,
15✔
34
});
15✔
35

36
type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
37

38
/**
39
 * Data source for loading graph data from JSON files.
40
 * Supports JMESPath queries for extracting nodes and edges from complex JSON structures.
41
 */
42
export class JsonDataSource extends DataSource {
15✔
43
    static type = "json";
21✔
44
    opts: JsonDataSourceConfigType;
45

46
    /**
47
     * Creates a new JsonDataSource instance.
48
     * @param anyOpts - Configuration options for JSON parsing and data extraction
49
     */
50
    constructor(anyOpts: object) {
21✔
51
        const opts = JsonDataSourceConfig.parse(anyOpts);
121✔
52

53
        // Pass errorLimit and chunkSize to base class
54
        super(opts.errorLimit ?? 100, opts.chunkSize ?? DataSource.DEFAULT_CHUNK_SIZE);
121✔
55

56
        this.opts = opts;
121✔
57
        if (opts.node.schema) {
121!
58
            this.nodeSchema = opts.node.schema;
4✔
59
        }
4✔
60

61
        if (opts.edge.schema) {
121!
62
            this.edgeSchema = opts.edge.schema;
1✔
63
        }
1✔
64
    }
121✔
65

66
    protected getConfig(): BaseDataSourceConfig {
21✔
67
        // JsonDataSource has special handling for 'data' field:
68
        // If data starts with http/https/data:, treat it as URL
69
        // Otherwise treat it as inline JSON
70
        const isUrl =
120✔
71
            (this.opts.data?.startsWith("http://") ?? false) ||
120✔
72
            (this.opts.data?.startsWith("https://") ?? false) ||
120!
73
            (this.opts.data?.startsWith("data:") ?? false);
34!
74

75
        return {
120✔
76
            data: isUrl ? undefined : this.opts.data,
120!
77
            file: this.opts.file,
120✔
78
            url: isUrl ? this.opts.data : this.opts.url,
120!
79
            chunkSize: this.opts.chunkSize,
120✔
80
            errorLimit: this.opts.errorLimit,
120✔
81
        };
120✔
82
    }
120✔
83

84
    /**
85
     * Fetches and parses JSON data into graph chunks.
86
     * Uses JMESPath to extract nodes and edges from the JSON structure.
87
     * @yields DataSourceChunk objects containing parsed nodes and edges
88
     */
89
    async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {
21✔
90
        let data: unknown;
120✔
91

92
        // Get JSON content (could be from data, file, or url)
93
        const jsonString = await this.getContent();
120✔
94

95
        // Handle empty content gracefully
96
        if (jsonString.trim() === "") {
119!
97
            yield* this.chunkData([], []);
1✔
98
            return;
1✔
99
        }
1✔
100

101
        // Parse JSON
102
        try {
118✔
103
            data = JSON.parse(jsonString);
118✔
104
        } catch (error) {
119!
105
            const canContinue = this.errorAggregator.addError({
2✔
106
                message: `Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`,
2!
107
                category: "parse-error",
2✔
108
            });
2✔
109

110
            if (!canContinue) {
2!
111
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
112
            }
×
113

114
            yield* this.chunkData([], []);
2✔
115
            return;
2✔
116
        }
2✔
117

118
        // Extract nodes using JMESPath
119
        let rawNodes: unknown[] = [];
116✔
120
        try {
116✔
121
            const nodes = jmespath.search(data, this.opts.node.path);
116✔
122
            if (Array.isArray(nodes)) {
119✔
123
                rawNodes = nodes;
113✔
124
            } else if (nodes !== null && nodes !== undefined) {
119!
125
                // Log error but continue with empty nodes
126
                const canContinue = this.errorAggregator.addError({
1✔
127
                    message: `Expected 'nodes' at path '${this.opts.node.path}' to be an array, got ${typeof nodes}`,
1✔
128
                    category: "validation-error",
1✔
129
                    field: "nodes",
1✔
130
                });
1✔
131

132
                if (!canContinue) {
1!
133
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
134
                }
×
135
            }
1✔
136
            // If nodes is null/undefined, just use empty array
137
        } catch (error) {
120!
138
            const canContinue = this.errorAggregator.addError({
×
139
                message: `Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`,
×
140
                category: "parse-error",
×
141
                field: "nodes",
×
142
            });
×
143

144
            if (!canContinue) {
×
145
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
146
            }
×
147
        }
✔
148

149
        // Extract edges using JMESPath
150
        let rawEdges: unknown[] = [];
116✔
151
        try {
116✔
152
            const edges = jmespath.search(data, this.opts.edge.path);
116✔
153
            if (Array.isArray(edges)) {
119✔
154
                rawEdges = edges;
113✔
155
            } else if (edges !== null && edges !== undefined) {
119!
156
                // Log error but continue with empty edges
157
                const canContinue = this.errorAggregator.addError({
1✔
158
                    message: `Expected 'edges' at path '${this.opts.edge.path}' to be an array, got ${typeof edges}`,
1✔
159
                    category: "validation-error",
1✔
160
                    field: "edges",
1✔
161
                });
1✔
162

163
                if (!canContinue) {
1!
164
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
165
                }
×
166
            }
1✔
167
            // If edges is null/undefined, just use empty array
168
        } catch (error) {
120!
169
            const canContinue = this.errorAggregator.addError({
×
170
                message: `Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`,
×
171
                category: "parse-error",
×
172
                field: "edges",
×
173
            });
×
174

175
            if (!canContinue) {
×
176
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
177
            }
×
178
        }
✔
179

180
        // Validate individual nodes and filter out invalid ones
181
        const validNodes: AdHocData[] = [];
116✔
182
        for (let i = 0; i < rawNodes.length; i++) {
119✔
183
            const node = rawNodes[i];
5,948✔
184
            if (this.isValidNode(node, i)) {
5,948✔
185
                validNodes.push(node as AdHocData);
5,947✔
186
            }
5,947✔
187
        }
5,948✔
188

189
        // Validate individual edges and filter out invalid ones
190
        const validEdges: AdHocData[] = [];
116✔
191
        for (let i = 0; i < rawEdges.length; i++) {
119✔
192
            const edge = rawEdges[i];
7,518✔
193
            if (this.isValidEdge(edge, i)) {
7,518✔
194
                validEdges.push(edge as AdHocData);
7,517✔
195
            }
7,517✔
196
        }
7,518✔
197

198
        // Yield data in chunks using inherited helper
199
        yield* this.chunkData(validNodes, validEdges);
116✔
200
    }
120✔
201

202
    /**
203
     * Validates a node object and logs errors if invalid.
204
     * Returns true if the node is valid and should be included.
205
     * @param node - The node object to validate
206
     * @param index - Index of the node in the array
207
     * @returns True if valid, false otherwise
208
     */
209
    private isValidNode(node: unknown, index: number): boolean {
21✔
210
        if (node === null || node === undefined) {
5,948!
211
            const canContinue = this.errorAggregator.addError({
×
212
                message: `Node at index ${index} is null or undefined`,
×
213
                category: "validation-error",
×
214
                field: "nodes",
×
215
                line: index,
×
216
            });
×
217

218
            if (!canContinue) {
×
219
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
220
            }
×
221

222
            return false;
×
223
        }
×
224

225
        if (typeof node !== "object") {
5,948!
226
            const canContinue = this.errorAggregator.addError({
×
227
                message: `Node at index ${index} is not an object (got ${typeof node})`,
×
228
                category: "validation-error",
×
229
                field: "nodes",
×
230
                line: index,
×
231
            });
×
232

233
            if (!canContinue) {
×
234
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
235
            }
×
236

237
            return false;
×
238
        }
×
239

240
        // Check for id field (common requirement - accept common identifier field names)
241
        // If a custom nodeIdPath is specified, also accept that field
242
        const nodeObj = node as Record<string, unknown>;
5,948✔
243
        const customIdPath = this.opts.nodeIdPath;
5,948✔
244
        const hasId =
5,948✔
245
            "id" in nodeObj ||
5,948!
246
            "name" in nodeObj ||
90!
247
            "key" in nodeObj ||
13!
248
            "label" in nodeObj ||
3!
249
            (customIdPath !== undefined && customIdPath in nodeObj);
3!
250
        if (!hasId) {
5,948!
251
            const expectedFields = customIdPath
1!
252
                ? `'id', 'name', 'key', 'label', or '${customIdPath}'`
×
253
                : "'id', 'name', 'key', or 'label'";
1✔
254
            const canContinue = this.errorAggregator.addError({
1✔
255
                message: `Node at index ${index} is missing identifier field (expected ${expectedFields})`,
1✔
256
                category: "missing-value",
1✔
257
                field: "nodes.id",
1✔
258
                line: index,
1✔
259
            });
1✔
260

261
            if (!canContinue) {
1!
262
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
263
            }
×
264

265
            return false;
1✔
266
        }
1✔
267

268
        return true;
5,947✔
269
    }
5,948✔
270

271
    /**
272
     * Validates an edge object and logs errors if invalid.
273
     * Returns true if the edge is valid and should be included.
274
     * @param edge - The edge object to validate
275
     * @param index - Index of the edge in the array
276
     * @returns True if valid, false otherwise
277
     */
278
    private isValidEdge(edge: unknown, index: number): boolean {
21✔
279
        if (edge === null || edge === undefined) {
7,518!
280
            const canContinue = this.errorAggregator.addError({
×
281
                message: `Edge at index ${index} is null or undefined`,
×
282
                category: "validation-error",
×
283
                field: "edges",
×
284
                line: index,
×
285
            });
×
286

287
            if (!canContinue) {
×
288
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
289
            }
×
290

291
            return false;
×
292
        }
×
293

294
        if (typeof edge !== "object") {
7,518!
295
            const canContinue = this.errorAggregator.addError({
×
296
                message: `Edge at index ${index} is not an object (got ${typeof edge})`,
×
297
                category: "validation-error",
×
298
                field: "edges",
×
299
                line: index,
×
300
            });
×
301

302
            if (!canContinue) {
×
303
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
304
            }
×
305

306
            return false;
×
307
        }
×
308

309
        // Check for source/target fields (common requirement)
310
        const edgeObj = edge as Record<string, unknown>;
7,518✔
311
        const hasSource = "source" in edgeObj || "src" in edgeObj || "from" in edgeObj;
7,518!
312
        const hasTarget = "target" in edgeObj || "dst" in edgeObj || "to" in edgeObj;
7,518!
313

314
        if (!hasSource) {
7,518!
315
            const canContinue = this.errorAggregator.addError({
1✔
316
                message: `Edge at index ${index} is missing source field (expected 'source', 'src', or 'from')`,
1✔
317
                category: "missing-value",
1✔
318
                field: "edges.source",
1✔
319
                line: index,
1✔
320
            });
1✔
321

322
            if (!canContinue) {
1!
323
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
324
            }
×
325

326
            return false;
1✔
327
        }
1✔
328

329
        if (!hasTarget) {
7,518!
330
            const canContinue = this.errorAggregator.addError({
×
331
                message: `Edge at index ${index} is missing target field (expected 'target', 'dst', or 'to')`,
×
332
                category: "missing-value",
×
333
                field: "edges.target",
×
334
                line: index,
×
335
            });
×
336

337
            if (!canContinue) {
×
338
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
339
            }
×
340

341
            return false;
×
342
        }
✔
343

344
        return true;
7,517✔
345
    }
7,518✔
346
}
21✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc