• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

graphty-org / graphty-element / 20514590651

26 Dec 2025 02:37AM UTC coverage: 70.559% (-0.3%) from 70.836%
20514590651

push

github

apowers313
ci: fix npm ci

9591 of 13363 branches covered (71.77%)

Branch coverage included in aggregate %.

25136 of 35854 relevant lines covered (70.11%)

6233.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.13
/src/data/JsonDataSource.ts
1
import jmespath from "jmespath";
15!
2
import {z} from "zod/v4";
15✔
3
import * as z4 from "zod/v4/core";
4

5
// import {JSONParser} from "@streamparser/json";
6
import type {AdHocData, PartiallyOptional} from "../config/common";
7
import {BaseDataSourceConfig, DataSource, DataSourceChunk} from "./DataSource";
15✔
8

9
const JsonNodeConfig = z.strictObject({
15✔
10
    path: z.string().default("nodes"),
15✔
11
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
12
}).prefault({});
15✔
13

14
const JsonEdgeConfig = z.strictObject({
15✔
15
    path: z.string().default("edges"),
15✔
16
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
15✔
17
}).prefault({});
15✔
18

19
export const JsonDataSourceConfig = z.object({
15✔
20
    data: z.string().optional(),
15✔
21
    file: z.instanceof(File).optional(),
15✔
22
    url: z.string().optional(),
15✔
23
    chunkSize: z.number().optional(),
15✔
24
    errorLimit: z.number().optional(),
15✔
25
    nodeIdPath: z.string().optional(),
15✔
26
    edgeSrcIdPath: z.string().optional(),
15✔
27
    edgeDstIdPath: z.string().optional(),
15✔
28
    node: JsonNodeConfig,
15✔
29
    edge: JsonEdgeConfig,
15✔
30
});
15✔
31

32
export type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
33
export type JsonDataSourceConfigOpts = PartiallyOptional<JsonDataSourceConfigType, "node" | "edge">;
34

35
/**
36
 * Data source for loading graph data from JSON files.
37
 * Supports JMESPath queries for extracting nodes and edges from complex JSON structures.
38
 */
39
export class JsonDataSource extends DataSource {
15✔
40
    static type = "json";
21✔
41
    opts: JsonDataSourceConfigType;
42

43
    /**
44
     * Creates a new JsonDataSource instance.
45
     * @param anyOpts - Configuration options for JSON parsing and data extraction
46
     */
47
    constructor(anyOpts: object) {
21✔
48
        const opts = JsonDataSourceConfig.parse(anyOpts);
121✔
49

50
        // Pass errorLimit and chunkSize to base class
51
        super(opts.errorLimit ?? 100, opts.chunkSize ?? DataSource.DEFAULT_CHUNK_SIZE);
121✔
52

53
        this.opts = opts;
121✔
54
        if (opts.node.schema) {
121!
55
            this.nodeSchema = opts.node.schema;
4✔
56
        }
4✔
57

58
        if (opts.edge.schema) {
121!
59
            this.edgeSchema = opts.edge.schema;
1✔
60
        }
1✔
61
    }
121✔
62

63
    protected getConfig(): BaseDataSourceConfig {
21✔
64
        // JsonDataSource has special handling for 'data' field:
65
        // If data starts with http/https/data:, treat it as URL
66
        // Otherwise treat it as inline JSON
67
        const isUrl = (this.opts.data?.startsWith("http://") ?? false) ||
120✔
68
                     (this.opts.data?.startsWith("https://") ?? false) ||
120!
69
                     (this.opts.data?.startsWith("data:") ?? false);
34!
70

71
        return {
120✔
72
            data: isUrl ? undefined : this.opts.data,
120!
73
            file: this.opts.file,
120✔
74
            url: isUrl ? this.opts.data : this.opts.url,
120!
75
            chunkSize: this.opts.chunkSize,
120✔
76
            errorLimit: this.opts.errorLimit,
120✔
77
        };
120✔
78
    }
120✔
79

80
    /**
81
     * Fetches and parses JSON data into graph chunks.
82
     * Uses JMESPath to extract nodes and edges from the JSON structure.
83
     * @yields DataSourceChunk objects containing parsed nodes and edges
84
     */
85
    async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {
21✔
86
        let data: unknown;
120✔
87

88
        // Get JSON content (could be from data, file, or url)
89
        const jsonString = await this.getContent();
120✔
90

91
        // Handle empty content gracefully
92
        if (jsonString.trim() === "") {
119!
93
            yield* this.chunkData([], []);
1✔
94
            return;
1✔
95
        }
1✔
96

97
        // Parse JSON
98
        try {
118✔
99
            data = JSON.parse(jsonString);
118✔
100
        } catch (error) {
119!
101
            const canContinue = this.errorAggregator.addError({
2✔
102
                message: `Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`,
2!
103
                category: "parse-error",
2✔
104
            });
2✔
105

106
            if (!canContinue) {
2!
107
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
108
            }
×
109

110
            yield* this.chunkData([], []);
2✔
111
            return;
2✔
112
        }
2✔
113

114
        // Extract nodes using JMESPath
115
        let rawNodes: unknown[] = [];
116✔
116
        try {
116✔
117
            const nodes = jmespath.search(data, this.opts.node.path);
116✔
118
            if (Array.isArray(nodes)) {
119✔
119
                rawNodes = nodes;
113✔
120
            } else if (nodes !== null && nodes !== undefined) {
119!
121
                // Log error but continue with empty nodes
122
                const canContinue = this.errorAggregator.addError({
1✔
123
                    message: `Expected 'nodes' at path '${this.opts.node.path}' to be an array, got ${typeof nodes}`,
1✔
124
                    category: "validation-error",
1✔
125
                    field: "nodes",
1✔
126
                });
1✔
127

128
                if (!canContinue) {
1!
129
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
130
                }
×
131
            }
1✔
132
            // If nodes is null/undefined, just use empty array
133
        } catch (error) {
120!
134
            const canContinue = this.errorAggregator.addError({
×
135
                message: `Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`,
×
136
                category: "parse-error",
×
137
                field: "nodes",
×
138
            });
×
139

140
            if (!canContinue) {
×
141
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
142
            }
×
143
        }
✔
144

145
        // Extract edges using JMESPath
146
        let rawEdges: unknown[] = [];
116✔
147
        try {
116✔
148
            const edges = jmespath.search(data, this.opts.edge.path);
116✔
149
            if (Array.isArray(edges)) {
119✔
150
                rawEdges = edges;
113✔
151
            } else if (edges !== null && edges !== undefined) {
119!
152
                // Log error but continue with empty edges
153
                const canContinue = this.errorAggregator.addError({
1✔
154
                    message: `Expected 'edges' at path '${this.opts.edge.path}' to be an array, got ${typeof edges}`,
1✔
155
                    category: "validation-error",
1✔
156
                    field: "edges",
1✔
157
                });
1✔
158

159
                if (!canContinue) {
1!
160
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
161
                }
×
162
            }
1✔
163
            // If edges is null/undefined, just use empty array
164
        } catch (error) {
120!
165
            const canContinue = this.errorAggregator.addError({
×
166
                message: `Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`,
×
167
                category: "parse-error",
×
168
                field: "edges",
×
169
            });
×
170

171
            if (!canContinue) {
×
172
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
173
            }
×
174
        }
✔
175

176
        // Validate individual nodes and filter out invalid ones
177
        const validNodes: AdHocData[] = [];
116✔
178
        for (let i = 0; i < rawNodes.length; i++) {
119✔
179
            const node = rawNodes[i];
5,948✔
180
            if (this.isValidNode(node, i)) {
5,948✔
181
                validNodes.push(node as AdHocData);
5,947✔
182
            }
5,947✔
183
        }
5,948✔
184

185
        // Validate individual edges and filter out invalid ones
186
        const validEdges: AdHocData[] = [];
116✔
187
        for (let i = 0; i < rawEdges.length; i++) {
119✔
188
            const edge = rawEdges[i];
7,518✔
189
            if (this.isValidEdge(edge, i)) {
7,518✔
190
                validEdges.push(edge as AdHocData);
7,517✔
191
            }
7,517✔
192
        }
7,518✔
193

194
        // Yield data in chunks using inherited helper
195
        yield* this.chunkData(validNodes, validEdges);
116✔
196
    }
120✔
197

198
    /**
199
     * Validates a node object and logs errors if invalid.
200
     * Returns true if the node is valid and should be included.
201
     * @param node - The node object to validate
202
     * @param index - Index of the node in the array
203
     * @returns True if valid, false otherwise
204
     */
205
    private isValidNode(node: unknown, index: number): boolean {
21✔
206
        if (node === null || node === undefined) {
5,948!
207
            const canContinue = this.errorAggregator.addError({
×
208
                message: `Node at index ${index} is null or undefined`,
×
209
                category: "validation-error",
×
210
                field: "nodes",
×
211
                line: index,
×
212
            });
×
213

214
            if (!canContinue) {
×
215
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
216
            }
×
217

218
            return false;
×
219
        }
×
220

221
        if (typeof node !== "object") {
5,948!
222
            const canContinue = this.errorAggregator.addError({
×
223
                message: `Node at index ${index} is not an object (got ${typeof node})`,
×
224
                category: "validation-error",
×
225
                field: "nodes",
×
226
                line: index,
×
227
            });
×
228

229
            if (!canContinue) {
×
230
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
231
            }
×
232

233
            return false;
×
234
        }
×
235

236
        // Check for id field (common requirement - accept common identifier field names)
237
        // If a custom nodeIdPath is specified, also accept that field
238
        const nodeObj = node as Record<string, unknown>;
5,948✔
239
        const customIdPath = this.opts.nodeIdPath;
5,948✔
240
        const hasId = "id" in nodeObj || "name" in nodeObj || "key" in nodeObj || "label" in nodeObj ||
5,948!
241
                      (customIdPath !== undefined && customIdPath in nodeObj);
3!
242
        if (!hasId) {
5,948!
243
            const expectedFields = customIdPath ?
1!
244
                `'id', 'name', 'key', 'label', or '${customIdPath}'` :
×
245
                "'id', 'name', 'key', or 'label'";
1✔
246
            const canContinue = this.errorAggregator.addError({
1✔
247
                message: `Node at index ${index} is missing identifier field (expected ${expectedFields})`,
1✔
248
                category: "missing-value",
1✔
249
                field: "nodes.id",
1✔
250
                line: index,
1✔
251
            });
1✔
252

253
            if (!canContinue) {
1!
254
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
255
            }
×
256

257
            return false;
1✔
258
        }
1✔
259

260
        return true;
5,947✔
261
    }
5,948✔
262

263
    /**
264
     * Validates an edge object and logs errors if invalid.
265
     * Returns true if the edge is valid and should be included.
266
     * @param edge - The edge object to validate
267
     * @param index - Index of the edge in the array
268
     * @returns True if valid, false otherwise
269
     */
270
    private isValidEdge(edge: unknown, index: number): boolean {
21✔
271
        if (edge === null || edge === undefined) {
7,518!
272
            const canContinue = this.errorAggregator.addError({
×
273
                message: `Edge at index ${index} is null or undefined`,
×
274
                category: "validation-error",
×
275
                field: "edges",
×
276
                line: index,
×
277
            });
×
278

279
            if (!canContinue) {
×
280
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
281
            }
×
282

283
            return false;
×
284
        }
×
285

286
        if (typeof edge !== "object") {
7,518!
287
            const canContinue = this.errorAggregator.addError({
×
288
                message: `Edge at index ${index} is not an object (got ${typeof edge})`,
×
289
                category: "validation-error",
×
290
                field: "edges",
×
291
                line: index,
×
292
            });
×
293

294
            if (!canContinue) {
×
295
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
296
            }
×
297

298
            return false;
×
299
        }
×
300

301
        // Check for source/target fields (common requirement)
302
        const edgeObj = edge as Record<string, unknown>;
7,518✔
303
        const hasSource = "source" in edgeObj || "src" in edgeObj || "from" in edgeObj;
7,518!
304
        const hasTarget = "target" in edgeObj || "dst" in edgeObj || "to" in edgeObj;
7,518!
305

306
        if (!hasSource) {
7,518!
307
            const canContinue = this.errorAggregator.addError({
1✔
308
                message: `Edge at index ${index} is missing source field (expected 'source', 'src', or 'from')`,
1✔
309
                category: "missing-value",
1✔
310
                field: "edges.source",
1✔
311
                line: index,
1✔
312
            });
1✔
313

314
            if (!canContinue) {
1!
315
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
316
            }
×
317

318
            return false;
1✔
319
        }
1✔
320

321
        if (!hasTarget) {
7,518!
322
            const canContinue = this.errorAggregator.addError({
×
323
                message: `Edge at index ${index} is missing target field (expected 'target', 'dst', or 'to')`,
×
324
                category: "missing-value",
×
325
                field: "edges.target",
×
326
                line: index,
×
327
            });
×
328

329
            if (!canContinue) {
×
330
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
331
            }
×
332

333
            return false;
×
334
        }
✔
335

336
        return true;
7,517✔
337
    }
7,518✔
338
}
21✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc