• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

graphty-org / graphty-element / 20219217725

15 Dec 2025 03:10AM UTC coverage: 83.666% (-2.7%) from 86.405%
20219217725

push

github

apowers313
chore: delint

4072 of 4771 branches covered (85.35%)

Branch coverage included in aggregate %.

15 of 26 new or added lines in 1 file covered. (57.69%)

890 existing lines in 12 files now uncovered.

18814 of 22583 relevant lines covered (83.31%)

8168.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.06
/src/data/JsonDataSource.ts
1
import jmespath from "jmespath";
3!
2
import {z} from "zod/v4";
3✔
3
import * as z4 from "zod/v4/core";
4

5
// import {JSONParser} from "@streamparser/json";
6
import type {AdHocData, PartiallyOptional} from "../config/common";
7
import {BaseDataSourceConfig, DataSource, DataSourceChunk} from "./DataSource";
3✔
8

9
const JsonNodeConfig = z.strictObject({
3✔
10
    path: z.string().default("nodes"),
3✔
11
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
3✔
12
}).prefault({});
3✔
13

14
const JsonEdgeConfig = z.strictObject({
3✔
15
    path: z.string().default("edges"),
3✔
16
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
3✔
17
}).prefault({});
3✔
18

19
export const JsonDataSourceConfig = z.object({
3✔
20
    data: z.string().optional(),
3✔
21
    file: z.instanceof(File).optional(),
3✔
22
    url: z.string().optional(),
3✔
23
    chunkSize: z.number().optional(),
3✔
24
    errorLimit: z.number().optional(),
3✔
25
    nodeIdPath: z.string().optional(),
3✔
26
    edgeSrcIdPath: z.string().optional(),
3✔
27
    edgeDstIdPath: z.string().optional(),
3✔
28
    node: JsonNodeConfig,
3✔
29
    edge: JsonEdgeConfig,
3✔
30
});
3✔
31

32
export type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
33
export type JsonDataSourceConfigOpts = PartiallyOptional<JsonDataSourceConfigType, "node" | "edge">;
34

35
export class JsonDataSource extends DataSource {
3✔
36
    static type = "json";
9✔
37
    opts: JsonDataSourceConfigType;
38

39
    constructor(anyOpts: object) {
9✔
40
        const opts = JsonDataSourceConfig.parse(anyOpts);
121✔
41

42
        // Pass errorLimit and chunkSize to base class
43
        super(opts.errorLimit ?? 100, opts.chunkSize ?? DataSource.DEFAULT_CHUNK_SIZE);
121✔
44

45
        this.opts = opts;
121✔
46
        if (opts.node.schema) {
121✔
47
            this.nodeSchema = opts.node.schema;
4✔
48
        }
4✔
49

50
        if (opts.edge.schema) {
121✔
51
            this.edgeSchema = opts.edge.schema;
1✔
52
        }
1✔
53
    }
121✔
54

55
    protected getConfig(): BaseDataSourceConfig {
9✔
56
        // JsonDataSource has special handling for 'data' field:
57
        // If data starts with http/https/data:, treat it as URL
58
        // Otherwise treat it as inline JSON
59
        const isUrl = (this.opts.data?.startsWith("http://") ?? false) ||
120✔
60
                     (this.opts.data?.startsWith("https://") ?? false) ||
120✔
61
                     (this.opts.data?.startsWith("data:") ?? false);
34✔
62

63
        return {
120✔
64
            data: isUrl ? undefined : this.opts.data,
120✔
65
            file: this.opts.file,
120✔
66
            url: isUrl ? this.opts.data : this.opts.url,
120✔
67
            chunkSize: this.opts.chunkSize,
120✔
68
            errorLimit: this.opts.errorLimit,
120✔
69
        };
120✔
70
    }
120✔
71

72
    async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {
9✔
73
        let data: unknown;
120✔
74

75
        // Get JSON content (could be from data, file, or url)
76
        const jsonString = await this.getContent();
120✔
77

78
        // Handle empty content gracefully
79
        if (jsonString.trim() === "") {
119✔
80
            yield* this.chunkData([], []);
1✔
81
            return;
1✔
82
        }
1✔
83

84
        // Parse JSON
85
        try {
118✔
86
            data = JSON.parse(jsonString);
118✔
87
        } catch (error) {
119✔
88
            const canContinue = this.errorAggregator.addError({
2✔
89
                message: `Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`,
2!
90
                category: "parse-error",
2✔
91
            });
2✔
92

93
            if (!canContinue) {
2!
UNCOV
94
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
95
            }
×
96

97
            yield* this.chunkData([], []);
2✔
98
            return;
2✔
99
        }
2✔
100

101
        // Extract nodes using JMESPath
102
        let rawNodes: unknown[] = [];
116✔
103
        try {
116✔
104
            const nodes = jmespath.search(data, this.opts.node.path);
116✔
105
            if (Array.isArray(nodes)) {
119✔
106
                rawNodes = nodes;
113✔
107
            } else if (nodes !== null && nodes !== undefined) {
119✔
108
                // Log error but continue with empty nodes
109
                const canContinue = this.errorAggregator.addError({
1✔
110
                    message: `Expected 'nodes' at path '${this.opts.node.path}' to be an array, got ${typeof nodes}`,
1✔
111
                    category: "validation-error",
1✔
112
                    field: "nodes",
1✔
113
                });
1✔
114

115
                if (!canContinue) {
1!
UNCOV
116
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
117
                }
×
118
            }
1✔
119
            // If nodes is null/undefined, just use empty array
120
        } catch (error) {
120!
UNCOV
121
            const canContinue = this.errorAggregator.addError({
×
UNCOV
122
                message: `Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`,
×
UNCOV
123
                category: "parse-error",
×
UNCOV
124
                field: "nodes",
×
UNCOV
125
            });
×
126

UNCOV
127
            if (!canContinue) {
×
UNCOV
128
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
129
            }
×
UNCOV
130
        }
✔
131

132
        // Extract edges using JMESPath
133
        let rawEdges: unknown[] = [];
116✔
134
        try {
116✔
135
            const edges = jmespath.search(data, this.opts.edge.path);
116✔
136
            if (Array.isArray(edges)) {
119✔
137
                rawEdges = edges;
113✔
138
            } else if (edges !== null && edges !== undefined) {
119✔
139
                // Log error but continue with empty edges
140
                const canContinue = this.errorAggregator.addError({
1✔
141
                    message: `Expected 'edges' at path '${this.opts.edge.path}' to be an array, got ${typeof edges}`,
1✔
142
                    category: "validation-error",
1✔
143
                    field: "edges",
1✔
144
                });
1✔
145

146
                if (!canContinue) {
1!
UNCOV
147
                    throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
148
                }
×
149
            }
1✔
150
            // If edges is null/undefined, just use empty array
151
        } catch (error) {
120!
UNCOV
152
            const canContinue = this.errorAggregator.addError({
×
UNCOV
153
                message: `Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`,
×
UNCOV
154
                category: "parse-error",
×
UNCOV
155
                field: "edges",
×
UNCOV
156
            });
×
157

UNCOV
158
            if (!canContinue) {
×
UNCOV
159
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
160
            }
×
UNCOV
161
        }
✔
162

163
        // Validate individual nodes and filter out invalid ones
164
        const validNodes: AdHocData[] = [];
116✔
165
        for (let i = 0; i < rawNodes.length; i++) {
119✔
166
            const node = rawNodes[i];
5,948✔
167
            if (this.isValidNode(node, i)) {
5,948✔
168
                validNodes.push(node as AdHocData);
5,947✔
169
            }
5,947✔
170
        }
5,948✔
171

172
        // Validate individual edges and filter out invalid ones
173
        const validEdges: AdHocData[] = [];
116✔
174
        for (let i = 0; i < rawEdges.length; i++) {
119✔
175
            const edge = rawEdges[i];
7,518✔
176
            if (this.isValidEdge(edge, i)) {
7,518✔
177
                validEdges.push(edge as AdHocData);
7,517✔
178
            }
7,517✔
179
        }
7,518✔
180

181
        // Yield data in chunks using inherited helper
182
        yield* this.chunkData(validNodes, validEdges);
116✔
183
    }
120✔
184

185
    /**
186
     * Validates a node object and logs errors if invalid.
187
     * Returns true if the node is valid and should be included.
188
     */
189
    private isValidNode(node: unknown, index: number): boolean {
9✔
190
        if (node === null || node === undefined) {
5,948!
UNCOV
191
            const canContinue = this.errorAggregator.addError({
×
UNCOV
192
                message: `Node at index ${index} is null or undefined`,
×
UNCOV
193
                category: "validation-error",
×
UNCOV
194
                field: "nodes",
×
UNCOV
195
                line: index,
×
UNCOV
196
            });
×
197

UNCOV
198
            if (!canContinue) {
×
UNCOV
199
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
200
            }
×
201

UNCOV
202
            return false;
×
UNCOV
203
        }
×
204

205
        if (typeof node !== "object") {
5,948!
UNCOV
206
            const canContinue = this.errorAggregator.addError({
×
UNCOV
207
                message: `Node at index ${index} is not an object (got ${typeof node})`,
×
UNCOV
208
                category: "validation-error",
×
UNCOV
209
                field: "nodes",
×
UNCOV
210
                line: index,
×
UNCOV
211
            });
×
212

UNCOV
213
            if (!canContinue) {
×
UNCOV
214
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
215
            }
×
216

UNCOV
217
            return false;
×
UNCOV
218
        }
×
219

220
        // Check for id field (common requirement - accept common identifier field names)
221
        // If a custom nodeIdPath is specified, also accept that field
222
        const nodeObj = node as Record<string, unknown>;
5,948✔
223
        const customIdPath = this.opts.nodeIdPath;
5,948✔
224
        const hasId = "id" in nodeObj || "name" in nodeObj || "key" in nodeObj || "label" in nodeObj ||
5,948✔
225
                      (customIdPath !== undefined && customIdPath in nodeObj);
3✔
226
        if (!hasId) {
5,948✔
227
            const expectedFields = customIdPath ?
1!
UNCOV
228
                `'id', 'name', 'key', 'label', or '${customIdPath}'` :
×
229
                "'id', 'name', 'key', or 'label'";
1✔
230
            const canContinue = this.errorAggregator.addError({
1✔
231
                message: `Node at index ${index} is missing identifier field (expected ${expectedFields})`,
1✔
232
                category: "missing-value",
1✔
233
                field: "nodes.id",
1✔
234
                line: index,
1✔
235
            });
1✔
236

237
            if (!canContinue) {
1!
UNCOV
238
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
239
            }
×
240

241
            return false;
1✔
242
        }
1✔
243

244
        return true;
5,947✔
245
    }
5,948✔
246

247
    /**
248
     * Validates an edge object and logs errors if invalid.
249
     * Returns true if the edge is valid and should be included.
250
     */
251
    private isValidEdge(edge: unknown, index: number): boolean {
9✔
252
        if (edge === null || edge === undefined) {
7,518!
UNCOV
253
            const canContinue = this.errorAggregator.addError({
×
UNCOV
254
                message: `Edge at index ${index} is null or undefined`,
×
UNCOV
255
                category: "validation-error",
×
UNCOV
256
                field: "edges",
×
UNCOV
257
                line: index,
×
UNCOV
258
            });
×
259

UNCOV
260
            if (!canContinue) {
×
UNCOV
261
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
262
            }
×
263

UNCOV
264
            return false;
×
UNCOV
265
        }
×
266

267
        if (typeof edge !== "object") {
7,518!
UNCOV
268
            const canContinue = this.errorAggregator.addError({
×
UNCOV
269
                message: `Edge at index ${index} is not an object (got ${typeof edge})`,
×
UNCOV
270
                category: "validation-error",
×
UNCOV
271
                field: "edges",
×
UNCOV
272
                line: index,
×
UNCOV
273
            });
×
274

UNCOV
275
            if (!canContinue) {
×
UNCOV
276
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
277
            }
×
278

UNCOV
279
            return false;
×
UNCOV
280
        }
×
281

282
        // Check for source/target fields (common requirement)
283
        const edgeObj = edge as Record<string, unknown>;
7,518✔
284
        const hasSource = "source" in edgeObj || "src" in edgeObj || "from" in edgeObj;
7,518✔
285
        const hasTarget = "target" in edgeObj || "dst" in edgeObj || "to" in edgeObj;
7,518✔
286

287
        if (!hasSource) {
7,518✔
288
            const canContinue = this.errorAggregator.addError({
1✔
289
                message: `Edge at index ${index} is missing source field (expected 'source', 'src', or 'from')`,
1✔
290
                category: "missing-value",
1✔
291
                field: "edges.source",
1✔
292
                line: index,
1✔
293
            });
1✔
294

295
            if (!canContinue) {
1!
UNCOV
296
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
297
            }
×
298

299
            return false;
1✔
300
        }
1✔
301

302
        if (!hasTarget) {
7,518!
UNCOV
303
            const canContinue = this.errorAggregator.addError({
×
UNCOV
304
                message: `Edge at index ${index} is missing target field (expected 'target', 'dst', or 'to')`,
×
UNCOV
305
                category: "missing-value",
×
UNCOV
306
                field: "edges.target",
×
UNCOV
307
                line: index,
×
UNCOV
308
            });
×
309

UNCOV
310
            if (!canContinue) {
×
UNCOV
311
                throw new Error(`Too many errors (${this.errorAggregator.getErrorCount()}), aborting parse`);
×
UNCOV
312
            }
×
313

UNCOV
314
            return false;
×
UNCOV
315
        }
✔
316

317
        return true;
7,517✔
318
    }
7,518✔
319
}
9✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc