18770573775

Committed 24 Oct 2025 05:27AM UTC coverage: 82.365% (+0.7%) from 81.618%

Build # 18770573775

Build Type

push

github

Committed by

apowers313

Commit Message

ci: fix ci, take 2

Run Details

1221 of 1491 branches covered (81.89%)

Branch coverage included in aggregate %.

6588 of 7990 relevant lines covered (82.45%)

601.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.2

/src/data/JsonDataSource.ts

import jmespath from "jmespath";
import {z} from "zod/v4";
import * as z4 from "zod/v4/core";

// import {JSONParser} from "@streamparser/json";
import type {PartiallyOptional} from "../config/common";
import {DataSource, DataSourceChunk} from "./DataSource";

const JsonNodeConfig = z.strictObject({
    path: z.string().default("nodes"),
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
}).prefault({});

const JsonEdgeConfig = z.strictObject({
    path: z.string().default("edges"),
    schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),
}).prefault({});

export const JsonDataSourceConfig = z.object({
    data: z.string(),
    node: JsonNodeConfig,
    edge: JsonEdgeConfig,
});

export type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
export type JsonDataSourceConfigOpts = PartiallyOptional<JsonDataSourceConfigType, "node" | "edge">;

export class JsonDataSource extends DataSource {
    static type = "json";
    url: string;
    opts: JsonDataSourceConfigType;

    constructor(anyOpts: object) {
        super();

        const opts = JsonDataSourceConfig.parse(anyOpts);
        this.opts = opts;
        if (opts.node.schema) {
            this.nodeSchema = opts.node.schema;
        }

        if (opts.edge.schema) {
            this.edgeSchema = opts.edge.schema;
        }

        this.url = opts.data;
    }

    private async fetchWithRetry(url: string, retries = 3, timeout = 30000): Promise<Response> {
        for (let attempt = 0; attempt < retries; attempt++) {
            try {
                // Create an AbortController for timeout
                const controller = new AbortController();
                const timeoutId = setTimeout(() => {
                    controller.abort();
                }, timeout);

                try {
                    const response = await fetch(url, {signal: controller.signal});
                    clearTimeout(timeoutId);

                    if (!response.ok) {
                        throw new Error(`HTTP error! status: ${response.status}`);
                    }

                    return response;
                } catch (error) {
                    clearTimeout(timeoutId);

                    if (error instanceof Error && error.name === "AbortError") {
                        throw new Error(`Request timeout after ${timeout}ms`);
                    }

                    throw error;
                }
            } catch (error) {
                const isLastAttempt = attempt === retries - 1;

                if (isLastAttempt) {
                    throw new Error(`Failed to fetch data after ${retries} attempts: ${error instanceof Error ? error.message : String(error)}`);
                }

                // Exponential backoff: wait 1s, 2s, 4s...
                const delay = Math.pow(2, attempt) * 1000;
                await new Promise((resolve) => setTimeout(resolve, delay));
            }
        }

        // This should never be reached due to the throw in the last attempt
        throw new Error("Unexpected error in fetchWithRetry");
    }

    async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {
        let response: Response;
        let data: unknown;

        try {
            response = await this.fetchWithRetry(this.url);
        } catch (error) {
            throw new Error(`Failed to fetch data from ${this.url}: ${error instanceof Error ? error.message : String(error)}`);
        }

        if (!response.body) {
            throw new Error("JSON response had no body");
        }

        try {
            data = await response.json();
        } catch (error) {
            throw new Error(`Failed to parse JSON response: ${error instanceof Error ? error.message : String(error)}`);
        }

        let nodes: unknown;
        let edges: unknown;

        try {
            nodes = jmespath.search(data, this.opts.node.path);
        } catch (error) {
            throw new Error(`Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`);
        }

        if (!Array.isArray(nodes)) {
            throw new TypeError(`JsonDataProvider expected 'nodes' at path '${this.opts.node.path}' to be an array of objects, got ${typeof nodes}`);
        }

        try {
            edges = jmespath.search(data, this.opts.edge.path);
        } catch (error) {
            throw new Error(`Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`);
        }

        if (!Array.isArray(edges)) {
            throw new TypeError(`JsonDataProvider expected 'edges' at path '${this.opts.edge.path}' to be an array of objects, got ${typeof edges}`);
        }

        yield {nodes, edges};
    }
}

1	import jmespath from "jmespath";	3!
2	import {z} from "zod/v4";	3✔
3	import * as z4 from "zod/v4/core";
4
5	// import {JSONParser} from "@streamparser/json";
6	import type {PartiallyOptional} from "../config/common";
7	import {DataSource, DataSourceChunk} from "./DataSource";	3✔
8
9	const JsonNodeConfig = z.strictObject({	3✔
10	path: z.string().default("nodes"),	3✔
11	schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),	3✔
12	}).prefault({});	3✔
13
14	const JsonEdgeConfig = z.strictObject({	3✔
15	path: z.string().default("edges"),	3✔
16	schema: z.custom<z4.$ZodObject>().or(z.null()).default(null),	3✔
17	}).prefault({});	3✔
18
19	export const JsonDataSourceConfig = z.object({	3✔
20	data: z.string(),	3✔
21	node: JsonNodeConfig,	3✔
22	edge: JsonEdgeConfig,	3✔
23	});	3✔
24
25	export type JsonDataSourceConfigType = z.infer<typeof JsonDataSourceConfig>;
26	export type JsonDataSourceConfigOpts = PartiallyOptional<JsonDataSourceConfigType, "node" \| "edge">;
27
28	export class JsonDataSource extends DataSource {	3✔
29	static type = "json";	3✔
30	url: string;
31	opts: JsonDataSourceConfigType;
32
33	constructor(anyOpts: object) {	3✔
34	super();	66✔
35
36	const opts = JsonDataSourceConfig.parse(anyOpts);	66✔
37	this.opts = opts;	66✔
38	if (opts.node.schema) {	66✔
39	this.nodeSchema = opts.node.schema;	2✔
40	}	2✔
41
42	if (opts.edge.schema) {	66!
43	this.edgeSchema = opts.edge.schema;	×
44	}	×
45
46	this.url = opts.data;	66✔
47	}	66✔
48
49	private async fetchWithRetry(url: string, retries = 3, timeout = 30000): Promise<Response> {	3✔
50	for (let attempt = 0; attempt < retries; attempt++) {	65✔
51	try {	65✔
52	// Create an AbortController for timeout
53	const controller = new AbortController();	65✔
54	const timeoutId = setTimeout(() => {	65✔
55	controller.abort();	×
56	}, timeout);	65✔
57
58	try {	65✔
59	const response = await fetch(url, {signal: controller.signal});	65✔
60	clearTimeout(timeoutId);	65✔
61
62	if (!response.ok) {	65!
63	throw new Error(`HTTP error! status: ${response.status}`);	×
64	}	×
65
66	return response;	65✔
67	} catch (error) {	65!
68	clearTimeout(timeoutId);	×
69
70	if (error instanceof Error && error.name === "AbortError") {	×
71	throw new Error(`Request timeout after ${timeout}ms`);	×
72	}	×
73
74	throw error;	×
75	}	×
76	} catch (error) {	65!
77	const isLastAttempt = attempt === retries - 1;	×
78
79	if (isLastAttempt) {	×
80	throw new Error(`Failed to fetch data after ${retries} attempts: ${error instanceof Error ? error.message : String(error)}`);	×
81	}	×
82
83	// Exponential backoff: wait 1s, 2s, 4s...
84	const delay = Math.pow(2, attempt) * 1000;	×
85	await new Promise((resolve) => setTimeout(resolve, delay));	×
86	}	×
87	}	65!
88
89	// This should never be reached due to the throw in the last attempt
90	throw new Error("Unexpected error in fetchWithRetry");	×
91	}	65✔
92
93	async *sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown> {	3✔
94	let response: Response;	65✔
95	let data: unknown;	65✔
96
97	try {	65✔
98	response = await this.fetchWithRetry(this.url);	65✔
99	} catch (error) {	65!
100	throw new Error(`Failed to fetch data from ${this.url}: ${error instanceof Error ? error.message : String(error)}`);	×
101	}	×
102
103	if (!response.body) {	65!
104	throw new Error("JSON response had no body");	×
105	}	×
106
107	try {	65✔
108	data = await response.json();	65✔
109	} catch (error) {	65!
110	throw new Error(`Failed to parse JSON response: ${error instanceof Error ? error.message : String(error)}`);	×
111	}	✔
112
113	let nodes: unknown;	65✔
114	let edges: unknown;	65✔
115
116	try {	65✔
117	nodes = jmespath.search(data, this.opts.node.path);	65✔
118	} catch (error) {	65!
119	throw new Error(`Failed to extract nodes using path '${this.opts.node.path}': ${error instanceof Error ? error.message : String(error)}`);	×
120	}	✔
121
122	if (!Array.isArray(nodes)) {	65!
123	throw new TypeError(`JsonDataProvider expected 'nodes' at path '${this.opts.node.path}' to be an array of objects, got ${typeof nodes}`);	×
124	}	✔
125
126	try {	65✔
127	edges = jmespath.search(data, this.opts.edge.path);	65✔
128	} catch (error) {	65!
129	throw new Error(`Failed to extract edges using path '${this.opts.edge.path}': ${error instanceof Error ? error.message : String(error)}`);	×
130	}	✔
131
132	if (!Array.isArray(edges)) {	65!
133	throw new TypeError(`JsonDataProvider expected 'edges' at path '${this.opts.edge.path}' to be an array of objects, got ${typeof edges}`);	×
134	}	✔
135
136	yield {nodes, edges};	65✔
137	}	65✔
138	}	3✔

graphty-org / graphty-element / 18770573775

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous