19792929756

Committed 30 Nov 2025 02:57AM UTC coverage: 86.308% (+3.9%) from 82.377%

Build # 19792929756

Build Type

push

github

Committed by

apowers313

Commit Message

docs: fix stories for chromatic

Run Details

3676 of 4303 branches covered (85.43%)

Branch coverage included in aggregate %.

17 of 17 new or added lines in 2 files covered. (100.0%)

1093 existing lines in 30 files now uncovered.

17371 of 20083 relevant lines covered (86.5%)

7075.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.86

/src/data/DataSource.ts

import {z} from "zod/v4";
import * as z4 from "zod/v4/core";

import {AdHocData} from "../config";
import {ErrorAggregator} from "./ErrorAggregator.js";

// Base configuration interface
export interface BaseDataSourceConfig {
    data?: string;
    file?: File;
    url?: string;
    chunkSize?: number;
    errorLimit?: number;
}

type DataSourceClass = new (opts: object) => DataSource;
const dataSourceRegistry = new Map<string, DataSourceClass>();
export interface DataSourceChunk {
    nodes: AdHocData[];
    edges: AdHocData[];
}

export abstract class DataSource {
    static readonly type: string;
    static readonly DEFAULT_CHUNK_SIZE = 1000;

    edgeSchema: z4.$ZodObject | null = null;
    nodeSchema: z4.$ZodObject | null = null;
    protected errorAggregator: ErrorAggregator;
    protected chunkSize: number;

    constructor(errorLimit = 100, chunkSize = DataSource.DEFAULT_CHUNK_SIZE) {
        this.errorAggregator = new ErrorAggregator(errorLimit);
        this.chunkSize = chunkSize;
    }

    // abstract init(): Promise<void>;
    abstract sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown>;

    /**
     * Subclasses must implement this to expose their config
     * Used by getContent() and other shared methods
     */
    protected abstract getConfig(): BaseDataSourceConfig;

    /**
     * Standardized error message templates
     */
    protected get errorMessages(): {
        missingInput: () => string;
        fetchFailed: (url: string, attempts: number, error: string) => string;
        parseFailed: (error: string) => string;
        invalidFormat: (reason: string) => string;
        extractionFailed: (path: string, error: string) => string;
    } {
        return {
            missingInput: () =>
                `${this.type}DataSource requires data, file, or url`,

            fetchFailed: (url: string, attempts: number, error: string) =>
                `Failed to fetch ${this.type} from ${url} after ${attempts} attempts: ${error}`,

            parseFailed: (error: string) =>
                `Failed to parse ${this.type}: ${error}`,

            invalidFormat: (reason: string) =>
                `Invalid ${this.type} format: ${reason}`,

            extractionFailed: (path: string, error: string) =>
                `Failed to extract data using path '${path}': ${error}`,
        };
    }

    /**
     * Fetch with retry logic and timeout
     * Protected method for use by all DataSources
     */
    protected async fetchWithRetry(
        url: string,
        retries = 3,
        timeout = 30000,
    ): Promise<Response> {
        // Data URLs don't need retries or timeouts
        if (url.startsWith("data:")) {
            return await fetch(url);
        }

        for (let attempt = 0; attempt < retries; attempt++) {
            try {
                // Create AbortController for timeout
                const controller = new AbortController();
                const timeoutId = setTimeout(() => {
                    controller.abort();
                }, timeout);

                try {
                    const response = await fetch(url, {signal: controller.signal});
                    clearTimeout(timeoutId);

                    if (!response.ok) {
                        throw new Error(`HTTP error! status: ${response.status}`);
                    }

                    return response;
                } catch (error) {
                    clearTimeout(timeoutId);

                    if (error instanceof Error && error.name === "AbortError") {
                        throw new Error(`Request timeout after ${timeout}ms`);
                    }

                    throw error;
                }
            } catch (error) {
                const isLastAttempt = attempt === retries - 1;

                if (isLastAttempt) {
                    const errorMsg = error instanceof Error ? error.message : String(error);
                    throw new Error(
                        `Failed to fetch from ${url} after ${retries} attempts: ${errorMsg}`,
                    );
                }

                // Exponential backoff: wait 1s, 2s, 4s...
                const delay = Math.pow(2, attempt) * 1000;
                await new Promise((resolve) => setTimeout(resolve, delay));
            }
        }

        // Should never reach here
        throw new Error("Unexpected error in fetchWithRetry");
    }

    /**
     * Shared method to get content from data, file, or URL
     * Subclasses should call this instead of implementing their own
     */
    protected async getContent(): Promise<string> {
        const config = this.getConfig();

        if (config.data !== undefined) {
            return config.data;
        }

        if (config.file) {
            return await config.file.text();
        }

        if (config.url) {
            const response = await this.fetchWithRetry(config.url);
            return await response.text();
        }

        throw new Error(this.errorMessages.missingInput());
    }

    /**
     * Shared chunking helper
     * Yields nodes in chunks, with all edges in the first chunk
     */
    protected *chunkData(
        nodes: AdHocData[],
        edges: AdHocData[],
    ): Generator<DataSourceChunk, void, unknown> {
        // Yield nodes in chunks
        for (let i = 0; i < nodes.length; i += this.chunkSize) {
            const nodeChunk = nodes.slice(i, i + this.chunkSize);
            const edgeChunk = i === 0 ? edges : [];
            yield {nodes: nodeChunk, edges: edgeChunk};
        }

        // If no nodes but edges exist, yield edges-only chunk
        if (nodes.length === 0 && edges.length > 0) {
            yield {nodes: [], edges};
        }
    }

    /**
     * Get the error aggregator for this data source
     */
    getErrorAggregator(): ErrorAggregator {
        return this.errorAggregator;
    }

    async *getData(): AsyncGenerator<DataSourceChunk, void, unknown> {
        for await (const chunk of this.sourceFetchData()) {
            // Filter out invalid nodes
            const validNodes: AdHocData[] = [];
            if (this.nodeSchema) {
                for (const n of chunk.nodes) {
                    const isValid = await this.dataValidator(this.nodeSchema, n);
                    if (isValid) {
                        validNodes.push(n);
                    }
                    // Invalid nodes are logged to errorAggregator but skipped
                }
            } else {
                validNodes.push(... chunk.nodes);
            }

            // Filter out invalid edges
            const validEdges: AdHocData[] = [];
            if (this.edgeSchema) {
                for (const e of chunk.edges) {
                    const isValid = await this.dataValidator(this.edgeSchema, e);
                    if (isValid) {
                        validEdges.push(e);
                    }
                }
            } else {
                validEdges.push(... chunk.edges);
            }

            // Only yield if we have data (or if we're not filtering)
            if (validNodes.length > 0 || validEdges.length > 0) {
                yield {nodes: validNodes, edges: validEdges};
            }

            // Stop if we've hit the error limit
            if (this.errorAggregator.hasReachedLimit()) {
                break;
            }
        }
    }

    /**
     * Validate data against schema
     * Returns false if validation fails (and adds error to aggregator)
     * Returns true if validation succeeds
     */
    async dataValidator(schema: z4.$ZodObject, obj: object): Promise<boolean> {
        const res = await z4.safeParseAsync(schema, obj);

        if (!res.success) {
            const errMsg = z.prettifyError(res.error);

            this.errorAggregator.addError({
                message: `Validation failed: ${errMsg}`,
                category: "validation-error",
            });

            return false; // Validation failed
        }

        return true; // Validation passed
    }

    get type(): string {
        return (this.constructor as typeof DataSource).type;
    }

    static register<T extends DataSourceClass>(cls: T): T {
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const t: string = (cls as any).type;
        dataSourceRegistry.set(t, cls);
        return cls;
    }

    static get(type: string, opts: object = {}): DataSource | null {
        const SourceClass = dataSourceRegistry.get(type);
        if (SourceClass) {
            return new SourceClass(opts);
        }

        return null;
    }
}

1	import {z} from "zod/v4";	3✔
2	import * as z4 from "zod/v4/core";	3✔
3
4	import {AdHocData} from "../config";
5	import {ErrorAggregator} from "./ErrorAggregator.js";	3✔
6
7	// Base configuration interface
8	export interface BaseDataSourceConfig {
9	data?: string;
10	file?: File;
11	url?: string;
12	chunkSize?: number;
13	errorLimit?: number;
14	}
15
16	type DataSourceClass = new (opts: object) => DataSource;
17	const dataSourceRegistry = new Map<string, DataSourceClass>();	3✔
18	export interface DataSourceChunk {
19	nodes: AdHocData[];
20	edges: AdHocData[];
21	}
22
23	export abstract class DataSource {	3✔
24	static readonly type: string;
25	static readonly DEFAULT_CHUNK_SIZE = 1000;	20✔
26
27	edgeSchema: z4.$ZodObject \| null = null;	20✔
28	nodeSchema: z4.$ZodObject \| null = null;	20✔
29	protected errorAggregator: ErrorAggregator;
30	protected chunkSize: number;
31
32	constructor(errorLimit = 100, chunkSize = DataSource.DEFAULT_CHUNK_SIZE) {	20✔
33	this.errorAggregator = new ErrorAggregator(errorLimit);	228✔
34	this.chunkSize = chunkSize;	228✔
35	}	228✔
36
37	// abstract init(): Promise<void>;
38	abstract sourceFetchData(): AsyncGenerator<DataSourceChunk, void, unknown>;
39
40	/**
41	* Subclasses must implement this to expose their config
42	* Used by getContent() and other shared methods
43	*/
44	protected abstract getConfig(): BaseDataSourceConfig;
45
46	/**
47	* Standardized error message templates
48	*/
49	protected get errorMessages(): {	20✔
50	missingInput: () => string;
51	fetchFailed: (url: string, attempts: number, error: string) => string;
52	parseFailed: (error: string) => string;
53	invalidFormat: (reason: string) => string;
54	extractionFailed: (path: string, error: string) => string;
55	} {	9✔
56	return {	9✔
57	missingInput: () =>	9✔
58	`${this.type}DataSource requires data, file, or url`,	9✔
59
60	fetchFailed: (url: string, attempts: number, error: string) =>	9✔
UNCOV 61	`Failed to fetch ${this.type} from ${url} after ${attempts} attempts: ${error}`,	×
62
63	parseFailed: (error: string) =>	9✔
64	`Failed to parse ${this.type}: ${error}`,	×
65
66	invalidFormat: (reason: string) =>	9✔
UNCOV 67	`Invalid ${this.type} format: ${reason}`,	×
68
69	extractionFailed: (path: string, error: string) =>	9✔
UNCOV 70	`Failed to extract data using path '${path}': ${error}`,	×
71	};	9✔
72	}	9✔
73
74	/**
75	* Fetch with retry logic and timeout
76	* Protected method for use by all DataSources
77	*/
78	protected async fetchWithRetry(	20✔
79	url: string,	98✔
80	retries = 3,	98✔
81	timeout = 30000,	98✔
82	): Promise<Response> {	98✔
83	// Data URLs don't need retries or timeouts
84	if (url.startsWith("data:")) {	98✔
85	return await fetch(url);	9✔
86	}	9✔
87
88	for (let attempt = 0; attempt < retries; attempt++) {	89✔
89	try {	93✔
90	// Create AbortController for timeout
91	const controller = new AbortController();	93✔
92	const timeoutId = setTimeout(() => {	93✔
UNCOV 93	controller.abort();	×
94	}, timeout);	93✔
95
96	try {	93✔
97	const response = await fetch(url, {signal: controller.signal});	93✔
98	clearTimeout(timeoutId);	87✔
99
100	if (!response.ok) {	86!
UNCOV 101	throw new Error(`HTTP error! status: ${response.status}`);	×
UNCOV 102	}	✔
103
104	return response;	87✔
105	} catch (error) {	92✔
106	clearTimeout(timeoutId);	6✔
107
108	if (error instanceof Error && error.name === "AbortError") {	6!
UNCOV 109	throw new Error(`Request timeout after ${timeout}ms`);	×
UNCOV 110	}	×
111
112	throw error;	6✔
113	}	6✔
114	} catch (error) {	93✔
115	const isLastAttempt = attempt === retries - 1;	6✔
116
117	if (isLastAttempt) {	6✔
118	const errorMsg = error instanceof Error ? error.message : String(error);	2!
119	throw new Error(	2✔
120	`Failed to fetch from ${url} after ${retries} attempts: ${errorMsg}`,	2✔
121	);	2✔
122	}	2✔
123
124	// Exponential backoff: wait 1s, 2s, 4s...
125	const delay = Math.pow(2, attempt) * 1000;	4✔
126	await new Promise((resolve) => setTimeout(resolve, delay));	4✔
127	}	4✔
128	}	93✔
129
130	// Should never reach here
131	throw new Error("Unexpected error in fetchWithRetry");	4✔
132	}	98✔
133
134	/**
135	* Shared method to get content from data, file, or URL
136	* Subclasses should call this instead of implementing their own
137	*/
138	protected async getContent(): Promise<string> {	20✔
139	const config = this.getConfig();	217✔
140
141	if (config.data !== undefined) {	217✔
142	return config.data;	112✔
143	}	112✔
144
145	if (config.file) {	141!
UNCOV 146	return await config.file.text();	×
UNCOV 147	}	✔
148
149	if (config.url) {	129✔
150	const response = await this.fetchWithRetry(config.url);	96✔
151	return await response.text();	94✔
152	}	94✔
153
154	throw new Error(this.errorMessages.missingInput());	9✔
155	}	217✔
156
157	/**
158	* Shared chunking helper
159	* Yields nodes in chunks, with all edges in the first chunk
160	*/
161	protected *chunkData(	20✔
162	nodes: AdHocData[],	181✔
163	edges: AdHocData[],	181✔
164	): Generator<DataSourceChunk, void, unknown> {	181✔
165	// Yield nodes in chunks
166	for (let i = 0; i < nodes.length; i += this.chunkSize) {	181✔
167	const nodeChunk = nodes.slice(i, i + this.chunkSize);	205✔
168	const edgeChunk = i === 0 ? edges : [];	205✔
169	yield {nodes: nodeChunk, edges: edgeChunk};	205✔
170	}	182✔
171
172	// If no nodes but edges exist, yield edges-only chunk
173	if (nodes.length === 0 && edges.length > 0) {	181✔
174	yield {nodes: [], edges};	3✔
175	}	3✔
176	}	181✔
177
178	/**
179	* Get the error aggregator for this data source
180	*/
181	getErrorAggregator(): ErrorAggregator {	20✔
182	return this.errorAggregator;	215✔
183	}	215✔
184
185	async *getData(): AsyncGenerator<DataSourceChunk, void, unknown> {	20✔
186	for await (const chunk of this.sourceFetchData()) {	199✔
187	// Filter out invalid nodes
188	const validNodes: AdHocData[] = [];	214✔
189	if (this.nodeSchema) {	214✔
190	for (const n of chunk.nodes) {	4✔
191	const isValid = await this.dataValidator(this.nodeSchema, n);	307✔
192	if (isValid) {	307✔
193	validNodes.push(n);	78✔
194	}	78✔
195	// Invalid nodes are logged to errorAggregator but skipped
196	}	307✔
197	} else {	214✔
198	validNodes.push(... chunk.nodes);	210✔
199	}	210✔
200
201	// Filter out invalid edges
202	const validEdges: AdHocData[] = [];	214✔
203	if (this.edgeSchema) {	214✔
204	for (const e of chunk.edges) {	1✔
205	const isValid = await this.dataValidator(this.edgeSchema, e);	3✔
206	if (isValid) {	3✔
207	validEdges.push(e);	2✔
208	}	2✔
209	}	3✔
210	} else {	214✔
211	validEdges.push(... chunk.edges);	213✔
212	}	213✔
213
214	// Only yield if we have data (or if we're not filtering)
215	if (validNodes.length > 0 \|\| validEdges.length > 0) {	214✔
216	yield {nodes: validNodes, edges: validEdges};	213✔
217	}	212✔
218
219	// Stop if we've hit the error limit
220	if (this.errorAggregator.hasReachedLimit()) {	214✔
221	break;	1✔
222	}	1✔
223	}	214✔
224	}	199✔
225
226	/**
227	* Validate data against schema
228	* Returns false if validation fails (and adds error to aggregator)
229	* Returns true if validation succeeds
230	*/
231	async dataValidator(schema: z4.$ZodObject, obj: object): Promise<boolean> {	20✔
232	const res = await z4.safeParseAsync(schema, obj);	310✔
233
234	if (!res.success) {	310✔
235	const errMsg = z.prettifyError(res.error);	230✔
236
237	this.errorAggregator.addError({	230✔
238	message: `Validation failed: ${errMsg}`,	230✔
239	category: "validation-error",	230✔
240	});	230✔
241
242	return false; // Validation failed	230✔
243	}	230✔
244
245	return true; // Validation passed	80✔
246	}	310✔
247
248	get type(): string {	20✔
249	return (this.constructor as typeof DataSource).type;	11✔
250	}	11✔
251
252	static register<T extends DataSourceClass>(cls: T): T {	20✔
253	// eslint-disable-next-line @typescript-eslint/no-explicit-any
254	const t: string = (cls as any).type;	546✔
255	dataSourceRegistry.set(t, cls);	546✔
256	return cls;	546✔
257	}	546✔
258
259	static get(type: string, opts: object = {}): DataSource \| null {	20✔
260	const SourceClass = dataSourceRegistry.get(type);	104✔
261	if (SourceClass) {	104✔
262	return new SourceClass(opts);	104✔
263	}	104✔
264
UNCOV 265	return null;	×
266	}	104✔
267	}	20✔

graphty-org / graphty-element / 19792929756

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous