#252645670

Committed 15 Nov 2023 11:12PM UTC coverage: 71.066% (-1.2%) from 72.267%

Build Type

Committed by

Commit Message

trying

Pull Request Pull Request #456: staging version 1.9.2491

Run Details

1760 of 3422 branches covered (0.0%)

Branch coverage included in aggregate %.

131 of 261 new or added lines in 18 files covered. (50.19%)

60 existing lines in 10 files now uncovered.

8084 of 10430 relevant lines covered (77.51%)

27.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.1

/lib/thinx/repository.js

/*
 * This THiNX Device Management API module is responsible for managing repositories.
 */

var Globals = require("./globals.js");
var app_config = Globals.app_config();
var finder = require("fs-finder");

var Devices = require("./devices"); 
var Sources = require("./sources"); var sources = new Sources();
var Platform = require("./platform");

let fs = require("fs-extra");
var crypto = require('crypto');
module.exports = class Repository {

        constructor(messenger, redis, queue) {
                if ((typeof(redis) === "undefined") || (redis === null)) {
            throw new Error("Redis client missing in Repository.js:17");
        }
                this.queue = queue;
                this.platform = new Platform();
                this.devices = new Devices(messenger, redis); // we need Devices but without messenger, only the DB access for udids with source...
        }

        static findAllRepositories() {
                var repositories_path = app_config.data_root + app_config.build_root;
                var repo_gits = finder.from(repositories_path).showSystemFiles().findDirectories(".git");
                console.log(`ℹ️ [info] [findAllRepositories] Search completed with ${repo_gits.length} repos: ${repositories_path}`);
                return repo_gits;
        }

        static findAllRepositoriesWithFullname(full_name) {
                var repo_gits = Repository.findAllRepositories();
                var repositories = [];
                console.log(`ℹ️ [info] [findAllRepositoriesWithFullname] Searching repos with fullname '${full_name}'`);
                let full_name_array = full_name.split("/");
                let repo_name = full_name_array[1];
                for (var dindex in repo_gits) {
                        const repo = repo_gits[dindex];
                        if (repo.indexOf(repo_name) == -1) continue;
                        if ((repo.length > 1) && (repo.indexOf("/.git") !== -1 && repo.indexOf("/.git") == repo.length - 5)) {
                                repositories.push(repo.replace("/.git", ""));
                        }
                }
                console.log(`ℹ️ [info] [findAllRepositoriesWithFullname] Searching repos completed with ${repositories.length} repos.`);
                return repositories;
        }

        // if the folder in device is not latest, cal be deleted
        isNotLatest(path, latest) {
                return (path.indexOf("/" + latest) == -1);
        }

        purgeRepositoriesWithFullName(repositories, name) {

                const deviceFolders = (repos) => {
                        if (typeof (repos) === "undefined") return [];
                        if (repos === null) return [];
                        return repos
                                .map((repo) => {
                                        let splitted = repo.split("/");
                                        return splitted.splice(0, splitted.length - 2).join("/");
                                });
                };

                const orderRecentDirectories = (repos) => {
                        return repos
                                .filter((repo) => fs.lstatSync(repo).isDirectory())
                                .map((repo) => ({ repo, mtime: fs.lstatSync(repo).mtime }))
                                .sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
                };

                const getMostRecentDirectory = (repos) => {
                        const files = orderRecentDirectories(repos);
                        return files.length ? files[0] : null;
                };

                const getDirectories = (source) => {
                        return fs.readdirSync(source, { withFileTypes: true }) // lgtm [js/path-injection]
                                .filter(dirent => dirent.isDirectory())
                                .map(dirent => source + "/" + dirent.name);
                };

                let device_folders = deviceFolders(repositories);
                for (let index in device_folders) {
                        let device_folder = device_folders[index];
                        console.log(`🔨 [debug] [repository] device_folder ${device_folder}`);
                        let all_in_device_folder = getDirectories(device_folder);
                        console.log(`🔨 [debug] [repository] all_in_device_folder ${JSON.stringify(all_in_device_folder)}`);
                        let latest_in_device_folder = getMostRecentDirectory(all_in_device_folder);
                        if (latest_in_device_folder === null) {
                                continue;
                        }
                        console.log(`🔨 [debug] [repository] latest_in_device_folder ${JSON.stringify(latest_in_device_folder)}`);
                        let latest_build_id_arr = latest_in_device_folder.toString().split("/");
                        if (typeof (latest_build_id_arr) === "undefined" || latest_build_id_arr === null) {
                                continue;
                        }
                        let latest_build_id = latest_build_id_arr[latest_build_id_arr.length - 1];
                        console.log(`🔨 [debug] [repository] latest_build_id ${JSON.stringify(latest_build_id)} in device folder ${device_folder} parsing ${JSON.stringify(all_in_device_folder)}`);
                        if ((typeof (latest_build_id) === "undefined") || (latest_build_id === "undefined")) {
                                continue;
                        }
                        for (let dindex in all_in_device_folder) {
                                let folder = all_in_device_folder[dindex];
                                let canBeDeleted = this.isNotLatest(folder, latest_build_id);
                                let repo_folder = getDirectories(folder)[0];
                                if (canBeDeleted && (repo_folder.indexOf(name) !== -1)) {
                                        console.log(`ℹ️ [info] Purging old repo folder ${folder}`);
                                        fs.rmdir(folder, { recursive: true });
                                }
                        }

                }
        }

        validateSignature(signature, body, secret) {
                if ((typeof (secret) === "undefined") || (secret === null)) return true;
                let bare_signature = signature.replace("sha256=", "");
                let hmac = crypto.createHmac('sha256', secret).update(body).digest("base64");
                return (hmac === bare_signature);
        }

        process_hook(req) {
                
                let info = req.body;
                let repository = info.repository;

                if (typeof (repository) === "undefined") {
                        console.log("🔆 [audit] Exiting, unknown webhook request: ", { info });
                        return false;
                }

                var name = repository.name;
                var full_name = repository.full_name;

                console.log(`☣️ [debug] [process_hook] [spec] Repository Name ${name} and Fullname ${full_name} (searching by name)`);

                var repositories = Repository.findAllRepositoriesWithFullname(name);
                if (repositories.length == 0) {
                        console.log(`⚠️ [warning] No repositories found with filter ${name}`);
                        return false;
                }

                this.purgeRepositoriesWithFullName(repositories, name);

                repositories = Repository.findAllRepositoriesWithFullname(name);
                console.log(`☣️ [debug] [process_hook] [spec] Repositories after purge: ${repositories}`);
                this.queue_updated_repos(repositories, info);
                return true;
        }

        queue_updated_repos(repositories, info) {
                const src = new Sources();
                for (let local_path of repositories) {
                        let owner_id = src.ownerIdFromPath(local_path);
                        src.withRepository(info, local_path, (source_ids) => {
                                if ((typeof (source_ids) === "undefined" || source_ids.length == 0)) {
                                        console.log(`[info] No sources to be queued with local_path ${local_path}`);
                                        return;
                                }
                                sources.list(owner_id, (/*success, repos*/) => {
                                        for (let source_id of source_ids) {
                                                console.log(`ℹ️ [info] Fetching matching devices for changed source_id ${source_id}`);
                                                /* TODO FIXME, but where do we get the signature from?
                                                let repo = repos[source_id];
                                                if (signature !== null) {
                                                        if (!this.validateSignature(signature, info, repo.secret)) {
                                                                console.log("☣️ [error] GitHub Secret validation failed for repo", { repo });
                                                                continue; // next source_id, this one is signed but signature does not compute
                                                        }
                                                }
                                                */
                                                this.devices.udidsWithSource(owner_id, source_id, (udids) => {
                                                        console.log("udids with source", source_id, ":", udids);
                                                        for (let udid of udids) {
                                                                console.log(`ℹ️ [info] Adding device ${udid} with this source.`);
                                                                if (typeof(this.queue) !== "undefined") {
                                                                        this.queue.add(udid, source_id, owner_id);
                                                                } else {
                                                                        console.log(`[warning] no queue defined in ${process.env.ENVIRONMENT} environment`);
                                                                }
                                                        }
                                                });
                                        }
                                });
                        });
                }
        }
};

1	/*
2	* This THiNX Device Management API module is responsible for managing repositories.
3	*/
4
5	var Globals = require("./globals.js");	1✔
6	var app_config = Globals.app_config();	1✔
7	var finder = require("fs-finder");	1✔
8
9	var Devices = require("./devices");	1✔
10	var Sources = require("./sources"); var sources = new Sources();	1✔
11	var Platform = require("./platform");	1✔
12
13	let fs = require("fs-extra");	1✔
14	var crypto = require('crypto');	1✔
15	module.exports = class Repository {	1✔
16
17	constructor(messenger, redis, queue) {
18	if ((typeof(redis) === "undefined") \|\| (redis === null)) {	20!
19	throw new Error("Redis client missing in Repository.js:17");	×
20	}
21	this.queue = queue;	20✔
22	this.platform = new Platform();	20✔
23	this.devices = new Devices(messenger, redis); // we need Devices but without messenger, only the DB access for udids with source...	20✔
24	}
25
26	static findAllRepositories() {
27	var repositories_path = app_config.data_root + app_config.build_root;	4✔
28	var repo_gits = finder.from(repositories_path).showSystemFiles().findDirectories(".git");	4✔
29	console.log(`ℹ️ [info] [findAllRepositories] Search completed with ${repo_gits.length} repos: ${repositories_path}`);	4✔
30	return repo_gits;	4✔
31	}
32
33	static findAllRepositoriesWithFullname(full_name) {
34	var repo_gits = Repository.findAllRepositories();	4✔
35	var repositories = [];	4✔
36	console.log(`ℹ️ [info] [findAllRepositoriesWithFullname] Searching repos with fullname '${full_name}'`);	4✔
37	let full_name_array = full_name.split("/");	4✔
38	let repo_name = full_name_array[1];	4✔
39	for (var dindex in repo_gits) {	4✔
40	const repo = repo_gits[dindex];	×
41	if (repo.indexOf(repo_name) == -1) continue;	×
42	if ((repo.length > 1) && (repo.indexOf("/.git") !== -1 && repo.indexOf("/.git") == repo.length - 5)) {	×
43	repositories.push(repo.replace("/.git", ""));	×
44	}
45	}
46	console.log(`ℹ️ [info] [findAllRepositoriesWithFullname] Searching repos completed with ${repositories.length} repos.`);	4✔
47	return repositories;	4✔
48	}
49
50	// if the folder in device is not latest, cal be deleted
51	isNotLatest(path, latest) {
52	return (path.indexOf("/" + latest) == -1);	×
53	}
54
55	purgeRepositoriesWithFullName(repositories, name) {
56
57	const deviceFolders = (repos) => {	1✔
58	if (typeof (repos) === "undefined") return [];	1!
59	if (repos === null) return [];	1!
60	return repos	1✔
61	.map((repo) => {
62	let splitted = repo.split("/");	×
63	return splitted.splice(0, splitted.length - 2).join("/");	×
64	});
65	};
66
67	const orderRecentDirectories = (repos) => {	1✔
68	return repos	×
69	.filter((repo) => fs.lstatSync(repo).isDirectory())	×
70	.map((repo) => ({ repo, mtime: fs.lstatSync(repo).mtime }))	×
71	.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());	×
72	};
73
74	const getMostRecentDirectory = (repos) => {	1✔
NEW 75	const files = orderRecentDirectories(repos);	×
NEW 76	return files.length ? files[0] : null;	×
77	};
78
79	const getDirectories = (source) => {	1✔
80	return fs.readdirSync(source, { withFileTypes: true }) // lgtm [js/path-injection]	×
81	.filter(dirent => dirent.isDirectory())	×
82	.map(dirent => source + "/" + dirent.name);	×
83	};
84
85	let device_folders = deviceFolders(repositories);	1✔
86	for (let index in device_folders) {	1✔
87	let device_folder = device_folders[index];	×
88	console.log(`🔨 [debug] [repository] device_folder ${device_folder}`);	×
89	let all_in_device_folder = getDirectories(device_folder);	×
90	console.log(`🔨 [debug] [repository] all_in_device_folder ${JSON.stringify(all_in_device_folder)}`);	×
91	let latest_in_device_folder = getMostRecentDirectory(all_in_device_folder);	×
NEW 92	if (latest_in_device_folder === null) {	×
93	continue;	×
94	}
95	console.log(`🔨 [debug] [repository] latest_in_device_folder ${JSON.stringify(latest_in_device_folder)}`);	×
96	let latest_build_id_arr = latest_in_device_folder.toString().split("/");	×
97	if (typeof (latest_build_id_arr) === "undefined" \|\| latest_build_id_arr === null) {	×
98	continue;	×
99	}
100	let latest_build_id = latest_build_id_arr[latest_build_id_arr.length - 1];	×
101	console.log(`🔨 [debug] [repository] latest_build_id ${JSON.stringify(latest_build_id)} in device folder ${device_folder} parsing ${JSON.stringify(all_in_device_folder)}`);	×
102	if ((typeof (latest_build_id) === "undefined") \|\| (latest_build_id === "undefined")) {	×
103	continue;	×
104	}
105	for (let dindex in all_in_device_folder) {	×
106	let folder = all_in_device_folder[dindex];	×
107	let canBeDeleted = this.isNotLatest(folder, latest_build_id);	×
108	let repo_folder = getDirectories(folder)[0];	×
109	if (canBeDeleted && (repo_folder.indexOf(name) !== -1)) {	×
110	console.log(`ℹ️ [info] Purging old repo folder ${folder}`);	×
111	fs.rmdir(folder, { recursive: true });	×
112	}
113	}
114
115	}
116	}
117
118	validateSignature(signature, body, secret) {
119	if ((typeof (secret) === "undefined") \|\| (secret === null)) return true;	1!
120	let bare_signature = signature.replace("sha256=", "");	1✔
121	let hmac = crypto.createHmac('sha256', secret).update(body).digest("base64");	1✔
122	return (hmac === bare_signature);	1✔
123	}
124
125	process_hook(req) {
126
127	let info = req.body;	4✔
128	let repository = info.repository;	4✔
129
130	if (typeof (repository) === "undefined") {	4✔
131	console.log("🔆 [audit] Exiting, unknown webhook request: ", { info });	3✔
132	return false;	3✔
133	}
134
135	var name = repository.name;	1✔
136	var full_name = repository.full_name;	1✔
137
138	console.log(`☣️ [debug] [process_hook] [spec] Repository Name ${name} and Fullname ${full_name} (searching by name)`);	1✔
139
140	var repositories = Repository.findAllRepositoriesWithFullname(name);	1✔
141	if (repositories.length == 0) {	1!
142	console.log(`⚠️ [warning] No repositories found with filter ${name}`);	1✔
143	return false;	1✔
144	}
145
NEW 146	this.purgeRepositoriesWithFullName(repositories, name);	×
147
148	repositories = Repository.findAllRepositoriesWithFullname(name);	×
149	console.log(`☣️ [debug] [process_hook] [spec] Repositories after purge: ${repositories}`);	×
NEW 150	this.queue_updated_repos(repositories, info);	×
151	return true;	×
152	}
153
154	queue_updated_repos(repositories, info) {
NEW 155	const src = new Sources();	×
156	for (let local_path of repositories) {	×
NEW 157	let owner_id = src.ownerIdFromPath(local_path);	×
NEW 158	src.withRepository(info, local_path, (source_ids) => {	×
159	if ((typeof (source_ids) === "undefined" \|\| source_ids.length == 0)) {	×
160	console.log(`[info] No sources to be queued with local_path ${local_path}`);	×
161	return;	×
162	}
NEW 163	sources.list(owner_id, (/success, repos/) => {	×
164	for (let source_id of source_ids) {	×
165	console.log(`ℹ️ [info] Fetching matching devices for changed source_id ${source_id}`);	×
166	/* TODO FIXME, but where do we get the signature from?
167	let repo = repos[source_id];
168	if (signature !== null) {
169	if (!this.validateSignature(signature, info, repo.secret)) {
170	console.log("☣️ [error] GitHub Secret validation failed for repo", { repo });
171	continue; // next source_id, this one is signed but signature does not compute
172	}
173	}
174	*/
175	this.devices.udidsWithSource(owner_id, source_id, (udids) => {	×
176	console.log("udids with source", source_id, ":", udids);	×
177	for (let udid of udids) {	×
178	console.log(`ℹ️ [info] Adding device ${udid} with this source.`);	×
179	if (typeof(this.queue) !== "undefined") {	×
180	this.queue.add(udid, source_id, owner_id);	×
181	} else {
182	console.log(`[warning] no queue defined in ${process.env.ENVIRONMENT} environment`);	×
183	}
184	}
185	});
186	}
187	});
188	});
189	}
190	}
191	};

suculent / thinx-device-api / #252645670

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous