#243

Committed 26 Oct 2025 07:37PM UTC coverage: 87.36% (-0.9%) from 88.257%

Build # #243

Build Type

screwdriver

Committed by ppaul

Commit Message

feat: Major rewrite of redis state logic using lua scripts

Pull Request Pull Request #80: feat: Major rewrite of redis state logic using lua scripts

Coverage Stats

338 of 414 branches covered (81.64%)

Branch coverage included in aggregate %.

167 of 179 new or added lines in 5 files covered. (93.3%)

1065 of 1192 relevant lines covered (89.35%)

26.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.0

/plugins/worker/lib/LuaScriptLoader.js

'use strict';

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const logger = require('screwdriver-logger');

/**
 * LuaScriptLoader - Manages loading and execution of Lua scripts in Redis
 *
 * Features:
 * - Load Lua scripts into Redis and cache their SHAs
 * - Auto-reload scripts if Redis loses them (NOSCRIPT error)
 * - Execute scripts by name with proper error handling
 * - Hash validation for cache consistency
 */
class LuaScriptLoader {
    /**
     * Constructor
     * @param {Object} redis - Redis client instance
     */
    constructor(redis) {
        this.redis = redis;
        this.scripts = new Map(); // Map<scriptName, {sha, content, hash}>
        this.scriptDir = path.join(__dirname, 'lua');
    }

    /**
     * Load a Lua script and return its SHA
     * @param {String} scriptName - Script filename (e.g., 'startBuild.lua')
     * @return {Promise<String>} SHA of loaded script
     */
    async loadScript(scriptName) {
        // Check if already loaded
        if (this.scripts.has(scriptName)) {
            const cached = this.scripts.get(scriptName);

            logger.info(`Lua script ${scriptName} already loaded (SHA: ${cached.sha.substring(0, 8)}...)`);

            return cached.sha;
        }

        const scriptPath = path.join(this.scriptDir, scriptName);

        // Check if file exists
        if (!fs.existsSync(scriptPath)) {
            throw new Error(`Lua script not found: ${scriptPath}`);
        }

        const scriptContent = fs.readFileSync(scriptPath, 'utf8');

        // Load script into Redis
        const sha = await this.redis.script('LOAD', scriptContent);

        // Cache script info
        this.scripts.set(scriptName, {
            sha,
            content: scriptContent,
            hash: this.hashScript(scriptContent)
        });

        logger.info(`Loaded Lua script: ${scriptName} (SHA: ${sha.substring(0, 8)}...)`);

        return sha;
    }

    /**
     * Get script SHA (load if not already loaded)
     * @param {String} scriptName
     * @return {Promise<String>}
     */
    async getScriptSha(scriptName) {
        if (!this.scripts.has(scriptName)) {
            await this.loadScript(scriptName);
        }

        return this.scripts.get(scriptName).sha;
    }

    /**
     * Execute a Lua script by name
     * @param {String} scriptName - Script filename
     * @param {Array} keys - KEYS array for script (usually empty for our scripts)
     * @param {Array} args - ARGV array for script
     * @return {Promise<Any>} Script result
     */
    async executeScript(scriptName, keys = [], args = []) {
        const sha = await this.getScriptSha(scriptName);

        try {
            // Execute script using EVALSHA (faster than EVAL)
            const result = await this.redis.evalsha(sha, keys.length, ...keys, ...args);

            return result;
        } catch (err) {
            // If script not found in Redis, reload and retry
            if (err.message && err.message.includes('NOSCRIPT')) {
                logger.warn(`Script ${scriptName} not found in Redis, reloading...`);

                // Remove from cache and reload
                this.scripts.delete(scriptName);
                await this.loadScript(scriptName);

                const newSha = this.scripts.get(scriptName).sha;

                // Retry execution
                return this.redis.evalsha(newSha, keys.length, ...keys, ...args);
            }

            // Re-throw other errors
            throw err;
        }
    }

    /**
     * Load all Lua scripts in the lua directory
     * @return {Promise<Number>} Number of scripts loaded
     */
    async loadAllScripts() {
        // Find all .lua files in lua directory (not in subdirectories)
        const files = fs.readdirSync(this.scriptDir);
        const luaFiles = files.filter(f => f.endsWith('.lua') && !f.startsWith('.'));

        let loadedCount = 0;

        for (const file of luaFiles) {
            try {
                await this.loadScript(file);
                loadedCount += 1;
            } catch (err) {
                logger.error(`Failed to load Lua script ${file}: ${err.message}`);
            }
        }

        logger.info(`Loaded ${loadedCount} Lua script(s)`);

        return loadedCount;
    }

    /**
     * Reload a specific script (useful for hot-reload during development)
     * @param {String} scriptName
     * @return {Promise<String>} New SHA
     */
    async reloadScript(scriptName) {
        this.scripts.delete(scriptName);

        return this.loadScript(scriptName);
    }

    /**
     * Reload all scripts
     * @return {Promise<Number>} Number of scripts reloaded
     */
    async reloadAllScripts() {
        this.scripts.clear();

        return this.loadAllScripts();
    }

    /**
     * Hash script content for cache validation
     * @param {String} content
     * @return {String} Hash (first 8 characters)
     */
    hashScript(content) {
        return crypto.createHash('sha256').update(content).digest('hex').substring(0, 8);
    }

    /**
     * Get info about loaded scripts
     * @return {Array<Object>} Script info
     */
    getLoadedScripts() {
        const scripts = [];

        for (const [name, info] of this.scripts.entries()) {
            scripts.push({
                name,
                sha: info.sha,
                hash: info.hash,
                size: info.content.length
            });
        }

        return scripts;
    }

    /**
     * Check if a script is loaded
     * @param {String} scriptName
     * @return {Boolean}
     */
    isScriptLoaded(scriptName) {
        return this.scripts.has(scriptName);
    }

    /**
     * Clear all cached scripts (does not remove from Redis)
     * @return {void}
     */
    clearCache() {
        this.scripts.clear();
        logger.info('Lua script cache cleared');
    }

    /**
     * Validate that a script exists in Redis
     * @param {String} scriptName
     * @return {Promise<Boolean>}
     */
    async validateScript(scriptName) {
        if (!this.scripts.has(scriptName)) {
            return false;
        }

        const { sha } = this.scripts.get(scriptName);

        try {
            // Check if script exists in Redis using SCRIPT EXISTS
            const exists = await this.redis.script('EXISTS', sha);

            return exists[0] === 1;
        } catch (err) {
            logger.error(`Error validating script ${scriptName}: ${err.message}`);

            return false;
        }
    }
}

module.exports = LuaScriptLoader;

1	'use strict';
2
3	const fs = require('fs');	21✔
4	const path = require('path');	21✔
5	const crypto = require('crypto');	21✔
6	const logger = require('screwdriver-logger');	21✔
7
8	/**
9	* LuaScriptLoader - Manages loading and execution of Lua scripts in Redis
10	*
11	* Features:
12	* - Load Lua scripts into Redis and cache their SHAs
13	* - Auto-reload scripts if Redis loses them (NOSCRIPT error)
14	* - Execute scripts by name with proper error handling
15	* - Hash validation for cache consistency
16	*/
17	class LuaScriptLoader {
18	/**
19	* Constructor
20	* @param {Object} redis - Redis client instance
21	*/
22	constructor(redis) {
23	this.redis = redis;	21✔
24	this.scripts = new Map(); // Map<scriptName, {sha, content, hash}>	21✔
25	this.scriptDir = path.join(__dirname, 'lua');	21✔
26	}
27
28	/**
29	* Load a Lua script and return its SHA
30	* @param {String} scriptName - Script filename (e.g., 'startBuild.lua')
31	* @return {Promise<String>} SHA of loaded script
32	*/
33	async loadScript(scriptName) {
34	// Check if already loaded
35	if (this.scripts.has(scriptName)) {	42✔
36	const cached = this.scripts.get(scriptName);	1✔
37
38	logger.info(`Lua script ${scriptName} already loaded (SHA: ${cached.sha.substring(0, 8)}...)`);	1✔
39
40	return cached.sha;	1✔
41	}
42
43	const scriptPath = path.join(this.scriptDir, scriptName);	41✔
44
45	// Check if file exists
46	if (!fs.existsSync(scriptPath)) {	41✔
47	throw new Error(`Lua script not found: ${scriptPath}`);	1✔
48	}
49
50	const scriptContent = fs.readFileSync(scriptPath, 'utf8');	40✔
51
52	// Load script into Redis
53	const sha = await this.redis.script('LOAD', scriptContent);	40✔
54
55	// Cache script info
56	this.scripts.set(scriptName, {	13✔
57	sha,
58	content: scriptContent,
59	hash: this.hashScript(scriptContent)
60	});
61
62	logger.info(`Loaded Lua script: ${scriptName} (SHA: ${sha.substring(0, 8)}...)`);	13✔
63
64	return sha;	13✔
65	}
66
67	/**
68	* Get script SHA (load if not already loaded)
69	* @param {String} scriptName
70	* @return {Promise<String>}
71	*/
72	async getScriptSha(scriptName) {
73	if (!this.scripts.has(scriptName)) {	18✔
74	await this.loadScript(scriptName);	3✔
75	}
76
77	return this.scripts.get(scriptName).sha;	18✔
78	}
79
80	/**
81	* Execute a Lua script by name
82	* @param {String} scriptName - Script filename
83	* @param {Array} keys - KEYS array for script (usually empty for our scripts)
84	* @param {Array} args - ARGV array for script
85	* @return {Promise<Any>} Script result
86	*/
87	async executeScript(scriptName, keys = [], args = []) {	×
88	const sha = await this.getScriptSha(scriptName);	17✔
89
90	try {	17✔
91	// Execute script using EVALSHA (faster than EVAL)
92	const result = await this.redis.evalsha(sha, keys.length, ...keys, ...args);	17✔
93
94	return result;	15✔
95	} catch (err) {
96	// If script not found in Redis, reload and retry
97	if (err.message && err.message.includes('NOSCRIPT')) {	2✔
98	logger.warn(`Script ${scriptName} not found in Redis, reloading...`);	1✔
99
100	// Remove from cache and reload
101	this.scripts.delete(scriptName);	1✔
102	await this.loadScript(scriptName);	1✔
103
104	const newSha = this.scripts.get(scriptName).sha;	1✔
105
106	// Retry execution
107	return this.redis.evalsha(newSha, keys.length, ...keys, ...args);	1✔
108	}
109
110	// Re-throw other errors
111	throw err;	1✔
112	}
113	}
114
115	/**
116	* Load all Lua scripts in the lua directory
117	* @return {Promise<Number>} Number of scripts loaded
118	*/
119	async loadAllScripts() {
120	// Find all .lua files in lua directory (not in subdirectories)
121	const files = fs.readdirSync(this.scriptDir);	10✔
122	const luaFiles = files.filter(f => f.endsWith('.lua') && !f.startsWith('.'));	40✔
123
124	let loadedCount = 0;	10✔
125
126	for (const file of luaFiles) {	10✔
127	try {	29✔
128	await this.loadScript(file);	29✔
129	loadedCount += 1;	2✔
130	} catch (err) {
131	logger.error(`Failed to load Lua script ${file}: ${err.message}`);	27✔
132	}
133	}
134
135	logger.info(`Loaded ${loadedCount} Lua script(s)`);	10✔
136
137	return loadedCount;	10✔
138	}
139
140	/**
141	* Reload a specific script (useful for hot-reload during development)
142	* @param {String} scriptName
143	* @return {Promise<String>} New SHA
144	*/
145	async reloadScript(scriptName) {
146	this.scripts.delete(scriptName);	1✔
147
148	return this.loadScript(scriptName);	1✔
149	}
150
151	/**
152	* Reload all scripts
153	* @return {Promise<Number>} Number of scripts reloaded
154	*/
155	async reloadAllScripts() {
NEW 156	this.scripts.clear();	×
157
NEW 158	return this.loadAllScripts();	×
159	}
160
161	/**
162	* Hash script content for cache validation
163	* @param {String} content
164	* @return {String} Hash (first 8 characters)
165	*/
166	hashScript(content) {
167	return crypto.createHash('sha256').update(content).digest('hex').substring(0, 8);	13✔
168	}
169
170	/**
171	* Get info about loaded scripts
172	* @return {Array<Object>} Script info
173	*/
174	getLoadedScripts() {
175	const scripts = [];	1✔
176
177	for (const [name, info] of this.scripts.entries()) {	1✔
178	scripts.push({	1✔
179	name,
180	sha: info.sha,
181	hash: info.hash,
182	size: info.content.length
183	});
184	}
185
186	return scripts;	1✔
187	}
188
189	/**
190	* Check if a script is loaded
191	* @param {String} scriptName
192	* @return {Boolean}
193	*/
194	isScriptLoaded(scriptName) {
195	return this.scripts.has(scriptName);	1✔
196	}
197
198	/**
199	* Clear all cached scripts (does not remove from Redis)
200	* @return {void}
201	*/
202	clearCache() {
NEW 203	this.scripts.clear();	×
NEW 204	logger.info('Lua script cache cleared');	×
205	}
206
207	/**
208	* Validate that a script exists in Redis
209	* @param {String} scriptName
210	* @return {Promise<Boolean>}
211	*/
212	async validateScript(scriptName) {
213	if (!this.scripts.has(scriptName)) {	2✔
214	return false;	1✔
215	}
216
217	const { sha } = this.scripts.get(scriptName);	1✔
218
219	try {	1✔
220	// Check if script exists in Redis using SCRIPT EXISTS
221	const exists = await this.redis.script('EXISTS', sha);	1✔
222
223	return exists[0] === 1;	1✔
224	} catch (err) {
NEW 225	logger.error(`Error validating script ${scriptName}: ${err.message}`);	×
226
NEW 227	return false;	×
228	}
229	}
230	}
231
232	module.exports = LuaScriptLoader;	21✔

screwdriver-cd / queue-service / #243

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous