4238184090

Build Type

github

Committed by GitHub

Commit Message

Merge 711dd38d8 into 3201845f8

Pull Request Pull Request #190: add waiter to executor to make sure Alias is active

Run Details

167 of 168 branches covered (99.4%)

Branch coverage included in aggregate %.

18 of 18 new or added lines in 2 files covered. (100.0%)

452 of 453 relevant lines covered (99.78%)

51.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.4

/lambda/utils.js

'use strict';

const AWS = require('aws-sdk');
const url = require('url');


// local reference to this module
const utils = module.exports;

// cost of 6+N state transitions (AWS Step Functions)
module.exports.stepFunctionsCost = (nPower) => +(this.stepFunctionsBaseCost() * (6 + nPower)).toFixed(5);

module.exports.stepFunctionsBaseCost = () => {
    const prices = JSON.parse(process.env.sfCosts);
    // assume the AWS_REGION variable is set for this function
    return this.baseCostForRegion(prices, process.env.AWS_REGION);
};

module.exports.lambdaBaseCost = (region, architecture) => {
    const prices = JSON.parse(process.env.baseCosts);
    const priceMap = prices[architecture];
    if (!priceMap){
        throw new Error('Unsupported architecture: ' + architecture);
    }
    return this.baseCostForRegion(priceMap, region);
};

module.exports.allPowerValues = () => {
    const increment = 64;
    const powerValues = [];
    for (let value = 128; value <= 3008; value += increment) {
        powerValues.push(value);
    }
    return powerValues;
};

/**
 * Check whether a Lambda Alias exists or not, and return its data.
 */
module.exports.getLambdaAlias = (lambdaARN, alias) => {
    console.log('Checking alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.getAlias(params).promise();
};

/**
 * Return true if alias exist, false if it doesn't.
 */
module.exports.verifyAliasExistance = async(lambdaARN, alias) => {
    try {
        await utils.getLambdaAlias(lambdaARN, alias);
        return true;
    } catch (error) {
        if (error.code === 'ResourceNotFoundException') {
            // OK, the alias isn't supposed to exist
            console.log('OK, even if missing alias ');
            return false;
        } else {
            console.log('error during alias check:');
            throw error; // a real error :)
        }
    }
};

/**
 * Update power, publish new version, and create/update alias.
 */
module.exports.createPowerConfiguration = async(lambdaARN, value, alias) => {
    try {
        await utils.setLambdaPower(lambdaARN, value);

        // wait for functoin update to complete
        await utils.waitForFunctionUpdate(lambdaARN);

        const {Version} = await utils.publishLambdaVersion(lambdaARN);
        const aliasExists = await utils.verifyAliasExistance(lambdaARN, alias);
        if (aliasExists) {
            await utils.updateLambdaAlias(lambdaARN, alias, Version);
        } else {
            await utils.createLambdaAlias(lambdaARN, alias, Version);
        }
    } catch (error) {
        if (error.message && error.message.includes('Alias already exists')) {
            // shouldn't happen, but nothing we can do in that case
            console.log('OK, even if: ', error);
        } else {
            console.log('error during config creation for value ' + value);
            throw error; // a real error :)
        }
    }
};

/**
 * Wait for the function's LastUpdateStatus to become Successful.
 * Documentation: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#functionUpdated-waiter
 * Why is this needed? https://aws.amazon.com/blogs/compute/coming-soon-expansion-of-aws-lambda-states-to-all-functions/
 */
module.exports.waitForFunctionUpdate = async(lambdaARN) => {
    console.log('Waiting for update to complete');
    const params = {
        FunctionName: lambdaARN,
        $waiter: { // override delay (5s by default)
            delay: 0.5,
        },
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.waitFor('functionUpdated', params).promise();
};

module.exports.waitForAliasActive = async(lambdaARN, alias, callback) => {
    console.log('Waiting for alias to be active');
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
        $waiter: {
            // https://aws.amazon.com/blogs/developer/waiters-in-modular-aws-sdk-for-javascript/
            // "In v2, there is no direct way to provide maximum wait time for a waiter.
            // You need to configure delay and maxAttempts to indirectly suggest the maximum time you want the waiter to run for."
            // 5s * 24 is ~2 minutes
            delay: 5,
            maxAttempts: 24,
        },
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.waitFor('functionActive', params).promise();
};

/**
 * Retrieve a given Lambda Function's memory size (always $LATEST version)
 */
module.exports.getLambdaPower = async(lambdaARN) => {
    console.log('Getting current power value');
    const params = {
        FunctionName: lambdaARN,
        Qualifier: '$LATEST',
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    const config = await lambda.getFunctionConfiguration(params).promise();
    return config.MemorySize;
};

/**
 * Retrieve a given Lambda Function's architecture and whether its state is Pending
 */
module.exports.getLambdaConfig = async(lambdaARN, alias) => {
    console.log('Getting current function config');
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
    };
    let architecture, isPending;
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    const config = await lambda.getFunctionConfiguration(params).promise();
    if (typeof config.Architectures !== 'undefined') {
        architecture = config.Architectures[0];
    } else {
        architecture = 'x86_64';
    }
    if (typeof config.State !== 'undefined') {
        isPending = config.State === 'Pending';
    } else {
        isPending = false;
    }
    return {architecture, isPending};
};

/**
 * Update a given Lambda Function's memory size (always $LATEST version).
 */
module.exports.setLambdaPower = (lambdaARN, value) => {
    console.log('Setting power to ', value);
    const params = {
        FunctionName: lambdaARN,
        MemorySize: parseInt(value, 10),
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.updateFunctionConfiguration(params).promise();
};

/**
 * Publish a new Lambda Version (version number will be returned).
 */
module.exports.publishLambdaVersion = (lambdaARN /*, alias*/) => {
    console.log('Publishing new version');
    const params = {
        FunctionName: lambdaARN,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.publishVersion(params).promise();
};

/**
 * Delete a given Lambda Version.
 */
module.exports.deleteLambdaVersion = (lambdaARN, version) => {
    console.log('Deleting version ', version);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: version,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.deleteFunction(params).promise();
};

/**
 * Create a new Lambda Alias and associate it with the given Lambda Version.
 */
module.exports.createLambdaAlias = (lambdaARN, alias, version) => {
    console.log('Creating Alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        FunctionVersion: version,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.createAlias(params).promise();
};

/**
 * Create a new Lambda Alias and associate it with the given Lambda Version.
 */
module.exports.updateLambdaAlias = (lambdaARN, alias, version) => {
    console.log('Updating Alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        FunctionVersion: version,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.updateAlias(params).promise();
};

/**
 * Delete a given Lambda Alias.
 */
module.exports.deleteLambdaAlias = (lambdaARN, alias) => {
    console.log('Deleting alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.deleteAlias(params).promise();
};

/**
 * Invoke a (pre/post-)processor Lambda function and return its output (data.Payload).
 */
module.exports.invokeLambdaProcessor = async(processorARN, payload, preOrPost = 'Pre') => {
    const processorData = await utils.invokeLambda(processorARN, null, payload);
    if (processorData.FunctionError) {
        throw new Error(`${preOrPost}Processor ${processorARN} failed with error ${processorData.Payload} and payload ${JSON.stringify(payload)}`);
    }
    return processorData.Payload;
};

/**
 * Wrapper around Lambda function invocation with pre/post-processor functions.
 */
module.exports.invokeLambdaWithProcessors = async(lambdaARN, alias, payload, preARN, postARN) => {

    var actualPayload = payload; // might change based on pre-processor

    // first invoke pre-processor, if provided
    if (preARN) {
        console.log('Invoking pre-processor');
        // overwrite payload with pre-processor's output (only if not empty)
        const preProcessorOutput = await utils.invokeLambdaProcessor(preARN, payload, 'Pre');
        if (preProcessorOutput) {
            actualPayload = preProcessorOutput;
        }
    }

    // invoke function to be power-tuned
    const invocationResults = await utils.invokeLambda(lambdaARN, alias, actualPayload);

    // then invoke post-processor, if provided
    if (postARN) {
        console.log('Invoking post-processor');
        // note: invocation may have failed (invocationResults.FunctionError)
        await utils.invokeLambdaProcessor(postARN, invocationResults.Payload, 'Post');
    }

    return {
        actualPayload,
        invocationResults,
    };
};

/**
 * Invoke a given Lambda Function:Alias with payload and return its logs.
 */
module.exports.invokeLambda = (lambdaARN, alias, payload) => {
    console.log(`Invoking function ${lambdaARN}:${alias || '$LATEST'} with payload ${JSON.stringify(payload)}`);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
        Payload: payload,
        LogType: 'Tail', // will return logs
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.invoke(params).promise();
};

/**
 * Fetch the body of an S3 object, given an S3 path such as s3://BUCKET/KEY
 */
module.exports.fetchPayloadFromS3 = async(s3Path) => {
    console.log('Fetch payload from S3', s3Path);

    if (typeof s3Path !== 'string' || s3Path.indexOf('s3://') === -1) {
        throw new Error('Invalid S3 path, not a string in the format s3://BUCKET/KEY');
    }

    const URI = url.parse(s3Path);
    URI.pathname = decodeURIComponent(URI.pathname || '');

    const bucket = URI.hostname;
    const key = URI.pathname.slice(1);

    if (!bucket || !key) {
        throw new Error(`Invalid S3 path: "${s3Path}" (bucket: ${bucket}, key: ${key})`);
    }

    const data = await utils._fetchS3Object(bucket, key);

    try {
        // try to parse into JSON object
        return JSON.parse(data);
    } catch (_) {
        // otherwise return as is
        return data;
    }


};

module.exports._fetchS3Object = async(bucket, key) => {
    const s3 = new AWS.S3();
    try {
        const response = await s3.getObject({
            Bucket: bucket,
            Key: key,
        }).promise();
        return response.Body.toString('utf-8');
    } catch (err) {
        if (err.statusCode === 403) {
            throw new Error(
                `Permission denied when trying to read s3://${bucket}/${key}. ` +
                'You might need to re-deploy the app with the correct payloadS3Bucket parameter.',
            );
        } else if (err.statusCode === 404) {
            throw new Error(
                `The object s3://${bucket}/${key} does not exist. ` +
                'Make sure you are trying to access an existing object in the correct bucket.',
            );
        } else {
            throw new Error(`Unknown error when trying to read s3://${bucket}/${key}. ${err.message}`);
        }
    }
};

/**
 * Generate a list of `num` payloads (repeated or weighted)
 */
module.exports.generatePayloads = (num, payloadInput) => {
    if (Array.isArray(payloadInput)) {
        // if array, generate a list of payloads based on weights

        // fail if empty list or missing weight/payload
        if (payloadInput.length === 0 || payloadInput.some(p => !p.weight || !p.payload)) {
            throw new Error('Invalid weighted payload structure');
        }

        if (num < payloadInput.length) {
            throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);
        }

        // we use relative weights (not %), so here we compute the total weight
        const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);

        // generate an array of num items (to be filled)
        const payloads = utils.range(num);

        // iterate over weighted payloads and fill the array based on relative weight
        let done = 0;
        for (let i = 0; i < payloadInput.length; i++) {
            const p = payloadInput[i];
            var howMany = Math.floor(p.weight * num / total);
            if (howMany < 1) {
                throw new Error('Invalid payload weight (num is too small)');
            }

            // make sure the last item fills the remaining gap
            if (i === payloadInput.length - 1) {
                howMany = num - done;
            }

            // finally fill the list with howMany items
            payloads.fill(utils.convertPayload(p.payload), done, done + howMany);
            done += howMany;
        }

        return payloads;
    } else {
        // if not an array, always use the same payload (still generate a list)
        const payloads = utils.range(num);
        payloads.fill(utils.convertPayload(payloadInput), 0, num);
        return payloads;
    }
};

/**
 * Convert payload to string, if it's not a string already
 */
module.exports.convertPayload = (payload) => {
    /**
     * Return true only if the input is a JSON-encoded string.
     * For example, '"test"' or '{"key": "value"}'.
     */
    const isJsonString = (s) => {
        if (typeof s !== 'string')
            return false;

        try {
            JSON.parse(s);
        } catch (e) {
            return false;
        }
        return true;
    };

    // optionally convert everything into string
    if (typeof payload !== 'undefined' && !isJsonString(payload)) {
        // note: 'just a string' becomes '"just a string"'
        console.log('Converting payload to JSON string from ', typeof payload);
        payload = JSON.stringify(payload);
    }
    return payload;
};

/**
 * Compute average price, given average duration.
 */
module.exports.computePrice = (minCost, minRAM, value, duration) => {
    // it's just proportional to ms (ceiled) and memory value
    return Math.ceil(duration) * minCost * (value / minRAM);
};

module.exports.parseLogAndExtractDurations = (data) => {
    return data.map(log => {
        const logString = utils.base64decode(log.LogResult || '');
        return utils.extractDuration(logString);
    });
};

/**
 * Compute total cost
 */
module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {
    if (!durations || !durations.length) {
        return 0;
    }

    // compute corresponding cost for each duration
    const costs = durations.map(duration => utils.computePrice(minCost, minRAM, value, duration));

    // sum all together
    return costs.reduce((a, b) => a + b, 0);
};

/**
 * Compute average duration
 */
module.exports.computeAverageDuration = (durations, discardTopBottom) => {
    if (!durations || !durations.length) {
        return 0;
    }

    // a percentage of durations will be discarded (trimmed mean)
    const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);

    if (discardTopBottom > 0 && toBeDiscarded === 0) {
        // not an error, but worth logging
        // this happens when you have less than 5 invocations
        // (only happens if dryrun or in tests)
        console.log('not enough results to discard');
    }

    const newN = durations.length - 2 * toBeDiscarded;

    // compute trimmed mean (discard a percentage of low/high values)
    const averageDuration = durations
        .sort(function(a, b) { return a - b; }) // sort numerically
        .slice(toBeDiscarded, toBeDiscarded > 0 ? -toBeDiscarded : durations.length) // discard first/last values
        .reduce((a, b) => a + b, 0) // sum all together
        / newN
    ;

    return averageDuration;
};

/**
 * Extract duration (in ms) from a given Lambda's CloudWatch log.
 */
module.exports.extractDuration = (log) => {
    // extract duration from log (anyone can suggest a regex?)
    const durationSplit = log.split('\tDuration: ');
    if (durationSplit.length < 2) return 0;

    const durationStr = durationSplit[1].split(' ms')[0];
    return parseFloat(durationStr);
};

/**
 * Encode a given string to base64.
 */
module.exports.base64decode = (str) => {
    return Buffer.from(str, 'base64').toString();
};

/**
 * Generate a list of size n.
 */
module.exports.range = (n) => {
    if (n === null || typeof n === 'undefined') {
        n = -1;
    }
    return Array.from(Array(n).keys());
};

module.exports.regionFromARN = (arn) => {
    if (typeof arn !== 'string' || arn.split(':').length !== 7) {
        throw new Error('Invalid ARN: ' + arn);
    }
    return arn.split(':')[3];
};

module.exports.lambdaClientFromARN = (lambdaARN) => {
    const region = this.regionFromARN(lambdaARN);
    return new AWS.Lambda({region});
};

/**
 * Generate a URL with encoded stats.
 * Note: the URL hash is never sent to the server.
 */
module.exports.buildVisualizationURL = (stats, baseURL) => {

    function encode(inputList, EncodeType = null) {
        EncodeType = EncodeType || Float32Array;
        inputList = new EncodeType(inputList);
        inputList = new Uint8Array(inputList.buffer);
        return Buffer.from(inputList).toString('base64');
    }

    // sort by power
    stats.sort((p1, p2) => {
        return p1.power - p2.power;
    });

    const sizes = stats.map(p => p.power);
    const times = stats.map(p => p.duration);
    const costs = stats.map(p => p.cost);

    const hash = [
        encode(sizes, Int16Array),
        encode(times),
        encode(costs),
    ].join(';');

    return baseURL + '#' + hash;
};

/**
 * Using the prices supplied,
 * to figure what the base price is for the
 * supplied region.
 */
module.exports.baseCostForRegion = (priceMap, region) => {
    if (priceMap[region]) {
        return priceMap[region];
    }
    console.log(region + ' not found in base price map, using default: ' + priceMap['default']);
    return priceMap['default'];
};


module.exports.sleep = async(sleepBetweenRunsMs) => {
    await new Promise(resolve => setTimeout(resolve, sleepBetweenRunsMs));
};

1	'use strict';
2
3	const AWS = require('aws-sdk');	1✔
4	const url = require('url');	1✔
5
6
7	// local reference to this module
8	const utils = module.exports;	1✔
9
10	// cost of 6+N state transitions (AWS Step Functions)
11	module.exports.stepFunctionsCost = (nPower) => +(this.stepFunctionsBaseCost() * (6 + nPower)).toFixed(5);	22✔
12
13	module.exports.stepFunctionsBaseCost = () => {	1✔
14	const prices = JSON.parse(process.env.sfCosts);	24✔
15	// assume the AWS_REGION variable is set for this function
16	return this.baseCostForRegion(prices, process.env.AWS_REGION);	24✔
17	};
18
19	module.exports.lambdaBaseCost = (region, architecture) => {	1✔
20	const prices = JSON.parse(process.env.baseCosts);	35✔
21	const priceMap = prices[architecture];	35✔
22	if (!priceMap){	35✔
23	throw new Error('Unsupported architecture: ' + architecture);	1✔
24	}
25	return this.baseCostForRegion(priceMap, region);	34✔
26	};
27
28	module.exports.allPowerValues = () => {	1✔
29	const increment = 64;	3✔
30	const powerValues = [];	3✔
31	for (let value = 128; value <= 3008; value += increment) {	3✔
32	powerValues.push(value);	138✔
33	}
34	return powerValues;	3✔
35	};
36
37	/**
38	* Check whether a Lambda Alias exists or not, and return its data.
39	*/
40	module.exports.getLambdaAlias = (lambdaARN, alias) => {	1✔
41	console.log('Checking alias ', alias);	1✔
42	const params = {	1✔
43	FunctionName: lambdaARN,
44	Name: alias,
45	};
46	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
47	return lambda.getAlias(params).promise();	1✔
48	};
49
50	/**
51	* Return true if alias exist, false if it doesn't.
52	*/
53	module.exports.verifyAliasExistance = async(lambdaARN, alias) => {	1✔
54	try {	74✔
55	await utils.getLambdaAlias(lambdaARN, alias);	74✔
56	return true;	3✔
57	} catch (error) {
58	if (error.code === 'ResourceNotFoundException') {	71✔
59	// OK, the alias isn't supposed to exist
60	console.log('OK, even if missing alias ');	70✔
61	return false;	70✔
62	} else {
63	console.log('error during alias check:');	1✔
64	throw error; // a real error :)	1✔
65	}
66	}
67	};
68
69	/**
70	* Update power, publish new version, and create/update alias.
71	*/
72	module.exports.createPowerConfiguration = async(lambdaARN, value, alias) => {	1✔
73	try {	73✔
74	await utils.setLambdaPower(lambdaARN, value);	73✔
75
76	// wait for functoin update to complete
77	await utils.waitForFunctionUpdate(lambdaARN);	72✔
78
79	const {Version} = await utils.publishLambdaVersion(lambdaARN);	72✔
80	const aliasExists = await utils.verifyAliasExistance(lambdaARN, alias);	72✔
81	if (aliasExists) {	71✔
82	await utils.updateLambdaAlias(lambdaARN, alias, Version);	2✔
83	} else {
84	await utils.createLambdaAlias(lambdaARN, alias, Version);	69✔
85	}
86	} catch (error) {
87	if (error.message && error.message.includes('Alias already exists')) {	6✔
88	// shouldn't happen, but nothing we can do in that case
89	console.log('OK, even if: ', error);	4✔
90	} else {
91	console.log('error during config creation for value ' + value);	2✔
92	throw error; // a real error :)	2✔
93	}
94	}
95	};
96
97	/**
98	* Wait for the function's LastUpdateStatus to become Successful.
99	* Documentation: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#functionUpdated-waiter
100	* Why is this needed? https://aws.amazon.com/blogs/compute/coming-soon-expansion-of-aws-lambda-states-to-all-functions/
101	*/
102	module.exports.waitForFunctionUpdate = async(lambdaARN) => {	1✔
103	console.log('Waiting for update to complete');	2✔
104	const params = {	2✔
105	FunctionName: lambdaARN,
106	$waiter: { // override delay (5s by default)
107	delay: 0.5,
108	},
109	};
110	const lambda = utils.lambdaClientFromARN(lambdaARN);	2✔
111	return lambda.waitFor('functionUpdated', params).promise();	2✔
112	};
113
114	module.exports.waitForAliasActive = async(lambdaARN, alias, callback) => {	1✔
115	console.log('Waiting for alias to be active');	1✔
116	const params = {	1✔
117	FunctionName: lambdaARN,
118	Qualifier: alias,
119	$waiter: {
120	// https://aws.amazon.com/blogs/developer/waiters-in-modular-aws-sdk-for-javascript/
121	// "In v2, there is no direct way to provide maximum wait time for a waiter.
122	// You need to configure delay and maxAttempts to indirectly suggest the maximum time you want the waiter to run for."
123	// 5s * 24 is ~2 minutes
124	delay: 5,
125	maxAttempts: 24,
126	},
127	};
128	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
129	return lambda.waitFor('functionActive', params).promise();	1✔
130	};
131
132	/**
133	* Retrieve a given Lambda Function's memory size (always $LATEST version)
134	*/
135	module.exports.getLambdaPower = async(lambdaARN) => {	1✔
136	console.log('Getting current power value');	1✔
137	const params = {	1✔
138	FunctionName: lambdaARN,
139	Qualifier: '$LATEST',
140	};
141	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
142	const config = await lambda.getFunctionConfiguration(params).promise();	1✔
143	return config.MemorySize;	1✔
144	};
145
146	/**
147	* Retrieve a given Lambda Function's architecture and whether its state is Pending
148	*/
149	module.exports.getLambdaConfig = async(lambdaARN, alias) => {	1✔
150	console.log('Getting current function config');	5✔
151	const params = {	5✔
152	FunctionName: lambdaARN,
153	Qualifier: alias,
154	};
155	let architecture, isPending;
156	const lambda = utils.lambdaClientFromARN(lambdaARN);	5✔
157	const config = await lambda.getFunctionConfiguration(params).promise();	5✔
158	if (typeof config.Architectures !== 'undefined') {	5✔
159	architecture = config.Architectures[0];	2✔
160	} else {
161	architecture = 'x86_64';	3✔
162	}
163	if (typeof config.State !== 'undefined') {	5!
164	isPending = config.State === 'Pending';	5✔
165	} else {
166	isPending = false;	×
167	}
168	return {architecture, isPending};	5✔
169	};
170
171	/**
172	* Update a given Lambda Function's memory size (always $LATEST version).
173	*/
174	module.exports.setLambdaPower = (lambdaARN, value) => {	1✔
175	console.log('Setting power to ', value);	1✔
176	const params = {	1✔
177	FunctionName: lambdaARN,
178	MemorySize: parseInt(value, 10),
179	};
180	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
181	return lambda.updateFunctionConfiguration(params).promise();	1✔
182	};
183
184	/**
185	* Publish a new Lambda Version (version number will be returned).
186	*/
187	module.exports.publishLambdaVersion = (lambdaARN /, alias/) => {	1✔
188	console.log('Publishing new version');	1✔
189	const params = {	1✔
190	FunctionName: lambdaARN,
191	};
192	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
193	return lambda.publishVersion(params).promise();	1✔
194	};
195
196	/**
197	* Delete a given Lambda Version.
198	*/
199	module.exports.deleteLambdaVersion = (lambdaARN, version) => {	1✔
200	console.log('Deleting version ', version);	1✔
201	const params = {	1✔
202	FunctionName: lambdaARN,
203	Qualifier: version,
204	};
205	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
206	return lambda.deleteFunction(params).promise();	1✔
207	};
208
209	/**
210	* Create a new Lambda Alias and associate it with the given Lambda Version.
211	*/
212	module.exports.createLambdaAlias = (lambdaARN, alias, version) => {	1✔
213	console.log('Creating Alias ', alias);	1✔
214	const params = {	1✔
215	FunctionName: lambdaARN,
216	FunctionVersion: version,
217	Name: alias,
218	};
219	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
220	return lambda.createAlias(params).promise();	1✔
221	};
222
223	/**
224	* Create a new Lambda Alias and associate it with the given Lambda Version.
225	*/
226	module.exports.updateLambdaAlias = (lambdaARN, alias, version) => {	1✔
227	console.log('Updating Alias ', alias);	1✔
228	const params = {	1✔
229	FunctionName: lambdaARN,
230	FunctionVersion: version,
231	Name: alias,
232	};
233	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
234	return lambda.updateAlias(params).promise();	1✔
235	};
236
237	/**
238	* Delete a given Lambda Alias.
239	*/
240	module.exports.deleteLambdaAlias = (lambdaARN, alias) => {	1✔
241	console.log('Deleting alias ', alias);	1✔
242	const params = {	1✔
243	FunctionName: lambdaARN,
244	Name: alias,
245	};
246	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
247	return lambda.deleteAlias(params).promise();	1✔
248	};
249
250	/**
251	* Invoke a (pre/post-)processor Lambda function and return its output (data.Payload).
252	*/
253	module.exports.invokeLambdaProcessor = async(processorARN, payload, preOrPost = 'Pre') => {	1✔
254	const processorData = await utils.invokeLambda(processorARN, null, payload);	3✔
255	if (processorData.FunctionError) {	3✔
256	throw new Error(`${preOrPost}Processor ${processorARN} failed with error ${processorData.Payload} and payload ${JSON.stringify(payload)}`);	1✔
257	}
258	return processorData.Payload;	2✔
259	};
260
261	/**
262	* Wrapper around Lambda function invocation with pre/post-processor functions.
263	*/
264	module.exports.invokeLambdaWithProcessors = async(lambdaARN, alias, payload, preARN, postARN) => {	1✔
265
266	var actualPayload = payload; // might change based on pre-processor	790✔
267
268	// first invoke pre-processor, if provided
269	if (preARN) {	790✔
270	console.log('Invoking pre-processor');	52✔
271	// overwrite payload with pre-processor's output (only if not empty)
272	const preProcessorOutput = await utils.invokeLambdaProcessor(preARN, payload, 'Pre');	52✔
273	if (preProcessorOutput) {	51✔
274	actualPayload = preProcessorOutput;	41✔
275	}
276	}
277
278	// invoke function to be power-tuned
279	const invocationResults = await utils.invokeLambda(lambdaARN, alias, actualPayload);	789✔
280
281	// then invoke post-processor, if provided
282	if (postARN) {	789✔
283	console.log('Invoking post-processor');	21✔
284	// note: invocation may have failed (invocationResults.FunctionError)
285	await utils.invokeLambdaProcessor(postARN, invocationResults.Payload, 'Post');	21✔
286	}
287
288	return {	788✔
289	actualPayload,
290	invocationResults,
291	};
292	};
293
294	/**
295	* Invoke a given Lambda Function:Alias with payload and return its logs.
296	*/
297	module.exports.invokeLambda = (lambdaARN, alias, payload) => {	1✔
298	console.log(`Invoking function ${lambdaARN}:${alias \|\| '$LATEST'} with payload ${JSON.stringify(payload)}`);	3✔
299	const params = {	3✔
300	FunctionName: lambdaARN,
301	Qualifier: alias,
302	Payload: payload,
303	LogType: 'Tail', // will return logs
304	};
305	const lambda = utils.lambdaClientFromARN(lambdaARN);	3✔
306	return lambda.invoke(params).promise();	3✔
307	};
308
309	/**
310	* Fetch the body of an S3 object, given an S3 path such as s3://BUCKET/KEY
311	*/
312	module.exports.fetchPayloadFromS3 = async(s3Path) => {	1✔
313	console.log('Fetch payload from S3', s3Path);	17✔
314
315	if (typeof s3Path !== 'string' \|\| s3Path.indexOf('s3://') === -1) {	17✔
316	throw new Error('Invalid S3 path, not a string in the format s3://BUCKET/KEY');	6✔
317	}
318
319	const URI = url.parse(s3Path);	11✔
320	URI.pathname = decodeURIComponent(URI.pathname \|\| '');	11✔
321
322	const bucket = URI.hostname;	11✔
323	const key = URI.pathname.slice(1);	11✔
324
325	if (!bucket \|\| !key) {	11✔
326	throw new Error(`Invalid S3 path: "${s3Path}" (bucket: ${bucket}, key: ${key})`);	3✔
327	}
328
329	const data = await utils._fetchS3Object(bucket, key);	8✔
330
331	try {	5✔
332	// try to parse into JSON object
333	return JSON.parse(data);	5✔
334	} catch (_) {
335	// otherwise return as is
336	return data;	1✔
337	}
338
339
340	};
341
342	module.exports._fetchS3Object = async(bucket, key) => {	1✔
343	const s3 = new AWS.S3();	8✔
344	try {	8✔
345	const response = await s3.getObject({	8✔
346	Bucket: bucket,
347	Key: key,
348	}).promise();
349	return response.Body.toString('utf-8');	5✔
350	} catch (err) {
351	if (err.statusCode === 403) {	3✔
352	throw new Error(	1✔
353	`Permission denied when trying to read s3://${bucket}/${key}. ` +
354	'You might need to re-deploy the app with the correct payloadS3Bucket parameter.',
355	);
356	} else if (err.statusCode === 404) {	2✔
357	throw new Error(	1✔
358	`The object s3://${bucket}/${key} does not exist. ` +
359	'Make sure you are trying to access an existing object in the correct bucket.',
360	);
361	} else {
362	throw new Error(`Unknown error when trying to read s3://${bucket}/${key}. ${err.message}`);	1✔
363	}
364	}
365	};
366
367	/**
368	* Generate a list of `num` payloads (repeated or weighted)
369	*/
370	module.exports.generatePayloads = (num, payloadInput) => {	1✔
371	if (Array.isArray(payloadInput)) {	64✔
372	// if array, generate a list of payloads based on weights
373
374	// fail if empty list or missing weight/payload
375	if (payloadInput.length === 0 \|\| payloadInput.some(p => !p.weight \|\| !p.payload)) {	89✔
376	throw new Error('Invalid weighted payload structure');	10✔
377	}
378
379	if (num < payloadInput.length) {	16✔
380	throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);	2✔
381	}
382
383	// we use relative weights (not %), so here we compute the total weight
384	const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);	63✔
385
386	// generate an array of num items (to be filled)
387	const payloads = utils.range(num);	14✔
388
389	// iterate over weighted payloads and fill the array based on relative weight
390	let done = 0;	14✔
391	for (let i = 0; i < payloadInput.length; i++) {	14✔
392	const p = payloadInput[i];	61✔
393	var howMany = Math.floor(p.weight * num / total);	61✔
394	if (howMany < 1) {	61✔
395	throw new Error('Invalid payload weight (num is too small)');	1✔
396	}
397
398	// make sure the last item fills the remaining gap
399	if (i === payloadInput.length - 1) {	60✔
400	howMany = num - done;	13✔
401	}
402
403	// finally fill the list with howMany items
404	payloads.fill(utils.convertPayload(p.payload), done, done + howMany);	60✔
405	done += howMany;	60✔
406	}
407
408	return payloads;	13✔
409	} else {
410	// if not an array, always use the same payload (still generate a list)
411	const payloads = utils.range(num);	38✔
412	payloads.fill(utils.convertPayload(payloadInput), 0, num);	38✔
413	return payloads;	38✔
414	}
415	};
416
417	/**
418	* Convert payload to string, if it's not a string already
419	*/
420	module.exports.convertPayload = (payload) => {	1✔
421	/**
422	* Return true only if the input is a JSON-encoded string.
423	* For example, '"test"' or '{"key": "value"}'.
424	*/
425	const isJsonString = (s) => {	120✔
426	if (typeof s !== 'string')	118✔
427	return false;	99✔
428
429	try {	19✔
430	JSON.parse(s);	19✔
431	} catch (e) {
432	return false;	11✔
433	}
434	return true;	8✔
435	};
436
437	// optionally convert everything into string
438	if (typeof payload !== 'undefined' && !isJsonString(payload)) {	120✔
439	// note: 'just a string' becomes '"just a string"'
440	console.log('Converting payload to JSON string from ', typeof payload);	110✔
441	payload = JSON.stringify(payload);	110✔
442	}
443	return payload;	120✔
444	};
445
446	/**
447	* Compute average price, given average duration.
448	*/
449	module.exports.computePrice = (minCost, minRAM, value, duration) => {	1✔
450	// it's just proportional to ms (ceiled) and memory value
451	return Math.ceil(duration) * minCost * (value / minRAM);	781✔
452	};
453
454	module.exports.parseLogAndExtractDurations = (data) => {	1✔
455	return data.map(log => {	35✔
456	const logString = utils.base64decode(log.LogResult \|\| '');	749✔
457	return utils.extractDuration(logString);	749✔
458	});
459	};
460
461	/**
462	* Compute total cost
463	*/
464	module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {	1✔
465	if (!durations \|\| !durations.length) {	34✔
466	return 0;	1✔
467	}
468
469	// compute corresponding cost for each duration
470	const costs = durations.map(duration => utils.computePrice(minCost, minRAM, value, duration));	748✔
471
472	// sum all together
473	return costs.reduce((a, b) => a + b, 0);	748✔
474	};
475
476	/**
477	* Compute average duration
478	*/
479	module.exports.computeAverageDuration = (durations, discardTopBottom) => {	1✔
480	if (!durations \|\| !durations.length) {	37✔
481	return 0;	1✔
482	}
483
484	// a percentage of durations will be discarded (trimmed mean)
485	const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);	36✔
486
487	if (discardTopBottom > 0 && toBeDiscarded === 0) {	36✔
488	// not an error, but worth logging
489	// this happens when you have less than 5 invocations
490	// (only happens if dryrun or in tests)
491	console.log('not enough results to discard');	4✔
492	}
493
494	const newN = durations.length - 2 * toBeDiscarded;	36✔
495
496	// compute trimmed mean (discard a percentage of low/high values)
497	const averageDuration = durations	36✔
498	.sort(function(a, b) { return a - b; }) // sort numerically	723✔
499	.slice(toBeDiscarded, toBeDiscarded > 0 ? -toBeDiscarded : durations.length) // discard first/last values	36✔
500	.reduce((a, b) => a + b, 0) // sum all together	453✔
501	/ newN
502	;
503
504	return averageDuration;	36✔
505	};
506
507	/**
508	* Extract duration (in ms) from a given Lambda's CloudWatch log.
509	*/
510	module.exports.extractDuration = (log) => {	1✔
511	// extract duration from log (anyone can suggest a regex?)
512	const durationSplit = log.split('\tDuration: ');	752✔
513	if (durationSplit.length < 2) return 0;	752✔
514
515	const durationStr = durationSplit[1].split(' ms')[0];	749✔
516	return parseFloat(durationStr);	749✔
517	};
518
519	/**
520	* Encode a given string to base64.
521	*/
522	module.exports.base64decode = (str) => {	1✔
523	return Buffer.from(str, 'base64').toString();	754✔
524	};
525
526	/**
527	* Generate a list of size n.
528	*/
529	module.exports.range = (n) => {	1✔
530	if (n === null \|\| typeof n === 'undefined') {	72✔
531	n = -1;	2✔
532	}
533	return Array.from(Array(n).keys());	72✔
534	};
535
536	module.exports.regionFromARN = (arn) => {	1✔
537	if (typeof arn !== 'string' \|\| arn.split(':').length !== 7) {	27✔
538	throw new Error('Invalid ARN: ' + arn);	7✔
539	}
540	return arn.split(':')[3];	20✔
541	};
542
543	module.exports.lambdaClientFromARN = (lambdaARN) => {	1✔
544	const region = this.regionFromARN(lambdaARN);	27✔
545	return new AWS.Lambda({region});	20✔
546	};
547
548	/**
549	* Generate a URL with encoded stats.
550	* Note: the URL hash is never sent to the server.
551	*/
552	module.exports.buildVisualizationURL = (stats, baseURL) => {	1✔
553
554	function encode(inputList, EncodeType = null) {	24✔
555	EncodeType = EncodeType \|\| Float32Array;	36✔
556	inputList = new EncodeType(inputList);	36✔
557	inputList = new Uint8Array(inputList.buffer);	36✔
558	return Buffer.from(inputList).toString('base64');	36✔
559	}
560
561	// sort by power
562	stats.sort((p1, p2) => {	12✔
563	return p1.power - p2.power;	41✔
564	});
565
566	const sizes = stats.map(p => p.power);	37✔
567	const times = stats.map(p => p.duration);	37✔
568	const costs = stats.map(p => p.cost);	37✔
569
570	const hash = [	12✔
571	encode(sizes, Int16Array),
572	encode(times),
573	encode(costs),
574	].join(';');
575
576	return baseURL + '#' + hash;	12✔
577	};
578
579	/**
580	* Using the prices supplied,
581	* to figure what the base price is for the
582	* supplied region.
583	*/
584	module.exports.baseCostForRegion = (priceMap, region) => {	1✔
585	if (priceMap[region]) {	7✔
586	return priceMap[region];	3✔
587	}
588	console.log(region + ' not found in base price map, using default: ' + priceMap['default']);	4✔
589	return priceMap['default'];	4✔
590	};
591
592
593	module.exports.sleep = async(sleepBetweenRunsMs) => {	1✔
594	await new Promise(resolve => setTimeout(resolve, sleepBetweenRunsMs));	1✔
595	};

alexcasalboni / aws-lambda-power-tuning / 4238184090

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous