4345884293

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/alexcasalboni/aws-lambda-power-tuning/commit/<a class=hub.com/alexcasalboni/aws-lambda-power-tuning/commit/fd841d1b72d65ce294bc101ff104ec12843c80b1">fd841d1b7<a href="https://github.com/alexcasalboni/aws-lambda-power-tuning/commit/fd841d1b72d65ce294bc101ff104ec12843c80b1">">Merge </a><a class="double-link" href="https://github.com/alexcasalboni/aws-lambda-power-tuning/commit/<a class="double-link" href="https://github.com/alexcasalboni/aws-lambda-power-tuning/commit/389479c42ba778034a9fcfb43f8e27461fb4145e">389479c42</a>">389479c42</a><a href="https://github.com/alexcasalboni/aws-lambda-power-tuning/commit/fd841d1b72d65ce294bc101ff104ec12843c80b1"> into 01d5277ec">01d5277ec</a>

Pull Request Pull Request #194: Add sam build to GH actions

Run Details

168 of 168 branches covered (100.0%)

Branch coverage included in aggregate %.

453 of 453 relevant lines covered (100.0%)

51.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/lambda/utils.js

'use strict';

const AWS = require('aws-sdk');
const url = require('url');


// local reference to this module
const utils = module.exports;

// cost of 6+N state transitions (AWS Step Functions)
module.exports.stepFunctionsCost = (nPower) => +(this.stepFunctionsBaseCost() * (6 + nPower)).toFixed(5);

module.exports.stepFunctionsBaseCost = () => {
    const prices = JSON.parse(process.env.sfCosts);
    // assume the AWS_REGION variable is set for this function
    return this.baseCostForRegion(prices, process.env.AWS_REGION);
};

module.exports.lambdaBaseCost = (region, architecture) => {
    const prices = JSON.parse(process.env.baseCosts);
    const priceMap = prices[architecture];
    if (!priceMap){
        throw new Error('Unsupported architecture: ' + architecture);
    }
    return this.baseCostForRegion(priceMap, region);
};

module.exports.allPowerValues = () => {
    const increment = 64;
    const powerValues = [];
    for (let value = 128; value <= 3008; value += increment) {
        powerValues.push(value);
    }
    return powerValues;
};

/**
 * Check whether a Lambda Alias exists or not, and return its data.
 */
module.exports.getLambdaAlias = (lambdaARN, alias) => {
    console.log('Checking alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.getAlias(params).promise();
};

/**
 * Return true if alias exist, false if it doesn't.
 */
module.exports.verifyAliasExistance = async(lambdaARN, alias) => {
    try {
        await utils.getLambdaAlias(lambdaARN, alias);
        return true;
    } catch (error) {
        if (error.code === 'ResourceNotFoundException') {
            // OK, the alias isn't supposed to exist
            console.log('OK, even if missing alias ');
            return false;
        } else {
            console.log('error during alias check:');
            throw error; // a real error :)
        }
    }
};

/**
 * Update power, publish new version, and create/update alias.
 */
module.exports.createPowerConfiguration = async(lambdaARN, value, alias) => {
    try {
        await utils.setLambdaPower(lambdaARN, value);

        // wait for functoin update to complete
        await utils.waitForFunctionUpdate(lambdaARN);

        const {Version} = await utils.publishLambdaVersion(lambdaARN);
        const aliasExists = await utils.verifyAliasExistance(lambdaARN, alias);
        if (aliasExists) {
            await utils.updateLambdaAlias(lambdaARN, alias, Version);
        } else {
            await utils.createLambdaAlias(lambdaARN, alias, Version);
        }
    } catch (error) {
        if (error.message && error.message.includes('Alias already exists')) {
            // shouldn't happen, but nothing we can do in that case
            console.log('OK, even if: ', error);
        } else {
            console.log('error during config creation for value ' + value);
            throw error; // a real error :)
        }
    }
};

/**
 * Wait for the function's LastUpdateStatus to become Successful.
 * Documentation: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#functionUpdated-waiter
 * Why is this needed? https://aws.amazon.com/blogs/compute/coming-soon-expansion-of-aws-lambda-states-to-all-functions/
 */
module.exports.waitForFunctionUpdate = async(lambdaARN) => {
    console.log('Waiting for update to complete');
    const params = {
        FunctionName: lambdaARN,
        $waiter: { // override delay (5s by default)
            delay: 0.5,
        },
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.waitFor('functionUpdated', params).promise();
};

module.exports.waitForAliasActive = async(lambdaARN, alias) => {
    console.log(`Waiting for alias ${alias} to be active`);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
        $waiter: {
            // https://aws.amazon.com/blogs/developer/waiters-in-modular-aws-sdk-for-javascript/
            // "In v2, there is no direct way to provide maximum wait time for a waiter.
            // You need to configure delay and maxAttempts to indirectly suggest the maximum time you want the waiter to run for."
            // 10s * 90 is ~15 minutes (max invocation time)
            delay: 10,
            maxAttempts: 90,
        },
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.waitFor('functionActive', params).promise();
};

/**
 * Retrieve a given Lambda Function's memory size (always $LATEST version)
 */
module.exports.getLambdaPower = async(lambdaARN) => {
    console.log('Getting current power value');
    const params = {
        FunctionName: lambdaARN,
        Qualifier: '$LATEST',
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    const config = await lambda.getFunctionConfiguration(params).promise();
    return config.MemorySize;
};

/**
 * Retrieve a given Lambda Function's architecture and whether its state is Pending
 */
module.exports.getLambdaConfig = async(lambdaARN, alias) => {
    console.log(`Getting current function config for alias ${alias}`);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
    };
    let architecture, isPending;
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    const config = await lambda.getFunctionConfiguration(params).promise();
    if (typeof config.Architectures !== 'undefined') {
        architecture = config.Architectures[0];
    } else {
        architecture = 'x86_64';
    }
    if (typeof config.State !== 'undefined') {
        // see https://docs.aws.amazon.com/lambda/latest/dg/functions-states.html
        // the most likely state here is Pending, but it could also be 
        // - Failed: it means the version creation failed (can't do much about it, the invocation will fail anyway)
        // - Inactive: it means the version hasn't been invoked for 14 days (can't happen because we always create new versions)
        isPending = config.State === 'Pending';
    } else {
        isPending = false;
    }
    return {architecture, isPending};
};

/**
 * Update a given Lambda Function's memory size (always $LATEST version).
 */
module.exports.setLambdaPower = (lambdaARN, value) => {
    console.log('Setting power to ', value);
    const params = {
        FunctionName: lambdaARN,
        MemorySize: parseInt(value, 10),
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.updateFunctionConfiguration(params).promise();
};

/**
 * Publish a new Lambda Version (version number will be returned).
 */
module.exports.publishLambdaVersion = (lambdaARN /*, alias*/) => {
    console.log('Publishing new version');
    const params = {
        FunctionName: lambdaARN,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.publishVersion(params).promise();
};

/**
 * Delete a given Lambda Version.
 */
module.exports.deleteLambdaVersion = (lambdaARN, version) => {
    console.log('Deleting version ', version);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: version,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.deleteFunction(params).promise();
};

/**
 * Create a new Lambda Alias and associate it with the given Lambda Version.
 */
module.exports.createLambdaAlias = (lambdaARN, alias, version) => {
    console.log('Creating Alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        FunctionVersion: version,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.createAlias(params).promise();
};

/**
 * Create a new Lambda Alias and associate it with the given Lambda Version.
 */
module.exports.updateLambdaAlias = (lambdaARN, alias, version) => {
    console.log('Updating Alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        FunctionVersion: version,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.updateAlias(params).promise();
};

/**
 * Delete a given Lambda Alias.
 */
module.exports.deleteLambdaAlias = (lambdaARN, alias) => {
    console.log('Deleting alias ', alias);
    const params = {
        FunctionName: lambdaARN,
        Name: alias,
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.deleteAlias(params).promise();
};

/**
 * Invoke a (pre/post-)processor Lambda function and return its output (data.Payload).
 */
module.exports.invokeLambdaProcessor = async(processorARN, payload, preOrPost = 'Pre') => {
    const processorData = await utils.invokeLambda(processorARN, null, payload);
    if (processorData.FunctionError) {
        throw new Error(`${preOrPost}Processor ${processorARN} failed with error ${processorData.Payload} and payload ${JSON.stringify(payload)}`);
    }
    return processorData.Payload;
};

/**
 * Wrapper around Lambda function invocation with pre/post-processor functions.
 */
module.exports.invokeLambdaWithProcessors = async(lambdaARN, alias, payload, preARN, postARN) => {

    var actualPayload = payload; // might change based on pre-processor

    // first invoke pre-processor, if provided
    if (preARN) {
        console.log('Invoking pre-processor');
        // overwrite payload with pre-processor's output (only if not empty)
        const preProcessorOutput = await utils.invokeLambdaProcessor(preARN, payload, 'Pre');
        if (preProcessorOutput) {
            actualPayload = preProcessorOutput;
        }
    }

    // invoke function to be power-tuned
    const invocationResults = await utils.invokeLambda(lambdaARN, alias, actualPayload);

    // then invoke post-processor, if provided
    if (postARN) {
        console.log('Invoking post-processor');
        // note: invocation may have failed (invocationResults.FunctionError)
        await utils.invokeLambdaProcessor(postARN, invocationResults.Payload, 'Post');
    }

    return {
        actualPayload,
        invocationResults,
    };
};

/**
 * Invoke a given Lambda Function:Alias with payload and return its logs.
 */
module.exports.invokeLambda = (lambdaARN, alias, payload) => {
    console.log(`Invoking function ${lambdaARN}:${alias || '$LATEST'} with payload ${JSON.stringify(payload)}`);
    const params = {
        FunctionName: lambdaARN,
        Qualifier: alias,
        Payload: payload,
        LogType: 'Tail', // will return logs
    };
    const lambda = utils.lambdaClientFromARN(lambdaARN);
    return lambda.invoke(params).promise();
};

/**
 * Fetch the body of an S3 object, given an S3 path such as s3://BUCKET/KEY
 */
module.exports.fetchPayloadFromS3 = async(s3Path) => {
    console.log('Fetch payload from S3', s3Path);

    if (typeof s3Path !== 'string' || s3Path.indexOf('s3://') === -1) {
        throw new Error('Invalid S3 path, not a string in the format s3://BUCKET/KEY');
    }

    const URI = url.parse(s3Path);
    URI.pathname = decodeURIComponent(URI.pathname || '');

    const bucket = URI.hostname;
    const key = URI.pathname.slice(1);

    if (!bucket || !key) {
        throw new Error(`Invalid S3 path: "${s3Path}" (bucket: ${bucket}, key: ${key})`);
    }

    const data = await utils._fetchS3Object(bucket, key);

    try {
        // try to parse into JSON object
        return JSON.parse(data);
    } catch (_) {
        // otherwise return as is
        return data;
    }


};

module.exports._fetchS3Object = async(bucket, key) => {
    const s3 = new AWS.S3();
    try {
        const response = await s3.getObject({
            Bucket: bucket,
            Key: key,
        }).promise();
        return response.Body.toString('utf-8');
    } catch (err) {
        if (err.statusCode === 403) {
            throw new Error(
                `Permission denied when trying to read s3://${bucket}/${key}. ` +
                'You might need to re-deploy the app with the correct payloadS3Bucket parameter.',
            );
        } else if (err.statusCode === 404) {
            throw new Error(
                `The object s3://${bucket}/${key} does not exist. ` +
                'Make sure you are trying to access an existing object in the correct bucket.',
            );
        } else {
            throw new Error(`Unknown error when trying to read s3://${bucket}/${key}. ${err.message}`);
        }
    }
};

/**
 * Generate a list of `num` payloads (repeated or weighted)
 */
module.exports.generatePayloads = (num, payloadInput) => {
    if (Array.isArray(payloadInput)) {
        // if array, generate a list of payloads based on weights

        // fail if empty list or missing weight/payload
        if (payloadInput.length === 0 || payloadInput.some(p => !p.weight || !p.payload)) {
            throw new Error('Invalid weighted payload structure');
        }

        if (num < payloadInput.length) {
            throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);
        }

        // we use relative weights (not %), so here we compute the total weight
        const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);

        // generate an array of num items (to be filled)
        const payloads = utils.range(num);

        // iterate over weighted payloads and fill the array based on relative weight
        let done = 0;
        for (let i = 0; i < payloadInput.length; i++) {
            const p = payloadInput[i];
            var howMany = Math.floor(p.weight * num / total);
            if (howMany < 1) {
                throw new Error('Invalid payload weight (num is too small)');
            }

            // make sure the last item fills the remaining gap
            if (i === payloadInput.length - 1) {
                howMany = num - done;
            }

            // finally fill the list with howMany items
            payloads.fill(utils.convertPayload(p.payload), done, done + howMany);
            done += howMany;
        }

        return payloads;
    } else {
        // if not an array, always use the same payload (still generate a list)
        const payloads = utils.range(num);
        payloads.fill(utils.convertPayload(payloadInput), 0, num);
        return payloads;
    }
};

/**
 * Convert payload to string, if it's not a string already
 */
module.exports.convertPayload = (payload) => {
    /**
     * Return true only if the input is a JSON-encoded string.
     * For example, '"test"' or '{"key": "value"}'.
     */
    const isJsonString = (s) => {
        if (typeof s !== 'string')
            return false;

        try {
            JSON.parse(s);
        } catch (e) {
            return false;
        }
        return true;
    };

    // optionally convert everything into string
    if (typeof payload !== 'undefined' && !isJsonString(payload)) {
        // note: 'just a string' becomes '"just a string"'
        console.log('Converting payload to JSON string from ', typeof payload);
        payload = JSON.stringify(payload);
    }
    return payload;
};

/**
 * Compute average price, given average duration.
 */
module.exports.computePrice = (minCost, minRAM, value, duration) => {
    // it's just proportional to ms (ceiled) and memory value
    return Math.ceil(duration) * minCost * (value / minRAM);
};

module.exports.parseLogAndExtractDurations = (data) => {
    return data.map(log => {
        const logString = utils.base64decode(log.LogResult || '');
        return utils.extractDuration(logString);
    });
};

/**
 * Compute total cost
 */
module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {
    if (!durations || !durations.length) {
        return 0;
    }

    // compute corresponding cost for each duration
    const costs = durations.map(duration => utils.computePrice(minCost, minRAM, value, duration));

    // sum all together
    return costs.reduce((a, b) => a + b, 0);
};

/**
 * Compute average duration
 */
module.exports.computeAverageDuration = (durations, discardTopBottom) => {
    if (!durations || !durations.length) {
        return 0;
    }

    // a percentage of durations will be discarded (trimmed mean)
    const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);

    if (discardTopBottom > 0 && toBeDiscarded === 0) {
        // not an error, but worth logging
        // this happens when you have less than 5 invocations
        // (only happens if dryrun or in tests)
        console.log('not enough results to discard');
    }

    const newN = durations.length - 2 * toBeDiscarded;

    // compute trimmed mean (discard a percentage of low/high values)
    const averageDuration = durations
        .sort(function(a, b) { return a - b; }) // sort numerically
        .slice(toBeDiscarded, toBeDiscarded > 0 ? -toBeDiscarded : durations.length) // discard first/last values
        .reduce((a, b) => a + b, 0) // sum all together
        / newN
    ;

    return averageDuration;
};

/**
 * Extract duration (in ms) from a given Lambda's CloudWatch log.
 */
module.exports.extractDuration = (log) => {
    // extract duration from log (anyone can suggest a regex?)
    const durationSplit = log.split('\tDuration: ');
    if (durationSplit.length < 2) return 0;

    const durationStr = durationSplit[1].split(' ms')[0];
    return parseFloat(durationStr);
};

/**
 * Encode a given string to base64.
 */
module.exports.base64decode = (str) => {
    return Buffer.from(str, 'base64').toString();
};

/**
 * Generate a list of size n.
 */
module.exports.range = (n) => {
    if (n === null || typeof n === 'undefined') {
        n = -1;
    }
    return Array.from(Array(n).keys());
};

module.exports.regionFromARN = (arn) => {
    if (typeof arn !== 'string' || arn.split(':').length !== 7) {
        throw new Error('Invalid ARN: ' + arn);
    }
    return arn.split(':')[3];
};

module.exports.lambdaClientFromARN = (lambdaARN) => {
    const region = this.regionFromARN(lambdaARN);
    return new AWS.Lambda({region});
};

/**
 * Generate a URL with encoded stats.
 * Note: the URL hash is never sent to the server.
 */
module.exports.buildVisualizationURL = (stats, baseURL) => {

    function encode(inputList, EncodeType = null) {
        EncodeType = EncodeType || Float32Array;
        inputList = new EncodeType(inputList);
        inputList = new Uint8Array(inputList.buffer);
        return Buffer.from(inputList).toString('base64');
    }

    // sort by power
    stats.sort((p1, p2) => {
        return p1.power - p2.power;
    });

    const sizes = stats.map(p => p.power);
    const times = stats.map(p => p.duration);
    const costs = stats.map(p => p.cost);

    const hash = [
        encode(sizes, Int16Array),
        encode(times),
        encode(costs),
    ].join(';');

    return baseURL + '#' + hash;
};

/**
 * Using the prices supplied,
 * to figure what the base price is for the
 * supplied region.
 */
module.exports.baseCostForRegion = (priceMap, region) => {
    if (priceMap[region]) {
        return priceMap[region];
    }
    console.log(region + ' not found in base price map, using default: ' + priceMap['default']);
    return priceMap['default'];
};


module.exports.sleep = async(sleepBetweenRunsMs) => {
    await new Promise(resolve => setTimeout(resolve, sleepBetweenRunsMs));
};

1	'use strict';
2
3	const AWS = require('aws-sdk');	1✔
4	const url = require('url');	1✔
5
6
7	// local reference to this module
8	const utils = module.exports;	1✔
9
10	// cost of 6+N state transitions (AWS Step Functions)
11	module.exports.stepFunctionsCost = (nPower) => +(this.stepFunctionsBaseCost() * (6 + nPower)).toFixed(5);	22✔
12
13	module.exports.stepFunctionsBaseCost = () => {	1✔
14	const prices = JSON.parse(process.env.sfCosts);	24✔
15	// assume the AWS_REGION variable is set for this function
16	return this.baseCostForRegion(prices, process.env.AWS_REGION);	24✔
17	};
18
19	module.exports.lambdaBaseCost = (region, architecture) => {	1✔
20	const prices = JSON.parse(process.env.baseCosts);	35✔
21	const priceMap = prices[architecture];	35✔
22	if (!priceMap){	35✔
23	throw new Error('Unsupported architecture: ' + architecture);	1✔
24	}
25	return this.baseCostForRegion(priceMap, region);	34✔
26	};
27
28	module.exports.allPowerValues = () => {	1✔
29	const increment = 64;	3✔
30	const powerValues = [];	3✔
31	for (let value = 128; value <= 3008; value += increment) {	3✔
32	powerValues.push(value);	138✔
33	}
34	return powerValues;	3✔
35	};
36
37	/**
38	* Check whether a Lambda Alias exists or not, and return its data.
39	*/
40	module.exports.getLambdaAlias = (lambdaARN, alias) => {	1✔
41	console.log('Checking alias ', alias);	1✔
42	const params = {	1✔
43	FunctionName: lambdaARN,
44	Name: alias,
45	};
46	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
47	return lambda.getAlias(params).promise();	1✔
48	};
49
50	/**
51	* Return true if alias exist, false if it doesn't.
52	*/
53	module.exports.verifyAliasExistance = async(lambdaARN, alias) => {	1✔
54	try {	74✔
55	await utils.getLambdaAlias(lambdaARN, alias);	74✔
56	return true;	3✔
57	} catch (error) {
58	if (error.code === 'ResourceNotFoundException') {	71✔
59	// OK, the alias isn't supposed to exist
60	console.log('OK, even if missing alias ');	70✔
61	return false;	70✔
62	} else {
63	console.log('error during alias check:');	1✔
64	throw error; // a real error :)	1✔
65	}
66	}
67	};
68
69	/**
70	* Update power, publish new version, and create/update alias.
71	*/
72	module.exports.createPowerConfiguration = async(lambdaARN, value, alias) => {	1✔
73	try {	73✔
74	await utils.setLambdaPower(lambdaARN, value);	73✔
75
76	// wait for functoin update to complete
77	await utils.waitForFunctionUpdate(lambdaARN);	72✔
78
79	const {Version} = await utils.publishLambdaVersion(lambdaARN);	72✔
80	const aliasExists = await utils.verifyAliasExistance(lambdaARN, alias);	72✔
81	if (aliasExists) {	71✔
82	await utils.updateLambdaAlias(lambdaARN, alias, Version);	2✔
83	} else {
84	await utils.createLambdaAlias(lambdaARN, alias, Version);	69✔
85	}
86	} catch (error) {
87	if (error.message && error.message.includes('Alias already exists')) {	6✔
88	// shouldn't happen, but nothing we can do in that case
89	console.log('OK, even if: ', error);	4✔
90	} else {
91	console.log('error during config creation for value ' + value);	2✔
92	throw error; // a real error :)	2✔
93	}
94	}
95	};
96
97	/**
98	* Wait for the function's LastUpdateStatus to become Successful.
99	* Documentation: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#functionUpdated-waiter
100	* Why is this needed? https://aws.amazon.com/blogs/compute/coming-soon-expansion-of-aws-lambda-states-to-all-functions/
101	*/
102	module.exports.waitForFunctionUpdate = async(lambdaARN) => {	1✔
103	console.log('Waiting for update to complete');	2✔
104	const params = {	2✔
105	FunctionName: lambdaARN,
106	$waiter: { // override delay (5s by default)
107	delay: 0.5,
108	},
109	};
110	const lambda = utils.lambdaClientFromARN(lambdaARN);	2✔
111	return lambda.waitFor('functionUpdated', params).promise();	2✔
112	};
113
114	module.exports.waitForAliasActive = async(lambdaARN, alias) => {	1✔
115	console.log(`Waiting for alias ${alias} to be active`);	1✔
116	const params = {	1✔
117	FunctionName: lambdaARN,
118	Qualifier: alias,
119	$waiter: {
120	// https://aws.amazon.com/blogs/developer/waiters-in-modular-aws-sdk-for-javascript/
121	// "In v2, there is no direct way to provide maximum wait time for a waiter.
122	// You need to configure delay and maxAttempts to indirectly suggest the maximum time you want the waiter to run for."
123	// 10s * 90 is ~15 minutes (max invocation time)
124	delay: 10,
125	maxAttempts: 90,
126	},
127	};
128	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
129	return lambda.waitFor('functionActive', params).promise();	1✔
130	};
131
132	/**
133	* Retrieve a given Lambda Function's memory size (always $LATEST version)
134	*/
135	module.exports.getLambdaPower = async(lambdaARN) => {	1✔
136	console.log('Getting current power value');	1✔
137	const params = {	1✔
138	FunctionName: lambdaARN,
139	Qualifier: '$LATEST',
140	};
141	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
142	const config = await lambda.getFunctionConfiguration(params).promise();	1✔
143	return config.MemorySize;	1✔
144	};
145
146	/**
147	* Retrieve a given Lambda Function's architecture and whether its state is Pending
148	*/
149	module.exports.getLambdaConfig = async(lambdaARN, alias) => {	1✔
150	console.log(`Getting current function config for alias ${alias}`);	6✔
151	const params = {	6✔
152	FunctionName: lambdaARN,
153	Qualifier: alias,
154	};
155	let architecture, isPending;
156	const lambda = utils.lambdaClientFromARN(lambdaARN);	6✔
157	const config = await lambda.getFunctionConfiguration(params).promise();	6✔
158	if (typeof config.Architectures !== 'undefined') {	6✔
159	architecture = config.Architectures[0];	2✔
160	} else {
161	architecture = 'x86_64';	4✔
162	}
163	if (typeof config.State !== 'undefined') {	6✔
164	// see https://docs.aws.amazon.com/lambda/latest/dg/functions-states.html
165	// the most likely state here is Pending, but it could also be
166	// - Failed: it means the version creation failed (can't do much about it, the invocation will fail anyway)
167	// - Inactive: it means the version hasn't been invoked for 14 days (can't happen because we always create new versions)
168	isPending = config.State === 'Pending';	5✔
169	} else {
170	isPending = false;	1✔
171	}
172	return {architecture, isPending};	6✔
173	};
174
175	/**
176	* Update a given Lambda Function's memory size (always $LATEST version).
177	*/
178	module.exports.setLambdaPower = (lambdaARN, value) => {	1✔
179	console.log('Setting power to ', value);	1✔
180	const params = {	1✔
181	FunctionName: lambdaARN,
182	MemorySize: parseInt(value, 10),
183	};
184	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
185	return lambda.updateFunctionConfiguration(params).promise();	1✔
186	};
187
188	/**
189	* Publish a new Lambda Version (version number will be returned).
190	*/
191	module.exports.publishLambdaVersion = (lambdaARN /, alias/) => {	1✔
192	console.log('Publishing new version');	1✔
193	const params = {	1✔
194	FunctionName: lambdaARN,
195	};
196	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
197	return lambda.publishVersion(params).promise();	1✔
198	};
199
200	/**
201	* Delete a given Lambda Version.
202	*/
203	module.exports.deleteLambdaVersion = (lambdaARN, version) => {	1✔
204	console.log('Deleting version ', version);	1✔
205	const params = {	1✔
206	FunctionName: lambdaARN,
207	Qualifier: version,
208	};
209	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
210	return lambda.deleteFunction(params).promise();	1✔
211	};
212
213	/**
214	* Create a new Lambda Alias and associate it with the given Lambda Version.
215	*/
216	module.exports.createLambdaAlias = (lambdaARN, alias, version) => {	1✔
217	console.log('Creating Alias ', alias);	1✔
218	const params = {	1✔
219	FunctionName: lambdaARN,
220	FunctionVersion: version,
221	Name: alias,
222	};
223	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
224	return lambda.createAlias(params).promise();	1✔
225	};
226
227	/**
228	* Create a new Lambda Alias and associate it with the given Lambda Version.
229	*/
230	module.exports.updateLambdaAlias = (lambdaARN, alias, version) => {	1✔
231	console.log('Updating Alias ', alias);	1✔
232	const params = {	1✔
233	FunctionName: lambdaARN,
234	FunctionVersion: version,
235	Name: alias,
236	};
237	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
238	return lambda.updateAlias(params).promise();	1✔
239	};
240
241	/**
242	* Delete a given Lambda Alias.
243	*/
244	module.exports.deleteLambdaAlias = (lambdaARN, alias) => {	1✔
245	console.log('Deleting alias ', alias);	1✔
246	const params = {	1✔
247	FunctionName: lambdaARN,
248	Name: alias,
249	};
250	const lambda = utils.lambdaClientFromARN(lambdaARN);	1✔
251	return lambda.deleteAlias(params).promise();	1✔
252	};
253
254	/**
255	* Invoke a (pre/post-)processor Lambda function and return its output (data.Payload).
256	*/
257	module.exports.invokeLambdaProcessor = async(processorARN, payload, preOrPost = 'Pre') => {	1✔
258	const processorData = await utils.invokeLambda(processorARN, null, payload);	3✔
259	if (processorData.FunctionError) {	3✔
260	throw new Error(`${preOrPost}Processor ${processorARN} failed with error ${processorData.Payload} and payload ${JSON.stringify(payload)}`);	1✔
261	}
262	return processorData.Payload;	2✔
263	};
264
265	/**
266	* Wrapper around Lambda function invocation with pre/post-processor functions.
267	*/
268	module.exports.invokeLambdaWithProcessors = async(lambdaARN, alias, payload, preARN, postARN) => {	1✔
269
270	var actualPayload = payload; // might change based on pre-processor	790✔
271
272	// first invoke pre-processor, if provided
273	if (preARN) {	790✔
274	console.log('Invoking pre-processor');	52✔
275	// overwrite payload with pre-processor's output (only if not empty)
276	const preProcessorOutput = await utils.invokeLambdaProcessor(preARN, payload, 'Pre');	52✔
277	if (preProcessorOutput) {	51✔
278	actualPayload = preProcessorOutput;	41✔
279	}
280	}
281
282	// invoke function to be power-tuned
283	const invocationResults = await utils.invokeLambda(lambdaARN, alias, actualPayload);	789✔
284
285	// then invoke post-processor, if provided
286	if (postARN) {	789✔
287	console.log('Invoking post-processor');	21✔
288	// note: invocation may have failed (invocationResults.FunctionError)
289	await utils.invokeLambdaProcessor(postARN, invocationResults.Payload, 'Post');	21✔
290	}
291
292	return {	788✔
293	actualPayload,
294	invocationResults,
295	};
296	};
297
298	/**
299	* Invoke a given Lambda Function:Alias with payload and return its logs.
300	*/
301	module.exports.invokeLambda = (lambdaARN, alias, payload) => {	1✔
302	console.log(`Invoking function ${lambdaARN}:${alias \|\| '$LATEST'} with payload ${JSON.stringify(payload)}`);	3✔
303	const params = {	3✔
304	FunctionName: lambdaARN,
305	Qualifier: alias,
306	Payload: payload,
307	LogType: 'Tail', // will return logs
308	};
309	const lambda = utils.lambdaClientFromARN(lambdaARN);	3✔
310	return lambda.invoke(params).promise();	3✔
311	};
312
313	/**
314	* Fetch the body of an S3 object, given an S3 path such as s3://BUCKET/KEY
315	*/
316	module.exports.fetchPayloadFromS3 = async(s3Path) => {	1✔
317	console.log('Fetch payload from S3', s3Path);	17✔
318
319	if (typeof s3Path !== 'string' \|\| s3Path.indexOf('s3://') === -1) {	17✔
320	throw new Error('Invalid S3 path, not a string in the format s3://BUCKET/KEY');	6✔
321	}
322
323	const URI = url.parse(s3Path);	11✔
324	URI.pathname = decodeURIComponent(URI.pathname \|\| '');	11✔
325
326	const bucket = URI.hostname;	11✔
327	const key = URI.pathname.slice(1);	11✔
328
329	if (!bucket \|\| !key) {	11✔
330	throw new Error(`Invalid S3 path: "${s3Path}" (bucket: ${bucket}, key: ${key})`);	3✔
331	}
332
333	const data = await utils._fetchS3Object(bucket, key);	8✔
334
335	try {	5✔
336	// try to parse into JSON object
337	return JSON.parse(data);	5✔
338	} catch (_) {
339	// otherwise return as is
340	return data;	1✔
341	}
342
343
344	};
345
346	module.exports._fetchS3Object = async(bucket, key) => {	1✔
347	const s3 = new AWS.S3();	8✔
348	try {	8✔
349	const response = await s3.getObject({	8✔
350	Bucket: bucket,
351	Key: key,
352	}).promise();
353	return response.Body.toString('utf-8');	5✔
354	} catch (err) {
355	if (err.statusCode === 403) {	3✔
356	throw new Error(	1✔
357	`Permission denied when trying to read s3://${bucket}/${key}. ` +
358	'You might need to re-deploy the app with the correct payloadS3Bucket parameter.',
359	);
360	} else if (err.statusCode === 404) {	2✔
361	throw new Error(	1✔
362	`The object s3://${bucket}/${key} does not exist. ` +
363	'Make sure you are trying to access an existing object in the correct bucket.',
364	);
365	} else {
366	throw new Error(`Unknown error when trying to read s3://${bucket}/${key}. ${err.message}`);	1✔
367	}
368	}
369	};
370
371	/**
372	* Generate a list of `num` payloads (repeated or weighted)
373	*/
374	module.exports.generatePayloads = (num, payloadInput) => {	1✔
375	if (Array.isArray(payloadInput)) {	64✔
376	// if array, generate a list of payloads based on weights
377
378	// fail if empty list or missing weight/payload
379	if (payloadInput.length === 0 \|\| payloadInput.some(p => !p.weight \|\| !p.payload)) {	89✔
380	throw new Error('Invalid weighted payload structure');	10✔
381	}
382
383	if (num < payloadInput.length) {	16✔
384	throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);	2✔
385	}
386
387	// we use relative weights (not %), so here we compute the total weight
388	const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);	63✔
389
390	// generate an array of num items (to be filled)
391	const payloads = utils.range(num);	14✔
392
393	// iterate over weighted payloads and fill the array based on relative weight
394	let done = 0;	14✔
395	for (let i = 0; i < payloadInput.length; i++) {	14✔
396	const p = payloadInput[i];	61✔
397	var howMany = Math.floor(p.weight * num / total);	61✔
398	if (howMany < 1) {	61✔
399	throw new Error('Invalid payload weight (num is too small)');	1✔
400	}
401
402	// make sure the last item fills the remaining gap
403	if (i === payloadInput.length - 1) {	60✔
404	howMany = num - done;	13✔
405	}
406
407	// finally fill the list with howMany items
408	payloads.fill(utils.convertPayload(p.payload), done, done + howMany);	60✔
409	done += howMany;	60✔
410	}
411
412	return payloads;	13✔
413	} else {
414	// if not an array, always use the same payload (still generate a list)
415	const payloads = utils.range(num);	38✔
416	payloads.fill(utils.convertPayload(payloadInput), 0, num);	38✔
417	return payloads;	38✔
418	}
419	};
420
421	/**
422	* Convert payload to string, if it's not a string already
423	*/
424	module.exports.convertPayload = (payload) => {	1✔
425	/**
426	* Return true only if the input is a JSON-encoded string.
427	* For example, '"test"' or '{"key": "value"}'.
428	*/
429	const isJsonString = (s) => {	120✔
430	if (typeof s !== 'string')	118✔
431	return false;	99✔
432
433	try {	19✔
434	JSON.parse(s);	19✔
435	} catch (e) {
436	return false;	11✔
437	}
438	return true;	8✔
439	};
440
441	// optionally convert everything into string
442	if (typeof payload !== 'undefined' && !isJsonString(payload)) {	120✔
443	// note: 'just a string' becomes '"just a string"'
444	console.log('Converting payload to JSON string from ', typeof payload);	110✔
445	payload = JSON.stringify(payload);	110✔
446	}
447	return payload;	120✔
448	};
449
450	/**
451	* Compute average price, given average duration.
452	*/
453	module.exports.computePrice = (minCost, minRAM, value, duration) => {	1✔
454	// it's just proportional to ms (ceiled) and memory value
455	return Math.ceil(duration) * minCost * (value / minRAM);	781✔
456	};
457
458	module.exports.parseLogAndExtractDurations = (data) => {	1✔
459	return data.map(log => {	35✔
460	const logString = utils.base64decode(log.LogResult \|\| '');	749✔
461	return utils.extractDuration(logString);	749✔
462	});
463	};
464
465	/**
466	* Compute total cost
467	*/
468	module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {	1✔
469	if (!durations \|\| !durations.length) {	34✔
470	return 0;	1✔
471	}
472
473	// compute corresponding cost for each duration
474	const costs = durations.map(duration => utils.computePrice(minCost, minRAM, value, duration));	748✔
475
476	// sum all together
477	return costs.reduce((a, b) => a + b, 0);	748✔
478	};
479
480	/**
481	* Compute average duration
482	*/
483	module.exports.computeAverageDuration = (durations, discardTopBottom) => {	1✔
484	if (!durations \|\| !durations.length) {	37✔
485	return 0;	1✔
486	}
487
488	// a percentage of durations will be discarded (trimmed mean)
489	const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);	36✔
490
491	if (discardTopBottom > 0 && toBeDiscarded === 0) {	36✔
492	// not an error, but worth logging
493	// this happens when you have less than 5 invocations
494	// (only happens if dryrun or in tests)
495	console.log('not enough results to discard');	4✔
496	}
497
498	const newN = durations.length - 2 * toBeDiscarded;	36✔
499
500	// compute trimmed mean (discard a percentage of low/high values)
501	const averageDuration = durations	36✔
502	.sort(function(a, b) { return a - b; }) // sort numerically	723✔
503	.slice(toBeDiscarded, toBeDiscarded > 0 ? -toBeDiscarded : durations.length) // discard first/last values	36✔
504	.reduce((a, b) => a + b, 0) // sum all together	453✔
505	/ newN
506	;
507
508	return averageDuration;	36✔
509	};
510
511	/**
512	* Extract duration (in ms) from a given Lambda's CloudWatch log.
513	*/
514	module.exports.extractDuration = (log) => {	1✔
515	// extract duration from log (anyone can suggest a regex?)
516	const durationSplit = log.split('\tDuration: ');	752✔
517	if (durationSplit.length < 2) return 0;	752✔
518
519	const durationStr = durationSplit[1].split(' ms')[0];	749✔
520	return parseFloat(durationStr);	749✔
521	};
522
523	/**
524	* Encode a given string to base64.
525	*/
526	module.exports.base64decode = (str) => {	1✔
527	return Buffer.from(str, 'base64').toString();	754✔
528	};
529
530	/**
531	* Generate a list of size n.
532	*/
533	module.exports.range = (n) => {	1✔
534	if (n === null \|\| typeof n === 'undefined') {	72✔
535	n = -1;	2✔
536	}
537	return Array.from(Array(n).keys());	72✔
538	};
539
540	module.exports.regionFromARN = (arn) => {	1✔
541	if (typeof arn !== 'string' \|\| arn.split(':').length !== 7) {	28✔
542	throw new Error('Invalid ARN: ' + arn);	7✔
543	}
544	return arn.split(':')[3];	21✔
545	};
546
547	module.exports.lambdaClientFromARN = (lambdaARN) => {	1✔
548	const region = this.regionFromARN(lambdaARN);	28✔
549	return new AWS.Lambda({region});	21✔
550	};
551
552	/**
553	* Generate a URL with encoded stats.
554	* Note: the URL hash is never sent to the server.
555	*/
556	module.exports.buildVisualizationURL = (stats, baseURL) => {	1✔
557
558	function encode(inputList, EncodeType = null) {	24✔
559	EncodeType = EncodeType \|\| Float32Array;	36✔
560	inputList = new EncodeType(inputList);	36✔
561	inputList = new Uint8Array(inputList.buffer);	36✔
562	return Buffer.from(inputList).toString('base64');	36✔
563	}
564
565	// sort by power
566	stats.sort((p1, p2) => {	12✔
567	return p1.power - p2.power;	41✔
568	});
569
570	const sizes = stats.map(p => p.power);	37✔
571	const times = stats.map(p => p.duration);	37✔
572	const costs = stats.map(p => p.cost);	37✔
573
574	const hash = [	12✔
575	encode(sizes, Int16Array),
576	encode(times),
577	encode(costs),
578	].join(';');
579
580	return baseURL + '#' + hash;	12✔
581	};
582
583	/**
584	* Using the prices supplied,
585	* to figure what the base price is for the
586	* supplied region.
587	*/
588	module.exports.baseCostForRegion = (priceMap, region) => {	1✔
589	if (priceMap[region]) {	7✔
590	return priceMap[region];	3✔
591	}
592	console.log(region + ' not found in base price map, using default: ' + priceMap['default']);	4✔
593	return priceMap['default'];	4✔
594	};
595
596
597	module.exports.sleep = async(sleepBetweenRunsMs) => {	1✔
598	await new Promise(resolve => setTimeout(resolve, sleepBetweenRunsMs));	1✔
599	};

alexcasalboni / aws-lambda-power-tuning / 4345884293

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous