3543

pending completion

Build # 3543

Build Type

Pull #1106

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/wikimedia/restbase/commit/<a class=hub.com/wikimedia/restbase/commit/<a class="double-link" href="https://git"><a class=hub.com/wikimedia/restbase/commit/<a class="double-link" href="https://git"><a class=hub.com/wikimedia/restbase/commit/<a class="double-link" href="https://git"><a class=hub.com/wikimedia/restbase/commit/471adf184e7b553df873a472d519f3a34fce872a">471adf184">&lt;a href=&quot;https://github.com/wikimedia/restbase/commit/</a><a class="double-link" href="https://github.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/wikimedia/restbase/commit/471adf184e7b553df873a472d519f3a34fce872a">471adf184</a><a href="https://github.com/wikimedia/restbase/commit/471adf184e7b553df873a472d519f3a34fce872a">&lt;a href=&quot;https://github.com/wikimedia/restbase/commit/471adf184e7b553df873a472d519f3a34fce872a&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/wikimedia/restbase/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/wikimedia/restbase/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/wikimedia/restbase/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/wikimedia/restbase/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/wikimedia/restbase/commit/471adf184e7b553df873a472d519f3a34fce872a&amp;quot;&amp;gt;471adf... (continued)

Pull Request Pull Request #1106: Simplify mcs storage

Run Details

845 of 1118 branches covered (75.58%)

32 of 32 new or added lines in 3 files covered. (100.0%)

1519 of 1789 relevant lines covered (84.91%)

183.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.16

/sys/key_value.js

'use strict';

/**
 * Key-value bucket handler
 */

const crypto = require('crypto');
const stringify = require('fast-json-stable-stringify');
const TimeUUID = require('cassandra-uuid').TimeUuid;
const mwUtil = require('../lib/mwUtil');
const HyperSwitch = require('hyperswitch');
const HTTPError = HyperSwitch.HTTPError;
const URI = HyperSwitch.URI;

const spec = HyperSwitch.utils.loadSpec(`${__dirname}/key_value.yaml`);

class KVBucket {
    createBucket(hyper, req) {
        const schema = this.makeSchema(req.body || {});
        schema.table = req.params.bucket;
        const rp = req.params;
        const storeRequest = {
            uri: new URI([rp.domain, 'sys', 'table', rp.bucket]),
            body: schema
        };
        return hyper.put(storeRequest);
    }

    makeSchema(opts) {
        const schemaVersionMajor = 6;

        return {
            // Combine option & bucket version into a monotonically increasing
            // combined schema version. By multiplying the bucket version by 1000,
            // we increase the chance of catching a reset in the option version.
            version: schemaVersionMajor * 1000 + (opts.version || 0),
            attributes: {
                key: opts.keyType || 'string',
                // Both TID and ETAG are added in case we ever want to support
                // CAS using lightweight transactions to support proper
                // conditional HTTP requests with `if-modified-since` or `if-match`
                tid: 'timeuuid',
                etag: 'string',
                headers: 'json',
                value: opts.valueType || 'blob'
            },
            index: [
                { attribute: 'key', type: 'hash' }
            ]
        };
    }

    getRevision(hyper, req) {
        if (mwUtil.isNoCacheRequest(req)) {
            throw new HTTPError({
                status: 404,
                body: {
                    type: 'not_found',
                    description: 'Not attempting to fetch content for no-cache request'
                }
            });
        }

        const rp = req.params;
        const storeReq = {
            uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
            body: {
                table: rp.bucket,
                attributes: {
                    key: rp.key
                }
            }
        };
        return hyper.get(storeReq).then((dbResult) => {
            if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {
                const row = dbResult.body.items[0];
                return {
                    status: 200,
                    headers: row.headers,
                    body: row.value
                };
            } else {
                throw new HTTPError({
                    status: 404,
                    body: {
                        type: 'not_found',
                        uri: req.uri,
                        method: req.method
                    }
                });
            }
        });
    }

    listRevisions(hyper, req) {
        const rp = req.params;
        const storeRequest = {
            uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
            body: {
                table: req.params.bucket,
                attributes: {
                    key: req.params.key
                },
                limit: 1000
            }
        };
        return hyper.get(storeRequest)
        .then((res) => ({
            status: 200,
            headers: {
                'content-type': 'application/json'
            },
            body: {
                items: res.body.items.map((row) => row.tid)
            }
        }));
    }

    putRevision(hyper, req) {
        if (mwUtil.isNoStoreRequest(req)) {
            return { status: 202 };
        }

        const rp = req.params;
        req.headers = req.headers || {};

        const tid = TimeUUID.now().toString();
        if (!req.headers.etag) {
            hyper.logger.log('fatal/kv/putRevision', {
                msg: 'No etag header provided to key-value bucket'
            });
            req.headers.etag = crypto.createHash('sha256')
                .update(stringify(req.body))
                .digest('hex');
        }

        const doPut = () => hyper.put({
            uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
            body: {
                table: rp.bucket,
                attributes: {
                    key: rp.key,
                    tid,
                    etag: req.headers.etag,
                    value: req.body,
                    headers: req.headers
                }
            }
        })
        .then((res) => {
            if (res.status === 201) {
                return {
                    status: 201,
                    body: {
                        message: 'Created.'
                    }
                };
            } else {
                throw res;
            }
        })
        .tapCatch((error) => hyper.logger.log('error/kv/putRevision', error));

        // TODO: Respect the stored ETag and allow matching on etag - either one provided
        // by the client or auto-generated one.
        if (req.headers['if-none-hash-match']) {
            delete req.headers['if-none-hash-match'];
            return hyper.get({
                uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key])
            })
            .then((oldContent) => {
                if (stringify(req.body) === stringify(oldContent.body) &&
                        (!req.headers['content-type'] ||
                        req.headers['content-type'] === oldContent.headers['content-type'])) {
                    hyper.metrics.increment(`sys_kv_${req.params.bucket}.unchanged_rev_render`);
                    throw new HTTPError({
                        status: 412,
                        body: {
                            type: 'precondition_failed',
                            description: 'Not replacing existing content'
                        }
                    });
                }
                return doPut();
            })
            .catch({ status: 404 }, doPut);
        } else {
            return doPut();
        }
    }
}
module.exports = (options) => {
    const kvBucket = new KVBucket(options);

    return {
        spec, // Re-export from spec module
        operations: {
            createBucket: kvBucket.createBucket.bind(kvBucket),
            listRevisions: kvBucket.listRevisions.bind(kvBucket),
            getRevision: kvBucket.getRevision.bind(kvBucket),
            putRevision: kvBucket.putRevision.bind(kvBucket)
        }
    };
};

1	'use strict';
2
3	/**
4	* Key-value bucket handler
5	*/
6
7	const crypto = require('crypto');	2✔
8	const stringify = require('fast-json-stable-stringify');	2✔
9	const TimeUUID = require('cassandra-uuid').TimeUuid;	2✔
10	const mwUtil = require('../lib/mwUtil');	2✔
11	const HyperSwitch = require('hyperswitch');	2✔
12	const HTTPError = HyperSwitch.HTTPError;	2✔
13	const URI = HyperSwitch.URI;	2✔
14
15	const spec = HyperSwitch.utils.loadSpec(`${__dirname}/key_value.yaml`);	2✔
16
17	class KVBucket {
18	createBucket(hyper, req) {
19	const schema = this.makeSchema(req.body \|\| {});	5,210✔
20	schema.table = req.params.bucket;	5,210✔
21	const rp = req.params;	5,210✔
22	const storeRequest = {	5,210✔
23	uri: new URI([rp.domain, 'sys', 'table', rp.bucket]),
24	body: schema
25	};
26	return hyper.put(storeRequest);	5,210✔
27	}
28
29	makeSchema(opts) {
30	const schemaVersionMajor = 6;	5,210✔
31
32	return {	5,210✔
33	// Combine option & bucket version into a monotonically increasing
34	// combined schema version. By multiplying the bucket version by 1000,
35	// we increase the chance of catching a reset in the option version.
36	version: schemaVersionMajor * 1000 + (opts.version \|\| 0),	10,420✔
37	attributes: {
38	key: opts.keyType \|\| 'string',	7,712✔
39	// Both TID and ETAG are added in case we ever want to support
40	// CAS using lightweight transactions to support proper
41	// conditional HTTP requests with `if-modified-since` or `if-match`
42	tid: 'timeuuid',
43	etag: 'string',
44	headers: 'json',
45	value: opts.valueType \|\| 'blob'	6,412✔
46	},
47	index: [
48	{ attribute: 'key', type: 'hash' }
49	]
50	};
51	}
52
53	getRevision(hyper, req) {
54	if (mwUtil.isNoCacheRequest(req)) {	186✔
55	throw new HTTPError({	4✔
56	status: 404,
57	body: {
58	type: 'not_found',
59	description: 'Not attempting to fetch content for no-cache request'
60	}
61	});
62	}
63
64	const rp = req.params;	182✔
65	const storeReq = {	182✔
66	uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
67	body: {
68	table: rp.bucket,
69	attributes: {
70	key: rp.key
71	}
72	}
73	};
74	return hyper.get(storeReq).then((dbResult) => {	182✔
75	if (dbResult.body && dbResult.body.items && dbResult.body.items.length) {	54!
76	const row = dbResult.body.items[0];	54✔
77	return {	54✔
78	status: 200,
79	headers: row.headers,
80	body: row.value
81	};
82	} else {
83	throw new HTTPError({	×
84	status: 404,
85	body: {
86	type: 'not_found',
87	uri: req.uri,
88	method: req.method
89	}
90	});
91	}
92	});
93	}
94
95	listRevisions(hyper, req) {
96	const rp = req.params;	2✔
97	const storeRequest = {	2✔
98	uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
99	body: {
100	table: req.params.bucket,
101	attributes: {
102	key: req.params.key
103	},
104	limit: 1000
105	}
106	};
107	return hyper.get(storeRequest)	2✔
108	.then((res) => ({	2✔
109	status: 200,
110	headers: {
111	'content-type': 'application/json'
112	},
113	body: {
114	items: res.body.items.map((row) => row.tid)	2✔
115	}
116	}));
117	}
118
119	putRevision(hyper, req) {
120	if (mwUtil.isNoStoreRequest(req)) {	94✔
121	return { status: 202 };	2✔
122	}
123
124	const rp = req.params;	92✔
125	req.headers = req.headers \|\| {};	92!
126
127	const tid = TimeUUID.now().toString();	92✔
128	if (!req.headers.etag) {	92✔
129	hyper.logger.log('fatal/kv/putRevision', {	36✔
130	msg: 'No etag header provided to key-value bucket'
131	});
132	req.headers.etag = crypto.createHash('sha256')	36✔
133	.update(stringify(req.body))
134	.digest('hex');
135	}
136
137	const doPut = () => hyper.put({	92✔
138	uri: new URI([rp.domain, 'sys', 'table', rp.bucket, '']),
139	body: {
140	table: rp.bucket,
141	attributes: {
142	key: rp.key,
143	tid,
144	etag: req.headers.etag,
145	value: req.body,
146	headers: req.headers
147	}
148	}
149	})
150	.then((res) => {
151	if (res.status === 201) {	88!
152	return {	88✔
153	status: 201,
154	body: {
155	message: 'Created.'
156	}
157	};
158	} else {
159	throw res;	×
160	}
161	})
162	.tapCatch((error) => hyper.logger.log('error/kv/putRevision', error));	×
163
164	// TODO: Respect the stored ETag and allow matching on etag - either one provided
165	// by the client or auto-generated one.
166	if (req.headers['if-none-hash-match']) {	92✔
167	delete req.headers['if-none-hash-match'];	54✔
168	return hyper.get({	54✔
169	uri: new URI([rp.domain, 'sys', 'key_value', rp.bucket, rp.key])
170	})
171	.then((oldContent) => {
172	if (stringify(req.body) === stringify(oldContent.body) &&	4!
173	(!req.headers['content-type'] \|\|
174	req.headers['content-type'] === oldContent.headers['content-type'])) {
175	hyper.metrics.increment(`sys_kv_${req.params.bucket}.unchanged_rev_render`);	4✔
176	throw new HTTPError({	4✔
177	status: 412,
178	body: {
179	type: 'precondition_failed',
180	description: 'Not replacing existing content'
181	}
182	});
183	}
184	return doPut();	×
185	})
186	.catch({ status: 404 }, doPut);
187	} else {
188	return doPut();	38✔
189	}
190	}
191	}
192	module.exports = (options) => {	2✔
193	const kvBucket = new KVBucket(options);	672✔
194
195	return {	672✔
196	spec, // Re-export from spec module
197	operations: {
198	createBucket: kvBucket.createBucket.bind(kvBucket),
199	listRevisions: kvBucket.listRevisions.bind(kvBucket),
200	getRevision: kvBucket.getRevision.bind(kvBucket),
201	putRevision: kvBucket.putRevision.bind(kvBucket)
202	}
203	};
204	};

wikimedia / restbase / 3543

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous