15200

pending completion

Build # 15200

Build Type

push

travis-ci

Committed by

locker

Commit Message

<a href="https://github.com/tarantool/tarantool/commit/<a class=hub.com/tarantool/tarantool/commit/21f87c30d0353727e01149ba49846ae6ef669302">21f87c30d<a href="https://github.com/tarantool/tarantool/commit/21f87c30d0353727e01149ba49846ae6ef669302">&quot;&gt;wal: remove old xlog files asynchronously

In contrast to TX thread, WAL thread performs garbage collection
synchronously, blocking all concurrent writes. We expected file removal
to happen instantly so we didn&amp;#39;t bother to offload this job to eio
threads. However, it turned out that sometimes removal of a single xlog
file can take 50 or even 100 ms. If there are a dozen files to be
removed, this means a second delay and &amp;#39;too long WAL write&amp;#39; warnings.

To fix this issue, let&amp;#39;s make WAL garbage collection fully asynchronous.
Simply submit a jobs to eio and assume it will successfully complete
sooner or later.  This means that if unlink() fails for some reason, we
will log an error and never retry file removal until the server is
restarted. Not a big deal. We can live with it assuming unlink() doesn&amp;#39;t
normally fail.

Closes #3938

(cherry picked from commit </a><a class="double-link" href="https://github.com/tarantool/tarantool/commit/<a class="double-link" href="https://github.com/tarantool/tarantool/commit/8e429f4b7a335ff318ca86b1147b194d1ede9c02">8e429f4b7</a>">8e429f4b7</a><a href="https://github.com/tarantool/tarantool/commit/21f87c30d0353727e01149ba49846ae6ef669302">)

Run Details

15 of 15 new or added lines in 3 files covered. (100.0%)

41156 of 50005 relevant lines covered (82.3%)

848720.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.8

/src/box/vy_read_set.c

/*
 * Copyright 2010-2017, Tarantool AUTHORS, please see AUTHORS file.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * 1. Redistributions of source code must retain the above
 *    copyright notice, this list of conditions and the
 *    following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above
 *    copyright notice, this list of conditions and the following
 *    disclaimer in the documentation and/or other materials
 *    provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
 * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include "vy_read_set.h"

#include <assert.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

#include "trivia/util.h"
#include "tuple.h"
#include "vy_lsm.h"
#include "vy_stmt.h"

int
vy_read_interval_cmpl(const struct vy_read_interval *a,
                      const struct vy_read_interval *b)
{
        assert(a->lsm == b->lsm);
        struct key_def *cmp_def = a->lsm->cmp_def;
        int cmp = vy_stmt_compare(a->left, b->left, cmp_def);
        if (cmp != 0)
                return cmp;
        if (a->left_belongs && !b->left_belongs)
                return -1;
        if (!a->left_belongs && b->left_belongs)
                return 1;
        uint32_t a_parts = tuple_field_count(a->left);
        uint32_t b_parts = tuple_field_count(b->left);
        a_parts = MIN(a_parts, cmp_def->part_count);
        b_parts = MIN(b_parts, cmp_def->part_count);
        if (a->left_belongs)
                return a_parts < b_parts ? -1 : a_parts > b_parts;
        else
                return a_parts > b_parts ? -1 : a_parts < b_parts;
}

int
vy_read_interval_cmpr(const struct vy_read_interval *a,
                      const struct vy_read_interval *b)
{
        assert(a->lsm == b->lsm);
        struct key_def *cmp_def = a->lsm->cmp_def;
        int cmp = vy_stmt_compare(a->right, b->right, cmp_def);
        if (cmp != 0)
                return cmp;
        if (a->right_belongs && !b->right_belongs)
                return 1;
        if (!a->right_belongs && b->right_belongs)
                return -1;
        uint32_t a_parts = tuple_field_count(a->right);
        uint32_t b_parts = tuple_field_count(b->right);
        a_parts = MIN(a_parts, cmp_def->part_count);
        b_parts = MIN(b_parts, cmp_def->part_count);
        if (a->right_belongs)
                return a_parts > b_parts ? -1 : a_parts < b_parts;
        else
                return a_parts < b_parts ? -1 : a_parts > b_parts;
}

bool
vy_read_interval_should_merge(const struct vy_read_interval *l,
                              const struct vy_read_interval *r)
{
        assert(l->lsm == r->lsm);
        assert(vy_read_interval_cmpl(l, r) <= 0);
        struct key_def *cmp_def = l->lsm->cmp_def;
        int cmp = vy_stmt_compare(l->right, r->left, cmp_def);
        if (cmp > 0)
                return true;
        if (cmp < 0)
                return false;
        if (l->right_belongs && r->left_belongs)
                return true;
        if (!l->right_belongs && !r->left_belongs)
                return false;
        uint32_t l_parts = tuple_field_count(l->right);
        uint32_t r_parts = tuple_field_count(r->left);
        l_parts = MIN(l_parts, cmp_def->part_count);
        r_parts = MIN(r_parts, cmp_def->part_count);
        if (l->right_belongs)
                return l_parts <= r_parts;
        else
                return l_parts >= r_parts;
}

struct vy_tx *
vy_tx_conflict_iterator_next(struct vy_tx_conflict_iterator *it)
{
        struct vy_read_interval *curr, *left, *right;
        while ((curr = vy_lsm_read_set_walk_next(&it->tree_walk, it->tree_dir,
                                                 &left, &right)) != NULL) {
                struct key_def *cmp_def = curr->lsm->cmp_def;
                const struct vy_read_interval *last = curr->subtree_last;

                assert(left == NULL || left->lsm == curr->lsm);
                assert(right == NULL || right->lsm == curr->lsm);

                int cmp_right = vy_stmt_compare(it->stmt, last->right, cmp_def);
                if (cmp_right == 0 && !last->right_belongs)
                        cmp_right = 1;

                if (cmp_right > 0) {
                        /*
                         * The point is to the right of the rightmost
                         * interval in the subtree so there cannot be
                         * any conflicts in this subtree.
                         */
                        it->tree_dir = 0;
                        continue;
                }

                int cmp_left;
                if (curr->left == last->right) {
                        /* Optimize comparison out. */
                        cmp_left = cmp_right;
                } else {
                        cmp_left = vy_stmt_compare(it->stmt, curr->left, cmp_def);
                        if (cmp_left == 0 && !curr->left_belongs)
                                cmp_left = -1;
                }

                if (cmp_left < 0) {
                        /*
                         * The point is to the left of the current interval
                         * so an intersection can only be found in the left
                         * subtree.
                         */
                        it->tree_dir = RB_WALK_LEFT;
                } else {
                        /*
                         * Both subtrees can have intervals that contain the
                         * given point.
                         */
                        it->tree_dir = RB_WALK_LEFT | RB_WALK_RIGHT;
                }

                /*
                 * Check if the point is within the current interval.
                 */
                if (curr->left == curr->right) {
                        /* Optimize comparison out. */
                        cmp_right = cmp_left;
                } else if (curr != last) {
                        cmp_right = vy_stmt_compare(it->stmt, curr->right,
                                                    cmp_def);
                        if (cmp_right == 0 && !curr->right_belongs)
                                cmp_right = 1;
                }

                if (cmp_left >= 0 && cmp_right <= 0) {
                        /*
                         * The point is within the current interval.
                         * Return the conflicting transaction before
                         * continuing tree traversal.
                         */
                        break;
                }
        }
        return curr != NULL ? curr->tx : NULL;
}

1	/*
2	* Copyright 2010-2017, Tarantool AUTHORS, please see AUTHORS file.
3	*
4	* Redistribution and use in source and binary forms, with or
5	* without modification, are permitted provided that the following
6	* conditions are met:
7	*
8	* 1. Redistributions of source code must retain the above
9	* copyright notice, this list of conditions and the
10	* following disclaimer.
11	*
12	* 2. Redistributions in binary form must reproduce the above
13	* copyright notice, this list of conditions and the following
14	* disclaimer in the documentation and/or other materials
15	* provided with the distribution.
16	*
17	* THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND
18	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19	* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
21	* AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
22	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
25	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26	* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
28	* THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29	* SUCH DAMAGE.
30	*/
31	#include "vy_read_set.h"
32
33	#include <assert.h>
34	#include <stdbool.h>
35	#include <stddef.h>
36	#include <stdint.h>
37
38	#include "trivia/util.h"
39	#include "tuple.h"
40	#include "vy_lsm.h"
41	#include "vy_stmt.h"
42
43	int
44	vy_read_interval_cmpl(const struct vy_read_interval *a,	27,585,380✔
45	const struct vy_read_interval *b)
46	{
47	assert(a->lsm == b->lsm);	27,585,380✔
48	struct key_def *cmp_def = a->lsm->cmp_def;	27,585,380✔
49	int cmp = vy_stmt_compare(a->left, b->left, cmp_def);	27,585,380✔
50	if (cmp != 0)	27,585,380✔
51	return cmp;	11,887,822✔
52	if (a->left_belongs && !b->left_belongs)	15,697,558✔
53	return -1;	319✔
54	if (!a->left_belongs && b->left_belongs)	15,697,239✔
55	return 1;	1,049✔
56	uint32_t a_parts = tuple_field_count(a->left);	15,696,190✔
57	uint32_t b_parts = tuple_field_count(b->left);	15,696,190✔
58	a_parts = MIN(a_parts, cmp_def->part_count);	15,696,190✔
59	b_parts = MIN(b_parts, cmp_def->part_count);	15,696,190✔
60	if (a->left_belongs)	15,696,190✔
61	return a_parts < b_parts ? -1 : a_parts > b_parts;	15,688,991✔
62	else
63	return a_parts > b_parts ? -1 : a_parts < b_parts;	7,199✔
64	}
65
66	int
67	vy_read_interval_cmpr(const struct vy_read_interval *a,	10,319,974✔
68	const struct vy_read_interval *b)
69	{
70	assert(a->lsm == b->lsm);	10,319,974✔
71	struct key_def *cmp_def = a->lsm->cmp_def;	10,319,974✔
72	int cmp = vy_stmt_compare(a->right, b->right, cmp_def);	10,319,974✔
73	if (cmp != 0)	10,319,974✔
74	return cmp;	8,199,304✔
75	if (a->right_belongs && !b->right_belongs)	2,120,670✔
76	return 1;	3,505✔
77	if (!a->right_belongs && b->right_belongs)	2,117,165✔
78	return -1;	1,400✔
79	uint32_t a_parts = tuple_field_count(a->right);	2,115,765✔
80	uint32_t b_parts = tuple_field_count(b->right);	2,115,765✔
81	a_parts = MIN(a_parts, cmp_def->part_count);	2,115,765✔
82	b_parts = MIN(b_parts, cmp_def->part_count);	2,115,765✔
83	if (a->right_belongs)	2,115,765✔
84	return a_parts > b_parts ? -1 : a_parts < b_parts;	2,113,742✔
85	else
86	return a_parts < b_parts ? -1 : a_parts > b_parts;	2,023✔
87	}
88
89	bool
90	vy_read_interval_should_merge(const struct vy_read_interval *l,	4,258,650✔
91	const struct vy_read_interval *r)
92	{
93	assert(l->lsm == r->lsm);	4,258,650✔
94	assert(vy_read_interval_cmpl(l, r) <= 0);	4,258,650✔
95	struct key_def *cmp_def = l->lsm->cmp_def;	4,258,650✔
96	int cmp = vy_stmt_compare(l->right, r->left, cmp_def);	4,258,650✔
97	if (cmp > 0)	4,258,650✔
98	return true;	3,854,208✔
99	if (cmp < 0)	404,442✔
100	return false;	87,704✔
101	if (l->right_belongs && r->left_belongs)	316,738✔
102	return true;	316,645✔
103	if (!l->right_belongs && !r->left_belongs)	93✔
104	return false;	26✔
105	uint32_t l_parts = tuple_field_count(l->right);	67✔
106	uint32_t r_parts = tuple_field_count(r->left);	67✔
107	l_parts = MIN(l_parts, cmp_def->part_count);	67✔
108	r_parts = MIN(r_parts, cmp_def->part_count);	67✔
109	if (l->right_belongs)	67✔
110	return l_parts <= r_parts;	12✔
111	else
112	return l_parts >= r_parts;	55✔
113	}
114
115	struct vy_tx *
116	vy_tx_conflict_iterator_next(struct vy_tx_conflict_iterator *it)	939,462✔
117	{
118	struct vy_read_interval curr, left, *right;
119	while ((curr = vy_lsm_read_set_walk_next(&it->tree_walk, it->tree_dir,	1,964,163✔
120	&left, &right)) != NULL) {
121	struct key_def *cmp_def = curr->lsm->cmp_def;	338,233✔
122	const struct vy_read_interval *last = curr->subtree_last;	338,233✔
123
124	assert(left == NULL \|\| left->lsm == curr->lsm);	338,233✔
125	assert(right == NULL \|\| right->lsm == curr->lsm);	338,233✔
126
127	int cmp_right = vy_stmt_compare(it->stmt, last->right, cmp_def);	338,233✔
128	if (cmp_right == 0 && !last->right_belongs)	338,233✔
129	cmp_right = 1;	6✔
130
131	if (cmp_right > 0) {	338,233✔
132	/*
133	* The point is to the right of the rightmost
134	* interval in the subtree so there cannot be
135	* any conflicts in this subtree.
136	*/
137	it->tree_dir = 0;	31,660✔
138	continue;	31,660✔
139	}
140
141	int cmp_left;
142	if (curr->left == last->right) {	306,573✔
143	/* Optimize comparison out. */
144	cmp_left = cmp_right;	266,488✔
145	} else {
146	cmp_left = vy_stmt_compare(it->stmt, curr->left, cmp_def);	40,085✔
147	if (cmp_left == 0 && !curr->left_belongs)	40,085✔
148	cmp_left = -1;	6✔
149	}
150
151	if (cmp_left < 0) {	306,573✔
152	/*
153	* The point is to the left of the current interval
154	* so an intersection can only be found in the left
155	* subtree.
156	*/
157	it->tree_dir = RB_WALK_LEFT;	39,908✔
158	} else {
159	/*
160	* Both subtrees can have intervals that contain the
161	* given point.
162	*/
163	it->tree_dir = RB_WALK_LEFT \| RB_WALK_RIGHT;	266,665✔
164	}
165
166	/*
167	* Check if the point is within the current interval.
168	*/
169	if (curr->left == curr->right) {	306,573✔
170	/* Optimize comparison out. */
171	cmp_right = cmp_left;	294,670✔
172	} else if (curr != last) {	11,903✔
173	cmp_right = vy_stmt_compare(it->stmt, curr->right,	8,016✔
174	cmp_def);
175	if (cmp_right == 0 && !curr->right_belongs)	8,016✔
176	cmp_right = 1;	×
177	}
178
179	if (cmp_left >= 0 && cmp_right <= 0) {	306,573✔
180	/*
181	* The point is within the current interval.
182	* Return the conflicting transaction before
183	* continuing tree traversal.
184	*/
185	break;	252,994✔
186	}
187	}
188	return curr != NULL ? curr->tx : NULL;	939,462✔
189	}

tarantool / tarantool / 15200

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous