4488104784

Build Type

push

github

Committed by Georgy Moiseev

Commit Message

select: yield on storage tuple lookup

Run Details

17 of 17 new or added lines in 3 files covered. (100.0%)

4348 of 4669 relevant lines covered (93.12%)

47870.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.0

/crud/select/executor.lua

local errors = require('errors')
local fiber = require('fiber')
local fun = require('fun')

local dev_checks = require('crud.common.dev_checks')
local select_comparators = require('crud.compare.comparators')
local compat = require('crud.common.compat')
local has_keydef = compat.exists('tuple.keydef', 'key_def')

local keydef_lib
if has_keydef then
    keydef_lib = compat.require('tuple.keydef', 'key_def')
end

local utils = require('crud.common.utils')

local ExecuteSelectError = errors.new_class('ExecuteSelectError')

local executor = {}

local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, after_tuple, yield_every)
    local primary_index = space.index[0]

    local scroll_key_parts = utils.merge_primary_key_parts(scan_index.parts, primary_index.parts)

    local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)
    local scroll_comparator = select_comparators.gen_tuples_comparator(cmp_operator, scroll_key_parts)

    local looked_up_tuples = 0
    while true do
        local tuple
        gen.state, tuple = gen(gen.param, gen.state)
        looked_up_tuples = looked_up_tuples + 1

        if yield_every ~= nil and looked_up_tuples % yield_every == 0 then
            fiber.yield()
        end

        if tuple == nil then
            return nil
        end

        if scroll_comparator(tuple, after_tuple) then
            return tuple
        end
    end
end

local generate_value

if has_keydef then
    generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter)
        local key_def = keydef_lib.new(index_parts)
        if #scan_value == 0 and after_tuple ~= nil then
            return key_def:extract_key(after_tuple)
        end
        local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)
        local cmp = key_def:compare_with_key(after_tuple, scan_value)
        if (cmp_operator == '<' and cmp < 0) or (cmp_operator == '>' and cmp > 0) then
            return key_def:extract_key(after_tuple)
        end
    end
else
    generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter)
        local after_tuple_key = utils.extract_key(after_tuple, index_parts)
        if #scan_value == 0 and after_tuple ~= nil then
            return after_tuple_key
        end
        local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)
        local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts)
        if scan_comparator(after_tuple_key, scan_value) then
            return after_tuple_key
        end
    end
end

function executor.execute(space, index, filter_func, opts)
    dev_checks('table', 'table', 'function', {
        scan_value = 'table',
        after_tuple = '?table',
        tarantool_iter = 'number',
        limit = '?number',
        yield_every = '?number',
    })

    opts = opts or {}

    local resp = { tuples_fetched = 0, tuples_lookup = 0, tuples = {} }

    if opts.limit == 0 then
        return resp
    end

    local value = opts.scan_value
    if opts.after_tuple ~= nil then
        local iter = opts.tarantool_iter
        if iter == box.index.EQ or iter == box.index.REQ then
            -- we need to make sure that the keys are equal
            -- the code is correct even if value is a partial key
            local parts = {}
            for i, _ in ipairs(value) do
                -- the code required for tarantool 1.10.6 at least
                table.insert(parts, index.parts[i])
            end

            local is_eq = iter == box.index.EQ
            local is_after_bigger
            if has_keydef then
                local key_def = keydef_lib.new(parts)
                local cmp = key_def:compare_with_key(opts.after_tuple, value)
                is_after_bigger = (is_eq and cmp > 0) or (not is_eq and cmp < 0)
            else
                local comparator
                if is_eq then
                    comparator = select_comparators.gen_func('<=', parts)
                else
                    comparator = select_comparators.gen_func('>=', parts)
                end
                local after_key = utils.extract_key(opts.after_tuple, parts)
                is_after_bigger = not comparator(after_key, value)
            end
            if is_after_bigger then
                -- it makes no sence to continue
                return resp
            end
        else
            local new_value = generate_value(opts.after_tuple, value, index.parts, iter)
            if new_value ~= nil then
                value = new_value
            end
        end
    end

    local tuple
    local raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter})
    local gen = fun.wrap(function(param, state)
        local next_state, var = raw_gen(param, state)

        if var ~= nil then
            resp.tuples_lookup = resp.tuples_lookup + 1
        end

        return next_state, var
    end, param, state)

    if opts.after_tuple ~= nil then
        local err
        tuple, err = scroll_to_after_tuple(gen, space, index, opts.tarantool_iter, opts.after_tuple, opts.yield_every)
        if err ~= nil then
            return nil, ExecuteSelectError:new("Failed to scroll to the after_tuple: %s", err)
        end

        if tuple == nil then
            return resp
        end
    end

    if tuple == nil then
        gen.state, tuple = gen(gen.param, gen.state)
    end

    local looked_up_tuples = 0
    while true do
        if tuple == nil then
            break
        end

        local matched, early_exit = filter_func(tuple)

        if matched then
            table.insert(resp.tuples, tuple)
            resp.tuples_fetched = resp.tuples_fetched + 1

            if opts.limit ~= nil and resp.tuples_fetched >= opts.limit then
                break
            end
        elseif early_exit then
            break
        end

        gen.state, tuple = gen(gen.param, gen.state)
        looked_up_tuples = looked_up_tuples + 1

        if opts.yield_every ~= nil and looked_up_tuples % opts.yield_every == 0 then
            fiber.yield()
        end
    end

    return resp
end

return executor

1	local errors = require('errors')	808✔
2	local fiber = require('fiber')	808✔
3	local fun = require('fun')	808✔
4
5	local dev_checks = require('crud.common.dev_checks')	808✔
6	local select_comparators = require('crud.compare.comparators')	808✔
7	local compat = require('crud.common.compat')	808✔
8	local has_keydef = compat.exists('tuple.keydef', 'key_def')	808✔
9
10	local keydef_lib
11	if has_keydef then	808✔
12	keydef_lib = compat.require('tuple.keydef', 'key_def')	1,616✔
13	end
14
15	local utils = require('crud.common.utils')	808✔
16
17	local ExecuteSelectError = errors.new_class('ExecuteSelectError')	808✔
18
19	local executor = {}	808✔
20
21	local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, after_tuple, yield_every)
22	local primary_index = space.index[0]	38,451✔
23
24	local scroll_key_parts = utils.merge_primary_key_parts(scan_index.parts, primary_index.parts)	38,451✔
25
26	local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)	38,451✔
27	local scroll_comparator = select_comparators.gen_tuples_comparator(cmp_operator, scroll_key_parts)	38,451✔
28
29	local looked_up_tuples = 0	38,451✔
30	while true do
31	local tuple
32	gen.state, tuple = gen(gen.param, gen.state)	488,220✔
33	looked_up_tuples = looked_up_tuples + 1	244,110✔
34
35	if yield_every ~= nil and looked_up_tuples % yield_every == 0 then	244,110✔
36	fiber.yield()	×
37	end
38
39	if tuple == nil then	244,110✔
40	return nil	432✔
41	end
42
43	if scroll_comparator(tuple, after_tuple) then	487,356✔
44	return tuple	38,019✔
45	end
46	end
47	end
48
49	local generate_value
50
51	if has_keydef then	808✔
52	generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter)
53	local key_def = keydef_lib.new(index_parts)	2,611✔
54	if #scan_value == 0 and after_tuple ~= nil then	2,611✔
55	return key_def:extract_key(after_tuple)	336✔
56	end
57	local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)	2,443✔
58	local cmp = key_def:compare_with_key(after_tuple, scan_value)	4,886✔
59	if (cmp_operator == '<' and cmp < 0) or (cmp_operator == '>' and cmp > 0) then	2,443✔
60	return key_def:extract_key(after_tuple)	4,434✔
61	end
62	end
63	else
64	generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter)
65	local after_tuple_key = utils.extract_key(after_tuple, index_parts)	×
66	if #scan_value == 0 and after_tuple ~= nil then	×
67	return after_tuple_key	×
68	end
69	local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)	×
70	local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts)	×
71	if scan_comparator(after_tuple_key, scan_value) then	×
72	return after_tuple_key	×
73	end
74	end
75	end
76
77	function executor.execute(space, index, filter_func, opts)	808✔
78	dev_checks('table', 'table', 'function', {	265,873✔
79	scan_value = 'table',
80	after_tuple = '?table',
81	tarantool_iter = 'number',
82	limit = '?number',
83	yield_every = '?number',
84	})
85
86	opts = opts or {}	265,873✔
87
88	local resp = { tuples_fetched = 0, tuples_lookup = 0, tuples = {} }	265,873✔
89
90	if opts.limit == 0 then	265,873✔
91	return resp	10✔
92	end
93
94	local value = opts.scan_value	265,863✔
95	if opts.after_tuple ~= nil then	265,863✔
96	local iter = opts.tarantool_iter	38,467✔
97	if iter == box.index.EQ or iter == box.index.REQ then	38,467✔
98	-- we need to make sure that the keys are equal
99	-- the code is correct even if value is a partial key
100	local parts = {}	35,856✔
101	for i, _ in ipairs(value) do	71,732✔
102	-- the code required for tarantool 1.10.6 at least
103	table.insert(parts, index.parts[i])	35,876✔
104	end
105
106	local is_eq = iter == box.index.EQ	35,856✔
107	local is_after_bigger
108	if has_keydef then	35,856✔
109	local key_def = keydef_lib.new(parts)	35,856✔
110	local cmp = key_def:compare_with_key(opts.after_tuple, value)	71,712✔
111	is_after_bigger = (is_eq and cmp > 0) or (not is_eq and cmp < 0)	35,856✔
112	else
113	local comparator
114	if is_eq then	×
115	comparator = select_comparators.gen_func('<=', parts)	×
116	else
117	comparator = select_comparators.gen_func('>=', parts)	×
118	end
119	local after_key = utils.extract_key(opts.after_tuple, parts)	×
120	is_after_bigger = not comparator(after_key, value)	×
121	end
122	if is_after_bigger then	35,856✔
123	-- it makes no sence to continue
124	return resp	16✔
125	end
126	else
127	local new_value = generate_value(opts.after_tuple, value, index.parts, iter)	2,611✔
128	if new_value ~= nil then	2,611✔
129	value = new_value	2,385✔
130	end
131	end
132	end
133
134	local tuple
135	local raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter})	265,847✔
136	local gen = fun.wrap(function(param, state)	531,566✔
137	local next_state, var = raw_gen(param, state)	899,146✔
138
139	if var ~= nil then	899,146✔
140	resp.tuples_lookup = resp.tuples_lookup + 1	699,215✔
141	end
142
143	return next_state, var	899,146✔
144	end, param, state)	265,783✔
145
146	if opts.after_tuple ~= nil then	265,783✔
147	local err
148	tuple, err = scroll_to_after_tuple(gen, space, index, opts.tarantool_iter, opts.after_tuple, opts.yield_every)	76,902✔
149	if err ~= nil then	38,451✔
150	return nil, ExecuteSelectError:new("Failed to scroll to the after_tuple: %s", err)	×
151	end
152
153	if tuple == nil then	38,451✔
154	return resp	432✔
155	end
156	end
157
158	if tuple == nil then	265,351✔
159	gen.state, tuple = gen(gen.param, gen.state)	454,664✔
160	end
161
162	local looked_up_tuples = 0	265,351✔
163	while true do
164	if tuple == nil then	693,055✔
165	break	199,499✔
166	end
167
168	local matched, early_exit = filter_func(tuple)	493,556✔
169
170	if matched then	493,556✔
171	table.insert(resp.tuples, tuple)	492,902✔
172	resp.tuples_fetched = resp.tuples_fetched + 1	492,902✔
173
174	if opts.limit ~= nil and resp.tuples_fetched >= opts.limit then	492,902✔
175	break	65,814✔
176	end
177	elseif early_exit then	654✔
178	break	38✔
179	end
180
181	gen.state, tuple = gen(gen.param, gen.state)	855,408✔
182	looked_up_tuples = looked_up_tuples + 1	427,704✔
183
184	if opts.yield_every ~= nil and looked_up_tuples % opts.yield_every == 0 then	427,704✔
185	fiber.yield()	8,000✔
186	end
187	end
188
189	return resp	265,351✔
190	end
191
192	return executor	808✔

tarantool / crud / 4488104784

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous