19545995168

Committed 20 Nov 2025 05:33PM UTC coverage: 88.525% (+0.06%) from 88.469%

Build # 19545995168

Build Type

Pull #464

github

Committed by

ita-sammann

Commit Message

TNTP-2109: Add rebalance related metrics

Pull Request Pull Request #464: TNTP-2109: Add rebalance related metrics

Run Details

17 of 22 new or added lines in 1 file covered. (77.27%)

8 existing lines in 1 file now uncovered.

5076 of 5734 relevant lines covered (88.52%)

6266.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.92

/crud/common/rebalance.lua

local fiber = require('fiber')
local log = require('log')
local metrics = require('metrics')
local vshard_consts = require('vshard.consts')
local utils = require('crud.common.utils')

local SETTINGS_SPACE_NAME = '_crud_settings'
local SAFE_MOD_ENABLE_EVENT = '_crud.safe_mode_enable'

local M = {
    safe_mode = false,
    safe_mode_enable_hooks = {},
    safe_mode_disable_hooks = {},
    _router_cache_last_clear_ts = fiber.time()
}

local function create_space()
    local settings_space = box.schema.space.create(SETTINGS_SPACE_NAME, {
        engine = 'memtx',
        format = {
            { name = 'key', type = 'string' },
            { name = 'value', type = 'any' },
        },
        if_not_exists = true,
    })
    settings_space:create_index('primary', { parts = { 'key' }, if_not_exists = true })
end

local function safe_mode_trigger(_, new, space, op)
    if space ~= '_bucket' then
        return
    end
    if (op == 'INSERT' and new.status == vshard_consts.BUCKET.RECEIVING) or
            (op == 'REPLACE' and new.status == vshard_consts.BUCKET.SENDING) then
        box.broadcast(SAFE_MOD_ENABLE_EVENT, true)
    end
end

local function register_enable_hook(func)
    M.safe_mode_enable_hooks[func] = true
end

local function remove_enable_hook(func)
    M.safe_mode_enable_hooks[func] = nil
end

local function register_disable_hook(func)
    M.safe_mode_disable_hooks[func] = true
end

local function remove_disable_hook(func)
    M.safe_mode_disable_hooks[func] = nil
end

local function safe_mode_status()
    return M.safe_mode
end

local function safe_mode_enable()
    if not box.info.ro then
        box.space[SETTINGS_SPACE_NAME]:replace{ 'safe_mode', true }
        for _, trig in pairs(box.space._bucket:on_replace()) do
            if trig == safe_mode_trigger then
                box.space._bucket:on_replace(nil, safe_mode_trigger)
            end
        end
    end
    M.safe_mode = true

    for hook, _ in pairs(M.safe_mode_enable_hooks) do
        hook()
    end
end

local function safe_mode_disable()
    if not box.info.ro then
        box.space[SETTINGS_SPACE_NAME]:replace{ 'safe_mode', false }
        box.space._bucket:on_replace(safe_mode_trigger)
    end
    M.safe_mode = false

    for hook, _ in pairs(M.safe_mode_disable_hooks) do
        hook()
    end
end

local function rebalance_init()
    M.metrics.enable_storage_metrics()

    -- box.watch was introduced in tarantool 2.10.0
    if not utils.tarantool_supports_box_watch() then
        log.warn('This version of tarantool does not support autoswitch to safe mode during rebalance. '
            .. 'Update to newer version or use `_crud.rebalance_safe_mode_enable()` to enable safe mode manually.')
        return
    end

    box.watch('box.status', function()
        if box.info.ro then
            return
        end

        local stored_safe_mode
        if box.space[SETTINGS_SPACE_NAME] == nil then
            create_space()
            box.space[SETTINGS_SPACE_NAME]:insert{ 'safe_mode', false }
        else
            stored_safe_mode = box.space[SETTINGS_SPACE_NAME]:get{ 'safe_mode' }
        end
        M.safe_mode = stored_safe_mode and stored_safe_mode.value or false

        if M.safe_mode then
            for hook, _ in pairs(M.safe_mode_enable_hooks) do
                hook()
            end
        else
            box.space._bucket:on_replace(safe_mode_trigger)
            for hook, _ in pairs(M.safe_mode_disable_hooks) do
                hook()
            end
        end
    end)

    box.watch(SAFE_MOD_ENABLE_EVENT, function(_, do_enable)
        if box.info.ro or not do_enable then
            return
        end
        safe_mode_enable()
    end)
end

local function rebalance_stop()
    M.safe_mode_disable()
end

local function router_cache_clear()
    M._router_cache_last_clear_ts = fiber.time()
    return utils.get_vshard_router_instance():_route_map_clear()
end

local function router_cache_length()
    return utils.get_vshard_router_instance().known_bucket_count
end

local function router_cache_last_clear_ts()
    return M._router_cache_last_clear_ts
end

-- Rebalance related metrics
local function enable_storage_metrics()
    local safe_mode_enabled_gauge = metrics.gauge(
            'tnt_crud_storage_safe_mode_enabled',
            "is safe mode enabled on this storage instance"
    )

    metrics.register_callback(function()
        safe_mode_enabled_gauge:set(safe_mode_status() and 1 or 0)
    end)
end

local function enable_router_metrics()
    local router_cache_length_gauge = metrics.gauge(
            'tnt_crud_router_cache_length',
            "number of bucket routes in vshard router cache"
    )
    local router_cache_last_clear_ts_gauge = metrics.gauge(
            'tnt_crud_router_cache_last_clear_ts',
            "when vshard router cache was cleared last time"
    )

    metrics.register_callback(function()
        router_cache_length_gauge:set(router_cache_length())
        router_cache_last_clear_ts_gauge:set(router_cache_last_clear_ts())
    end)
end

M.init = rebalance_init
M.stop = rebalance_stop
M.safe_mode_status = safe_mode_status
M.safe_mode_enable = safe_mode_enable
M.safe_mode_disable = safe_mode_disable
M.register_safe_mode_enable_hook = register_enable_hook
M.remove_safe_mode_enable_hook = remove_enable_hook
M.register_safe_mode_disable_hook = register_disable_hook
M.remove_safe_mode_disable_hook = remove_disable_hook

M.router = {
    cache_clear = router_cache_clear,
    cache_length = router_cache_length,
    cache_last_clear_ts = router_cache_last_clear_ts,
}

M.storage_api = {
    rebalance_safe_mode_status = safe_mode_status,
    rebalance_safe_mode_enable = safe_mode_enable,
    rebalance_safe_mode_disable = safe_mode_disable,
}

M.metrics = {
    enable_storage_metrics = enable_storage_metrics,
    enable_router_metrics = enable_router_metrics,
}

return M

1	local fiber = require('fiber')	258✔
2	local log = require('log')	258✔
3	local metrics = require('metrics')	258✔
4	local vshard_consts = require('vshard.consts')	258✔
5	local utils = require('crud.common.utils')	258✔
6
7	local SETTINGS_SPACE_NAME = '_crud_settings'	258✔
8	local SAFE_MOD_ENABLE_EVENT = '_crud.safe_mode_enable'	258✔
9
10	local M = {	258✔
11	safe_mode = false,
12	safe_mode_enable_hooks = {},	258✔
13	safe_mode_disable_hooks = {},	258✔
14	_router_cache_last_clear_ts = fiber.time()	516✔
15	}
16
17	local function create_space()
18	local settings_space = box.schema.space.create(SETTINGS_SPACE_NAME, {	208✔
19	engine = 'memtx',
20	format = {	104✔
21	{ name = 'key', type = 'string' },	104✔
22	{ name = 'value', type = 'any' },	104✔
23	},	104✔
24	if_not_exists = true,
25	})
26	settings_space:create_index('primary', { parts = { 'key' }, if_not_exists = true })	104✔
27	end
28
29	local function safe_mode_trigger(_, new, space, op)
30	if space ~= '_bucket' then	320,000✔
UNCOV 31	return	×
32	end
33	if (op == 'INSERT' and new.status == vshard_consts.BUCKET.RECEIVING) or	480,000✔
34	(op == 'REPLACE' and new.status == vshard_consts.BUCKET.SENDING) then	320,000✔
35	box.broadcast(SAFE_MOD_ENABLE_EVENT, true)	160,000✔
36	end
37	end
38
39	local function register_enable_hook(func)
40	M.safe_mode_enable_hooks[func] = true	258✔
41	end
42
43	local function remove_enable_hook(func)
UNCOV 44	M.safe_mode_enable_hooks[func] = nil	×
45	end
46
47	local function register_disable_hook(func)
48	M.safe_mode_disable_hooks[func] = true	258✔
49	end
50
51	local function remove_disable_hook(func)
UNCOV 52	M.safe_mode_disable_hooks[func] = nil	×
53	end
54
55	local function safe_mode_status()
UNCOV 56	return M.safe_mode	×
57	end
58
59	local function safe_mode_enable()
60	if not box.info.ro then	204✔
61	box.space[SETTINGS_SPACE_NAME]:replace{ 'safe_mode', true }	204✔
62	for _, trig in pairs(box.space._bucket:on_replace()) do	720✔
63	if trig == safe_mode_trigger then	312✔
64	box.space._bucket:on_replace(nil, safe_mode_trigger)	106✔
65	end
66	end
67	end
68	M.safe_mode = true	204✔
69
70	for hook, _ in pairs(M.safe_mode_enable_hooks) do	612✔
71	hook()	204✔
72	end
73	end
74
75	local function safe_mode_disable()
76	if not box.info.ro then	1✔
77	box.space[SETTINGS_SPACE_NAME]:replace{ 'safe_mode', false }	1✔
78	box.space._bucket:on_replace(safe_mode_trigger)	1✔
79	end
80	M.safe_mode = false	1✔
81
82	for hook, _ in pairs(M.safe_mode_disable_hooks) do	3✔
83	hook()	1✔
84	end
85	end
86
87	local function rebalance_init()
88	M.metrics.enable_storage_metrics()	200✔
89
90	-- box.watch was introduced in tarantool 2.10.0
91	if not utils.tarantool_supports_box_watch() then	400✔
UNCOV 92	log.warn('This version of tarantool does not support autoswitch to safe mode during rebalance. '	×
93	.. 'Update to newer version or use `_crud.rebalance_safe_mode_enable()` to enable safe mode manually.')	×
94	return	×
95	end
96
97	box.watch('box.status', function()	400✔
98	if box.info.ro then	219✔
99	return	101✔
100	end
101
102	local stored_safe_mode
103	if box.space[SETTINGS_SPACE_NAME] == nil then	118✔
104	create_space()	104✔
105	box.space[SETTINGS_SPACE_NAME]:insert{ 'safe_mode', false }	208✔
106	else
107	stored_safe_mode = box.space[SETTINGS_SPACE_NAME]:get{ 'safe_mode' }	28✔
108	end
109	M.safe_mode = stored_safe_mode and stored_safe_mode.value or false	132✔
110
111	if M.safe_mode then	118✔
112	for hook, _ in pairs(M.safe_mode_enable_hooks) do	30✔
113	hook()	10✔
114	end
115	else
116	box.space._bucket:on_replace(safe_mode_trigger)	108✔
117	for hook, _ in pairs(M.safe_mode_disable_hooks) do	324✔
118	hook()	108✔
119	end
120	end
121	end)
122
123	box.watch(SAFE_MOD_ENABLE_EVENT, function(_, do_enable)	400✔
124	if box.info.ro or not do_enable then	403✔
125	return	199✔
126	end
127	safe_mode_enable()	204✔
128	end)
129	end
130
131	local function rebalance_stop()
132	M.safe_mode_disable()	1✔
133	end
134
135	local function router_cache_clear()
UNCOV 136	M._router_cache_last_clear_ts = fiber.time()	×
NEW 137	return utils.get_vshard_router_instance():_route_map_clear()	×
138	end
139
140	local function router_cache_length()
NEW UNCOV 141	return utils.get_vshard_router_instance().known_bucket_count	×
142	end
143
144	local function router_cache_last_clear_ts()
UNCOV 145	return M._router_cache_last_clear_ts	×
146	end
147
148	-- Rebalance related metrics
149	local function enable_storage_metrics()
150	local safe_mode_enabled_gauge = metrics.gauge(	400✔
151	'tnt_crud_storage_safe_mode_enabled',	200✔
152	"is safe mode enabled on this storage instance"
153	)	200✔
154
155	metrics.register_callback(function()	400✔
NEW 156	safe_mode_enabled_gauge:set(safe_mode_status() and 1 or 0)	×
157	end)
158	end
159
160	local function enable_router_metrics()
161	local router_cache_length_gauge = metrics.gauge(	102✔
162	'tnt_crud_router_cache_length',	51✔
163	"number of bucket routes in vshard router cache"
164	)	51✔
165	local router_cache_last_clear_ts_gauge = metrics.gauge(	102✔
166	'tnt_crud_router_cache_last_clear_ts',	51✔
167	"when vshard router cache was cleared last time"
168	)	51✔
169
170	metrics.register_callback(function()	102✔
NEW 171	router_cache_length_gauge:set(router_cache_length())	×
NEW 172	router_cache_last_clear_ts_gauge:set(router_cache_last_clear_ts())	×
173	end)
174	end
175
176	M.init = rebalance_init	258✔
177	M.stop = rebalance_stop	258✔
178	M.safe_mode_status = safe_mode_status	258✔
179	M.safe_mode_enable = safe_mode_enable	258✔
180	M.safe_mode_disable = safe_mode_disable	258✔
181	M.register_safe_mode_enable_hook = register_enable_hook	258✔
182	M.remove_safe_mode_enable_hook = remove_enable_hook	258✔
183	M.register_safe_mode_disable_hook = register_disable_hook	258✔
184	M.remove_safe_mode_disable_hook = remove_disable_hook	258✔
185
186	M.router = {	258✔
187	cache_clear = router_cache_clear,	258✔
188	cache_length = router_cache_length,	258✔
189	cache_last_clear_ts = router_cache_last_clear_ts,	258✔
190	}	258✔
191
192	M.storage_api = {	258✔
193	rebalance_safe_mode_status = safe_mode_status,	258✔
194	rebalance_safe_mode_enable = safe_mode_enable,	258✔
195	rebalance_safe_mode_disable = safe_mode_disable,	258✔
196	}	258✔
197
198	M.metrics = {	258✔
199	enable_storage_metrics = enable_storage_metrics,	258✔
200	enable_router_metrics = enable_router_metrics,	258✔
201	}	258✔
202
203	return M	258✔

tarantool / crud / 19545995168

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous