13457957190

Committed 21 Feb 2025 01:35PM CUT coverage: 78.516%. Remained the same

Build # 13457957190

Build Type

Pull #6312

github

Committed by

Vincent-lau

Commit Message

CA-407033: Call `receive_finalize2` synchronously

`Remote.receive_finalize2` is called at the end of SXM to clean things
up and compose the base and leaf images together. The compose operation
should only be called while the VDI is deactivated. Currently a thread
is created to call `receive_finalize2`, which could caused problems
where the VM itself gets started while the `receive_finalize2`/`VDI.compose`
is still in progress. This is not a safe operation to do.

The fix here is to simply remove the thread and make the whole operation
sequential.

Signed-off-by: Vincent Liu <shuntian.liu2@cloud.com>

Pull Request Pull Request #6312: CA-407033: Call `receive_finalize2` synchronously

Run Details

3512 of 4473 relevant lines covered (78.52%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.05

/python3/libexec/nbd_client_manager.py

#!/usr/bin/python3

"""
Provides functions and a CLI for safely connecting to and disconnecting from
NBD devices.
"""

import argparse
import fcntl
import json
import logging
import logging.handlers
import os
import re
import subprocess
import time
from datetime import datetime, timedelta

LOGGER = logging.getLogger("nbd_client_manager")
LOGGER.setLevel(logging.DEBUG)

LOCK_FILE = "/var/run/nonpersistent/nbd_client_manager"

# Don't wait more than 10 minutes for the NBD device
MAX_DEVICE_WAIT_MINUTES = 10

# According to https://github.com/thom311/libnl/blob/main/include/netlink/errno.h#L38
NLE_BUSY = 25

class InvalidNbdDevName(Exception):
    """
    The NBD device should be in this format: nbd{0-1000}
    If we cannot match this pattern, raise this exception
    """

class NbdConnStateTimeout(Exception):
    """
    If we cannot get the connection status of a nbd device,
    raise this exception.
    """

class NbdDeviceNotFound(Exception):
    """
    The NBD device file does not exist. Raised when there are no free NBD
    devices.
    """

    def __init__(self, nbd_device):
        super().__init__(
            "NBD device '{}' does not exist".format(nbd_device)
        )
        self.nbd_device = nbd_device

class FileLock: # pragma: no cover
    """Container for data relating to a file lock"""

    def __init__(self, path):
        self._path = path
        self._lock_file = None

    def _lock(self):
        """Acquire the lock"""
        flags = fcntl.LOCK_EX
        # pylint: disable=consider-using-with
        self._lock_file = open(self._path, "w+", encoding="utf8")
        fcntl.flock(self._lock_file, flags)

    def _unlock(self):
        """Unlock and remove the lock file"""
        if self._lock_file:
            fcntl.flock(self._lock_file, fcntl.LOCK_UN)
            self._lock_file.close()
            self._lock_file = None

    def __enter__(self):
        self._lock()

    def __exit__(self, *args):
        self._unlock()


FILE_LOCK = FileLock(path=LOCK_FILE)


def _call(cmd_args, raise_err=True, log_err=True):
    """
    [call cmd_args] executes [cmd_args] and returns the exit code.
    If [error] and exit code != 0, log and throws a CalledProcessError.
    """
    LOGGER.debug("Running cmd %s", cmd_args)
    # pylint: disable=consider-using-with
    proc = subprocess.Popen(
        cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True,
        universal_newlines=True
    )

    _, stderr = proc.communicate()

    if proc.returncode != 0:
        if log_err:
            LOGGER.error(
                "%s exited with code %d: %s", " ".join(cmd_args), proc.returncode, stderr
            )

        if raise_err:
            raise subprocess.CalledProcessError(
                returncode=proc.returncode, cmd=cmd_args, output=stderr
            )

    return proc.returncode


def _is_nbd_device_connected(nbd_device):
    """
    Checks whether the specified nbd device is connected according to
    nbd-client.
    """
    # First check if the file exists, because "nbd-client -c" returns
    # 1 for a non-existent file.
    if not os.path.exists(nbd_device):
        raise NbdDeviceNotFound(nbd_device)
    cmd = ["nbd-client", "-check", nbd_device]
    returncode = _call(cmd, raise_err=False, log_err=False)
    if returncode == 0:
        return True
    if returncode == 1:
        return False
    raise subprocess.CalledProcessError(returncode=returncode, cmd=cmd)


def _find_unused_nbd_device():
    """
    Returns the path of the first /dev/nbdX device that is not
    connected according to nbd-client.
    Raises NbdDeviceNotFound if no devices are available.
    """
    for device_no in range(0, 1000):
        nbd_device = "/dev/nbd{}".format(device_no)
        if not _is_nbd_device_connected(nbd_device=nbd_device):
            return nbd_device

    # If there are 1000 nbd devices (unlikely) and all are connected
    raise NbdDeviceNotFound(nbd_device)  # pyright:ignore[reportPossiblyUnboundVariable]

def _wait_for_nbd_device(nbd_device, connected):
    deadline = datetime.now() + timedelta(minutes=MAX_DEVICE_WAIT_MINUTES)

    while _is_nbd_device_connected(nbd_device=nbd_device) != connected:
        if datetime.now() > deadline:
            raise NbdConnStateTimeout(
                "Timed out waiting for connection state of device %s to be %s"
                % (nbd_device, connected)
            )

        LOGGER.debug(
            "Connection status of NBD device %s not yet %s, waiting",
            nbd_device,
            connected,
        )
        time.sleep(0.1)


PERSISTENT_INFO_DIR = "/var/run/nonpersistent/nbd"


def _get_persistent_connect_info_filename(device):
    """
    Return the full path for the persistent file containing
    the connection details. This is based on the device
    name, so /dev/nbd0 -> /var/run/nonpersistent/nbd/0
    """
    matched = re.search("/dev/nbd([0-9]+)", device)
    if not matched:
        raise InvalidNbdDevName("Can not get the nbd number")
    number = matched.group(1)
    return PERSISTENT_INFO_DIR + "/" + number


def _persist_connect_info(device, path, exportname):
    if not os.path.exists(PERSISTENT_INFO_DIR):
        os.makedirs(PERSISTENT_INFO_DIR)
    filename = _get_persistent_connect_info_filename(device)
    with open(filename, "w", encoding="utf-8") as info_file:
        info_file.write(json.dumps({"path": path, "exportname": exportname}))


def _remove_persistent_connect_info(device):
    try:
        os.remove(_get_persistent_connect_info_filename(device))
    except OSError:
        pass


def connect_nbd(path, exportname):
    """Connects to a free NBD device using nbd-client and returns its path"""
    # We should not ask for too many nbds, as we might not have enough memory
    _call(["modprobe", "nbd", "nbds_max=24"])
    # Wait for systemd-udevd to process the udev rules
    _call(["udevadm", "settle", "--timeout=30"])
    retries = 0
    while True:
        try:
            with FILE_LOCK:
                nbd_device = _find_unused_nbd_device()
                cmd = [
                    "nbd-client",
                    "-unix",
                    path,
                    nbd_device,
                    "-timeout",
                    "60",
                    "-name",
                    exportname,
                ]
                ret = _call(cmd, raise_err=False, log_err=True)
                if NLE_BUSY == ret:
                    # Although _find_unused_nbd_device tell us the nbd devcie is
                    # not connected by other nbd-client, it may be opened and locked
                    # by other process like systemd-udev, raise NbdDeviceNotFound to retry
                    LOGGER.warning("Device %s is busy, will retry", nbd_device)
                    raise NbdDeviceNotFound(nbd_device)

                if 0 != ret:
                    raise subprocess.CalledProcessError(returncode=ret, cmd=cmd)

                _wait_for_nbd_device(nbd_device=nbd_device, connected=True)
                _persist_connect_info(nbd_device, path, exportname)
                nbd = (
                    nbd_device[len("/dev/") :]
                    if nbd_device.startswith("/dev/")
                    else nbd_device
                )
                with open("/sys/block/" + nbd + "/queue/scheduler",
                          "w", encoding="utf-8") as fd:
                    fd.write("none")
                # Set the NBD queue size to the same as the qcow2 cluster size
                with open("/sys/block/" + nbd + "/queue/max_sectors_kb",
                          "w", encoding="utf-8") as fd:
                    fd.write("512")
                with open("/sys/block/" + nbd + "/queue/nr_requests",
                          "w", encoding="utf-8") as fd:
                    fd.write("8")

            return nbd_device
        except NbdDeviceNotFound as exn:
            LOGGER.warning("Failed to find free nbd device: %s", exn)
            retries = retries + 1
            if retries == 1:
                # We sleep for a shorter period first, in case an nbd device
                # will become available soon (e.g. during PV Linux guest bootstorm):
                time.sleep(10)
            elif retries < 30:
                time.sleep(60)
            else:
                raise exn


def disconnect_nbd_device(nbd_device):
    """
    Disconnects the given device using nbd-client.
    This function is idempotent: calling it on an already disconnected device
    does nothing.
    """
    try:
        if _is_nbd_device_connected(nbd_device=nbd_device):
            _remove_persistent_connect_info(nbd_device)
            cmd = ["nbd-client", "-disconnect", nbd_device]
            _call(cmd)
            _wait_for_nbd_device(nbd_device=nbd_device, connected=False)
    except NbdDeviceNotFound:
        # Device gone, no-op
        pass


def _connect_cli(args):
    device = connect_nbd(path=args.path, exportname=args.exportname)
    print(device)


def _disconnect_cli(args):
    disconnect_nbd_device(nbd_device=args.device)

# The main function is covered by manual test and XenRT test
# Exclude it from unit test coverage
def _main(): # pragma: no cover
    # Configure the root logger to log into syslog
    # (Specifically, into /var/log/user.log)
    syslog_handler = logging.handlers.SysLogHandler(
        address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_USER
    )
    # Ensure the program name is included in the log messages:
    formatter = logging.Formatter("%(name)s: [%(levelname)s] %(message)s")
    syslog_handler.setFormatter(formatter)
    logging.getLogger().addHandler(syslog_handler)

    try:
        parser = argparse.ArgumentParser(
            description="Connect to and disconnect from an NBD device"
        )

        subparsers = parser.add_subparsers(dest="command_name")

        parser_connect = subparsers.add_parser(
            "connect", help="Connect to a free NBD device and return its path"
        )
        parser_connect.add_argument(
            "--path",
            required=True,
            help="The path of the Unix domain socket of the NBD server",
        )
        parser_connect.add_argument(
            "--exportname",
            required=True,
            help="The export name of the device to connect to",
        )
        parser_connect.set_defaults(func=_connect_cli)

        parser_disconnect = subparsers.add_parser(
            "disconnect", help="Disconnect from the given NBD device"
        )
        parser_disconnect.add_argument(
            "--device", required=True, help="The path of the NBD device to disconnect"
        )
        parser_disconnect.set_defaults(func=_disconnect_cli)

        args = parser.parse_args()
        args.func(args)
    except Exception as exn:
        LOGGER.exception(exn)
        raise


if __name__ == "__main__":
    _main()

1	#!/usr/bin/python3
2
3	"""
4	Provides functions and a CLI for safely connecting to and disconnecting from
5	NBD devices.
6	"""
7
8	import argparse	1✔
9	import fcntl	1✔
10	import json	1✔
11	import logging	1✔
12	import logging.handlers	1✔
13	import os	1✔
14	import re	1✔
15	import subprocess	1✔
16	import time	1✔
17	from datetime import datetime, timedelta	1✔
18
19	LOGGER = logging.getLogger("nbd_client_manager")	1✔
20	LOGGER.setLevel(logging.DEBUG)	1✔
21
22	LOCK_FILE = "/var/run/nonpersistent/nbd_client_manager"	1✔
23
24	# Don't wait more than 10 minutes for the NBD device
25	MAX_DEVICE_WAIT_MINUTES = 10	1✔
26
27	# According to https://github.com/thom311/libnl/blob/main/include/netlink/errno.h#L38
28	NLE_BUSY = 25	1✔
29
30	class InvalidNbdDevName(Exception):	1✔
31	"""
32	The NBD device should be in this format: nbd{0-1000}
33	If we cannot match this pattern, raise this exception
34	"""
35
36	class NbdConnStateTimeout(Exception):	1✔
37	"""
38	If we cannot get the connection status of a nbd device,
39	raise this exception.
40	"""
41
42	class NbdDeviceNotFound(Exception):	1✔
43	"""
44	The NBD device file does not exist. Raised when there are no free NBD
45	devices.
46	"""
47
48	def __init__(self, nbd_device):	1✔
49	super().__init__(	1✔
50	"NBD device '{}' does not exist".format(nbd_device)
51	)
52	self.nbd_device = nbd_device	1✔
53
54	class FileLock: # pragma: no cover
55	"""Container for data relating to a file lock"""
56
57	def __init__(self, path):
58	self._path = path
59	self._lock_file = None
60
61	def _lock(self):
62	"""Acquire the lock"""
63	flags = fcntl.LOCK_EX
64	# pylint: disable=consider-using-with
65	self._lock_file = open(self._path, "w+", encoding="utf8")
66	fcntl.flock(self._lock_file, flags)
67
68	def _unlock(self):
69	"""Unlock and remove the lock file"""
70	if self._lock_file:
71	fcntl.flock(self._lock_file, fcntl.LOCK_UN)
72	self._lock_file.close()
73	self._lock_file = None
74
75	def __enter__(self):
76	self._lock()
77
78	def __exit__(self, *args):
79	self._unlock()
80
81
82	FILE_LOCK = FileLock(path=LOCK_FILE)	1✔
83
84
85	def _call(cmd_args, raise_err=True, log_err=True):	1✔
86	"""
87	[call cmd_args] executes [cmd_args] and returns the exit code.
88	If [error] and exit code != 0, log and throws a CalledProcessError.
89	"""
90	LOGGER.debug("Running cmd %s", cmd_args)	1✔
91	# pylint: disable=consider-using-with
92	proc = subprocess.Popen(	1✔
93	cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True,
94	universal_newlines=True
95	)
96
97	_, stderr = proc.communicate()	1✔
98
99	if proc.returncode != 0:	1✔
100	if log_err:	1✔
101	LOGGER.error(	1✔
102	"%s exited with code %d: %s", " ".join(cmd_args), proc.returncode, stderr
103	)
104
105	if raise_err:	1✔
106	raise subprocess.CalledProcessError(	1✔
107	returncode=proc.returncode, cmd=cmd_args, output=stderr
108	)
109
110	return proc.returncode	1✔
111
112
113	def _is_nbd_device_connected(nbd_device):	1✔
114	"""
115	Checks whether the specified nbd device is connected according to
116	nbd-client.
117	"""
118	# First check if the file exists, because "nbd-client -c" returns
119	# 1 for a non-existent file.
120	if not os.path.exists(nbd_device):	1✔
121	raise NbdDeviceNotFound(nbd_device)
122	cmd = ["nbd-client", "-check", nbd_device]	1✔
123	returncode = _call(cmd, raise_err=False, log_err=False)	1✔
124	if returncode == 0:	1✔
125	return True	1✔
126	if returncode == 1:	1✔
127	return False	1✔
128	raise subprocess.CalledProcessError(returncode=returncode, cmd=cmd)	×
129
130
131	def _find_unused_nbd_device():	1✔
132	"""
133	Returns the path of the first /dev/nbdX device that is not
134	connected according to nbd-client.
135	Raises NbdDeviceNotFound if no devices are available.
136	"""
137	for device_no in range(0, 1000):	1✔
138	nbd_device = "/dev/nbd{}".format(device_no)	1✔
139	if not _is_nbd_device_connected(nbd_device=nbd_device):	1✔
140	return nbd_device	1✔
141
142	# If there are 1000 nbd devices (unlikely) and all are connected
143	raise NbdDeviceNotFound(nbd_device) # pyright:ignore[reportPossiblyUnboundVariable]
144
145	def _wait_for_nbd_device(nbd_device, connected):	1✔
146	deadline = datetime.now() + timedelta(minutes=MAX_DEVICE_WAIT_MINUTES)	1✔
147
148	while _is_nbd_device_connected(nbd_device=nbd_device) != connected:	1✔
149	if datetime.now() > deadline:	×
150	raise NbdConnStateTimeout(	×
151	"Timed out waiting for connection state of device %s to be %s"
152	% (nbd_device, connected)
153	)
154
155	LOGGER.debug(	×
156	"Connection status of NBD device %s not yet %s, waiting",
157	nbd_device,
158	connected,
159	)
160	time.sleep(0.1)	×
161
162
163	PERSISTENT_INFO_DIR = "/var/run/nonpersistent/nbd"	1✔
164
165
166	def _get_persistent_connect_info_filename(device):	1✔
167	"""
168	Return the full path for the persistent file containing
169	the connection details. This is based on the device
170	name, so /dev/nbd0 -> /var/run/nonpersistent/nbd/0
171	"""
172	matched = re.search("/dev/nbd([0-9]+)", device)	1✔
173	if not matched:	1✔
174	raise InvalidNbdDevName("Can not get the nbd number")	×
175	number = matched.group(1)	1✔
176	return PERSISTENT_INFO_DIR + "/" + number	1✔
177
178
179	def _persist_connect_info(device, path, exportname):	1✔
180	if not os.path.exists(PERSISTENT_INFO_DIR):	1✔
181	os.makedirs(PERSISTENT_INFO_DIR)	1✔
182	filename = _get_persistent_connect_info_filename(device)	1✔
183	with open(filename, "w", encoding="utf-8") as info_file:	1✔
184	info_file.write(json.dumps({"path": path, "exportname": exportname}))	1✔
185
186
187	def _remove_persistent_connect_info(device):	1✔
188	try:	1✔
189	os.remove(_get_persistent_connect_info_filename(device))	1✔
190	except OSError:	×
191	pass
192
193
194	def connect_nbd(path, exportname):	1✔
195	"""Connects to a free NBD device using nbd-client and returns its path"""
196	# We should not ask for too many nbds, as we might not have enough memory
197	_call(["modprobe", "nbd", "nbds_max=24"])	1✔
198	# Wait for systemd-udevd to process the udev rules
199	_call(["udevadm", "settle", "--timeout=30"])	1✔
200	retries = 0	1✔
201	while True:	1✔
202	try:	1✔
203	with FILE_LOCK:	1✔
204	nbd_device = _find_unused_nbd_device()	1✔
205	cmd = [	1✔
206	"nbd-client",
207	"-unix",
208	path,
209	nbd_device,
210	"-timeout",
211	"60",
212	"-name",
213	exportname,
214	]
215	ret = _call(cmd, raise_err=False, log_err=True)	1✔
216	if NLE_BUSY == ret:	1✔
217	# Although _find_unused_nbd_device tell us the nbd devcie is
218	# not connected by other nbd-client, it may be opened and locked
219	# by other process like systemd-udev, raise NbdDeviceNotFound to retry
220	LOGGER.warning("Device %s is busy, will retry", nbd_device)	×
221	raise NbdDeviceNotFound(nbd_device)
222
223	if 0 != ret:	1✔
224	raise subprocess.CalledProcessError(returncode=ret, cmd=cmd)	×
225
226	_wait_for_nbd_device(nbd_device=nbd_device, connected=True)	1✔
227	_persist_connect_info(nbd_device, path, exportname)	1✔
228	nbd = (	1✔
229	nbd_device[len("/dev/") :]
230	if nbd_device.startswith("/dev/")
231	else nbd_device
232	)
233	with open("/sys/block/" + nbd + "/queue/scheduler",	1✔
234	"w", encoding="utf-8") as fd:
235	fd.write("none")	1✔
236	# Set the NBD queue size to the same as the qcow2 cluster size
237	with open("/sys/block/" + nbd + "/queue/max_sectors_kb",	1✔
238	"w", encoding="utf-8") as fd:
239	fd.write("512")	1✔
240	with open("/sys/block/" + nbd + "/queue/nr_requests",	1✔
241	"w", encoding="utf-8") as fd:
242	fd.write("8")	1✔
243
244	return nbd_device	1✔
245	except NbdDeviceNotFound as exn:	×
246	LOGGER.warning("Failed to find free nbd device: %s", exn)	×
247	retries = retries + 1	×
248	if retries == 1:	×
249	# We sleep for a shorter period first, in case an nbd device
250	# will become available soon (e.g. during PV Linux guest bootstorm):
251	time.sleep(10)	×
252	elif retries < 30:	×
253	time.sleep(60)	×
254	else:
255	raise exn	×
256
257
258	def disconnect_nbd_device(nbd_device):	1✔
259	"""
260	Disconnects the given device using nbd-client.
261	This function is idempotent: calling it on an already disconnected device
262	does nothing.
263	"""
264	try:	1✔
265	if _is_nbd_device_connected(nbd_device=nbd_device):	1✔
266	_remove_persistent_connect_info(nbd_device)	1✔
267	cmd = ["nbd-client", "-disconnect", nbd_device]	1✔
268	_call(cmd)	1✔
269	_wait_for_nbd_device(nbd_device=nbd_device, connected=False)	1✔
270	except NbdDeviceNotFound:	1✔
271	# Device gone, no-op
272	pass
273
274
275	def _connect_cli(args):	1✔
276	device = connect_nbd(path=args.path, exportname=args.exportname)	×
277	print(device)	×
278
279
280	def _disconnect_cli(args):	1✔
281	disconnect_nbd_device(nbd_device=args.device)	×
282
283	# The main function is covered by manual test and XenRT test
284	# Exclude it from unit test coverage
285	def _main(): # pragma: no cover
286	# Configure the root logger to log into syslog
287	# (Specifically, into /var/log/user.log)
288	syslog_handler = logging.handlers.SysLogHandler(
289	address="/dev/log", facility=logging.handlers.SysLogHandler.LOG_USER
290	)
291	# Ensure the program name is included in the log messages:
292	formatter = logging.Formatter("%(name)s: [%(levelname)s] %(message)s")
293	syslog_handler.setFormatter(formatter)
294	logging.getLogger().addHandler(syslog_handler)
295
296	try:
297	parser = argparse.ArgumentParser(
298	description="Connect to and disconnect from an NBD device"
299	)
300
301	subparsers = parser.add_subparsers(dest="command_name")
302
303	parser_connect = subparsers.add_parser(
304	"connect", help="Connect to a free NBD device and return its path"
305	)
306	parser_connect.add_argument(
307	"--path",
308	required=True,
309	help="The path of the Unix domain socket of the NBD server",
310	)
311	parser_connect.add_argument(
312	"--exportname",
313	required=True,
314	help="The export name of the device to connect to",
315	)
316	parser_connect.set_defaults(func=_connect_cli)
317
318	parser_disconnect = subparsers.add_parser(
319	"disconnect", help="Disconnect from the given NBD device"
320	)
321	parser_disconnect.add_argument(
322	"--device", required=True, help="The path of the NBD device to disconnect"
323	)
324	parser_disconnect.set_defaults(func=_disconnect_cli)
325
326	args = parser.parse_args()
327	args.func(args)
328	except Exception as exn:
329	LOGGER.exception(exn)
330	raise
331
332
333	if __name__ == "__main__":	1✔
334	_main()	×

xapi-project / xen-api / 13457957190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous