324825643

Build Type

push

github

Committed by Unrud

Commit Message

Use renameat2 on Linux for atomic exchanging of files

Run Details

1225 of 1587 branches covered (77.19%)

Branch coverage included in aggregate %.

24 of 39 new or added lines in 2 files covered. (61.54%)

30 existing lines in 1 file now uncovered.

2934 of 3507 relevant lines covered (83.66%)

12.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.89

/radicale/pathutils.py

# This file is part of Radicale Server - Calendar Server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Radicale.  If not, see <http://www.gnu.org/licenses/>.

"""
Helper functions for working with the file system.

"""

import contextlib
import os
import posixpath
import sys
import threading
from tempfile import TemporaryDirectory

if os.name == "nt":
    import ctypes
    import ctypes.wintypes
    import msvcrt

    LOCKFILE_EXCLUSIVE_LOCK = 2
    if ctypes.sizeof(ctypes.c_void_p) == 4:
        ULONG_PTR = ctypes.c_uint32
    else:
        ULONG_PTR = ctypes.c_uint64

    class Overlapped(ctypes.Structure):
        _fields_ = [
            ("internal", ULONG_PTR),
            ("internal_high", ULONG_PTR),
            ("offset", ctypes.wintypes.DWORD),
            ("offset_high", ctypes.wintypes.DWORD),
            ("h_event", ctypes.wintypes.HANDLE)]

    kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
    lock_file_ex = kernel32.LockFileEx
    lock_file_ex.argtypes = [
        ctypes.wintypes.HANDLE,
        ctypes.wintypes.DWORD,
        ctypes.wintypes.DWORD,
        ctypes.wintypes.DWORD,
        ctypes.wintypes.DWORD,
        ctypes.POINTER(Overlapped)]
    lock_file_ex.restype = ctypes.wintypes.BOOL
    unlock_file_ex = kernel32.UnlockFileEx
    unlock_file_ex.argtypes = [
        ctypes.wintypes.HANDLE,
        ctypes.wintypes.DWORD,
        ctypes.wintypes.DWORD,
        ctypes.wintypes.DWORD,
        ctypes.POINTER(Overlapped)]
    unlock_file_ex.restype = ctypes.wintypes.BOOL
elif os.name == "posix":
    import fcntl

HAVE_RENAMEAT2 = False
if sys.platform == "linux":
    import ctypes

    RENAME_EXCHANGE = 2
    try:
        renameat2 = ctypes.CDLL(None, use_errno=True).renameat2
    except AttributeError:
        pass
    else:
        HAVE_RENAMEAT2 = True
        renameat2.argtypes = [
            ctypes.c_int, ctypes.c_char_p,
            ctypes.c_int, ctypes.c_char_p,
            ctypes.c_uint]
        renameat2.restype = ctypes.c_int


class RwLock:
    """A readers-Writer lock that locks a file."""

    def __init__(self, path):
        self._path = path
        self._readers = 0
        self._writer = False
        self._lock = threading.Lock()

    @property
    def locked(self):
        with self._lock:
            if self._readers > 0:
                return "r"
            if self._writer:
                return "w"
            return ""

    @contextlib.contextmanager
    def acquire(self, mode):
        if mode not in "rw":
            raise ValueError("Invalid mode: %r" % mode)
        with open(self._path, "w+") as lock_file:
            if os.name == "nt":
                handle = msvcrt.get_osfhandle(lock_file.fileno())
                flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0
                overlapped = Overlapped()
                try:
                    if not lock_file_ex(handle, flags, 0, 1, 0, overlapped):
                        raise ctypes.WinError()
                except OSError as e:
                    raise RuntimeError("Locking the storage failed: %s" %
                                       e) from e
            elif os.name == "posix":
                _cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH
                try:
                    fcntl.flock(lock_file.fileno(), _cmd)
                except OSError as e:
                    raise RuntimeError("Locking the storage failed: %s" %
                                       e) from e
            else:
                raise RuntimeError("Locking the storage failed: "
                                   "Unsupported operating system")
            with self._lock:
                if self._writer or mode == "w" and self._readers != 0:
                    raise RuntimeError("Locking the storage failed: "
                                       "Guarantees failed")
                if mode == "r":
                    self._readers += 1
                else:
                    self._writer = True
            try:
                yield
            finally:
                with self._lock:
                    if mode == "r":
                        self._readers -= 1
                    self._writer = False


def rename_exchange(src, dst):
    """Exchange the files or directories `src` and `dst`.

    Both `src` and `dst` must exist but may be of different types.

    On Linux with renameat2 the operation is atomic.
    On other platforms it's not atomic.

    """
    src_dir, src_base = os.path.split(src)
    dst_dir, dst_base = os.path.split(dst)
    src_dir = src_dir or os.curdir
    dst_dir = dst_dir or os.curdir
    if not src_base or not dst_base:
        raise ValueError("Invalid arguments: %r -> %r" % (src, dst))
    if HAVE_RENAMEAT2:
        src_base_bytes = os.fsencode(src_base)
        dst_base_bytes = os.fsencode(dst_base)
        src_dir_fd = os.open(src_dir, 0)
        try:
            dst_dir_fd = os.open(dst_dir, 0)
            try:
                if renameat2(src_dir_fd, src_base_bytes,
                             dst_dir_fd, dst_base_bytes,
                             RENAME_EXCHANGE) != 0:
                    errno = ctypes.get_errno()
                    raise OSError(errno, os.strerror(errno))
            finally:
                os.close(dst_dir_fd)
        finally:
            os.close(src_dir_fd)
    else:
        with TemporaryDirectory(
                prefix=".Radicale.tmp-", dir=src_dir) as tmp_dir:
            os.rename(dst, os.path.join(tmp_dir, "interim"))
            os.rename(src, dst)
            os.rename(os.path.join(tmp_dir, "interim"), src)


def fsync(fd):
    if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
        fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
    else:
        os.fsync(fd)


def strip_path(path):
    assert sanitize_path(path) == path
    return path.strip("/")


def unstrip_path(stripped_path, trailing_slash=False):
    assert strip_path(sanitize_path(stripped_path)) == stripped_path
    assert stripped_path or trailing_slash
    path = "/%s" % stripped_path
    if trailing_slash and not path.endswith("/"):
        path += "/"
    return path


def sanitize_path(path):
    """Make path absolute with leading slash to prevent access to other data.

    Preserve potential trailing slash.

    """
    trailing_slash = "/" if path.endswith("/") else ""
    path = posixpath.normpath(path)
    new_path = "/"
    for part in path.split("/"):
        if not is_safe_path_component(part):
            continue
        new_path = posixpath.join(new_path, part)
    trailing_slash = "" if new_path.endswith("/") else trailing_slash
    return new_path + trailing_slash


def is_safe_path_component(path):
    """Check if path is a single component of a path.

    Check that the path is safe to join too.

    """
    return path and "/" not in path and path not in (".", "..")


def is_safe_filesystem_path_component(path):
    """Check if path is a single component of a local and posix filesystem
       path.

    Check that the path is safe to join too.

    """
    return (
        path and not os.path.splitdrive(path)[0] and
        not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
        not path.startswith(".") and not path.endswith("~") and
        is_safe_path_component(path))


def path_to_filesystem(root, sane_path):
    """Convert `sane_path` to a local filesystem path relative to `root`.

    `root` must be a secure filesystem path, it will be prepend to the path.

    `sane_path` must be a sanitized path without leading or trailing ``/``.

    Conversion of `sane_path` is done in a secure manner,
    or raises ``ValueError``.

    """
    assert sane_path == strip_path(sanitize_path(sane_path))
    safe_path = root
    parts = sane_path.split("/") if sane_path else []
    for part in parts:
        if not is_safe_filesystem_path_component(part):
            raise UnsafePathError(part)
        safe_path_parent = safe_path
        safe_path = os.path.join(safe_path, part)
        # Check for conflicting files (e.g. case-insensitive file systems
        # or short names on Windows file systems)
        if (os.path.lexists(safe_path) and
                part not in (e.name for e in
                             os.scandir(safe_path_parent))):
            raise CollidingPathError(part)
    return safe_path


class UnsafePathError(ValueError):
    def __init__(self, path):
        message = "Can't translate name safely to filesystem: %r" % path
        super().__init__(message)


class CollidingPathError(ValueError):
    def __init__(self, path):
        message = "File name collision: %r" % path
        super().__init__(message)


def name_from_path(path, collection):
    """Return Radicale item name from ``path``."""
    assert sanitize_path(path) == path
    start = unstrip_path(collection.path, True)
    if not (path + "/").startswith(start):
        raise ValueError("%r doesn't start with %r" % (path, start))
    name = path[len(start):]
    if name and not is_safe_path_component(name):
        raise ValueError("%r is not a component in collection %r" %
                         (name, collection.path))
    return name

1	# This file is part of Radicale Server - Calendar Server
2	# Copyright © 2014 Jean-Marc Martins
3	# Copyright © 2012-2017 Guillaume Ayoub
4	# Copyright © 2017-2018 Unrud <unrud@outlook.com>
5	#
6	# This library is free software: you can redistribute it and/or modify
7	# it under the terms of the GNU General Public License as published by
8	# the Free Software Foundation, either version 3 of the License, or
9	# (at your option) any later version.
10	#
11	# This library is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
18
19	"""	15✔
20	Helper functions for working with the file system.
21
22	"""
23
24	import contextlib	15✔
25	import os	15✔
26	import posixpath	15✔
27	import sys	15✔
28	import threading	15✔
29	from tempfile import TemporaryDirectory	15✔
30
31	if os.name == "nt":	15!
UNCOV 32	import ctypes	×
UNCOV 33	import ctypes.wintypes	×
34	import msvcrt	×
35
36	LOCKFILE_EXCLUSIVE_LOCK = 2	×
UNCOV 37	if ctypes.sizeof(ctypes.c_void_p) == 4:	×
38	ULONG_PTR = ctypes.c_uint32	×
39	else:
40	ULONG_PTR = ctypes.c_uint64	×
41
42	class Overlapped(ctypes.Structure):	×
UNCOV 43	_fields_ = [	×
44	("internal", ULONG_PTR),
45	("internal_high", ULONG_PTR),
46	("offset", ctypes.wintypes.DWORD),
47	("offset_high", ctypes.wintypes.DWORD),
48	("h_event", ctypes.wintypes.HANDLE)]
49
UNCOV 50	kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)	×
UNCOV 51	lock_file_ex = kernel32.LockFileEx	×
52	lock_file_ex.argtypes = [	×
53	ctypes.wintypes.HANDLE,
54	ctypes.wintypes.DWORD,
55	ctypes.wintypes.DWORD,
56	ctypes.wintypes.DWORD,
57	ctypes.wintypes.DWORD,
58	ctypes.POINTER(Overlapped)]
UNCOV 59	lock_file_ex.restype = ctypes.wintypes.BOOL	×
UNCOV 60	unlock_file_ex = kernel32.UnlockFileEx	×
61	unlock_file_ex.argtypes = [	×
62	ctypes.wintypes.HANDLE,
63	ctypes.wintypes.DWORD,
64	ctypes.wintypes.DWORD,
65	ctypes.wintypes.DWORD,
66	ctypes.POINTER(Overlapped)]
UNCOV 67	unlock_file_ex.restype = ctypes.wintypes.BOOL	×
68	elif os.name == "posix":	15!
69	import fcntl	15✔
70
71	HAVE_RENAMEAT2 = False	15✔
72	if sys.platform == "linux":	15!
73	import ctypes	15✔
74
75	RENAME_EXCHANGE = 2	15✔
76	try:	15✔
77	renameat2 = ctypes.CDLL(None, use_errno=True).renameat2	15✔
78	except AttributeError:	15✔
79	pass	15✔
80	else:
NEW 81	HAVE_RENAMEAT2 = True	×
NEW 82	renameat2.argtypes = [	×
83	ctypes.c_int, ctypes.c_char_p,
84	ctypes.c_int, ctypes.c_char_p,
85	ctypes.c_uint]
NEW 86	renameat2.restype = ctypes.c_int	×
87
88
89	class RwLock:	15✔
90	"""A readers-Writer lock that locks a file."""
91
92	def __init__(self, path):	15✔
93	self._path = path	15✔
94	self._readers = 0	15✔
95	self._writer = False	15✔
96	self._lock = threading.Lock()	15✔
97
98	@property	15✔
99	def locked(self):	6✔
100	with self._lock:	15✔
101	if self._readers > 0:	15✔
102	return "r"	15✔
103	if self._writer:	15✔
104	return "w"	15✔
105	return ""	15✔
106
107	@contextlib.contextmanager	15✔
108	def acquire(self, mode):	6✔
109	if mode not in "rw":	15!
UNCOV 110	raise ValueError("Invalid mode: %r" % mode)	×
111	with open(self._path, "w+") as lock_file:	15✔
112	if os.name == "nt":	15!
UNCOV 113	handle = msvcrt.get_osfhandle(lock_file.fileno())	×
UNCOV 114	flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0	×
UNCOV 115	overlapped = Overlapped()	×
UNCOV 116	try:	×
UNCOV 117	if not lock_file_ex(handle, flags, 0, 1, 0, overlapped):	×
UNCOV 118	raise ctypes.WinError()	×
UNCOV 119	except OSError as e:	×
UNCOV 120	raise RuntimeError("Locking the storage failed: %s" %	×
121	e) from e
122	elif os.name == "posix":	15!
123	_cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH	15✔
124	try:	15✔
125	fcntl.flock(lock_file.fileno(), _cmd)	15✔
UNCOV 126	except OSError as e:	×
UNCOV 127	raise RuntimeError("Locking the storage failed: %s" %	×
128	e) from e
129	else:
UNCOV 130	raise RuntimeError("Locking the storage failed: "	×
131	"Unsupported operating system")
132	with self._lock:	15✔
133	if self._writer or mode == "w" and self._readers != 0:	15!
134	raise RuntimeError("Locking the storage failed: "	×
135	"Guarantees failed")
136	if mode == "r":	15✔
137	self._readers += 1	15✔
138	else:
139	self._writer = True	15✔
140	try:	15✔
141	yield	15✔
142	finally:
143	with self._lock:	15✔
144	if mode == "r":	15✔
145	self._readers -= 1	15✔
146	self._writer = False	15✔
147
148
149	def rename_exchange(src, dst):	15✔
150	"""Exchange the files or directories `src` and `dst`.
151
152	Both `src` and `dst` must exist but may be of different types.
153
154	On Linux with renameat2 the operation is atomic.
155	On other platforms it's not atomic.
156
157	"""
158	src_dir, src_base = os.path.split(src)	15✔
159	dst_dir, dst_base = os.path.split(dst)	15✔
160	src_dir = src_dir or os.curdir	15✔
161	dst_dir = dst_dir or os.curdir	15✔
162	if not src_base or not dst_base:	15!
NEW 163	raise ValueError("Invalid arguments: %r -> %r" % (src, dst))	×
164	if HAVE_RENAMEAT2:	15!
NEW 165	src_base_bytes = os.fsencode(src_base)	×
NEW 166	dst_base_bytes = os.fsencode(dst_base)	×
NEW 167	src_dir_fd = os.open(src_dir, 0)	×
NEW 168	try:	×
NEW 169	dst_dir_fd = os.open(dst_dir, 0)	×
NEW 170	try:	×
NEW 171	if renameat2(src_dir_fd, src_base_bytes,	×
172	dst_dir_fd, dst_base_bytes,
173	RENAME_EXCHANGE) != 0:
NEW 174	errno = ctypes.get_errno()	×
NEW 175	raise OSError(errno, os.strerror(errno))	×
176	finally:
NEW 177	os.close(dst_dir_fd)	×
178	finally:
NEW 179	os.close(src_dir_fd)	×
180	else:
181	with TemporaryDirectory(	15✔
182	prefix=".Radicale.tmp-", dir=src_dir) as tmp_dir:
183	os.rename(dst, os.path.join(tmp_dir, "interim"))	15✔
184	os.rename(src, dst)	15✔
185	os.rename(os.path.join(tmp_dir, "interim"), src)	15✔
186
187
188	def fsync(fd):	15✔
189	if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):	15!
UNCOV 190	fcntl.fcntl(fd, fcntl.F_FULLFSYNC)	×
191	else:
192	os.fsync(fd)	15✔
193
194
195	def strip_path(path):	15✔
196	assert sanitize_path(path) == path	15✔
197	return path.strip("/")	15✔
198
199
200	def unstrip_path(stripped_path, trailing_slash=False):	15✔
201	assert strip_path(sanitize_path(stripped_path)) == stripped_path	15✔
202	assert stripped_path or trailing_slash	15✔
203	path = "/%s" % stripped_path	15✔
204	if trailing_slash and not path.endswith("/"):	15✔
205	path += "/"	15✔
206	return path	15✔
207
208
209	def sanitize_path(path):	15✔
210	"""Make path absolute with leading slash to prevent access to other data.
211
212	Preserve potential trailing slash.
213
214	"""
215	trailing_slash = "/" if path.endswith("/") else ""	15✔
216	path = posixpath.normpath(path)	15✔
217	new_path = "/"	15✔
218	for part in path.split("/"):	15✔
219	if not is_safe_path_component(part):	15✔
220	continue	15✔
221	new_path = posixpath.join(new_path, part)	15✔
222	trailing_slash = "" if new_path.endswith("/") else trailing_slash	15✔
223	return new_path + trailing_slash	15✔
224
225
226	def is_safe_path_component(path):	15✔
227	"""Check if path is a single component of a path.
228
229	Check that the path is safe to join too.
230
231	"""
232	return path and "/" not in path and path not in (".", "..")	15✔
233
234
235	def is_safe_filesystem_path_component(path):	15✔
236	"""Check if path is a single component of a local and posix filesystem
237	path.
238
239	Check that the path is safe to join too.
240
241	"""
242	return (	15✔
243	path and not os.path.splitdrive(path)[0] and
244	not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
245	not path.startswith(".") and not path.endswith("~") and
246	is_safe_path_component(path))
247
248
249	def path_to_filesystem(root, sane_path):	15✔
250	"""Convert `sane_path` to a local filesystem path relative to `root`.
251
252	`root` must be a secure filesystem path, it will be prepend to the path.
253
254	`sane_path` must be a sanitized path without leading or trailing ``/``.
255
256	Conversion of `sane_path` is done in a secure manner,
257	or raises ``ValueError``.
258
259	"""
260	assert sane_path == strip_path(sanitize_path(sane_path))	15✔
261	safe_path = root	15✔
262	parts = sane_path.split("/") if sane_path else []	15✔
263	for part in parts:	15✔
264	if not is_safe_filesystem_path_component(part):	15!
UNCOV 265	raise UnsafePathError(part)	×
266	safe_path_parent = safe_path	15✔
267	safe_path = os.path.join(safe_path, part)	15✔
268	# Check for conflicting files (e.g. case-insensitive file systems
269	# or short names on Windows file systems)
270	if (os.path.lexists(safe_path) and	15!
271	part not in (e.name for e in
272	os.scandir(safe_path_parent))):
UNCOV 273	raise CollidingPathError(part)	×
274	return safe_path	15✔
275
276
277	class UnsafePathError(ValueError):	15✔
278	def __init__(self, path):	15✔
UNCOV 279	message = "Can't translate name safely to filesystem: %r" % path	×
UNCOV 280	super().__init__(message)	×
281
282
283	class CollidingPathError(ValueError):	15✔
284	def __init__(self, path):	15✔
UNCOV 285	message = "File name collision: %r" % path	×
UNCOV 286	super().__init__(message)	×
287
288
289	def name_from_path(path, collection):	15✔
290	"""Return Radicale item name from ``path``."""
291	assert sanitize_path(path) == path	15✔
292	start = unstrip_path(collection.path, True)	15✔
293	if not (path + "/").startswith(start):	15!
UNCOV 294	raise ValueError("%r doesn't start with %r" % (path, start))	×
295	name = path[len(start):]	15✔
296	if name and not is_safe_path_component(name):	15!
UNCOV 297	raise ValueError("%r is not a component in collection %r" %	×
298	(name, collection.path))
299	return name	15✔

Kozea / Radicale / 324825643

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous