2067407235

Build # 2067407235

Build Type

push

github

Committed by Unrud

Commit Message

Fix upload of calendars with colliding UIDs

Run Details

1699 of 2065 branches covered (82.28%)

Branch coverage included in aggregate %.

3 of 4 new or added lines in 1 file covered. (75.0%)

3 existing lines in 1 file now uncovered.

3425 of 3909 relevant lines covered (87.62%)

16.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.47

/radicale/storage/multifilesystem/upload.py

# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Radicale.  If not, see <http://www.gnu.org/licenses/>.

import errno
import os
import pickle
import sys
from typing import Iterable, Set, TextIO, cast

import radicale.item as radicale_item
from radicale import pathutils
from radicale.storage.multifilesystem.base import CollectionBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.get import CollectionPartGet
from radicale.storage.multifilesystem.history import CollectionPartHistory


class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
                           CollectionPartHistory, CollectionBase):

    def upload(self, href: str, item: radicale_item.Item
               ) -> radicale_item.Item:
        if not pathutils.is_safe_filesystem_path_component(href):
            raise pathutils.UnsafePathError(href)
        try:
            self._store_item_cache(href, item)
        except Exception as e:
            raise ValueError("Failed to store item %r in collection %r: %s" %
                             (href, self.path, e)) from e
        path = pathutils.path_to_filesystem(self._filesystem_path, href)
        with self._atomic_write(path, newline="") as fo:
            f = cast(TextIO, fo)
            f.write(item.serialize())
        # Clean the cache after the actual item is stored, or the cache entry
        # will be removed again.
        self._clean_item_cache()
        # Track the change
        self._update_history_etag(href, item)
        self._clean_history()
        uploaded_item = self._get(href, verify_href=False)
        if uploaded_item is None:
            raise RuntimeError("Storage modified externally")
        return uploaded_item

    def _upload_all_nonatomic(self, items: Iterable[radicale_item.Item],
                              suffix: str = "") -> None:
        """Upload a new set of items.

        This takes a list of vobject items and
        uploads them nonatomic and without existence checks.

        """
        cache_folder = os.path.join(self._filesystem_path,
                                    ".Radicale.cache", "item")
        self._storage._makedirs_synced(cache_folder)
        hrefs: Set[str] = set()
        for item in items:
            uid = item.uid
            try:
                cache_content = self._item_cache_content(item)
            except Exception as e:
                raise ValueError(
                    "Failed to store item %r in temporary collection %r: %s" %
                    (uid, self.path, e)) from e
            href_candidate_funtions = [
                lambda: uid if uid.lower().endswith(suffix.lower())
                else uid + suffix,
                lambda: radicale_item.get_etag(uid).strip('"') + suffix,
                lambda: radicale_item.find_available_uid(
                    hrefs.__contains__, suffix)]
            href = f = None
            while href_candidate_funtions:
                href = href_candidate_funtions.pop(0)()
                if href in hrefs:
                    continue
                if not pathutils.is_safe_filesystem_path_component(href):
                    if not href_candidate_funtions:
                        raise pathutils.UnsafePathError(href)
                    continue
                try:
                    f = open(pathutils.path_to_filesystem(
                        self._filesystem_path, href),
                        "w", newline="", encoding=self._encoding)
                    break
                except pathutils.CollidingPathError:
                    if href_candidate_funtions:
                        continue
                    raise
                except OSError as e:
                    if href_candidate_funtions and (
                            sys.platform != "win32" and
                            e.errno == errno.EINVAL or
                            sys.platform == "win32" and e.errno == 123):
                        continue
                    raise
            assert href is not None and f is not None
            with f:
                f.write(item.serialize())
                f.flush()
                self._storage._fsync(f)
            hrefs.add(href)
            with open(os.path.join(cache_folder, href), "wb") as fb:
                pickle.dump(cache_content, fb)
                fb.flush()
                self._storage._fsync(fb)
        self._storage._sync_directory(cache_folder)
        self._storage._sync_directory(self._filesystem_path)

1	# This file is part of Radicale - CalDAV and CardDAV server
2	# Copyright © 2014 Jean-Marc Martins
3	# Copyright © 2012-2017 Guillaume Ayoub
4	# Copyright © 2017-2018 Unrud <unrud@outlook.com>
5	#
6	# This library is free software: you can redistribute it and/or modify
7	# it under the terms of the GNU General Public License as published by
8	# the Free Software Foundation, either version 3 of the License, or
9	# (at your option) any later version.
10	#
11	# This library is distributed in the hope that it will be useful,
12	# but WITHOUT ANY WARRANTY; without even the implied warranty of
13	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	# GNU General Public License for more details.
15	#
16	# You should have received a copy of the GNU General Public License
17	# along with Radicale. If not, see <http://www.gnu.org/licenses/>.
18
19	import errno	19✔
20	import os	19✔
21	import pickle	19✔
22	import sys	19✔
23	from typing import Iterable, Set, TextIO, cast	19✔
24
25	import radicale.item as radicale_item	19✔
26	from radicale import pathutils	19✔
27	from radicale.storage.multifilesystem.base import CollectionBase	19✔
28	from radicale.storage.multifilesystem.cache import CollectionPartCache	19✔
29	from radicale.storage.multifilesystem.get import CollectionPartGet	19✔
30	from radicale.storage.multifilesystem.history import CollectionPartHistory	19✔
31
32
33	class CollectionPartUpload(CollectionPartGet, CollectionPartCache,	19✔
34	CollectionPartHistory, CollectionBase):
35
36	def upload(self, href: str, item: radicale_item.Item	19✔
37	) -> radicale_item.Item:
38	if not pathutils.is_safe_filesystem_path_component(href):	19!
39	raise pathutils.UnsafePathError(href)	×
40	try:	19✔
41	self._store_item_cache(href, item)	19✔
42	except Exception as e:	×
43	raise ValueError("Failed to store item %r in collection %r: %s" %	×
44	(href, self.path, e)) from e
45	path = pathutils.path_to_filesystem(self._filesystem_path, href)	19✔
46	with self._atomic_write(path, newline="") as fo:	19✔
47	f = cast(TextIO, fo)	19✔
48	f.write(item.serialize())	19✔
49	# Clean the cache after the actual item is stored, or the cache entry
50	# will be removed again.
51	self._clean_item_cache()	19✔
52	# Track the change
53	self._update_history_etag(href, item)	19✔
54	self._clean_history()	19✔
55	uploaded_item = self._get(href, verify_href=False)	19✔
56	if uploaded_item is None:	19!
57	raise RuntimeError("Storage modified externally")	×
58	return uploaded_item	19✔
59
60	def _upload_all_nonatomic(self, items: Iterable[radicale_item.Item],	19✔
61	suffix: str = "") -> None:
62	"""Upload a new set of items.
63
64	This takes a list of vobject items and
65	uploads them nonatomic and without existence checks.
66
67	"""
68	cache_folder = os.path.join(self._filesystem_path,	19✔
69	".Radicale.cache", "item")
70	self._storage._makedirs_synced(cache_folder)	19✔
71	hrefs: Set[str] = set()	19✔
72	for item in items:	19✔
73	uid = item.uid	19✔
74	try:	19✔
75	cache_content = self._item_cache_content(item)	19✔
76	except Exception as e:	×
77	raise ValueError(	×
78	"Failed to store item %r in temporary collection %r: %s" %
79	(uid, self.path, e)) from e
80	href_candidate_funtions = [	19✔
81	lambda: uid if uid.lower().endswith(suffix.lower())
82	else uid + suffix,
83	lambda: radicale_item.get_etag(uid).strip('"') + suffix,
84	lambda: radicale_item.find_available_uid(
85	hrefs.__contains__, suffix)]
86	href = f = None	19✔
87	while href_candidate_funtions:	19!
88	href = href_candidate_funtions.pop(0)()	19✔
89	if href in hrefs:	19!
90	continue	×
91	if not pathutils.is_safe_filesystem_path_component(href):	19!
92	if not href_candidate_funtions:	×
93	raise pathutils.UnsafePathError(href)	×
94	continue	×
95	try:	19✔
96	f = open(pathutils.path_to_filesystem(	19✔
97	self._filesystem_path, href),
98	"w", newline="", encoding=self._encoding)
99	break	19✔
100	except pathutils.CollidingPathError:	12!
101	if href_candidate_funtions:	12!
102	continue	12✔
NEW 103	raise	×
104	except OSError as e:	×
105	if href_candidate_funtions and (	×
106	sys.platform != "win32" and
107	e.errno == errno.EINVAL or
108	sys.platform == "win32" and e.errno == 123):
109	continue	×
110	raise	×
111	assert href is not None and f is not None	19✔
112	with f:	19✔
113	f.write(item.serialize())	19✔
114	f.flush()	19✔
115	self._storage._fsync(f)	19✔
116	hrefs.add(href)	19✔
117	with open(os.path.join(cache_folder, href), "wb") as fb:	19✔
118	pickle.dump(cache_content, fb)	19✔
119	fb.flush()	19✔
120	self._storage._fsync(fb)	19✔
121	self._storage._sync_directory(cache_folder)	19✔
122	self._storage._sync_directory(self._filesystem_path)	19✔

Kozea / Radicale / 2067407235

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous