20530599664

Committed 26 Dec 2025 10:35PM UTC coverage: 96.567%. First build

Build # 20530599664

Build Type

push

github

Committed by

funilrys

Commit Message

Add a locking mechanism when accessing CSV files.

This patch touches #435.

Indeed, as discovered by @Yuki2718 and @DandelionSprout, our processes sometimes
trigger a PermissionError exception on Windows.

While testing with one of the latest builds of Windows 11 (Pro), I was able to
reproduce the issue sometimes and therefore analyze it in more detail. It turns out
that in some rare cases, 2 processes are simultaneously attempting to read the same
CSV file.

This patch attempts to solve the problem by introducing a shared lock which - for now
- only affect Windows-Users. When running on Windows, all processes will share a lock
that blocks when 2 processes try to access the same ressource at the same time.

Contributors:
  * @Yuki2718
  * @DandelionSprout

Run Details

28 of 30 new or added lines in 11 files covered. (93.33%)

11730 of 12147 relevant lines covered (96.57%)

11.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.74

/PyFunceble/dataset/base.py

"""
The tool to check the availability or syntax of domain, IP or URL.

::


    ██████╗ ██╗   ██╗███████╗██╗   ██╗███╗   ██╗ ██████╗███████╗██████╗ ██╗     ███████╗
    ██╔══██╗╚██╗ ██╔╝██╔════╝██║   ██║████╗  ██║██╔════╝██╔════╝██╔══██╗██║     ██╔════╝
    ██████╔╝ ╚████╔╝ █████╗  ██║   ██║██╔██╗ ██║██║     █████╗  ██████╔╝██║     █████╗
    ██╔═══╝   ╚██╔╝  ██╔══╝  ██║   ██║██║╚██╗██║██║     ██╔══╝  ██╔══██╗██║     ██╔══╝
    ██║        ██║   ██║     ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
    ╚═╝        ╚═╝   ╚═╝      ╚═════╝ ╚═╝  ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝

Provides the base of all datasets classes.

Author:
    Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom

Special thanks:
    https://pyfunceble.github.io/#/special-thanks

Contributors:
    https://pyfunceble.github.io/#/contributors

Project link:
    https://github.com/funilrys/PyFunceble

Project documentation:
    https://docs.pyfunceble.com

Project homepage:
    https://pyfunceble.github.io/

License:
::


    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        https://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

import functools
from typing import Any, Optional

import PyFunceble.storage
from PyFunceble.downloader.base import DownloaderBase
from PyFunceble.helpers.dict import DictHelper
from PyFunceble.helpers.file import FileHelper
from PyFunceble.utils.platform import PlatformUtility


class DatasetBase:
    """
    Provides the base of all dataset.

    :param Any shared_lock:
        Optional, The shared lock to use to access shared resources.
    """

    STORAGE_INDEX: Optional[str] = None
    downloader: Optional[DownloaderBase] = None

    source_file: Optional[str] = None

    shared_lock: Optional[Any] = None

    def __init__(self, *, shared_lock: Optional[Any] = None) -> None:
        if shared_lock is not None:
            self.shared_lock = shared_lock

    def __contains__(self, value: Any):  # pragma: no cover
        raise NotImplementedError()

    def __getattr__(self, value: Any):  # pragma: no cover
        raise AttributeError(value)

    def __getitem__(self, value: Any):  # pragma: no cover
        raise KeyError(value)

    def __getstate__(self):  # pragma: no cover
        return vars(self)

    def __setstate__(self, state):  # pragma: no cover
        vars(self).update(state)

    def ensure_source_file_exists(func):  # pylint: disable=no-self-argument
        """
        Ensures that the source file exists before running the decorated
        method.

        :raise TypeError:
            When :code:`self.source_file` is not a :py:class:`str`.
        :raise ValueError:
            When :code:`self.source_file` is empty.
        """

        @functools.wraps(func)
        def wrapper(self, *args, **kwargs):
            if not isinstance(self.source_file, str):
                raise TypeError(
                    f"<self.source_file> should be {str}, "
                    f"{type(self.source_file)} given."
                )

            if not self.source_file:
                raise ValueError("<self.source_file> should not be empty.")

            return func(self, *args, **kwargs)  # pylint: disable=not-callable

        return wrapper

    def autolock(func):  # pylint: disable=no-self-argument
        """
        A decorator to automatically lock and unlock the shared lock.
        """

        @functools.wraps(func)
        def wrapper(self, *args, **kwargs):
            if PlatformUtility.is_windows() and self.shared_lock is not None:
                with self.shared_lock:
                    return func(self, *args, **kwargs)  # pylint: disable=not-callable
            return func(self, *args, **kwargs)  # pylint: disable=not-callable

        return wrapper

    @ensure_source_file_exists
    @autolock
    def get_content(self) -> Optional[dict]:
        """
        Provides the cached or the real contend of the dataset (after caching)

        :raise FileNotFoundError:
            When the declared file does not exists.
        """

        if (
            bool(self.STORAGE_INDEX)
            and hasattr(PyFunceble.storage, self.STORAGE_INDEX)
            and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX))
        ):
            return getattr(PyFunceble.storage, self.STORAGE_INDEX)

        file_helper = FileHelper(self.source_file)

        if not file_helper.exists() and bool(
            self.downloader
        ):  # pragma: no cover ## This is just a safety endpoint.
            self.downloader.start()

            if not file_helper.exists():
                raise FileNotFoundError(file_helper.path)

        content = DictHelper().from_json_file(
            self.source_file, return_dict_on_error=False
        )

        setattr(PyFunceble.storage, self.STORAGE_INDEX, content)

        return content or {}

1	"""
2	The tool to check the availability or syntax of domain, IP or URL.
3
4	::
5
6
7	██████╗ ██╗ ██╗███████╗██╗ ██╗███╗ ██╗ ██████╗███████╗██████╗ ██╗ ███████╗
8	██╔══██╗╚██╗ ██╔╝██╔════╝██║ ██║████╗ ██║██╔════╝██╔════╝██╔══██╗██║ ██╔════╝
9	██████╔╝ ╚████╔╝ █████╗ ██║ ██║██╔██╗ ██║██║ █████╗ ██████╔╝██║ █████╗
10	██╔═══╝ ╚██╔╝ ██╔══╝ ██║ ██║██║╚██╗██║██║ ██╔══╝ ██╔══██╗██║ ██╔══╝
11	██║ ██║ ██║ ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
12	╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝
13
14	Provides the base of all datasets classes.
15
16	Author:
17	Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom
18
19	Special thanks:
20	https://pyfunceble.github.io/#/special-thanks
21
22	Contributors:
23	https://pyfunceble.github.io/#/contributors
24
25	Project link:
26	https://github.com/funilrys/PyFunceble
27
28	Project documentation:
29	https://docs.pyfunceble.com
30
31	Project homepage:
32	https://pyfunceble.github.io/
33
34	License:
35	::
36
37
38	Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
39
40	Licensed under the Apache License, Version 2.0 (the "License");
41	you may not use this file except in compliance with the License.
42	You may obtain a copy of the License at
43
44	https://www.apache.org/licenses/LICENSE-2.0
45
46	Unless required by applicable law or agreed to in writing, software
47	distributed under the License is distributed on an "AS IS" BASIS,
48	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
49	See the License for the specific language governing permissions and
50	limitations under the License.
51	"""
52
53	import functools	12✔
54	from typing import Any, Optional	12✔
55
56	import PyFunceble.storage	12✔
57	from PyFunceble.downloader.base import DownloaderBase	12✔
58	from PyFunceble.helpers.dict import DictHelper	12✔
59	from PyFunceble.helpers.file import FileHelper	12✔
60	from PyFunceble.utils.platform import PlatformUtility	12✔
61
62
63	class DatasetBase:	12✔
64	"""
65	Provides the base of all dataset.
66
67	:param Any shared_lock:
68	Optional, The shared lock to use to access shared resources.
69	"""
70
71	STORAGE_INDEX: Optional[str] = None	12✔
72	downloader: Optional[DownloaderBase] = None	12✔
73
74	source_file: Optional[str] = None	12✔
75
76	shared_lock: Optional[Any] = None	12✔
77
78	def __init__(self, *, shared_lock: Optional[Any] = None) -> None:
79	if shared_lock is not None:
80	self.shared_lock = shared_lock
81
82	def __contains__(self, value: Any): # pragma: no cover
83	raise NotImplementedError()
84
85	def __getattr__(self, value: Any): # pragma: no cover
86	raise AttributeError(value)
87
88	def __getitem__(self, value: Any): # pragma: no cover
89	raise KeyError(value)
90
91	def __getstate__(self): # pragma: no cover
92	return vars(self)
93
94	def __setstate__(self, state): # pragma: no cover
95	vars(self).update(state)
96
97	def ensure_source_file_exists(func): # pylint: disable=no-self-argument	12✔
98	"""
99	Ensures that the source file exists before running the decorated
100	method.
101
102	:raise TypeError:
103	When :code:`self.source_file` is not a :py:class:`str`.
104	:raise ValueError:
105	When :code:`self.source_file` is empty.
106	"""
107
108	@functools.wraps(func)	12✔
109	def wrapper(self, args, *kwargs):	12✔
110	if not isinstance(self.source_file, str):	12✔
111	raise TypeError(	12✔
112	f"<self.source_file> should be {str}, "
113	f"{type(self.source_file)} given."
114	)
115
116	if not self.source_file:	12✔
117	raise ValueError("<self.source_file> should not be empty.")	12✔
118
119	return func(self, args, *kwargs) # pylint: disable=not-callable	12✔
120
121	return wrapper	12✔
122
123	def autolock(func): # pylint: disable=no-self-argument	12✔
124	"""
125	A decorator to automatically lock and unlock the shared lock.
126	"""
127
128	@functools.wraps(func)	12✔
129	def wrapper(self, args, *kwargs):	12✔
130	if PlatformUtility.is_windows() and self.shared_lock is not None:	12✔
NEW 131	with self.shared_lock:	×
NEW 132	return func(self, args, *kwargs) # pylint: disable=not-callable	×
133	return func(self, args, *kwargs) # pylint: disable=not-callable	12✔
134
135	return wrapper	12✔
136
137	@ensure_source_file_exists	12✔
138	@autolock	12✔
139	def get_content(self) -> Optional[dict]:	12✔
140	"""
141	Provides the cached or the real contend of the dataset (after caching)
142
143	:raise FileNotFoundError:
144	When the declared file does not exists.
145	"""
146
147	if (	12✔
148	bool(self.STORAGE_INDEX)
149	and hasattr(PyFunceble.storage, self.STORAGE_INDEX)
150	and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX))
151	):
152	return getattr(PyFunceble.storage, self.STORAGE_INDEX)	12✔
153
154	file_helper = FileHelper(self.source_file)	12✔
155
156	if not file_helper.exists() and bool(
157	self.downloader
158	): # pragma: no cover ## This is just a safety endpoint.
159	self.downloader.start()
160
161	if not file_helper.exists():
162	raise FileNotFoundError(file_helper.path)
163
164	content = DictHelper().from_json_file(	12✔
165	self.source_file, return_dict_on_error=False
166	)
167
168	setattr(PyFunceble.storage, self.STORAGE_INDEX, content)	12✔
169
170	return content or {}	12✔

funilrys / PyFunceble / 20530599664

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous