9058668052

Committed 13 May 2024 07:05AM UTC coverage: 77.713% (-8.4%) from 86.115%

Build # 9058668052

Build Type

Pull #3727

github

Committed by

web-flow

Commit Message

Merge 128d38387 into 050ed61bf

Pull Request Pull Request #3727: fix: don't fail session launch when gitlab couldn't be reached

Run Details

15 of 29 new or added lines in 3 files covered. (51.72%)

2594 existing lines in 125 files now uncovered.

23893 of 30745 relevant lines covered (77.71%)

3.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.45

/renku/domain_model/project_context.py

# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Project context."""

import contextlib
import threading
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Generator, List, NamedTuple, Optional, Union

from renku.core import errors
from renku.core.constant import (
    APP_NAME,
    CONFIG_NAME,
    DATA_DIR_CONFIG_KEY,
    DATABASE_PATH,
    DATASET_IMAGES,
    DEFAULT_DATA_DIR,
    DOCKERFILE,
    IMAGES,
    LOCK_SUFFIX,
    POINTERS,
    RENKU_HOME,
    TEMPLATE_CHECKSUMS,
)

if TYPE_CHECKING:
    from renku.domain_model.project import Project
    from renku.infrastructure.database import Database
    from renku.infrastructure.repository import Remote, Repository


class ProjectContext(threading.local):
    """A configuration class to hold global configuration."""

    external_storage_requested = True
    """External storage (e.g. LFS) requested for Renku command."""

    def __init__(self) -> None:
        self._context_stack: List[ProjectProperties] = []

    def __del__(self):
        self.clear()

    @property
    def database(self) -> "Database":
        """Current database."""
        if not self._top.database:
            from renku.infrastructure.database import Database

            self._top.database = Database.from_path(self.database_path)

        return self._top.database

    @property
    def database_path(self) -> Path:
        """Path to the metadata storage directory."""
        return self.metadata_path / DATABASE_PATH

    @property
    def datadir(self) -> str:
        """Define a name of the folder for storing datasets."""
        from renku.core.config import get_value
        from renku.domain_model.enums import ConfigFilter

        if not self._top.datadir:
            datadir = get_value("renku", DATA_DIR_CONFIG_KEY, config_filter=ConfigFilter.LOCAL_ONLY)
            self._top.datadir = datadir or DEFAULT_DATA_DIR

        return self._top.datadir

    @datadir.setter
    def datadir(self, value: str):
        """Set the current datadir."""
        self._top.datadir = value

    @property
    def dataset_images_path(self) -> Path:
        """Return a ``Path`` instance of Renku dataset metadata folder."""
        return self.path / RENKU_HOME / DATASET_IMAGES

    @property
    def project_image_pathname(self) -> Path:
        """Return the path to the project's image file."""
        return self.path / RENKU_HOME / IMAGES / "project" / "0.png"

    @property
    def dockerfile_path(self) -> Path:
        """Path to the Dockerfile."""
        return self.path / DOCKERFILE

    @property
    def ssh_authorized_keys_path(self) -> Path:
        """Path to SSH authorized keys."""
        return self.path / ".ssh" / "authorized_keys"

    @property
    def global_config_dir(self) -> str:
        """Return user's config directory."""
        import click

        return click.get_app_dir(APP_NAME, force_posix=True)

    @property
    def global_config_path(self) -> Path:
        """Renku global (user's) config path."""
        config = Path(self.global_config_dir)
        if not config.exists():
            config.mkdir(parents=True)

        return config / CONFIG_NAME

    @property
    def latest_agent(self) -> Optional[str]:
        """Returns latest agent version used in the repository."""
        try:
            return self.project.agent_version
        except ValueError as e:
            if "Project name not set" in str(e):
                return None
            raise

    @property
    def local_config_path(self) -> Path:
        """Renku local (project) config path."""
        return self.metadata_path / CONFIG_NAME

    @property
    def lock(self):
        """Create a Renku config lock."""
        from renku.core.util.contexts import Lock

        return Lock(filename=self.metadata_path.with_suffix(LOCK_SUFFIX), mode="exclusive")

    @property
    def metadata_path(self) -> Path:
        """Current project's metadata (RENKU_HOME) path."""
        return self.path / RENKU_HOME

    @property
    def path(self) -> Path:
        """Current project path."""
        return self._top.path

    @property
    def pointers_path(self) -> Path:
        """Return a ``Path`` instance of Renku pointer files folder."""
        path = self.path / RENKU_HOME / POINTERS
        path.mkdir(exist_ok=True)
        return path

    @property
    def project(self) -> "Project":
        """Return the Project instance."""
        from renku.command.command_builder.command import inject
        from renku.core.interface.project_gateway import IProjectGateway

        project_gateway = inject.instance(IProjectGateway)
        # NOTE: Don't cache the project since it can be updated in the ``ProjectGateway``
        return project_gateway.get_project()

    @property
    def remote(self) -> "ProjectRemote":
        """Return host, owner and name of the remote if it exists."""
        from renku.core.util.git import get_remote

        repository = self.repository

        remote = get_remote(repository=repository)

        if not remote and len(repository.remotes) > 1:
            remote = repository.remotes.get("origin")

        return ProjectRemote.from_remote(remote=remote)

    @property
    def repository(self) -> "Repository":
        """Return current context's repository."""
        if not self._top.repository:
            from renku.infrastructure.repository import Repository

            try:
                self._top.repository = Repository(project_context.path)
            except errors.GitError as e:
                raise ValueError from e

        return self._top.repository

    @repository.setter
    def repository(self, value: Optional["Repository"]):
        """Set the current repository."""
        self._top.repository = value

    @property
    def template_checksums_path(self):
        """Return a ``Path`` instance to the template checksums file."""
        return self.metadata_path / TEMPLATE_CHECKSUMS

    @property
    def transaction_id(self) -> str:
        """Get a transaction id for the current context to be used for grouping git commits."""
        if not self._top.transaction_id:
            self._top.transaction_id = uuid.uuid4().hex

        return f"\n\nrenku-transaction: {self._top.transaction_id}"

    @property
    def _top(self) -> "ProjectProperties":
        """Return current context."""
        if self._context_stack:
            return self._context_stack[-1]

        raise errors.ProjectContextError("No project context was pushed")

    def has_context(self, path: Optional[Union[Path, str]] = None) -> bool:
        """Return if at least one context which is equal to path (if not None) is pushed."""
        return True if self._context_stack and (path is None or self.path == Path(path).resolve()) else False

    def clear(self) -> None:
        """Remove all contexts and reset the state without committing intermediate changes.

        NOTE: This method should be used only in tests.
        """
        while self._context_stack:
            if self._top.repository:
                self._top.repository.close()
            self._context_stack.pop()

        self.external_storage_requested = True

    def pop_context(self) -> "ProjectProperties":
        """Pop current project context from stack.

        Returns:
            Path: the popped project path.
        """
        if self._context_stack:
            if self._top.save_changes and self._top.database:
                self._top.database.commit()

            if self._top.repository:
                self._top.repository.close()

            return self._context_stack.pop()
        else:
            raise IndexError("No more context to pop.")

    def push_path(self, path: Union[Path, str], save_changes: bool = False) -> None:
        """Push a new project path to the stack.

        Arguments:
            path(Union[Path, str]): The path to push.
            save_changes(bool): Whether to save changes to the database or not.
        """
        path = Path(path).resolve()
        self._context_stack.append(ProjectProperties(path=path, save_changes=save_changes))

    def replace_path(self, path: Union[Path, str]):
        """Replace the current project path with a new one if they are different.

        Arguments:
            path(Union[Path, str]): The path to replace with.
        """
        path = Path(path).resolve()

        if not self._context_stack:
            self.push_path(path)
        elif self._top.path != path:
            self._context_stack[-1] = ProjectProperties(path=path)

    @contextlib.contextmanager
    def with_path(
        self, path: Union[Path, str], save_changes: bool = False
    ) -> Generator["ProjectProperties", None, None]:
        """Temporarily push a new project path to the stack.

        Arguments:
            path(Union[Path, str]): The path to push.
            save_changes(bool): Whether to save changes to the database or not.
        """
        with self.with_rollback():
            self.push_path(path=path, save_changes=save_changes)
            yield self._top

    @contextlib.contextmanager
    def with_rollback(self) -> Generator[None, None, None]:
        """Rollback to the current state.

        NOTE: This won't work correctly if the current context is popped or swapped.
        """
        before_top = self._top if self._context_stack else None

        try:
            yield
        finally:
            could_rollback = False

            while self._context_stack:
                if self._top == before_top:
                    could_rollback = True
                    break

                self.pop_context()

            if not could_rollback and before_top is not None:
                raise errors.ProjectContextError(f"Cannot rollback to {before_top.path}.")


project_context: ProjectContext = ProjectContext()


def has_graph_files() -> bool:
    """Return true if database exists."""
    return project_context.database_path.exists() and any(
        f for f in project_context.database_path.iterdir() if f != project_context.database_path / "root"
    )


@dataclass
class ProjectProperties:
    """Various properties of the current project."""

    path: Path
    database: Optional["Database"] = None
    datadir: Optional[str] = None
    repository: Optional["Repository"] = None
    save_changes: bool = False
    transaction_id: Optional[str] = None


class ProjectRemote(NamedTuple):
    """Information about a project's remote."""

    name: Optional[str]
    owner: Optional[str]
    host: Optional[str]

    @classmethod
    def from_remote(cls, remote: Optional["Remote"]) -> "ProjectRemote":
        """Create an instance from a Repository remote."""
        from renku.domain_model.git import GitURL

        if not remote:
            return ProjectRemote(None, None, None)

        url = GitURL.parse(remote.url)

        # NOTE: Remove gitlab unless running on gitlab.com
        hostname_parts = url.hostname.split(".")
        if len(hostname_parts) > 2 and hostname_parts[0] == "gitlab":
            hostname_parts = hostname_parts[1:]

        return ProjectRemote(name=url.name, owner=url.owner, host=".".join(hostname_parts))

    def __bool__(self):
        return bool(self.name or self.owner or self.host)

1	# Copyright Swiss Data Science Center (SDSC). A partnership between
2	# École Polytechnique Fédérale de Lausanne (EPFL) and
3	# Eidgenössische Technische Hochschule Zürich (ETHZ).
4	#
5	# Licensed under the Apache License, Version 2.0 (the "License");
6	# you may not use this file except in compliance with the License.
7	# You may obtain a copy of the License at
8	#
9	# http://www.apache.org/licenses/LICENSE-2.0
10	#
11	# Unless required by applicable law or agreed to in writing, software
12	# distributed under the License is distributed on an "AS IS" BASIS,
13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	# See the License for the specific language governing permissions and
15	# limitations under the License.
16	"""Project context."""	7✔
17
18	import contextlib	7✔
19	import threading	7✔
20	import uuid	7✔
21	from dataclasses import dataclass	7✔
22	from pathlib import Path	7✔
23	from typing import TYPE_CHECKING, Generator, List, NamedTuple, Optional, Union	7✔
24
25	from renku.core import errors	7✔
26	from renku.core.constant import (	7✔
27	APP_NAME,
28	CONFIG_NAME,
29	DATA_DIR_CONFIG_KEY,
30	DATABASE_PATH,
31	DATASET_IMAGES,
32	DEFAULT_DATA_DIR,
33	DOCKERFILE,
34	IMAGES,
35	LOCK_SUFFIX,
36	POINTERS,
37	RENKU_HOME,
38	TEMPLATE_CHECKSUMS,
39	)
40
41	if TYPE_CHECKING:	7✔
42	from renku.domain_model.project import Project	×
43	from renku.infrastructure.database import Database	×
44	from renku.infrastructure.repository import Remote, Repository	×
45
46
47	class ProjectContext(threading.local):	7✔
48	"""A configuration class to hold global configuration."""
49
50	external_storage_requested = True	7✔
51	"""External storage (e.g. LFS) requested for Renku command."""	7✔
52
53	def __init__(self) -> None:	7✔
54	self._context_stack: List[ProjectProperties] = []	7✔
55
56	def __del__(self):	7✔
57	self.clear()	1✔
58
59	@property	7✔
60	def database(self) -> "Database":	7✔
61	"""Current database."""
62	if not self._top.database:	7✔
63	from renku.infrastructure.database import Database	7✔
64
65	self._top.database = Database.from_path(self.database_path)	7✔
66
67	return self._top.database	7✔
68
69	@property	7✔
70	def database_path(self) -> Path:	7✔
71	"""Path to the metadata storage directory."""
72	return self.metadata_path / DATABASE_PATH	7✔
73
74	@property	7✔
75	def datadir(self) -> str:	7✔
76	"""Define a name of the folder for storing datasets."""
77	from renku.core.config import get_value	6✔
78	from renku.domain_model.enums import ConfigFilter	6✔
79
80	if not self._top.datadir:	6✔
81	datadir = get_value("renku", DATA_DIR_CONFIG_KEY, config_filter=ConfigFilter.LOCAL_ONLY)	6✔
82	self._top.datadir = datadir or DEFAULT_DATA_DIR	6✔
83
84	return self._top.datadir	6✔
85
86	@datadir.setter	7✔
87	def datadir(self, value: str):	7✔
88	"""Set the current datadir."""
UNCOV 89	self._top.datadir = value	×
90
91	@property	7✔
92	def dataset_images_path(self) -> Path:	7✔
93	"""Return a ``Path`` instance of Renku dataset metadata folder."""
94	return self.path / RENKU_HOME / DATASET_IMAGES	1✔
95
96	@property	7✔
97	def project_image_pathname(self) -> Path:	7✔
98	"""Return the path to the project's image file."""
99	return self.path / RENKU_HOME / IMAGES / "project" / "0.png"	1✔
100
101	@property	7✔
102	def dockerfile_path(self) -> Path:	7✔
103	"""Path to the Dockerfile."""
104	return self.path / DOCKERFILE	5✔
105
106	@property	7✔
107	def ssh_authorized_keys_path(self) -> Path:	7✔
108	"""Path to SSH authorized keys."""
109	return self.path / ".ssh" / "authorized_keys"	1✔
110
111	@property	7✔
112	def global_config_dir(self) -> str:	7✔
113	"""Return user's config directory."""
114	import click	3✔
115
116	return click.get_app_dir(APP_NAME, force_posix=True)	3✔
117
118	@property	7✔
119	def global_config_path(self) -> Path:	7✔
120	"""Renku global (user's) config path."""
121	config = Path(self.global_config_dir)	6✔
122	if not config.exists():	6✔
123	config.mkdir(parents=True)	6✔
124
125	return config / CONFIG_NAME	6✔
126
127	@property	7✔
128	def latest_agent(self) -> Optional[str]:	7✔
129	"""Returns latest agent version used in the repository."""
130	try:	2✔
131	return self.project.agent_version	2✔
132	except ValueError as e:	1✔
133	if "Project name not set" in str(e):	1✔
134	return None	×
135	raise	1✔
136
137	@property	7✔
138	def local_config_path(self) -> Path:	7✔
139	"""Renku local (project) config path."""
140	return self.metadata_path / CONFIG_NAME	6✔
141
142	@property	7✔
143	def lock(self):	7✔
144	"""Create a Renku config lock."""
145	from renku.core.util.contexts import Lock	7✔
146
147	return Lock(filename=self.metadata_path.with_suffix(LOCK_SUFFIX), mode="exclusive")	7✔
148
149	@property	7✔
150	def metadata_path(self) -> Path:	7✔
151	"""Current project's metadata (RENKU_HOME) path."""
152	return self.path / RENKU_HOME	7✔
153
154	@property	7✔
155	def path(self) -> Path:	7✔
156	"""Current project path."""
157	return self._top.path	7✔
158
159	@property	7✔
160	def pointers_path(self) -> Path:	7✔
161	"""Return a ``Path`` instance of Renku pointer files folder."""
162	path = self.path / RENKU_HOME / POINTERS	6✔
163	path.mkdir(exist_ok=True)	6✔
164	return path	6✔
165
166	@property	7✔
167	def project(self) -> "Project":	7✔
168	"""Return the Project instance."""
169	from renku.command.command_builder.command import inject	7✔
170	from renku.core.interface.project_gateway import IProjectGateway	7✔
171
172	project_gateway = inject.instance(IProjectGateway)	7✔
173	# NOTE: Don't cache the project since it can be updated in the ``ProjectGateway``
174	return project_gateway.get_project()	7✔
175
176	@property	7✔
177	def remote(self) -> "ProjectRemote":	7✔
178	"""Return host, owner and name of the remote if it exists."""
179	from renku.core.util.git import get_remote	7✔
180
181	repository = self.repository	7✔
182
183	remote = get_remote(repository=repository)	7✔
184
185	if not remote and len(repository.remotes) > 1:	7✔
186	remote = repository.remotes.get("origin")	×
187
188	return ProjectRemote.from_remote(remote=remote)	7✔
189
190	@property	7✔
191	def repository(self) -> "Repository":	7✔
192	"""Return current context's repository."""
193	if not self._top.repository:	7✔
194	from renku.infrastructure.repository import Repository	7✔
195
196	try:	7✔
197	self._top.repository = Repository(project_context.path)	7✔
198	except errors.GitError as e:	3✔
199	raise ValueError from e	2✔
200
201	return self._top.repository	7✔
202
203	@repository.setter	7✔
204	def repository(self, value: Optional["Repository"]):	7✔
205	"""Set the current repository."""
206	self._top.repository = value	7✔
207
208	@property	7✔
209	def template_checksums_path(self):	7✔
210	"""Return a ``Path`` instance to the template checksums file."""
211	return self.metadata_path / TEMPLATE_CHECKSUMS	7✔
212
213	@property	7✔
214	def transaction_id(self) -> str:	7✔
215	"""Get a transaction id for the current context to be used for grouping git commits."""
216	if not self._top.transaction_id:	7✔
217	self._top.transaction_id = uuid.uuid4().hex	7✔
218
219	return f"\n\nrenku-transaction: {self._top.transaction_id}"	7✔
220
221	@property	7✔
222	def _top(self) -> "ProjectProperties":	7✔
223	"""Return current context."""
224	if self._context_stack:	7✔
225	return self._context_stack[-1]	7✔
226
227	raise errors.ProjectContextError("No project context was pushed")	1✔
228
229	def has_context(self, path: Optional[Union[Path, str]] = None) -> bool:	7✔
230	"""Return if at least one context which is equal to path (if not None) is pushed."""
231	return True if self._context_stack and (path is None or self.path == Path(path).resolve()) else False	7✔
232
233	def clear(self) -> None:	7✔
234	"""Remove all contexts and reset the state without committing intermediate changes.
235
236	NOTE: This method should be used only in tests.
237	"""
238	while self._context_stack:	6✔
239	if self._top.repository:	4✔
240	self._top.repository.close()	2✔
241	self._context_stack.pop()	4✔
242
243	self.external_storage_requested = True	6✔
244
245	def pop_context(self) -> "ProjectProperties":	7✔
246	"""Pop current project context from stack.
247
248	Returns:
249	Path: the popped project path.
250	"""
251	if self._context_stack:	7✔
252	if self._top.save_changes and self._top.database:	7✔
253	self._top.database.commit()	7✔
254
255	if self._top.repository:	7✔
256	self._top.repository.close()	7✔
257
258	return self._context_stack.pop()	7✔
259	else:
260	raise IndexError("No more context to pop.")	1✔
261
262	def push_path(self, path: Union[Path, str], save_changes: bool = False) -> None:	7✔
263	"""Push a new project path to the stack.
264
265	Arguments:
266	path(Union[Path, str]): The path to push.
267	save_changes(bool): Whether to save changes to the database or not.
268	"""
269	path = Path(path).resolve()	7✔
270	self._context_stack.append(ProjectProperties(path=path, save_changes=save_changes))	7✔
271
272	def replace_path(self, path: Union[Path, str]):	7✔
273	"""Replace the current project path with a new one if they are different.
274
275	Arguments:
276	path(Union[Path, str]): The path to replace with.
277	"""
278	path = Path(path).resolve()	3✔
279
280	if not self._context_stack:	3✔
281	self.push_path(path)	3✔
282	elif self._top.path != path:	2✔
283	self._context_stack[-1] = ProjectProperties(path=path)	×
284
285	@contextlib.contextmanager	7✔
286	def with_path(	7✔
287	self, path: Union[Path, str], save_changes: bool = False
288	) -> Generator["ProjectProperties", None, None]:
289	"""Temporarily push a new project path to the stack.
290
291	Arguments:
292	path(Union[Path, str]): The path to push.
293	save_changes(bool): Whether to save changes to the database or not.
294	"""
295	with self.with_rollback():	7✔
296	self.push_path(path=path, save_changes=save_changes)	7✔
297	yield self._top	7✔
298
299	@contextlib.contextmanager	7✔
300	def with_rollback(self) -> Generator[None, None, None]:	7✔
301	"""Rollback to the current state.
302
303	NOTE: This won't work correctly if the current context is popped or swapped.
304	"""
305	before_top = self._top if self._context_stack else None	7✔
306
307	try:	7✔
308	yield	7✔
309	finally:
310	could_rollback = False	7✔
311
312	while self._context_stack:	7✔
313	if self._top == before_top:	7✔
314	could_rollback = True	7✔
315	break	7✔
316
317	self.pop_context()	7✔
318
319	if not could_rollback and before_top is not None:	7✔
320	raise errors.ProjectContextError(f"Cannot rollback to {before_top.path}.")	2✔
321
322
323	project_context: ProjectContext = ProjectContext()	7✔
324
325
326	def has_graph_files() -> bool:	7✔
327	"""Return true if database exists."""
328	return project_context.database_path.exists() and any(	2✔
329	f for f in project_context.database_path.iterdir() if f != project_context.database_path / "root"
330	)
331
332
333	@dataclass	7✔
334	class ProjectProperties:	7✔
335	"""Various properties of the current project."""
336
337	path: Path	7✔
338	database: Optional["Database"] = None	7✔
339	datadir: Optional[str] = None	7✔
340	repository: Optional["Repository"] = None	7✔
341	save_changes: bool = False	7✔
342	transaction_id: Optional[str] = None	7✔
343
344
345	class ProjectRemote(NamedTuple):	7✔
346	"""Information about a project's remote."""
347
348	name: Optional[str]	7✔
349	owner: Optional[str]	7✔
350	host: Optional[str]	7✔
351
352	@classmethod	7✔
353	def from_remote(cls, remote: Optional["Remote"]) -> "ProjectRemote":	7✔
354	"""Create an instance from a Repository remote."""
355	from renku.domain_model.git import GitURL	7✔
356
357	if not remote:	7✔
358	return ProjectRemote(None, None, None)	7✔
359
360	url = GitURL.parse(remote.url)	3✔
361
362	# NOTE: Remove gitlab unless running on gitlab.com
363	hostname_parts = url.hostname.split(".")	3✔
364	if len(hostname_parts) > 2 and hostname_parts[0] == "gitlab":	3✔
365	hostname_parts = hostname_parts[1:]	3✔
366
367	return ProjectRemote(name=url.name, owner=url.owner, host=".".join(hostname_parts))	3✔
368
369	def __bool__(self):	7✔
370	return bool(self.name or self.owner or self.host)	7✔

SwissDataScienceCenter / renku-python / 9058668052

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous