5948296099

Committed 23 Aug 2023 07:23AM UTC coverage: 85.801% (+0.04%) from 85.766%

Build # 5948296099

Build Type

Pull #3601

github-actions

Committed by

olevski

Commit Message

chore: run poetry lock

Pull Request Pull Request #3601: hotfix: v2.6.1

Run Details

40 of 48 new or added lines in 10 files covered. (83.33%)

285 existing lines in 25 files now uncovered.

25875 of 30157 relevant lines covered (85.8%)

4.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.64

/renku/ui/service/gateways/gitlab_api_provider.py

#
# Copyright 2020 - Swiss Data Science Center (SDSC)
# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Git APi provider interface."""

import tarfile
import tempfile
from pathlib import Path
from typing import List, Optional, Union

import gitlab

from renku.core import errors
from renku.core.util.os import delete_dataset_file
from renku.domain_model.git import GitURL
from renku.ui.service.interfaces.git_api_provider import IGitAPIProvider


class GitlabAPIProvider(IGitAPIProvider):
    """GitLab API provider abstraction layer.

    Args:
        paths: List of files to download.
        target_folder: Folder to use to download the files.
        remote: Remote repository URL.
        token: User bearer token.
        ref: optional reference to checkout,
    Raises:
        errors.ProjectNotFound: If the remote URL is not accessible.
        errors.AuthenticationError: If the bearer token is invalid in any way.
    """

    def download_files_from_api(
        self,
        files: List[Union[Path, str]],
        folders: List[Union[Path, str]],
        target_folder: Union[Path, str],
        remote: str,
        token: str,
        branch: Optional[str] = None,
    ):
        """Download files through a remote Git API.

        Args:
            files(List[Union[Path, str]]): Files to download.
            folders(List[Union[Path, str]]): Folders to download.
            target_folder(Union[Path, str]): Destination to save downloads to.
            remote(str): Git remote URL.
            token(str): Gitlab API token.
            branch(Optional[str]): Git reference (Default value = None).
        """
        if not branch:
            branch = "HEAD"

        target_folder = Path(target_folder)

        git_data = GitURL.parse(remote)
        try:
            gl = gitlab.Gitlab(git_data.instance_url, oauth_token=token)
            project = gl.projects.get(f"{git_data.owner}/{git_data.name}")
        except gitlab.GitlabAuthenticationError:
            # NOTE: Invalid or expired tokens fail even on public projects. Let's give it a try without tokens
            try:
                gl = gitlab.Gitlab(git_data.instance_url)
                project = gl.projects.get(f"{git_data.owner}/{git_data.name}")
            except gitlab.GitlabAuthenticationError as e:
                raise errors.AuthenticationError from e
            except gitlab.GitlabGetError as e:
                # NOTE: better to re-raise this as a core error since it's a common case
                if "project not found" in getattr(e, "error_message", "").lower():
                    raise errors.ProjectNotFound from e
                else:
                    raise

        for file in files:
            full_path = target_folder / file

            full_path.parent.mkdir(parents=True, exist_ok=True)

            try:
                with open(full_path, "wb") as f:
                    project.files.raw(file_path=str(file), ref=branch, streamed=True, action=f.write)
            except gitlab.GitlabGetError:
                delete_dataset_file(full_path)
                continue

        for folder in folders:
            with tempfile.NamedTemporaryFile() as f:
                project.repository_archive(path=str(folder), sha=branch, streamed=True, action=f.write, format="tar.gz")
                f.seek(0)
                with tarfile.open(fileobj=f) as archive:
                    archive.extractall(path=target_folder)

1	#
2	# Copyright 2020 - Swiss Data Science Center (SDSC)
3	# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
4	# Eidgenössische Technische Hochschule Zürich (ETHZ).
5	#
6	# Licensed under the Apache License, Version 2.0 (the "License");
7	# you may not use this file except in compliance with the License.
8	# You may obtain a copy of the License at
9	#
10	# http://www.apache.org/licenses/LICENSE-2.0
11	#
12	# Unless required by applicable law or agreed to in writing, software
13	# distributed under the License is distributed on an "AS IS" BASIS,
14	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	# See the License for the specific language governing permissions and
16	# limitations under the License.
17	"""Git APi provider interface."""	4✔
18
19	import tarfile	4✔
20	import tempfile	4✔
21	from pathlib import Path	4✔
22	from typing import List, Optional, Union	4✔
23
24	import gitlab	4✔
25
26	from renku.core import errors	4✔
27	from renku.core.util.os import delete_dataset_file	4✔
28	from renku.domain_model.git import GitURL	4✔
29	from renku.ui.service.interfaces.git_api_provider import IGitAPIProvider	4✔
30
31
32	class GitlabAPIProvider(IGitAPIProvider):	4✔
33	"""GitLab API provider abstraction layer.
34
35	Args:
36	paths: List of files to download.
37	target_folder: Folder to use to download the files.
38	remote: Remote repository URL.
39	token: User bearer token.
40	ref: optional reference to checkout,
41	Raises:
42	errors.ProjectNotFound: If the remote URL is not accessible.
43	errors.AuthenticationError: If the bearer token is invalid in any way.
44	"""
45
46	def download_files_from_api(	4✔
47	self,
48	files: List[Union[Path, str]],
49	folders: List[Union[Path, str]],
50	target_folder: Union[Path, str],
51	remote: str,
52	token: str,
53	branch: Optional[str] = None,
54	):
55	"""Download files through a remote Git API.
56
57	Args:
58	files(List[Union[Path, str]]): Files to download.
59	folders(List[Union[Path, str]]): Folders to download.
60	target_folder(Union[Path, str]): Destination to save downloads to.
61	remote(str): Git remote URL.
62	token(str): Gitlab API token.
63	branch(Optional[str]): Git reference (Default value = None).
64	"""
65	if not branch:	1✔
66	branch = "HEAD"	1✔
67
68	target_folder = Path(target_folder)	1✔
69
70	git_data = GitURL.parse(remote)	1✔
71	try:	1✔
72	gl = gitlab.Gitlab(git_data.instance_url, oauth_token=token)	1✔
73	project = gl.projects.get(f"{git_data.owner}/{git_data.name}")	1✔
74	except gitlab.GitlabAuthenticationError:	1✔
75	# NOTE: Invalid or expired tokens fail even on public projects. Let's give it a try without tokens
76	try:	1✔
77	gl = gitlab.Gitlab(git_data.instance_url)	1✔
78	project = gl.projects.get(f"{git_data.owner}/{git_data.name}")	1✔
79	except gitlab.GitlabAuthenticationError as e:	1✔
80	raise errors.AuthenticationError from e	×
81	except gitlab.GitlabGetError as e:	1✔
82	# NOTE: better to re-raise this as a core error since it's a common case
83	if "project not found" in getattr(e, "error_message", "").lower():	1✔
84	raise errors.ProjectNotFound from e	1✔
85	else:
UNCOV 86	raise	×
87
88	for file in files:	1✔
89	full_path = target_folder / file	1✔
90
91	full_path.parent.mkdir(parents=True, exist_ok=True)	1✔
92
93	try:	1✔
94	with open(full_path, "wb") as f:	1✔
95	project.files.raw(file_path=str(file), ref=branch, streamed=True, action=f.write)	1✔
UNCOV 96	except gitlab.GitlabGetError:	×
UNCOV 97	delete_dataset_file(full_path)	×
UNCOV 98	continue	×
99
100	for folder in folders:	1✔
101	with tempfile.NamedTemporaryFile() as f:	1✔
102	project.repository_archive(path=str(folder), sha=branch, streamed=True, action=f.write, format="tar.gz")	1✔
103	f.seek(0)	1✔
104	with tarfile.open(fileobj=f) as archive:	1✔
105	archive.extractall(path=target_folder)	1✔

SwissDataScienceCenter / renku-python / 5948296099

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous