14377770991

Committed 10 Apr 2025 10:04AM UTC coverage: 86.244% (-0.1%) from 86.351%

Build # 14377770991

Build Type

Pull #785

github

Committed by

web-flow

Commit Message

Merge 5b3aea87d into 74eb7d965

Pull Request Pull Request #785: feat: add endpoint to get v1 project properties

Run Details

32 of 73 new or added lines in 6 files covered. (43.84%)

1 existing line in 1 file now uncovered.

20013 of 23205 relevant lines covered (86.24%)

1.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.81

/components/renku_data_services/project/core.py

"""Business logic for projects."""

from pathlib import PurePosixPath
from typing import cast
from urllib.parse import urljoin, urlparse

import httpx
from ulid import ULID

from renku_data_services import errors
from renku_data_services.authz.models import Visibility
from renku_data_services.base_models import RESET, APIUser, ResetType, Slug
from renku_data_services.base_models.core import GitlabAPIProtocol
from renku_data_services.data_connectors.db import DataConnectorRepository
from renku_data_services.project import apispec, models
from renku_data_services.project.db import ProjectRepository
from renku_data_services.session.db import SessionRepository


def validate_unsaved_project(body: apispec.ProjectPost, created_by: str) -> models.UnsavedProject:
    """Validate an unsaved project."""
    keywords = [kw.root for kw in body.keywords] if body.keywords is not None else []
    visibility = Visibility.PRIVATE if body.visibility is None else Visibility(body.visibility.value)
    secrets_mount_directory = PurePosixPath(body.secrets_mount_directory) if body.secrets_mount_directory else None
    repositories = _validate_repositories(body.repositories)
    return models.UnsavedProject(
        name=body.name,
        namespace=body.namespace,
        slug=body.slug or Slug.from_name(body.name).value,
        description=body.description,
        repositories=repositories or [],
        created_by=created_by,
        visibility=visibility,
        keywords=keywords,
        documentation=body.documentation,
        secrets_mount_directory=secrets_mount_directory,
    )


def validate_project_patch(patch: apispec.ProjectPatch) -> models.ProjectPatch:
    """Validate the update to a project."""
    keywords = [kw.root for kw in patch.keywords] if patch.keywords is not None else None
    secrets_mount_directory: PurePosixPath | ResetType | None
    match patch.secrets_mount_directory:
        case "":
            secrets_mount_directory = RESET
        case str():
            secrets_mount_directory = PurePosixPath(patch.secrets_mount_directory)
        case _:
            secrets_mount_directory = None
    repositories = _validate_repositories(patch.repositories)
    return models.ProjectPatch(
        name=patch.name,
        namespace=patch.namespace,
        slug=patch.slug,
        visibility=Visibility(patch.visibility.value) if patch.visibility is not None else None,
        repositories=repositories,
        description=patch.description,
        keywords=keywords,
        documentation=patch.documentation,
        template_id=None if patch.template_id is None else "",
        is_template=patch.is_template,
        secrets_mount_directory=secrets_mount_directory,
    )


async def copy_project(
    project_id: ULID,
    user: APIUser,
    name: str,
    namespace: str,
    slug: str | None,
    description: str | None,
    repositories: list[models.Repository] | None,
    visibility: Visibility | None,
    keywords: list[str],
    secrets_mount_directory: str | None,
    project_repo: ProjectRepository,
    session_repo: SessionRepository,
    data_connector_repo: DataConnectorRepository,
) -> models.Project:
    """Create a copy of a given project."""
    template = await project_repo.get_project(user=user, project_id=project_id, with_documentation=True)
    repositories_ = _validate_repositories(repositories)

    unsaved_project = models.UnsavedProject(
        name=name,
        namespace=namespace,
        slug=slug or Slug.from_name(name).value,
        description=description or template.description,
        repositories=repositories_ or template.repositories,
        created_by=user.id,  # type: ignore[arg-type]
        visibility=template.visibility if visibility is None else visibility,
        keywords=keywords or template.keywords,
        template_id=template.id,
        secrets_mount_directory=PurePosixPath(secrets_mount_directory) if secrets_mount_directory else None,
        documentation=template.documentation,
    )
    project = await project_repo.insert_project(user, unsaved_project)

    # NOTE: Copy session launchers
    launchers = await session_repo.get_project_launchers(user=user, project_id=project_id)
    for launcher in launchers:
        await session_repo.copy_launcher(user=user, project_id=project.id, launcher=launcher)

    # NOTE: Copy data connector links. If this operation fails due to lack of permission, still proceed to create the
    # copy but return an error code that reflects this
    uncopied_dc_ids: list[ULID] = []
    dc_links = await data_connector_repo.get_links_to(user=user, project_id=project_id)
    for dc_link in dc_links:
        try:
            await data_connector_repo.copy_link(user=user, project_id=project.id, link=dc_link)
        except errors.MissingResourceError:
            uncopied_dc_ids.append(dc_link.data_connector_id)

    if uncopied_dc_ids:
        data_connectors_names_ids = await data_connector_repo.get_data_connectors_names_and_ids(user, uncopied_dc_ids)
        dc_str = ", ".join([f"{name} ({id})" for name, id in data_connectors_names_ids])
        if len(data_connectors_names_ids) == 1:
            message = (
                f"The project was copied but data connector with name '{dc_str}' was not able to be linked to "
                "your copy of this project due to insufficient permissions. To make a copy that includes the data "
                "connector, ask its owner to make it public."
            )
        else:
            message = (
                f"The project was copied but data connectors with names '[{dc_str}]' were not able to be linked to "
                "your copy of this project due to insufficient permissions. To make a copy that includes the data "
                "connectors, ask their owners to make them public."
            )
        raise errors.CopyDataConnectorsError(message=message)

    return project


def validate_unsaved_session_secret_slot(
    body: apispec.SessionSecretSlotPost,
) -> models.UnsavedSessionSecretSlot:
    """Validate an unsaved secret slot."""
    _validate_session_launcher_secret_slot_filename(body.filename)
    return models.UnsavedSessionSecretSlot(
        project_id=ULID.from_str(body.project_id),
        name=body.name,
        description=body.description,
        filename=body.filename,
    )


def validate_session_secret_slot_patch(
    body: apispec.SessionSecretSlotPatch,
) -> models.SessionSecretSlotPatch:
    """Validate the update to a secret slot."""
    if body.filename is not None:
        _validate_session_launcher_secret_slot_filename(body.filename)
    return models.SessionSecretSlotPatch(
        name=body.name,
        description=body.description,
        filename=body.filename,
    )


def validate_session_secrets_patch(
    body: apispec.SessionSecretPatchList,
) -> list[models.SessionSecretPatchExistingSecret | models.SessionSecretPatchSecretValue]:
    """Validate the update to a session launcher's secrets."""
    result: list[models.SessionSecretPatchExistingSecret | models.SessionSecretPatchSecretValue] = []
    seen_slot_ids: set[str] = set()
    for item in body.root:
        if item.secret_slot_id in seen_slot_ids:
            raise errors.ValidationError(
                message=f"Found duplicate secret_slot_id '{item.secret_slot_id}' in the list of secrets."
            )
        seen_slot_ids.add(item.secret_slot_id)

        if isinstance(item, apispec.SessionSecretPatch2):
            result.append(
                models.SessionSecretPatchExistingSecret(
                    secret_slot_id=ULID.from_str(item.secret_slot_id),
                    secret_id=ULID.from_str(item.secret_id),
                )
            )
        else:
            result.append(
                models.SessionSecretPatchSecretValue(
                    secret_slot_id=ULID.from_str(item.secret_slot_id),
                    value=item.value,
                )
            )
    return result


def _validate_repositories(repositories: list[str] | None) -> list[str] | None:
    """Validate a list of git repositories."""
    if repositories is None:
        return None
    seen: set[str] = set()
    without_duplicates: list[str] = []
    for repo in repositories:
        repo = _validate_repository(repo)
        if repo not in seen:
            without_duplicates.append(repo)
            seen.add(repo)
    return without_duplicates


def _validate_repository(repository: str) -> str:
    """Validate a git repository."""
    stripped = repository.strip()
    parsed = urlparse(stripped)
    if parsed.scheme not in ["http", "https"]:
        raise errors.ValidationError(message=f'The repository URL "{repository}" is not a valid HTTP or HTTPS URL.')
    return stripped


def _validate_session_launcher_secret_slot_filename(filename: str) -> None:
    """Validate the filename field of a secret slot."""
    filename_candidate = PurePosixPath(filename)
    if filename_candidate.name != filename:
        raise errors.ValidationError(message=f"Filename {filename} is not valid.")


async def get_v1_project_info(
    user: APIUser,
    internal_gitlab_user: APIUser,
    project_path: str,
    gitlab_client: GitlabAPIProtocol,
    core_svc_url: str,
) -> dict[str, str | list[str] | int | None]:
    """Request project information from the core service for a Renku v1 project."""
    url = await gitlab_client.get_project_url_from_path(internal_gitlab_user, project_path)
    if not url:
        raise errors.MissingResourceError(
            message=f"The Renku v1 project with path {project_path} cannot be found "
            "in Gitlab or you do not have access to it"
        )

    body = {"git_url": url, "is_delayed": False, "migrate_project": False}
    headers = {}
    if user.access_token:
        headers["Authorization"] = user.access_token
    full_url = urljoin(core_svc_url + "/", "project.show")
    async with httpx.AsyncClient() as clnt:
        res = await clnt.post(full_url, json=body, headers=headers)
    if res.status_code != 200:
        raise errors.MissingResourceError(
            message=f"The core service responded with an unexpected code {res.status_code} when getting "
            f"information about project {project_path} and url {url}"
        )
    res_json = cast(dict[str, dict[str, str | int | list[str]]], res.json())
    if res_json.get("error") is not None:
        raise errors.MissingResourceError(
            message=f"The core service responded with an error when getting "
            f"information about project {project_path} and url {url}",
            detail=cast(str | None, res_json.get("error", {}).get("userMessage")),
        )

    kws = res_json.get("result", {}).get("keywords")
    desc = res_json.get("result", {}).get("description")
    id = res_json.get("result", {}).get("id")
    name = res_json.get("result", {}).get("name")
    output = {
        "name": name,
        "id": id,
        "keywords": kws,
        "description": desc,
    }
    return output

1	"""Business logic for projects."""
2
3	from pathlib import PurePosixPath	2✔
4	from typing import cast	2✔
5	from urllib.parse import urljoin, urlparse	2✔
6
7	import httpx	2✔
8	from ulid import ULID	2✔
9
10	from renku_data_services import errors	2✔
11	from renku_data_services.authz.models import Visibility	2✔
12	from renku_data_services.base_models import RESET, APIUser, ResetType, Slug	2✔
13	from renku_data_services.base_models.core import GitlabAPIProtocol	2✔
14	from renku_data_services.data_connectors.db import DataConnectorRepository	2✔
15	from renku_data_services.project import apispec, models	2✔
16	from renku_data_services.project.db import ProjectRepository	2✔
17	from renku_data_services.session.db import SessionRepository	2✔
18
19
20	def validate_unsaved_project(body: apispec.ProjectPost, created_by: str) -> models.UnsavedProject:	2✔
21	"""Validate an unsaved project."""
22	keywords = [kw.root for kw in body.keywords] if body.keywords is not None else []	2✔
23	visibility = Visibility.PRIVATE if body.visibility is None else Visibility(body.visibility.value)	2✔
24	secrets_mount_directory = PurePosixPath(body.secrets_mount_directory) if body.secrets_mount_directory else None	2✔
25	repositories = _validate_repositories(body.repositories)	2✔
26	return models.UnsavedProject(	2✔
27	name=body.name,
28	namespace=body.namespace,
29	slug=body.slug or Slug.from_name(body.name).value,
30	description=body.description,
31	repositories=repositories or [],
32	created_by=created_by,
33	visibility=visibility,
34	keywords=keywords,
35	documentation=body.documentation,
36	secrets_mount_directory=secrets_mount_directory,
37	)
38
39
40	def validate_project_patch(patch: apispec.ProjectPatch) -> models.ProjectPatch:	2✔
41	"""Validate the update to a project."""
42	keywords = [kw.root for kw in patch.keywords] if patch.keywords is not None else None	2✔
43	secrets_mount_directory: PurePosixPath \| ResetType \| None
44	match patch.secrets_mount_directory:	2✔
45	case "":	2✔
46	secrets_mount_directory = RESET	1✔
47	case str():	2✔
48	secrets_mount_directory = PurePosixPath(patch.secrets_mount_directory)	2✔
49	case _:	2✔
50	secrets_mount_directory = None	2✔
51	repositories = _validate_repositories(patch.repositories)	2✔
52	return models.ProjectPatch(	2✔
53	name=patch.name,
54	namespace=patch.namespace,
55	slug=patch.slug,
56	visibility=Visibility(patch.visibility.value) if patch.visibility is not None else None,
57	repositories=repositories,
58	description=patch.description,
59	keywords=keywords,
60	documentation=patch.documentation,
61	template_id=None if patch.template_id is None else "",
62	is_template=patch.is_template,
63	secrets_mount_directory=secrets_mount_directory,
64	)
65
66
67	async def copy_project(	2✔
68	project_id: ULID,
69	user: APIUser,
70	name: str,
71	namespace: str,
72	slug: str \| None,
73	description: str \| None,
74	repositories: list[models.Repository] \| None,
75	visibility: Visibility \| None,
76	keywords: list[str],
77	secrets_mount_directory: str \| None,
78	project_repo: ProjectRepository,
79	session_repo: SessionRepository,
80	data_connector_repo: DataConnectorRepository,
81	) -> models.Project:
82	"""Create a copy of a given project."""
83	template = await project_repo.get_project(user=user, project_id=project_id, with_documentation=True)	2✔
84	repositories_ = _validate_repositories(repositories)	1✔
85
86	unsaved_project = models.UnsavedProject(	1✔
87	name=name,
88	namespace=namespace,
89	slug=slug or Slug.from_name(name).value,
90	description=description or template.description,
91	repositories=repositories_ or template.repositories,
92	created_by=user.id, # type: ignore[arg-type]
93	visibility=template.visibility if visibility is None else visibility,
94	keywords=keywords or template.keywords,
95	template_id=template.id,
96	secrets_mount_directory=PurePosixPath(secrets_mount_directory) if secrets_mount_directory else None,
97	documentation=template.documentation,
98	)
99	project = await project_repo.insert_project(user, unsaved_project)	1✔
100
101	# NOTE: Copy session launchers
102	launchers = await session_repo.get_project_launchers(user=user, project_id=project_id)	1✔
103	for launcher in launchers:	1✔
104	await session_repo.copy_launcher(user=user, project_id=project.id, launcher=launcher)	1✔
105
106	# NOTE: Copy data connector links. If this operation fails due to lack of permission, still proceed to create the
107	# copy but return an error code that reflects this
108	uncopied_dc_ids: list[ULID] = []	1✔
109	dc_links = await data_connector_repo.get_links_to(user=user, project_id=project_id)	1✔
110	for dc_link in dc_links:	1✔
111	try:	1✔
112	await data_connector_repo.copy_link(user=user, project_id=project.id, link=dc_link)	1✔
113	except errors.MissingResourceError:	×
114	uncopied_dc_ids.append(dc_link.data_connector_id)	×
115
116	if uncopied_dc_ids:	1✔
117	data_connectors_names_ids = await data_connector_repo.get_data_connectors_names_and_ids(user, uncopied_dc_ids)	×
118	dc_str = ", ".join([f"{name} ({id})" for name, id in data_connectors_names_ids])	×
119	if len(data_connectors_names_ids) == 1:	×
120	message = (	×
121	f"The project was copied but data connector with name '{dc_str}' was not able to be linked to "
122	"your copy of this project due to insufficient permissions. To make a copy that includes the data "
123	"connector, ask its owner to make it public."
124	)
125	else:
126	message = (	×
127	f"The project was copied but data connectors with names '[{dc_str}]' were not able to be linked to "
128	"your copy of this project due to insufficient permissions. To make a copy that includes the data "
129	"connectors, ask their owners to make them public."
130	)
131	raise errors.CopyDataConnectorsError(message=message)	×
132
133	return project	1✔
134
135
136	def validate_unsaved_session_secret_slot(	2✔
137	body: apispec.SessionSecretSlotPost,
138	) -> models.UnsavedSessionSecretSlot:
139	"""Validate an unsaved secret slot."""
140	_validate_session_launcher_secret_slot_filename(body.filename)	2✔
141	return models.UnsavedSessionSecretSlot(	2✔
142	project_id=ULID.from_str(body.project_id),
143	name=body.name,
144	description=body.description,
145	filename=body.filename,
146	)
147
148
149	def validate_session_secret_slot_patch(	2✔
150	body: apispec.SessionSecretSlotPatch,
151	) -> models.SessionSecretSlotPatch:
152	"""Validate the update to a secret slot."""
153	if body.filename is not None:	2✔
154	_validate_session_launcher_secret_slot_filename(body.filename)	2✔
155	return models.SessionSecretSlotPatch(	2✔
156	name=body.name,
157	description=body.description,
158	filename=body.filename,
159	)
160
161
162	def validate_session_secrets_patch(	2✔
163	body: apispec.SessionSecretPatchList,
164	) -> list[models.SessionSecretPatchExistingSecret \| models.SessionSecretPatchSecretValue]:
165	"""Validate the update to a session launcher's secrets."""
166	result: list[models.SessionSecretPatchExistingSecret \| models.SessionSecretPatchSecretValue] = []	2✔
167	seen_slot_ids: set[str] = set()	2✔
168	for item in body.root:	2✔
169	if item.secret_slot_id in seen_slot_ids:	1✔
170	raise errors.ValidationError(	×
171	message=f"Found duplicate secret_slot_id '{item.secret_slot_id}' in the list of secrets."
172	)
173	seen_slot_ids.add(item.secret_slot_id)	1✔
174
175	if isinstance(item, apispec.SessionSecretPatch2):	1✔
176	result.append(	1✔
177	models.SessionSecretPatchExistingSecret(
178	secret_slot_id=ULID.from_str(item.secret_slot_id),
179	secret_id=ULID.from_str(item.secret_id),
180	)
181	)
182	else:
183	result.append(	1✔
184	models.SessionSecretPatchSecretValue(
185	secret_slot_id=ULID.from_str(item.secret_slot_id),
186	value=item.value,
187	)
188	)
189	return result	2✔
190
191
192	def _validate_repositories(repositories: list[str] \| None) -> list[str] \| None:	2✔
193	"""Validate a list of git repositories."""
194	if repositories is None:	2✔
195	return None	2✔
196	seen: set[str] = set()	2✔
197	without_duplicates: list[str] = []	2✔
198	for repo in repositories:	2✔
199	repo = _validate_repository(repo)	2✔
200	if repo not in seen:	2✔
201	without_duplicates.append(repo)	2✔
202	seen.add(repo)	2✔
203	return without_duplicates	2✔
204
205
206	def _validate_repository(repository: str) -> str:	2✔
207	"""Validate a git repository."""
208	stripped = repository.strip()	2✔
209	parsed = urlparse(stripped)	2✔
210	if parsed.scheme not in ["http", "https"]:	2✔
211	raise errors.ValidationError(message=f'The repository URL "{repository}" is not a valid HTTP or HTTPS URL.')	2✔
212	return stripped	2✔
213
214
215	def _validate_session_launcher_secret_slot_filename(filename: str) -> None:	2✔
216	"""Validate the filename field of a secret slot."""
217	filename_candidate = PurePosixPath(filename)	2✔
218	if filename_candidate.name != filename:	2✔
219	raise errors.ValidationError(message=f"Filename {filename} is not valid.")	×
220
221
222	async def get_v1_project_info(	2✔
223	user: APIUser,
224	internal_gitlab_user: APIUser,
225	project_path: str,
226	gitlab_client: GitlabAPIProtocol,
227	core_svc_url: str,
228	) -> dict[str, str \| list[str] \| int \| None]:
229	"""Request project information from the core service for a Renku v1 project."""
NEW 230	url = await gitlab_client.get_project_url_from_path(internal_gitlab_user, project_path)	×
NEW 231	if not url:	×
NEW 232	raise errors.MissingResourceError(	×
233	message=f"The Renku v1 project with path {project_path} cannot be found "
234	"in Gitlab or you do not have access to it"
235	)
236
NEW 237	body = {"git_url": url, "is_delayed": False, "migrate_project": False}	×
NEW 238	headers = {}	×
NEW 239	if user.access_token:	×
NEW 240	headers["Authorization"] = user.access_token	×
NEW 241	full_url = urljoin(core_svc_url + "/", "project.show")	×
NEW 242	async with httpx.AsyncClient() as clnt:	×
NEW 243	res = await clnt.post(full_url, json=body, headers=headers)	×
NEW 244	if res.status_code != 200:	×
NEW 245	raise errors.MissingResourceError(	×
246	message=f"The core service responded with an unexpected code {res.status_code} when getting "
247	f"information about project {project_path} and url {url}"
248	)
NEW 249	res_json = cast(dict[str, dict[str, str \| int \| list[str]]], res.json())	×
NEW 250	if res_json.get("error") is not None:	×
NEW 251	raise errors.MissingResourceError(	×
252	message=f"The core service responded with an error when getting "
253	f"information about project {project_path} and url {url}",
254	detail=cast(str \| None, res_json.get("error", {}).get("userMessage")),
255	)
256
NEW 257	kws = res_json.get("result", {}).get("keywords")	×
NEW 258	desc = res_json.get("result", {}).get("description")	×
NEW 259	id = res_json.get("result", {}).get("id")	×
NEW 260	name = res_json.get("result", {}).get("name")	×
NEW 261	output = {	×
262	"name": name,
263	"id": id,
264	"keywords": kws,
265	"description": desc,
266	}
NEW 267	return output	×

SwissDataScienceCenter / renku-data-services / 14377770991

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous