11288123511

Committed 11 Oct 2024 07:23AM UTC coverage: 90.667% (+0.2%) from 90.477%

Build # 11288123511

Build Type

Pull #407

github

Committed by

web-flow

Commit Message

Merge 20c6c8af6 into 5b095d795

Pull Request Pull Request #407: feat!: add data connectors

Run Details

1226 of 1325 new or added lines in 28 files covered. (92.53%)

3 existing lines in 3 files now uncovered.

10589 of 11679 relevant lines covered (90.67%)

1.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.67

/components/renku_data_services/data_connectors/core.py

"""Business logic for data connectors."""

from dataclasses import asdict
from typing import Any

from renku_data_services import base_models, errors
from renku_data_services.authz.models import Visibility
from renku_data_services.data_connectors import apispec, models
from renku_data_services.storage import models as storage_models
from renku_data_services.storage.rclone import RCloneValidator


def dump_storage_with_sensitive_fields(
    storage: models.CloudStorageCore, validator: RCloneValidator
) -> models.CloudStorageCoreWithSensitiveFields:
    """Add sensitive fields to a storage configuration."""
    return models.CloudStorageCoreWithSensitiveFields(
        sensitive_fields=list(validator.get_private_fields(storage.configuration)), **asdict(storage)
    )


def validate_unsaved_storage(
    storage: apispec.CloudStorageCorePost | apispec.CloudStorageUrlV2, validator: RCloneValidator
) -> models.CloudStorageCore:
    """Validate the storage configuration of an unsaved data connector."""

    configuration: dict[str, Any]
    source_path: str

    if isinstance(storage, apispec.CloudStorageUrlV2):
        cloud_storage = storage_models.UnsavedCloudStorage.from_url(
            project_id="FAKEPROJECTID",
            name="fake-storage-name",
            storage_url=storage.storage_url,
            target_path=storage.target_path,
            readonly=storage.readonly,
        )
        configuration = cloud_storage.configuration.config
        source_path = cloud_storage.source_path
    else:
        configuration = storage.configuration
        source_path = storage.source_path

    validator.validate(configuration)

    return models.CloudStorageCore(
        storage_type=configuration["type"],
        configuration=configuration,
        source_path=source_path,
        target_path=storage.target_path,
        readonly=storage.readonly,
    )


def validate_unsaved_data_connector(
    body: apispec.DataConnectorPost, validator: RCloneValidator
) -> models.UnsavedDataConnector:
    """Validate an unsaved data connector."""

    keywords = [kw.root for kw in body.keywords] if body.keywords is not None else []
    storage = validate_unsaved_storage(body.storage, validator=validator)

    return models.UnsavedDataConnector(
        name=body.name,
        namespace=body.namespace,
        slug=body.slug or base_models.Slug.from_name(body.name).value,
        visibility=Visibility(body.visibility.value),
        created_by="",
        storage=storage,
        description=body.description,
        keywords=keywords,
    )


def validate_storage_patch(
    storage: models.CloudStorageCore, patch: apispec.CloudStorageCorePatch, validator: RCloneValidator
) -> models.CloudStorageCorePatch:
    """Validate the update to a data connector's storage."""

    if patch.configuration is not None:
        # we need to apply the patch to the existing storage to properly validate it
        patch.configuration = {**storage.configuration, **patch.configuration}
        dict_items = list(patch.configuration.items())
        for k, v in dict_items:
            if v is None:
                # delete fields that were unset
                del patch.configuration[k]
        validator.validate(patch.configuration)

    return models.CloudStorageCorePatch(
        storage_type=patch.storage_type,
        configuration=patch.configuration,
        source_path=patch.source_path,
        target_path=patch.target_path,
        readonly=patch.readonly,
    )


def validate_data_connector_patch(
    data_connector: models.DataConnector,
    patch: apispec.DataConnectorPatch,
    validator: RCloneValidator,
) -> models.DataConnectorPatch:
    """Validate the update to a data connector."""

    keywords = [kw.root for kw in patch.keywords] if patch.keywords is not None else None
    storage = (
        validate_storage_patch(data_connector.storage, patch.storage, validator=validator)
        if patch.storage is not None
        else None
    )

    return models.DataConnectorPatch(
        name=patch.name,
        namespace=patch.namespace,
        slug=patch.slug,
        visibility=Visibility(patch.visibility.value) if patch.visibility is not None else None,
        description=patch.description,
        keywords=keywords,
        storage=storage,
    )


def validate_data_connector_secrets_patch(
    put: apispec.DataConnectorSecretPatchList,
) -> list[models.DataConnectorSecretUpdate]:
    """Validate the update to a data connector's secrets."""
    seen_names: set[str] = set()
    for secret in put.root:
        if secret.name in seen_names:
            raise errors.ValidationError(message=f"Found duplicate name '{secret.name}' in the list of secrets.")
        seen_names.add(secret.name)

    return [
        models.DataConnectorSecretUpdate(
            name=secret.name,
            value=secret.value,
        )
        for secret in put.root
    ]

1	"""Business logic for data connectors."""	2✔
2
3	from dataclasses import asdict	2✔
4	from typing import Any	2✔
5
6	from renku_data_services import base_models, errors	2✔
7	from renku_data_services.authz.models import Visibility	2✔
8	from renku_data_services.data_connectors import apispec, models	2✔
9	from renku_data_services.storage import models as storage_models	2✔
10	from renku_data_services.storage.rclone import RCloneValidator	2✔
11
12
13	def dump_storage_with_sensitive_fields(	2✔
14	storage: models.CloudStorageCore, validator: RCloneValidator
15	) -> models.CloudStorageCoreWithSensitiveFields:
16	"""Add sensitive fields to a storage configuration."""
17	return models.CloudStorageCoreWithSensitiveFields(	1✔
18	sensitive_fields=list(validator.get_private_fields(storage.configuration)), **asdict(storage)
19	)
20
21
22	def validate_unsaved_storage(	2✔
23	storage: apispec.CloudStorageCorePost \| apispec.CloudStorageUrlV2, validator: RCloneValidator
24	) -> models.CloudStorageCore:
25	"""Validate the storage configuration of an unsaved data connector."""
26
27	configuration: dict[str, Any]
28	source_path: str
29
30	if isinstance(storage, apispec.CloudStorageUrlV2):	2✔
31	cloud_storage = storage_models.UnsavedCloudStorage.from_url(	1✔
32	project_id="FAKEPROJECTID",
33	name="fake-storage-name",
34	storage_url=storage.storage_url,
35	target_path=storage.target_path,
36	readonly=storage.readonly,
37	)
38	configuration = cloud_storage.configuration.config	1✔
39	source_path = cloud_storage.source_path	1✔
40	else:
41	configuration = storage.configuration	2✔
42	source_path = storage.source_path	2✔
43
44	validator.validate(configuration)	2✔
45
46	return models.CloudStorageCore(	1✔
47	storage_type=configuration["type"],
48	configuration=configuration,
49	source_path=source_path,
50	target_path=storage.target_path,
51	readonly=storage.readonly,
52	)
53
54
55	def validate_unsaved_data_connector(	2✔
56	body: apispec.DataConnectorPost, validator: RCloneValidator
57	) -> models.UnsavedDataConnector:
58	"""Validate an unsaved data connector."""
59
60	keywords = [kw.root for kw in body.keywords] if body.keywords is not None else []	2✔
61	storage = validate_unsaved_storage(body.storage, validator=validator)	2✔
62
63	return models.UnsavedDataConnector(	1✔
64	name=body.name,
65	namespace=body.namespace,
66	slug=body.slug or base_models.Slug.from_name(body.name).value,
67	visibility=Visibility(body.visibility.value),
68	created_by="",
69	storage=storage,
70	description=body.description,
71	keywords=keywords,
72	)
73
74
75	def validate_storage_patch(	2✔
76	storage: models.CloudStorageCore, patch: apispec.CloudStorageCorePatch, validator: RCloneValidator
77	) -> models.CloudStorageCorePatch:
78	"""Validate the update to a data connector's storage."""
79
80	if patch.configuration is not None:	1✔
81	# we need to apply the patch to the existing storage to properly validate it
82	patch.configuration = {storage.configuration, patch.configuration}	1✔
83	dict_items = list(patch.configuration.items())	1✔
84	for k, v in dict_items:	1✔
85	if v is None:	1✔
86	# delete fields that were unset
87	del patch.configuration[k]	1✔
88	validator.validate(patch.configuration)	1✔
89
90	return models.CloudStorageCorePatch(	1✔
91	storage_type=patch.storage_type,
92	configuration=patch.configuration,
93	source_path=patch.source_path,
94	target_path=patch.target_path,
95	readonly=patch.readonly,
96	)
97
98
99	def validate_data_connector_patch(	2✔
100	data_connector: models.DataConnector,
101	patch: apispec.DataConnectorPatch,
102	validator: RCloneValidator,
103	) -> models.DataConnectorPatch:
104	"""Validate the update to a data connector."""
105
106	keywords = [kw.root for kw in patch.keywords] if patch.keywords is not None else None	1✔
107	storage = (	1✔
108	validate_storage_patch(data_connector.storage, patch.storage, validator=validator)
109	if patch.storage is not None
110	else None
111	)
112
113	return models.DataConnectorPatch(	1✔
114	name=patch.name,
115	namespace=patch.namespace,
116	slug=patch.slug,
117	visibility=Visibility(patch.visibility.value) if patch.visibility is not None else None,
118	description=patch.description,
119	keywords=keywords,
120	storage=storage,
121	)
122
123
124	def validate_data_connector_secrets_patch(	2✔
125	put: apispec.DataConnectorSecretPatchList,
126	) -> list[models.DataConnectorSecretUpdate]:
127	"""Validate the update to a data connector's secrets."""
128	seen_names: set[str] = set()	2✔
129	for secret in put.root:	2✔
130	if secret.name in seen_names:	2✔
NEW 131	raise errors.ValidationError(message=f"Found duplicate name '{secret.name}' in the list of secrets.")	×
132	seen_names.add(secret.name)	2✔
133
134	return [	2✔
135	models.DataConnectorSecretUpdate(
136	name=secret.name,
137	value=secret.value,
138	)
139	for secret in put.root
140	]

SwissDataScienceCenter / renku-data-services / 11288123511

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous