5388199646

Committed 27 Jun 2023 09:35AM UTC coverage: 88.917% (+0.4%) from 88.567%

Build # 5388199646

Build Type

push

gihub-action

Committed by

web-flow

Commit Message

chore: changes for Postgres and Helm (#9)

Co-authored-by: Johann-Michael Thiebaut <johann.thiebaut@gmail.com>
Co-authored-by: Alessandro Degano <40891147+aledegano@users.noreply.github.com>
Co-authored-by: Ralf Grubenmann <ralf.grubenmann@protonmail.com>

Run Details

654 of 654 new or added lines in 15 files covered. (100.0%)

1428 of 1606 relevant lines covered (88.92%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.38

/src/k8s/quota.py

"""The adapter used to create/delete/update/get resource quotas and priority classes in k8s."""
from dataclasses import dataclass, field
from typing import List, Optional

from kubernetes import client
from pydantic import ByteSize

import models
from k8s.client_interfaces import K8sCoreClientInterface, K8sSchedudlingClientInterface
from models import errors


@dataclass
class QuotaRepository:
    """Adapter for CRUD operations on resource quotas and prioirty classes in k8s."""

    core_client: K8sCoreClientInterface
    scheduling_client: K8sSchedudlingClientInterface
    namespace: str = "default"
    _label_name: str = field(init=False, default="app")
    _label_value: str = field(init=False, default="renku")

    def _quota_from_manifest(self, manifest: client.V1ResourceQuota) -> models.Quota:
        gpu = 0
        gpu_kind = models.GpuKind.NVIDIA
        for igpu_kind in models.GpuKind:
            key = f"requests.{igpu_kind}/gpu"
            if key in manifest.spec.hard:
                gpu = int(manifest.spec.hard.get(key))
                gpu_kind = igpu_kind
                break
        memory_raw = manifest.spec.hard.get("requests.memory")
        if memory_raw[-1] == "i":
            memory_raw += "b"
        return models.Quota(
            cpu=float(manifest.spec.hard.get("requests.cpu")),
            memory=round(ByteSize.validate(memory_raw).to("G")),
            gpu=gpu,
            gpu_kind=gpu_kind,
            id=manifest.metadata.name,
        )

    def _quota_to_manifest(self, quota: models.Quota) -> client.V1ResourceQuota:
        if quota.id is None:
            raise errors.ValidationError(message="The id of a quota has to be set when it is created.")
        return client.V1ResourceQuota(
            metadata=client.V1ObjectMeta(labels={self._label_name: self._label_value}, name=quota.id),
            spec=client.V1ResourceQuotaSpec(
                hard={
                    "requests.cpu": quota.cpu,
                    "requests.memory": str(quota.memory * 1_000_000_000),
                    f"requests.{quota.gpu_kind}/gpu": quota.gpu,
                },
                scope_selector=client.V1ScopeSelector(
                    match_expressions=[{"operator": "In", "scopeName": "PriorityClass", "values": [quota.id]}]
                ),
            ),
        )

    def _get_quota(self, name: str) -> Optional[models.Quota]:
        try:
            res_quota: client.V1ResourceQuota = self.core_client.read_namespaced_resource_quota(
                name=name, namespace=self.namespace
            )
        except client.ApiException as e:
            if e.status == 404:
                return None
            raise
        return self._quota_from_manifest(res_quota)

    def get_quotas(self, name: Optional[str] = None) -> List[models.Quota]:
        """Get a specific resource quota."""
        if name is not None:
            quota = self._get_quota(name)
            return [quota] if quota is not None else []
        quotas = self.core_client.list_namespaced_resource_quota(
            namespace=self.namespace, label_selector=f"{self._label_name}={self._label_value}"
        )
        return [self._quota_from_manifest(q) for q in quotas.items]

    def create_quota(self, quota: models.Quota):
        """Create a resource quota and priority class."""
        metadata = {"labels": {self._label_name: self._label_value}, "name": quota.id}
        quota_manifest = self._quota_to_manifest(quota)
        pc: client.V1PriorityClass = self.scheduling_client.create_priority_class(
            client.V1PriorityClass(
                global_default=False,
                value=100,
                preemption_policy="Never",
                description="Renku resource quota prioirty class",
                metadata=client.V1ObjectMeta(**metadata),
            ),
        )
        quota_manifest.owner_references = [
            client.V1OwnerReference(
                api_version=pc.api_version,
                block_owner_deletion=True,
                controller=False,
                kind=pc.kind,
                name=pc.metadata.name,
                uid=pc.metadata.uid,
            )
        ]
        self.core_client.create_namespaced_resource_quota(self.namespace, quota_manifest)

    def delete_quota(self, name: str):
        """Delete a resource quota and priority class."""
        self.scheduling_client.delete_priority_class(
            name=name, body=client.V1DeleteOptions(propagation_policy="Foreground")
        )
        try:
            self.core_client.delete_namespaced_resource_quota(name=name, namespace=self.namespace)
        except client.ApiException as e:
            if e.status == 404:
                # NOTE: The priorityclass is an owner of the resource quota so when the priority class is delete the
                # resource class is also deleted.
                pass
            raise

    def update_quota(self, quota: models.Quota):
        """Update a specific resource quota."""
        quota_manifest = self._quota_to_manifest(quota)
        self.core_client.patch_namespaced_resource_quota(name=quota.id, namespace=self.namespace, body=quota_manifest)

1	"""The adapter used to create/delete/update/get resource quotas and priority classes in k8s."""	1✔
2	from dataclasses import dataclass, field	1✔
3	from typing import List, Optional	1✔
4
5	from kubernetes import client	1✔
6	from pydantic import ByteSize	1✔
7
8	import models	1✔
9	from k8s.client_interfaces import K8sCoreClientInterface, K8sSchedudlingClientInterface	1✔
10	from models import errors	1✔
11
12
13	@dataclass	1✔
14	class QuotaRepository:	1✔
15	"""Adapter for CRUD operations on resource quotas and prioirty classes in k8s."""	1✔
16
17	core_client: K8sCoreClientInterface	1✔
18	scheduling_client: K8sSchedudlingClientInterface	1✔
19	namespace: str = "default"	1✔
20	_label_name: str = field(init=False, default="app")	1✔
21	_label_value: str = field(init=False, default="renku")	1✔
22
23	def _quota_from_manifest(self, manifest: client.V1ResourceQuota) -> models.Quota:	1✔
24	gpu = 0	1✔
25	gpu_kind = models.GpuKind.NVIDIA	1✔
26	for igpu_kind in models.GpuKind:	1✔
27	key = f"requests.{igpu_kind}/gpu"	1✔
28	if key in manifest.spec.hard:	1✔
29	gpu = int(manifest.spec.hard.get(key))	1✔
30	gpu_kind = igpu_kind	1✔
31	break	1✔
32	memory_raw = manifest.spec.hard.get("requests.memory")	1✔
33	if memory_raw[-1] == "i":	1✔
34	memory_raw += "b"	×
35	return models.Quota(	1✔
36	cpu=float(manifest.spec.hard.get("requests.cpu")),
37	memory=round(ByteSize.validate(memory_raw).to("G")),
38	gpu=gpu,
39	gpu_kind=gpu_kind,
40	id=manifest.metadata.name,
41	)
42
43	def _quota_to_manifest(self, quota: models.Quota) -> client.V1ResourceQuota:	1✔
44	if quota.id is None:	1✔
45	raise errors.ValidationError(message="The id of a quota has to be set when it is created.")	×
46	return client.V1ResourceQuota(	1✔
47	metadata=client.V1ObjectMeta(labels={self._label_name: self._label_value}, name=quota.id),
48	spec=client.V1ResourceQuotaSpec(
49	hard={
50	"requests.cpu": quota.cpu,
51	"requests.memory": str(quota.memory * 1_000_000_000),
52	f"requests.{quota.gpu_kind}/gpu": quota.gpu,
53	},
54	scope_selector=client.V1ScopeSelector(
55	match_expressions=[{"operator": "In", "scopeName": "PriorityClass", "values": [quota.id]}]
56	),
57	),
58	)
59
60	def _get_quota(self, name: str) -> Optional[models.Quota]:	1✔
61	try:	1✔
62	res_quota: client.V1ResourceQuota = self.core_client.read_namespaced_resource_quota(	1✔
63	name=name, namespace=self.namespace
64	)
65	except client.ApiException as e:	×
66	if e.status == 404:	×
67	return None	×
68	raise	×
69	return self._quota_from_manifest(res_quota)	1✔
70
71	def get_quotas(self, name: Optional[str] = None) -> List[models.Quota]:	1✔
72	"""Get a specific resource quota."""
73	if name is not None:	1✔
74	quota = self._get_quota(name)	1✔
75	return [quota] if quota is not None else []	1✔
76	quotas = self.core_client.list_namespaced_resource_quota(	1✔
77	namespace=self.namespace, label_selector=f"{self._label_name}={self._label_value}"
78	)
79	return [self._quota_from_manifest(q) for q in quotas.items]	1✔
80
81	def create_quota(self, quota: models.Quota):	1✔
82	"""Create a resource quota and priority class."""
83	metadata = {"labels": {self._label_name: self._label_value}, "name": quota.id}	1✔
84	quota_manifest = self._quota_to_manifest(quota)	1✔
85	pc: client.V1PriorityClass = self.scheduling_client.create_priority_class(	1✔
86	client.V1PriorityClass(
87	global_default=False,
88	value=100,
89	preemption_policy="Never",
90	description="Renku resource quota prioirty class",
91	metadata=client.V1ObjectMeta(**metadata),
92	),
93	)
94	quota_manifest.owner_references = [	1✔
95	client.V1OwnerReference(
96	api_version=pc.api_version,
97	block_owner_deletion=True,
98	controller=False,
99	kind=pc.kind,
100	name=pc.metadata.name,
101	uid=pc.metadata.uid,
102	)
103	]
104	self.core_client.create_namespaced_resource_quota(self.namespace, quota_manifest)	1✔
105
106	def delete_quota(self, name: str):	1✔
107	"""Delete a resource quota and priority class."""
108	self.scheduling_client.delete_priority_class(	1✔
109	name=name, body=client.V1DeleteOptions(propagation_policy="Foreground")
110	)
111	try:	1✔
112	self.core_client.delete_namespaced_resource_quota(name=name, namespace=self.namespace)	1✔
113	except client.ApiException as e:	×
114	if e.status == 404:	×
115	# NOTE: The priorityclass is an owner of the resource quota so when the priority class is delete the
116	# resource class is also deleted.
117	pass	×
118	raise	×
119
120	def update_quota(self, quota: models.Quota):	1✔
121	"""Update a specific resource quota."""
122	quota_manifest = self._quota_to_manifest(quota)	1✔
123	self.core_client.patch_namespaced_resource_quota(name=quota.id, namespace=self.namespace, body=quota_manifest)	1✔

SwissDataScienceCenter / renku-data-services / 5388199646

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous