5388199646

Committed 27 Jun 2023 09:35AM UTC coverage: 88.917% (+0.4%) from 88.567%

Build # 5388199646

Build Type

push

gihub-action

Committed by

web-flow

Commit Message

chore: changes for Postgres and Helm (#9)

Co-authored-by: Johann-Michael Thiebaut <johann.thiebaut@gmail.com>
Co-authored-by: Alessandro Degano <40891147+aledegano@users.noreply.github.com>
Co-authored-by: Ralf Grubenmann <ralf.grubenmann@protonmail.com>

Run Details

654 of 654 new or added lines in 15 files covered. (100.0%)

1428 of 1606 relevant lines covered (88.92%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.83

/src/renku_crc/server_options.py

"""Validation and parsing for the notebook service (old) style of server options and defaults.

The purpose of this is to be able to create resource
pools and classes based on the old server options until the admin UI interface
is added.
"""
from typing import Any, List, Set

from pydantic import BaseModel, ByteSize, Extra, Field, validator

import models


def _check_greater_than_zero(cls: Any, v: int | float) -> int | float:
    if v <= 0:
        raise ValueError(f"The provided value should be greater than zero, instead it was {v}.")
    return v


class ServerOptionsDefaults(BaseModel):
    """Used to parse the server option defaults passed to the notebook service in the Helm values."""

    cpu_request: float = Field(gt=0)
    mem_request: ByteSize
    disk_request: ByteSize
    gpu_request: int = Field(ge=0, default=0)

    class Config:
        """Configuration."""

        extra = Extra.ignore


class _ServerOptionsCpu(BaseModel):
    options: List[float] = Field(min_items=1)

    class Config:
        extra = Extra.ignore

    @validator("options", pre=False, each_item=True)
    def greater_than_zero(cls, val):
        return _check_greater_than_zero(cls, val)


class _ServerOptionsGpu(BaseModel):
    options: List[int] = Field(min_items=1)

    class Config:
        extra = Extra.ignore

    @validator("options", pre=False, each_item=True)
    def greater_than_or_equal_to_zero(cls, v):
        if v < 0:
            raise ValueError(f"The provided value should be greater than or equal to zero, instead it was {v}.")
        return v


class _ServerOptionsBytes(BaseModel):
    options: List[ByteSize] = Field(min_items=1)

    class Config:
        extra = Extra.ignore

    @validator("options", pre=True)
    def convert_units(cls, vals):
        for ival, val in enumerate(vals):
            if isinstance(val, str) and val.strip().endswith("i"):
                vals[ival] = val.strip() + "b"
        return vals

    @validator("options", pre=False, each_item=True)
    def greater_than_zero(cls, val):
        return _check_greater_than_zero(cls, val)


class ServerOptions(BaseModel):
    """Used to parse the server options passed to the notebook service in the Helm values."""

    cpu_request: _ServerOptionsCpu
    mem_request: _ServerOptionsBytes
    disk_request: _ServerOptionsBytes
    gpu_request: _ServerOptionsGpu = Field(default_factory=lambda: _ServerOptionsGpu(options=[0]))

    class Config:
        """Configuration."""

        extra = Extra.ignore

    def find_largest_attribute(self) -> str:
        """Find the attribute with the largest number of choices."""
        options = ((k, v.get("options", [])) for k, v in self.dict().items() if k != "disk_request")
        largest_list = max(options, key=lambda t: len(t[1]))[0]
        return largest_list


def _get_classname():
    yield "small"
    yield "medium"
    yield "large"
    count = 1
    while True:
        yield "x" * count + "large"
        count += 1


def generate_default_resource_pool(
    server_options: ServerOptions, defaults: ServerOptionsDefaults
) -> models.ResourcePool:
    """Generate a resource pool from the notebook service style server options."""
    clses: Set[models.ResourceClass] = set()
    largest_attribute = server_options.find_largest_attribute()
    options_xref = {
        "cpu_request": "cpu",
        "mem_request": "memory",
        "gpu_request": "gpu",
    }
    class_names = _get_classname()
    largest_attribute_options = getattr(getattr(server_options, largest_attribute), "options")
    max_storage = round(max(server_options.disk_request.options) / 1_000_000_000)
    for ival, val in enumerate(sorted(largest_attribute_options)):
        cls = {}
        for old_name, new_name in options_xref.items():
            if largest_attribute == old_name:
                cls[new_name] = val
            else:
                options = getattr(getattr(server_options, old_name), "options")
                try:
                    cls[new_name] = options[ival]
                except IndexError:
                    cls[new_name] = options[-1]
            if new_name == "memory":
                cls[new_name] = round(cls[new_name] / 1_000_000_000)
        cls["name"] = next(class_names)
        cls["max_storage"] = max_storage
        cls["default_storage"] = round(defaults.disk_request / 1_000_000_000)
        clses.add(models.ResourceClass.from_dict(cls))
    clses.add(
        models.ResourceClass(
            cpu=defaults.cpu_request,
            memory=round(defaults.mem_request / 1_000_000_000),
            gpu=defaults.gpu_request,
            name="default",
            default=True,
            max_storage=max_storage,
            default_storage=round(defaults.disk_request / 1_000_000_000),
        )
    )
    return models.ResourcePool(
        classes=clses,
        default=True,
        public=True,
        name="default",
    )

1	"""Validation and parsing for the notebook service (old) style of server options and defaults.	1✔
2
3	The purpose of this is to be able to create resource
4	pools and classes based on the old server options until the admin UI interface
5	is added.
6	"""
7	from typing import Any, List, Set	1✔
8
9	from pydantic import BaseModel, ByteSize, Extra, Field, validator	1✔
10
11	import models	1✔
12
13
14	def _check_greater_than_zero(cls: Any, v: int \| float) -> int \| float:	1✔
15	if v <= 0:	1✔
16	raise ValueError(f"The provided value should be greater than zero, instead it was {v}.")	×
17	return v	1✔
18
19
20	class ServerOptionsDefaults(BaseModel):	1✔
21	"""Used to parse the server option defaults passed to the notebook service in the Helm values."""	1✔
22
23	cpu_request: float = Field(gt=0)	1✔
24	mem_request: ByteSize	1✔
25	disk_request: ByteSize	1✔
26	gpu_request: int = Field(ge=0, default=0)	1✔
27
28	class Config:	1✔
29	"""Configuration."""	1✔
30
31	extra = Extra.ignore	1✔
32
33
34	class _ServerOptionsCpu(BaseModel):	1✔
35	options: List[float] = Field(min_items=1)	1✔
36
37	class Config:	1✔
38	extra = Extra.ignore	1✔
39
40	@validator("options", pre=False, each_item=True)	1✔
41	def greater_than_zero(cls, val):	1✔
42	return _check_greater_than_zero(cls, val)	1✔
43
44
45	class _ServerOptionsGpu(BaseModel):	1✔
46	options: List[int] = Field(min_items=1)	1✔
47
48	class Config:	1✔
49	extra = Extra.ignore	1✔
50
51	@validator("options", pre=False, each_item=True)	1✔
52	def greater_than_or_equal_to_zero(cls, v):	1✔
53	if v < 0:	1✔
54	raise ValueError(f"The provided value should be greater than or equal to zero, instead it was {v}.")	×
55	return v	1✔
56
57
58	class _ServerOptionsBytes(BaseModel):	1✔
59	options: List[ByteSize] = Field(min_items=1)	1✔
60
61	class Config:	1✔
62	extra = Extra.ignore	1✔
63
64	@validator("options", pre=True)	1✔
65	def convert_units(cls, vals):	1✔
66	for ival, val in enumerate(vals):	1✔
67	if isinstance(val, str) and val.strip().endswith("i"):	1✔
68	vals[ival] = val.strip() + "b"	1✔
69	return vals	1✔
70
71	@validator("options", pre=False, each_item=True)	1✔
72	def greater_than_zero(cls, val):	1✔
73	return _check_greater_than_zero(cls, val)	1✔
74
75
76	class ServerOptions(BaseModel):	1✔
77	"""Used to parse the server options passed to the notebook service in the Helm values."""	1✔
78
79	cpu_request: _ServerOptionsCpu	1✔
80	mem_request: _ServerOptionsBytes	1✔
81	disk_request: _ServerOptionsBytes	1✔
82	gpu_request: _ServerOptionsGpu = Field(default_factory=lambda: _ServerOptionsGpu(options=[0]))	1✔
83
84	class Config:	1✔
85	"""Configuration."""	1✔
86
87	extra = Extra.ignore	1✔
88
89	def find_largest_attribute(self) -> str:	1✔
90	"""Find the attribute with the largest number of choices."""
91	options = ((k, v.get("options", [])) for k, v in self.dict().items() if k != "disk_request")	1✔
92	largest_list = max(options, key=lambda t: len(t[1]))[0]	1✔
93	return largest_list	1✔
94
95
96	def _get_classname():	1✔
97	yield "small"	1✔
98	yield "medium"	1✔
99	yield "large"	1✔
100	count = 1	1✔
101	while True:	1✔
102	yield "x" * count + "large"	1✔
103	count += 1	1✔
104
105
106	def generate_default_resource_pool(	1✔
107	server_options: ServerOptions, defaults: ServerOptionsDefaults
108	) -> models.ResourcePool:
109	"""Generate a resource pool from the notebook service style server options."""
110	clses: Set[models.ResourceClass] = set()	1✔
111	largest_attribute = server_options.find_largest_attribute()	1✔
112	options_xref = {	1✔
113	"cpu_request": "cpu",
114	"mem_request": "memory",
115	"gpu_request": "gpu",
116	}
117	class_names = _get_classname()	1✔
118	largest_attribute_options = getattr(getattr(server_options, largest_attribute), "options")	1✔
119	max_storage = round(max(server_options.disk_request.options) / 1_000_000_000)	1✔
120	for ival, val in enumerate(sorted(largest_attribute_options)):	1✔
121	cls = {}	1✔
122	for old_name, new_name in options_xref.items():	1✔
123	if largest_attribute == old_name:	1✔
124	cls[new_name] = val	1✔
125	else:
126	options = getattr(getattr(server_options, old_name), "options")	1✔
127	try:	1✔
128	cls[new_name] = options[ival]	1✔
129	except IndexError:	1✔
130	cls[new_name] = options[-1]	1✔
131	if new_name == "memory":	1✔
132	cls[new_name] = round(cls[new_name] / 1_000_000_000)	1✔
133	cls["name"] = next(class_names)	1✔
134	cls["max_storage"] = max_storage	1✔
135	cls["default_storage"] = round(defaults.disk_request / 1_000_000_000)	1✔
136	clses.add(models.ResourceClass.from_dict(cls))	1✔
137	clses.add(	1✔
138	models.ResourceClass(
139	cpu=defaults.cpu_request,
140	memory=round(defaults.mem_request / 1_000_000_000),
141	gpu=defaults.gpu_request,
142	name="default",
143	default=True,
144	max_storage=max_storage,
145	default_storage=round(defaults.disk_request / 1_000_000_000),
146	)
147	)
148	return models.ResourcePool(	1✔
149	classes=clses,
150	default=True,
151	public=True,
152	name="default",
153	)

SwissDataScienceCenter / renku-data-services / 5388199646

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous