5388199646

Committed 27 Jun 2023 09:35AM UTC coverage: 88.917% (+0.4%) from 88.567%

Build # 5388199646

Build Type

push

gihub-action

Committed by

web-flow

Commit Message

chore: changes for Postgres and Helm (#9)

Co-authored-by: Johann-Michael Thiebaut <johann.thiebaut@gmail.com>
Co-authored-by: Alessandro Degano <40891147+aledegano@users.noreply.github.com>
Co-authored-by: Ralf Grubenmann <ralf.grubenmann@protonmail.com>

Run Details

654 of 654 new or added lines in 15 files covered. (100.0%)

1428 of 1606 relevant lines covered (88.92%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.62

/src/renku_crc/config.py

"""Configurations."""
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Optional

import httpx
from jwt import PyJWKClient
from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed
from yaml import safe_load

import models
from db.adapter import ResourcePoolRepository, UserRepository
from k8s.clients import DummyCoreClient, DummySchedulingClient, K8sCoreClient, K8sSchedulingClient
from k8s.quota import QuotaRepository
from models import errors
from renku_crc.server_options import ServerOptions, ServerOptionsDefaults, generate_default_resource_pool
from users.credentials import KeycloakAuthenticator
from users.dummy import DummyAuthenticator, DummyUserStore
from users.keycloak import KcUserStore


@retry(stop=(stop_after_attempt(20) | stop_after_delay(300)), wait=wait_fixed(2), reraise=True)
def _oidc_discovery(url: str, realm: str) -> Dict[str, Any]:
    url = f"{url}/realms/{realm}/.well-known/openid-configuration"
    res = httpx.get(url)
    if res.status_code == 200:
        return res.json()
    raise errors.ConfigurationError(message=f"Cannot successfully do OIDC discovery with url {url}.")


default_resource_pool = models.ResourcePool(
    name="default",
    classes=set(
        [
            models.ResourceClass(
                name="small",
                cpu=0.5,
                memory=1,
                max_storage=20,
                gpu=0,
                default=True,
            ),
            models.ResourceClass(
                name="large",
                cpu=1.0,
                memory=2,
                max_storage=20,
                gpu=0,
                default=False,
            ),
        ]
    ),
    quota=None,
    public=True,
    default=True,
)


@dataclass
class Config:
    """Configuration for the CRC service."""

    user_repo: UserRepository
    rp_repo: ResourcePoolRepository
    user_store: models.UserStore
    authenticator: models.Authenticator
    quota_repo: QuotaRepository
    spec: Dict[str, Any] = field(init=False, default_factory=dict)
    version: str = "0.0.1"
    app_name: str = "renku_crc"
    default_resource_pool_file: Optional[str] = None
    default_resource_pool: models.ResourcePool = default_resource_pool
    server_options_file: Optional[str] = None
    server_defaults_file: Optional[str] = None

    def __post_init__(self):
        spec_file = Path(__file__).resolve().parent / "api.spec.yaml"
        with open(spec_file, "r") as f:
            self.spec = safe_load(f)
        if self.default_resource_pool_file is not None:
            with open(self.default_resource_pool_file, "r") as f:
                self.default_resource_pool = models.ResourcePool.from_dict(safe_load(f))
        if self.server_defaults_file is not None and self.server_options_file is not None:
            with open(self.server_options_file, "r") as f:
                options = ServerOptions.parse_obj(safe_load(f))
            with open(self.server_defaults_file, "r") as f:
                defaults = ServerOptionsDefaults.parse_obj(safe_load(f))
            self.default_resource_pool = generate_default_resource_pool(options, defaults)

    @classmethod
    def from_env(cls):
        """Create a config from environment variables."""

        prefix = ""
        user_store: models.UserStore
        authenticator: models.Authenticator
        version = os.environ.get(f"{prefix}VERSION", "0.0.1")
        keycloak_url = None
        keycloak_realm = "Renku"
        server_options_file = os.environ.get("SERVER_OPTIONS")
        server_defaults_file = os.environ.get("SERVER_DEFAULTS")
        k8s_namespace = os.environ.get("K8S_NAMESPACE", "default")

        if os.environ.get(f"{prefix}DUMMY_STORES", "false").lower() == "true":
            async_sqlalchemy_url = os.environ.get(
                f"{prefix}ASYNC_SQLALCHEMY_URL", "sqlite+aiosqlite:///data_services.db"
            )
            sync_sqlalchemy_url = os.environ.get(f"{prefix}SYNC_SQLALCHEMY_URL", "sqlite:///data_services.db")
            authenticator = DummyAuthenticator(admin=True)
            user_always_exists = os.environ.get("DUMMY_USERSTORE_USER_ALWAYS_EXISTS", "true").lower() == "true"
            user_store = DummyUserStore(user_always_exists=user_always_exists)
            quota_repo = QuotaRepository(DummyCoreClient({}), DummySchedulingClient({}), namespace=k8s_namespace)
        else:
            pg_host = os.environ.get("DB_HOST", "localhost")
            pg_user = os.environ.get("DB_USER", "renku")
            pg_port = os.environ.get("DB_PORT", "5432")
            db_name = os.environ.get("DB_NAME", "renku")
            pg_password = os.environ.get("DB_PASSWORD")
            if pg_password is None:
                raise errors.ConfigurationError(
                    message="Please provide a database password in the 'DB_PASSWORD' environment variable."
                )
            async_sqlalchemy_url = f"postgresql+asyncpg://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{db_name}"
            sync_sqlalchemy_url = f"postgresql+psycopg2://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{db_name}"
            keycloak_url = os.environ.get(f"{prefix}KEYCLOAK_URL")
            if keycloak_url is None:
                raise errors.ConfigurationError(message="The Keycloak URL has to be specified.")
            keycloak_url = keycloak_url.rstrip("/")
            keycloak_realm = os.environ.get(f"{prefix}KEYCLOAK_REALM", "Renku")
            oidc_disc_data = _oidc_discovery(keycloak_url, keycloak_realm)
            jwks_url = oidc_disc_data.get("jwks_uri")
            if jwks_url is None:
                raise errors.ConfigurationError(
                    message="The JWKS url for Keycloak cannot be found from the OIDC discovery endpoint."
                )
            algorithms = os.environ.get(f"{prefix}KEYCLOAK_TOKEN_SIGNATURE_ALGS")
            if algorithms is None:
                raise errors.ConfigurationError(message="At least one token signature algorithm is requried.")
            algorithms_lst = [i.strip() for i in algorithms.split(",")]
            jwks = PyJWKClient(jwks_url)
            authenticator = KeycloakAuthenticator(jwks=jwks, algorithms=algorithms_lst)
            user_store = KcUserStore(keycloak_url=keycloak_url, realm=keycloak_realm)
            quota_repo = QuotaRepository(K8sCoreClient(), K8sSchedulingClient(), namespace=k8s_namespace)

        user_repo = UserRepository(sync_sqlalchemy_url=sync_sqlalchemy_url, async_sqlalchemy_url=async_sqlalchemy_url)
        rp_repo = ResourcePoolRepository(
            sync_sqlalchemy_url=sync_sqlalchemy_url, async_sqlalchemy_url=async_sqlalchemy_url
        )
        return cls(
            user_repo=user_repo,
            rp_repo=rp_repo,
            version=version,
            authenticator=authenticator,
            user_store=user_store,
            quota_repo=quota_repo,
            server_defaults_file=server_defaults_file,
            server_options_file=server_options_file,
        )

1	"""Configurations."""	1✔
2	import os	1✔
3	from dataclasses import dataclass, field	1✔
4	from pathlib import Path	1✔
5	from typing import Any, Dict, Optional	1✔
6
7	import httpx	1✔
8	from jwt import PyJWKClient	1✔
9	from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed	1✔
10	from yaml import safe_load	1✔
11
12	import models	1✔
13	from db.adapter import ResourcePoolRepository, UserRepository	1✔
14	from k8s.clients import DummyCoreClient, DummySchedulingClient, K8sCoreClient, K8sSchedulingClient	1✔
15	from k8s.quota import QuotaRepository	1✔
16	from models import errors	1✔
17	from renku_crc.server_options import ServerOptions, ServerOptionsDefaults, generate_default_resource_pool	1✔
18	from users.credentials import KeycloakAuthenticator	1✔
19	from users.dummy import DummyAuthenticator, DummyUserStore	1✔
20	from users.keycloak import KcUserStore	1✔
21
22
23	@retry(stop=(stop_after_attempt(20) \| stop_after_delay(300)), wait=wait_fixed(2), reraise=True)	1✔
24	def _oidc_discovery(url: str, realm: str) -> Dict[str, Any]:	1✔
25	url = f"{url}/realms/{realm}/.well-known/openid-configuration"	×
26	res = httpx.get(url)	×
27	if res.status_code == 200:	×
28	return res.json()	×
29	raise errors.ConfigurationError(message=f"Cannot successfully do OIDC discovery with url {url}.")	×
30
31
32	default_resource_pool = models.ResourcePool(	1✔
33	name="default",
34	classes=set(
35	[
36	models.ResourceClass(
37	name="small",
38	cpu=0.5,
39	memory=1,
40	max_storage=20,
41	gpu=0,
42	default=True,
43	),
44	models.ResourceClass(
45	name="large",
46	cpu=1.0,
47	memory=2,
48	max_storage=20,
49	gpu=0,
50	default=False,
51	),
52	]
53	),
54	quota=None,
55	public=True,
56	default=True,
57	)
58
59
60	@dataclass	1✔
61	class Config:	1✔
62	"""Configuration for the CRC service."""	1✔
63
64	user_repo: UserRepository	1✔
65	rp_repo: ResourcePoolRepository	1✔
66	user_store: models.UserStore	1✔
67	authenticator: models.Authenticator	1✔
68	quota_repo: QuotaRepository	1✔
69	spec: Dict[str, Any] = field(init=False, default_factory=dict)	1✔
70	version: str = "0.0.1"	1✔
71	app_name: str = "renku_crc"	1✔
72	default_resource_pool_file: Optional[str] = None	1✔
73	default_resource_pool: models.ResourcePool = default_resource_pool	1✔
74	server_options_file: Optional[str] = None	1✔
75	server_defaults_file: Optional[str] = None	1✔
76
77	def __post_init__(self):	1✔
78	spec_file = Path(__file__).resolve().parent / "api.spec.yaml"	1✔
79	with open(spec_file, "r") as f:	1✔
80	self.spec = safe_load(f)	1✔
81	if self.default_resource_pool_file is not None:	1✔
82	with open(self.default_resource_pool_file, "r") as f:	×
83	self.default_resource_pool = models.ResourcePool.from_dict(safe_load(f))	×
84	if self.server_defaults_file is not None and self.server_options_file is not None:	1✔
85	with open(self.server_options_file, "r") as f:	×
86	options = ServerOptions.parse_obj(safe_load(f))	×
87	with open(self.server_defaults_file, "r") as f:	×
88	defaults = ServerOptionsDefaults.parse_obj(safe_load(f))	×
89	self.default_resource_pool = generate_default_resource_pool(options, defaults)	×
90
91	@classmethod	1✔
92	def from_env(cls):	1✔
93	"""Create a config from environment variables."""
94
95	prefix = ""	1✔
96	user_store: models.UserStore
97	authenticator: models.Authenticator
98	version = os.environ.get(f"{prefix}VERSION", "0.0.1")	1✔
99	keycloak_url = None	1✔
100	keycloak_realm = "Renku"	1✔
101	server_options_file = os.environ.get("SERVER_OPTIONS")	1✔
102	server_defaults_file = os.environ.get("SERVER_DEFAULTS")	1✔
103	k8s_namespace = os.environ.get("K8S_NAMESPACE", "default")	1✔
104
105	if os.environ.get(f"{prefix}DUMMY_STORES", "false").lower() == "true":	1✔
106	async_sqlalchemy_url = os.environ.get(	1✔
107	f"{prefix}ASYNC_SQLALCHEMY_URL", "sqlite+aiosqlite:///data_services.db"
108	)
109	sync_sqlalchemy_url = os.environ.get(f"{prefix}SYNC_SQLALCHEMY_URL", "sqlite:///data_services.db")	1✔
110	authenticator = DummyAuthenticator(admin=True)	1✔
111	user_always_exists = os.environ.get("DUMMY_USERSTORE_USER_ALWAYS_EXISTS", "true").lower() == "true"	1✔
112	user_store = DummyUserStore(user_always_exists=user_always_exists)	1✔
113	quota_repo = QuotaRepository(DummyCoreClient({}), DummySchedulingClient({}), namespace=k8s_namespace)	1✔
114	else:
115	pg_host = os.environ.get("DB_HOST", "localhost")	×
116	pg_user = os.environ.get("DB_USER", "renku")	×
117	pg_port = os.environ.get("DB_PORT", "5432")	×
118	db_name = os.environ.get("DB_NAME", "renku")	×
119	pg_password = os.environ.get("DB_PASSWORD")	×
120	if pg_password is None:	×
121	raise errors.ConfigurationError(	×
122	message="Please provide a database password in the 'DB_PASSWORD' environment variable."
123	)
124	async_sqlalchemy_url = f"postgresql+asyncpg://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{db_name}"	×
125	sync_sqlalchemy_url = f"postgresql+psycopg2://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{db_name}"	×
126	keycloak_url = os.environ.get(f"{prefix}KEYCLOAK_URL")	×
127	if keycloak_url is None:	×
128	raise errors.ConfigurationError(message="The Keycloak URL has to be specified.")	×
129	keycloak_url = keycloak_url.rstrip("/")	×
130	keycloak_realm = os.environ.get(f"{prefix}KEYCLOAK_REALM", "Renku")	×
131	oidc_disc_data = _oidc_discovery(keycloak_url, keycloak_realm)	×
132	jwks_url = oidc_disc_data.get("jwks_uri")	×
133	if jwks_url is None:	×
134	raise errors.ConfigurationError(	×
135	message="The JWKS url for Keycloak cannot be found from the OIDC discovery endpoint."
136	)
137	algorithms = os.environ.get(f"{prefix}KEYCLOAK_TOKEN_SIGNATURE_ALGS")	×
138	if algorithms is None:	×
139	raise errors.ConfigurationError(message="At least one token signature algorithm is requried.")	×
140	algorithms_lst = [i.strip() for i in algorithms.split(",")]	×
141	jwks = PyJWKClient(jwks_url)	×
142	authenticator = KeycloakAuthenticator(jwks=jwks, algorithms=algorithms_lst)	×
143	user_store = KcUserStore(keycloak_url=keycloak_url, realm=keycloak_realm)	×
144	quota_repo = QuotaRepository(K8sCoreClient(), K8sSchedulingClient(), namespace=k8s_namespace)	×
145
146	user_repo = UserRepository(sync_sqlalchemy_url=sync_sqlalchemy_url, async_sqlalchemy_url=async_sqlalchemy_url)	1✔
147	rp_repo = ResourcePoolRepository(	1✔
148	sync_sqlalchemy_url=sync_sqlalchemy_url, async_sqlalchemy_url=async_sqlalchemy_url
149	)
150	return cls(	1✔
151	user_repo=user_repo,
152	rp_repo=rp_repo,
153	version=version,
154	authenticator=authenticator,
155	user_store=user_store,
156	quota_repo=quota_repo,
157	server_defaults_file=server_defaults_file,
158	server_options_file=server_options_file,
159	)

SwissDataScienceCenter / renku-data-services / 5388199646

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous