reanahub / reana-commons / 816

Committed 11 Jun 2020 - 14:22 coverage: 41.784% (-0.4%) from 42.138%

Build # 816

Build Type

Pull #192

travis-ci

Committed by

web-flow

Commit Message

Merge d358ecc0d into f79147a06

Pull Request Pull Request #192: config: centrally configured k8s namespace

Run Details

0 of 8 new or added lines in 1 file covered. (0.0%)

1 existing line in 1 file now uncovered.

267 of 639 relevant lines covered (41.78%)

1.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/reana_commons/config.py

# -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2018 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""REANA Commons configuration."""

import logging
import os

REANA_COMPONENT_PREFIX = os.getenv("REANA_COMPONENT_PREFIX", "reana")
"""REANA component naming prefix, i.e. my-prefix-job-controller.

Useful to find the correct fully qualified name of a infrastructure component
and to correctly create new runtime pods.
"""

REANA_COMPONENT_PREFIX_ENVIRONMENT = REANA_COMPONENT_PREFIX.upper().replace("-", "_")
"""Environment variable friendly REANA component prefix."""

REANA_COMPONENT_TYPES = [
    "run-batch",
    "run-session",
    "run-job",
    "secretsstore",
]
"""Type of REANA components.

Note: this list is used for validation of on demand created REANA components
names, this is why it doesn't contain REANA instrastructure components.

``run-batch``: An instance of reana-workflow-engine-_
``run-session``: An instance of an interactive session
``run-job``: An instance of a workflow's job
``secretsstore``: An instance of a user secret store
"""

REANA_INFRASTRUCTURE_COMPONENTS = [
    "ui",
    "server",
    "workflow-controller",
    "cache",
    "message-broker",
    "db",
]
"""REANA infrastructure pods."""

REANA_COMPONENT_NAMING_SCHEME = os.getenv(
    "REANA_COMPONENT_NAMING_SCHEME", "{prefix}-{component_type}-{id}"
)
"""The naming scheme the components created by REANA should follow.

It is a Python format string which take as arguments:
- ``prefix``: the ``REANA_COMPONENT_PREFIX``
- ``component_type``: one of ``REANA_COMPONENT_TYPES``
- ``id``: unique identifier for the component, by default UUID4.
"""

REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE = os.getenv(
    "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "default"
)
"""Kubernetes namespace in which REANA instrastructure is currently deployed."""

REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES = {
    component_name: (
        "{component_prefix}-{component_name}.{namespace}.svc.cluster.local"
    ).format(
        component_prefix=REANA_COMPONENT_PREFIX,
        component_name=component_name,
        namespace=REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE,
    )
    for component_name in REANA_INFRASTRUCTURE_COMPONENTS
}
"""REANA infrastructure pods hostnames.

Uses the FQDN of the infrastructure components (which should be behind a Kubernetes
service) following the
`Kubernetes DNS-Based Service Discovery <https://github.com/kubernetes/dns/blob/master/docs/specification.md>`_
"""

REANA_RUNTIME_KUBERNETES_NAMESPACE = os.getenv(
    "REANA_RUNTIME_KUBERNETES_NAMESPACE", REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE
)
"""Kubernetes namespace in which REANA runtime pods should be running in.

By default runtime pods will run in the same namespace as the instrastructure pods.
"""

MQ_HOST = REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["message-broker"]
"""Message queue (RabbitMQ) server host name."""

MQ_USER = os.getenv("RABBIT_MQ_USER", "test")
"""Message queue (RabbitMQ) user name."""

MQ_PASS = os.getenv("RABBIT_MQ_PASS", "1234")
"""Message queue (RabbitMQ) password."""

MQ_PORT = os.getenv("RABBIT_MQ_PORT", 5672)
"""Message queue (RabbitMQ) service port."""

MQ_CONNECTION_STRING = os.getenv(
    "RABBIT_MQ", "amqp://{0}:{1}@{2}//".format(MQ_USER, MQ_PASS, MQ_HOST)
)
"""Message queue (RabbitMQ) connection string."""

MQ_DEFAULT_FORMAT = "json"
"""Default serializing format (to consume/produce)."""

MQ_DEFAULT_EXCHANGE = ""
"""Message queue (RabbitMQ) exchange."""

MQ_DEFAULT_QUEUES = {
    "jobs-status": {
        "routing_key": "jobs-status",
        "exchange": MQ_DEFAULT_EXCHANGE,
        "durable": False,
    },
    "workflow-submission": {
        "routing_key": "workflow-submission",
        "exchange": MQ_DEFAULT_EXCHANGE,
        "durable": True,
    },
}
"""Default message queues."""

MQ_PRODUCER_MAX_RETRIES = 3
"""Max retries to send a message."""

OPENAPI_SPECS = {
    "reana-workflow-controller": (
        "http://{host}:{port}".format(
            host=REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["workflow-controller"],
            port="80",
        ),
        "reana_workflow_controller.json",
    ),
    "reana-server": (
        os.getenv("REANA_SERVER_URL", "http://0.0.0.0:80"),
        "reana_server.json",
    ),
    "reana-job-controller": (
        "http://{address}:{port}".format(address="0.0.0.0", port="5000"),
        "reana_job_controller.json",
    ),
}
"""REANA Workflow Controller address."""

REANA_MAX_CONCURRENT_BATCH_WORKFLOWS = int(
    os.getenv("REANA_MAX_CONCURRENT_BATCH_WORKFLOWS", "30")
)
"""Upper limit on concurrent REANA batch workflows running in the cluster."""

REANA_READY_CONDITIONS = {
    "reana_commons.tasks": [
        "check_predefined_conditions",
        "check_running_reana_batch_workflows_count",
    ]
}

REANA_LOG_LEVEL = logging.getLevelName(os.getenv("REANA_LOG_LEVEL", "INFO"))
"""Log verbosity level for REANA components."""

REANA_LOG_FORMAT = os.getenv(
    "REANA_LOG_FORMAT",
    "%(asctime)s | %(name)s | %(threadName)s | " "%(levelname)s | %(message)s",
)
"""REANA components log format."""

CVMFS_REPOSITORIES = {
    "alice.cern.ch": "alice",
    "alice-ocdb.cern.ch": "alice-ocdb",
    "ams.cern.ch": "ams",
    "atlas.cern.ch": "atlas",
    "atlas-condb.cern.ch": "atlas-condb",
    "atlas-nightlies.cern.ch": "atlas-nightlies",
    "cms.cern.ch": "cms",
    "cms-ib.cern.ch": "cms-ib",
    "cms-opendata-conddb.cern.ch": "cms-opendata-conddb",
    "compass.cern.ch": "compass",
    "compass-condb.cern.ch": "compass-condb",
    "cvmfs-config.cern.ch": "cvmfs-config",
    "fcc.cern.ch": "fcc",
    "geant4.cern.ch": "geant4",
    "ilc.desy.de": "ilc-desy",
    "lhcb.cern.ch": "lhcb",
    "lhcb-condb.cern.ch": "lhcb-condb",
    "na61.cern.ch": "na61",
    "na62.cern.ch": "na62",
    "projects.cern.ch": "projects",
    "sft.cern.ch": "sft",
    "unpacked.cern.ch": "unpacked",
}
"""CVMFS repositories available for mounting."""

REANA_CVMFS_PVC_TEMPLATE = {
    "metadata": {"name": ""},
    "spec": {
        "accessModes": ["ReadOnlyMany"],
        "storageClassName": "",
        "resources": {"requests": {"storage": "1G"}},
    },
}
"""CVMFS persistent volume claim template."""

REANA_CVMFS_SC_TEMPLATE = {
    "metadata": {"name": ""},
    "provisioner": "cvmfs.csi.cern.ch",
    "parameters": {"repository": ""},
}
"""CVMFS storage claim template."""

INTERACTIVE_SESSION_TYPES = ["jupyter"]
"""List of supported interactive systems."""

REANA_STORAGE_BACKEND = os.getenv("REANA_STORAGE_BACKEND", "local")
"""Storage backend deployed in current REANA cluster ['local'|'cephfs']."""

REANA_SHARED_PVC_NAME = os.getenv(
    "REANA_SHARED_PVC_NAME",
    "{}-shared-persistent-volume".format(REANA_COMPONENT_PREFIX),
)
"""Name of the shared CEPHFS PVC which will be used by all REANA jobs."""

REANA_WORKFLOW_UMASK = 0o0002
"""Umask used for workflow worksapce."""

WORKFLOW_RUNTIME_USER_NAME = os.getenv("WORKFLOW_RUNTIME_USER_NAME", "reana")
"""Default OS user name for running job controller."""

WORKFLOW_RUNTIME_USER_UID = os.getenv("WORKFLOW_RUNTIME_USER_UID", 1000)
"""Default user id for running job controller/workflow engine apps & jobs."""

WORKFLOW_RUNTIME_USER_GID = os.getenv("WORKFLOW_RUNTIME_USER_GID", 0)
"""Default group id for running job controller/workflow engine apps & jobs."""

REANA_USER_SECRET_MOUNT_PATH = os.getenv(
    "REANA_USER_SECRET_MOUNT_PATH", "/etc/reana/secrets"
)
"""Default mount path for user secrets which is mounted for job pod &
   workflow engines."""

SHARED_VOLUME_PATH = os.getenv("SHARED_VOLUME_PATH", "/var/reana")
"""Default shared volume path."""

K8S_CERN_EOS_MOUNT_CONFIGURATION = {
    "volume": {"name": "eos", "hostPath": {"path": "/var/eos"}},
    "volumeMounts": {
        "name": "eos",
        "mountPath": "/eos",
        "mountPropagation": "HostToContainer",
    },
}
"""Configuration to mount EOS in Kubernetes objects.

For more information see the official documentation at
https://clouddocs.web.cern.ch/containers/tutorials/eos.html.
"""

K8S_CERN_EOS_AVAILABLE = os.getenv("K8S_CERN_EOS_AVAILABLE")
"""Whether EOS is available in the current cluster or not.

This a configuration set by the system administrators through Helm values at
cluster creation time.
"""

REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME = os.getenv(
    "REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME"
)
"""REANA instrastructure service account."""

REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME = os.getenv(
    "REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME",
    REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME,
)
"""REANA runtime service account.

If no runtime namespace is deployed it will default to the infrastructure service
account.
"""

1	# -- coding: utf-8 --
2	#
3	# This file is part of REANA.
4	# Copyright (C) 2018 CERN.
5	#
6	# REANA is free software; you can redistribute it and/or modify it
7	# under the terms of the MIT License; see LICENSE file for more details.
8
9	"""REANA Commons configuration."""	!
10
11	import logging	!
12	import os	!
13
14	REANA_COMPONENT_PREFIX = os.getenv("REANA_COMPONENT_PREFIX", "reana")	!
15	"""REANA component naming prefix, i.e. my-prefix-job-controller.
16
17	Useful to find the correct fully qualified name of a infrastructure component
18	and to correctly create new runtime pods.
19	"""
20
21	REANA_COMPONENT_PREFIX_ENVIRONMENT = REANA_COMPONENT_PREFIX.upper().replace("-", "_")	!
22	"""Environment variable friendly REANA component prefix."""
23
24	REANA_COMPONENT_TYPES = [	!
25	"run-batch",
26	"run-session",
27	"run-job",
28	"secretsstore",
29	]
30	"""Type of REANA components.
31
32	Note: this list is used for validation of on demand created REANA components
33	names, this is why it doesn't contain REANA instrastructure components.
34
35	``run-batch``: An instance of reana-workflow-engine-_
36	``run-session``: An instance of an interactive session
37	``run-job``: An instance of a workflow's job
38	``secretsstore``: An instance of a user secret store
39	"""
40
NEW 41	REANA_INFRASTRUCTURE_COMPONENTS = [	!
42	"ui",
43	"server",
44	"workflow-controller",
45	"cache",
46	"message-broker",
47	"db",
48	]
49	"""REANA infrastructure pods."""
50
UNCOV 51	REANA_COMPONENT_NAMING_SCHEME = os.getenv(	!
52	"REANA_COMPONENT_NAMING_SCHEME", "{prefix}-{component_type}-{id}"
53	)
54	"""The naming scheme the components created by REANA should follow.
55
56	It is a Python format string which take as arguments:
57	- ``prefix``: the ``REANA_COMPONENT_PREFIX``
58	- ``component_type``: one of ``REANA_COMPONENT_TYPES``
59	- ``id``: unique identifier for the component, by default UUID4.
60	"""
61
NEW 62	REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE = os.getenv(	!
63	"REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE", "default"
64	)
65	"""Kubernetes namespace in which REANA instrastructure is currently deployed."""
66
NEW 67	REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES = {	!
68	component_name: (
69	"{component_prefix}-{component_name}.{namespace}.svc.cluster.local"
70	).format(
71	component_prefix=REANA_COMPONENT_PREFIX,
72	component_name=component_name,
73	namespace=REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE,
74	)
75	for component_name in REANA_INFRASTRUCTURE_COMPONENTS
76	}
77	"""REANA infrastructure pods hostnames.
78
79	Uses the FQDN of the infrastructure components (which should be behind a Kubernetes
80	service) following the
81	`Kubernetes DNS-Based Service Discovery <https://github.com/kubernetes/dns/blob/master/docs/specification.md>`_
82	"""
83
NEW 84	REANA_RUNTIME_KUBERNETES_NAMESPACE = os.getenv(	!
85	"REANA_RUNTIME_KUBERNETES_NAMESPACE", REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE
86	)
87	"""Kubernetes namespace in which REANA runtime pods should be running in.
88
89	By default runtime pods will run in the same namespace as the instrastructure pods.
90	"""
91
NEW 92	MQ_HOST = REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["message-broker"]	!
93	"""Message queue (RabbitMQ) server host name."""
94
95	MQ_USER = os.getenv("RABBIT_MQ_USER", "test")	!
96	"""Message queue (RabbitMQ) user name."""
97
98	MQ_PASS = os.getenv("RABBIT_MQ_PASS", "1234")	!
99	"""Message queue (RabbitMQ) password."""
100
101	MQ_PORT = os.getenv("RABBIT_MQ_PORT", 5672)	!
102	"""Message queue (RabbitMQ) service port."""
103
104	MQ_CONNECTION_STRING = os.getenv(	!
105	"RABBIT_MQ", "amqp://{0}:{1}@{2}//".format(MQ_USER, MQ_PASS, MQ_HOST)
106	)
107	"""Message queue (RabbitMQ) connection string."""
108
109	MQ_DEFAULT_FORMAT = "json"	!
110	"""Default serializing format (to consume/produce)."""
111
112	MQ_DEFAULT_EXCHANGE = ""	!
113	"""Message queue (RabbitMQ) exchange."""
114
115	MQ_DEFAULT_QUEUES = {	!
116	"jobs-status": {
117	"routing_key": "jobs-status",
118	"exchange": MQ_DEFAULT_EXCHANGE,
119	"durable": False,
120	},
121	"workflow-submission": {
122	"routing_key": "workflow-submission",
123	"exchange": MQ_DEFAULT_EXCHANGE,
124	"durable": True,
125	},
126	}
127	"""Default message queues."""
128
129	MQ_PRODUCER_MAX_RETRIES = 3	!
130	"""Max retries to send a message."""
131
132	OPENAPI_SPECS = {	!
133	"reana-workflow-controller": (
134	"http://{host}:{port}".format(
135	host=REANA_INFRASTRUCTURE_COMPONENTS_HOSTNAMES["workflow-controller"],
136	port="80",
137	),
138	"reana_workflow_controller.json",
139	),
140	"reana-server": (
141	os.getenv("REANA_SERVER_URL", "http://0.0.0.0:80"),
142	"reana_server.json",
143	),
144	"reana-job-controller": (
145	"http://{address}:{port}".format(address="0.0.0.0", port="5000"),
146	"reana_job_controller.json",
147	),
148	}
149	"""REANA Workflow Controller address."""
150
151	REANA_MAX_CONCURRENT_BATCH_WORKFLOWS = int(	!
152	os.getenv("REANA_MAX_CONCURRENT_BATCH_WORKFLOWS", "30")
153	)
154	"""Upper limit on concurrent REANA batch workflows running in the cluster."""
155
156	REANA_READY_CONDITIONS = {	!
157	"reana_commons.tasks": [
158	"check_predefined_conditions",
159	"check_running_reana_batch_workflows_count",
160	]
161	}
162
163	REANA_LOG_LEVEL = logging.getLevelName(os.getenv("REANA_LOG_LEVEL", "INFO"))	!
164	"""Log verbosity level for REANA components."""
165
166	REANA_LOG_FORMAT = os.getenv(	!
167	"REANA_LOG_FORMAT",
168	"%(asctime)s \| %(name)s \| %(threadName)s \| " "%(levelname)s \| %(message)s",
169	)
170	"""REANA components log format."""
171
172	CVMFS_REPOSITORIES = {	!
173	"alice.cern.ch": "alice",
174	"alice-ocdb.cern.ch": "alice-ocdb",
175	"ams.cern.ch": "ams",
176	"atlas.cern.ch": "atlas",
177	"atlas-condb.cern.ch": "atlas-condb",
178	"atlas-nightlies.cern.ch": "atlas-nightlies",
179	"cms.cern.ch": "cms",
180	"cms-ib.cern.ch": "cms-ib",
181	"cms-opendata-conddb.cern.ch": "cms-opendata-conddb",
182	"compass.cern.ch": "compass",
183	"compass-condb.cern.ch": "compass-condb",
184	"cvmfs-config.cern.ch": "cvmfs-config",
185	"fcc.cern.ch": "fcc",
186	"geant4.cern.ch": "geant4",
187	"ilc.desy.de": "ilc-desy",
188	"lhcb.cern.ch": "lhcb",
189	"lhcb-condb.cern.ch": "lhcb-condb",
190	"na61.cern.ch": "na61",
191	"na62.cern.ch": "na62",
192	"projects.cern.ch": "projects",
193	"sft.cern.ch": "sft",
194	"unpacked.cern.ch": "unpacked",
195	}
196	"""CVMFS repositories available for mounting."""
197
198	REANA_CVMFS_PVC_TEMPLATE = {	!
199	"metadata": {"name": ""},
200	"spec": {
201	"accessModes": ["ReadOnlyMany"],
202	"storageClassName": "",
203	"resources": {"requests": {"storage": "1G"}},
204	},
205	}
206	"""CVMFS persistent volume claim template."""
207
208	REANA_CVMFS_SC_TEMPLATE = {	!
209	"metadata": {"name": ""},
210	"provisioner": "cvmfs.csi.cern.ch",
211	"parameters": {"repository": ""},
212	}
213	"""CVMFS storage claim template."""
214
215	INTERACTIVE_SESSION_TYPES = ["jupyter"]	!
216	"""List of supported interactive systems."""
217
218	REANA_STORAGE_BACKEND = os.getenv("REANA_STORAGE_BACKEND", "local")	!
219	"""Storage backend deployed in current REANA cluster ['local'\|'cephfs']."""
220
221	REANA_SHARED_PVC_NAME = os.getenv(	!
222	"REANA_SHARED_PVC_NAME",
223	"{}-shared-persistent-volume".format(REANA_COMPONENT_PREFIX),
224	)
225	"""Name of the shared CEPHFS PVC which will be used by all REANA jobs."""
226
227	REANA_WORKFLOW_UMASK = 0o0002	!
228	"""Umask used for workflow worksapce."""
229
230	WORKFLOW_RUNTIME_USER_NAME = os.getenv("WORKFLOW_RUNTIME_USER_NAME", "reana")	!
231	"""Default OS user name for running job controller."""
232
233	WORKFLOW_RUNTIME_USER_UID = os.getenv("WORKFLOW_RUNTIME_USER_UID", 1000)	!
234	"""Default user id for running job controller/workflow engine apps & jobs."""
235
236	WORKFLOW_RUNTIME_USER_GID = os.getenv("WORKFLOW_RUNTIME_USER_GID", 0)	!
237	"""Default group id for running job controller/workflow engine apps & jobs."""
238
239	REANA_USER_SECRET_MOUNT_PATH = os.getenv(	!
240	"REANA_USER_SECRET_MOUNT_PATH", "/etc/reana/secrets"
241	)
242	"""Default mount path for user secrets which is mounted for job pod &
243	workflow engines."""
244
245	SHARED_VOLUME_PATH = os.getenv("SHARED_VOLUME_PATH", "/var/reana")	!
246	"""Default shared volume path."""
247
248	K8S_CERN_EOS_MOUNT_CONFIGURATION = {	!
249	"volume": {"name": "eos", "hostPath": {"path": "/var/eos"}},
250	"volumeMounts": {
251	"name": "eos",
252	"mountPath": "/eos",
253	"mountPropagation": "HostToContainer",
254	},
255	}
256	"""Configuration to mount EOS in Kubernetes objects.
257
258	For more information see the official documentation at
259	https://clouddocs.web.cern.ch/containers/tutorials/eos.html.
260	"""
261
262	K8S_CERN_EOS_AVAILABLE = os.getenv("K8S_CERN_EOS_AVAILABLE")	!
263	"""Whether EOS is available in the current cluster or not.
264
265	This a configuration set by the system administrators through Helm values at
266	cluster creation time.
267	"""
268
NEW 269	REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME = os.getenv(	!
270	"REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME"
271	)
272	"""REANA instrastructure service account."""
273
NEW 274	REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME = os.getenv(	!
275	"REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME",
276	REANA_INFRASTRUCTURE_KUBERNETES_SERVICEACCOUNT_NAME,
277	)
NEW 278	"""REANA runtime service account.	!
279
280	If no runtime namespace is deployed it will default to the infrastructure service
281	account.
282	"""

reanahub / reana-commons / 816

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous