9058668052

Committed 13 May 2024 07:05AM UTC coverage: 77.713% (-8.4%) from 86.115%

Build # 9058668052

Build Type

Pull #3727

github

Committed by

web-flow

Commit Message

Merge 128d38387 into 050ed61bf

Pull Request Pull Request #3727: fix: don't fail session launch when gitlab couldn't be reached

Run Details

15 of 29 new or added lines in 3 files covered. (51.72%)

2594 existing lines in 125 files now uncovered.

23893 of 30745 relevant lines covered (77.71%)

3.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.16

/renku/core/config.py

# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Configuration management."""

import configparser
import os
from io import StringIO
from pathlib import Path

from renku.core.constant import DATA_DIR_CONFIG_KEY
from renku.domain_model.enums import ConfigFilter
from renku.domain_model.project_context import project_context


def global_config_read_lock():
    """Create a user-level config read lock."""
    from renku.core.util.contexts import Lock

    lock_path = os.environ.get("RENKU_LOCK_PATH")
    if lock_path is not None:
        return Lock(Path(lock_path))

    return Lock(project_context.global_config_path)


def global_config_write_lock():
    """Create a user-level config write lock."""
    from renku.core.util.contexts import Lock

    lock_path = os.environ.get("RENKU_LOCK_PATH")
    if lock_path is not None:
        return Lock(Path(lock_path), mode="exclusive")

    return Lock(project_context.global_config_path, mode="exclusive")


def get_value(section, key, config_filter=ConfigFilter.ALL):
    """Get value from specified section and key."""
    config = load_config(config_filter=config_filter)
    return config.get(section, key, fallback=None)


def set_value(section, key, value, global_only=False):
    """Set value to specified section and key."""
    config_filter = ConfigFilter.GLOBAL_ONLY

    if not global_only:
        config_filter = ConfigFilter.LOCAL_ONLY
        _check_config_is_not_readonly(section, key)

    config = load_config(config_filter=config_filter)
    if section in config:
        config[section][key] = value
    else:
        config[section] = {key: value}

    store_config(config, global_only=global_only)


def remove_value(section, key, global_only=False):
    """Remove key from specified section or remove sections."""
    config_filter = ConfigFilter.GLOBAL_ONLY

    if not global_only:
        config_filter = ConfigFilter.LOCAL_ONLY
        _check_config_is_not_readonly(section, key)

    value = None
    config = load_config(config_filter=config_filter)
    removed_sections = []
    if section in config:
        if key == "*":
            value = config.pop(section)
        else:
            value = config[section].pop(key, None)

            if not config[section].keys():
                removed_sections.append(section)
    elif section.endswith("*"):
        section_prefix = section[:-1]
        for section in config:
            if section.startswith(section_prefix):
                value = config[section]
                removed_sections.append(section)

    for section in removed_sections:
        config.pop(section)
    if value is not None:
        store_config(config, global_only=global_only)
    return value


def load_config(config_filter=ConfigFilter.ALL):
    """Loads local, global or both configuration object."""
    try:
        import importlib_resources  # type:ignore
    except ImportError:
        import importlib.resources as importlib_resources  # type:ignore

    # NOTE: Use RawConfigParser because ConfigParser does non-standard INI interpolation of some values
    config = configparser.RawConfigParser()
    ref = importlib_resources.files("renku.data") / "defaults.ini"
    with importlib_resources.as_file(ref) as default_ini:
        config_files = [default_ini]

    if config_filter == ConfigFilter.LOCAL_ONLY:
        config_files += [project_context.local_config_path]
    elif config_filter == ConfigFilter.GLOBAL_ONLY:
        config_files += [project_context.global_config_path]
    elif config_filter == ConfigFilter.ALL:
        config_files += [
            project_context.global_config_path,
            project_context.local_config_path,
        ]

    if config_filter != ConfigFilter.LOCAL_ONLY:
        with global_config_read_lock():
            config.read(config_files)
    else:
        config.read(config_files)

    # NOTE: transform config section for backwards compatibility. Changes section names like
    # 'renku "interactive"' to just 'interactive' to be in line with python config conventions.
    for section in config.sections():
        if not section.startswith('renku "'):
            continue

        config[section[7:-1]] = dict(config.items(section))  # NOTE: Drop first 7 and last char
        config.pop(section)

    return config


def store_config(config, global_only):
    """Persists locally or globally configuration object.

    Global configuration is updated only when :global_only: is True,
    otherwise, updates are written to local project configuration
    """
    filepath = project_context.global_config_path if global_only else project_context.local_config_path

    if global_only:
        with global_config_write_lock():
            os.umask(0)
            fd = os.open(filepath, os.O_CREAT | os.O_RDWR | os.O_TRUNC, 0o600)
            write_config(fd, config)
    else:
        write_config(filepath, config)


def get_config(config_filter=ConfigFilter.ALL, as_string=True):
    """Read all configurations."""
    config = load_config(config_filter=config_filter)
    if as_string:
        with StringIO() as output:
            config.write(output)
            return output.getvalue()
    else:
        return {f"{s}.{k}": v for s in config.sections() for k, v in config.items(s)}


def _check_config_is_not_readonly(section, key):
    from renku.core import errors

    readonly_configs = {"renku": [DATA_DIR_CONFIG_KEY]}

    value = get_value(section, key, config_filter=ConfigFilter.LOCAL_ONLY)
    if not value:
        return

    if key in readonly_configs.get(section, []):
        raise errors.ParameterError(f"Configuration {key} cannot be modified.")


def write_config(filepath, config):
    """Write config value to a specified path."""
    with open(filepath, "w+") as file:
        config.write(file)

1	# Copyright Swiss Data Science Center (SDSC). A partnership between
2	# École Polytechnique Fédérale de Lausanne (EPFL) and
3	# Eidgenössische Technische Hochschule Zürich (ETHZ).
4	#
5	# Licensed under the Apache License, Version 2.0 (the "License");
6	# you may not use this file except in compliance with the License.
7	# You may obtain a copy of the License at
8	#
9	# http://www.apache.org/licenses/LICENSE-2.0
10	#
11	# Unless required by applicable law or agreed to in writing, software
12	# distributed under the License is distributed on an "AS IS" BASIS,
13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	# See the License for the specific language governing permissions and
15	# limitations under the License.
16	"""Configuration management."""	7✔
17
18	import configparser	7✔
19	import os	7✔
20	from io import StringIO	7✔
21	from pathlib import Path	7✔
22
23	from renku.core.constant import DATA_DIR_CONFIG_KEY	7✔
24	from renku.domain_model.enums import ConfigFilter	7✔
25	from renku.domain_model.project_context import project_context	7✔
26
27
28	def global_config_read_lock():	7✔
29	"""Create a user-level config read lock."""
30	from renku.core.util.contexts import Lock	6✔
31
32	lock_path = os.environ.get("RENKU_LOCK_PATH")	6✔
33	if lock_path is not None:	6✔
NEW 34	return Lock(Path(lock_path))	×
35
36	return Lock(project_context.global_config_path)	6✔
37
38
39	def global_config_write_lock():	7✔
40	"""Create a user-level config write lock."""
41	from renku.core.util.contexts import Lock	6✔
42
43	lock_path = os.environ.get("RENKU_LOCK_PATH")	6✔
44	if lock_path is not None:	6✔
NEW 45	return Lock(Path(lock_path), mode="exclusive")	×
46
47	return Lock(project_context.global_config_path, mode="exclusive")	6✔
48
49
50	def get_value(section, key, config_filter=ConfigFilter.ALL):	7✔
51	"""Get value from specified section and key."""
52	config = load_config(config_filter=config_filter)	6✔
53	return config.get(section, key, fallback=None)	6✔
54
55
56	def set_value(section, key, value, global_only=False):	7✔
57	"""Set value to specified section and key."""
58	config_filter = ConfigFilter.GLOBAL_ONLY	6✔
59
60	if not global_only:	6✔
61	config_filter = ConfigFilter.LOCAL_ONLY	4✔
62	_check_config_is_not_readonly(section, key)	4✔
63
64	config = load_config(config_filter=config_filter)	6✔
65	if section in config:	6✔
66	config[section][key] = value	6✔
67	else:
68	config[section] = {key: value}	4✔
69
70	store_config(config, global_only=global_only)	6✔
71
72
73	def remove_value(section, key, global_only=False):	7✔
74	"""Remove key from specified section or remove sections."""
75	config_filter = ConfigFilter.GLOBAL_ONLY	4✔
76
77	if not global_only:	4✔
78	config_filter = ConfigFilter.LOCAL_ONLY	3✔
79	_check_config_is_not_readonly(section, key)	3✔
80
81	value = None	4✔
82	config = load_config(config_filter=config_filter)	4✔
83	removed_sections = []	4✔
84	if section in config:	4✔
85	if key == "*":	4✔
UNCOV 86	value = config.pop(section)	×
87	else:
88	value = config[section].pop(key, None)	4✔
89
90	if not config[section].keys():	4✔
91	removed_sections.append(section)	1✔
92	elif section.endswith("*"):	1✔
UNCOV 93	section_prefix = section[:-1]	×
UNCOV 94	for section in config:	×
UNCOV 95	if section.startswith(section_prefix):	×
UNCOV 96	value = config[section]	×
UNCOV 97	removed_sections.append(section)	×
98
99	for section in removed_sections:	4✔
100	config.pop(section)	1✔
101	if value is not None:	4✔
102	store_config(config, global_only=global_only)	3✔
103	return value	4✔
104
105
106	def load_config(config_filter=ConfigFilter.ALL):	7✔
107	"""Loads local, global or both configuration object."""
108	try:	6✔
109	import importlib_resources # type:ignore	6✔
UNCOV 110	except ImportError:	×
UNCOV 111	import importlib.resources as importlib_resources # type:ignore	×
112
113	# NOTE: Use RawConfigParser because ConfigParser does non-standard INI interpolation of some values
114	config = configparser.RawConfigParser()	6✔
115	ref = importlib_resources.files("renku.data") / "defaults.ini"	6✔
116	with importlib_resources.as_file(ref) as default_ini:	6✔
117	config_files = [default_ini]	6✔
118
119	if config_filter == ConfigFilter.LOCAL_ONLY:	6✔
120	config_files += [project_context.local_config_path]	6✔
121	elif config_filter == ConfigFilter.GLOBAL_ONLY:	6✔
122	config_files += [project_context.global_config_path]	6✔
123	elif config_filter == ConfigFilter.ALL:	6✔
124	config_files += [	6✔
125	project_context.global_config_path,
126	project_context.local_config_path,
127	]
128
129	if config_filter != ConfigFilter.LOCAL_ONLY:	6✔
130	with global_config_read_lock():	6✔
131	config.read(config_files)	6✔
132	else:
133	config.read(config_files)	6✔
134
135	# NOTE: transform config section for backwards compatibility. Changes section names like
136	# 'renku "interactive"' to just 'interactive' to be in line with python config conventions.
137	for section in config.sections():	6✔
138	if not section.startswith('renku "'):	6✔
139	continue	6✔
140
141	config[section[7:-1]] = dict(config.items(section)) # NOTE: Drop first 7 and last char	2✔
142	config.pop(section)	2✔
143
144	return config	6✔
145
146
147	def store_config(config, global_only):	7✔
148	"""Persists locally or globally configuration object.
149
150	Global configuration is updated only when :global_only: is True,
151	otherwise, updates are written to local project configuration
152	"""
153	filepath = project_context.global_config_path if global_only else project_context.local_config_path	6✔
154
155	if global_only:	6✔
156	with global_config_write_lock():	6✔
157	os.umask(0)	6✔
158	fd = os.open(filepath, os.O_CREAT \| os.O_RDWR \| os.O_TRUNC, 0o600)	6✔
159	write_config(fd, config)	6✔
160	else:
161	write_config(filepath, config)	4✔
162
163
164	def get_config(config_filter=ConfigFilter.ALL, as_string=True):	7✔
165	"""Read all configurations."""
166	config = load_config(config_filter=config_filter)	1✔
167	if as_string:	1✔
UNCOV 168	with StringIO() as output:	×
UNCOV 169	config.write(output)	×
UNCOV 170	return output.getvalue()	×
171	else:
172	return {f"{s}.{k}": v for s in config.sections() for k, v in config.items(s)}	1✔
173
174
175	def _check_config_is_not_readonly(section, key):	7✔
176	from renku.core import errors	4✔
177
178	readonly_configs = {"renku": [DATA_DIR_CONFIG_KEY]}	4✔
179
180	value = get_value(section, key, config_filter=ConfigFilter.LOCAL_ONLY)	4✔
181	if not value:	4✔
182	return	4✔
183
184	if key in readonly_configs.get(section, []):	4✔
UNCOV 185	raise errors.ParameterError(f"Configuration {key} cannot be modified.")	×
186
187
188	def write_config(filepath, config):	7✔
189	"""Write config value to a specified path."""
190	with open(filepath, "w+") as file:	6✔
191	config.write(file)	6✔

SwissDataScienceCenter / renku-python / 9058668052

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous