14841109746

Committed 05 May 2025 04:18PM UTC coverage: 80.118% (-0.03%) from 80.149%

Build # 14841109746

Build Type

Pull #1642

github

Committed by

web-flow

Commit Message

Merge e9f1fc9de into 831e535f6

Pull Request Pull Request #1642: Performance to report accurate times based on end-to-end time() diffs, rather than accumulate cProfile numbers over methods that seemed relevant.

Run Details

1650 of 2043 branches covered (80.76%)

Branch coverage included in aggregate %.

10302 of 12875 relevant lines covered (80.02%)

0.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.17

src/unitxt/settings_utils.py

import importlib.metadata
import importlib.util
import os
from contextlib import contextmanager

from .version import version


def cast_to_type(value, value_type):
    if value_type is bool:
        if value not in ["True", "False", True, False]:
            raise ValueError(
                f"Value must be in ['True', 'False', True, False] got {value}"
            )
        if value == "True":
            return True
        if value == "False":
            return False
        return value
    if value_type is int:
        return int(value)
    if value_type is float:
        return float(value)

    raise ValueError("Unsupported type.")


class Settings:
    _instance = None
    _settings = {}
    _types = {}
    _logger = None

    @classmethod
    def is_uninitilized(cls):
        return cls._instance is None

    def __new__(cls):
        if cls.is_uninitilized():
            cls._instance = super().__new__(cls)
        return cls._instance

    def __setattr__(self, key, value):
        if key.endswith("_key") or key in {"_instance", "_settings"}:
            raise AttributeError(f"Modifying '{key}' is not allowed.")

        if isinstance(value, tuple) and len(value) == 2:
            value_type, value = value
            if value_type not in [int, float, bool]:
                raise ValueError(
                    f"Setting settings with tuple requires the first element to be either [int, float, bool], got {value_type}"
                )
            self._types[key] = value_type

        if key in self._types and value is not None:
            value_type = self._types[key]
            value = cast_to_type(value, value_type)

        if key in self._settings:
            if self._logger is not None:
                self._logger.info(
                    f"unitxt.settings.{key} changed: {self._settings[key]} -> {value}"
                )
        self._settings[key] = value

    def __getattr__(self, key):
        if key.endswith("_key"):
            actual_key = key[:-4]  # Remove the "_key" suffix
            return self.environment_variable_key_name(actual_key)

        key_name = self.environment_variable_key_name(key)
        env_value = os.getenv(key_name)

        if env_value is not None:
            if key in self._types:
                env_value = cast_to_type(env_value, self._types[key])
            return env_value

        if key in self._settings:
            return self._settings[key]

        raise AttributeError(f"'{key}' not found")

    def environment_variable_key_name(self, key):
        return "UNITXT_" + key.upper()

    def get_all_environment_variables(self):
        return [
            self.environment_variable_key_name(key) for key in self._settings.keys()
        ]

    @contextmanager
    def context(self, **kwargs):
        old_values = {key: self._settings.get(key, None) for key in kwargs}
        try:
            for key, value in kwargs.items():
                self.__setattr__(key, value)
            yield
        finally:
            for key, value in old_values.items():
                self.__setattr__(key, value)


class Constants:
    _instance = None
    _constants = {}

    @classmethod
    def is_uninitilized(cls):
        return cls._instance is None

    def __new__(cls):
        if cls.is_uninitilized():
            cls._instance = super().__new__(cls)
        return cls._instance

    def __setattr__(self, key, value):
        if key.endswith("_key") or key in {"_instance", "_constants"}:
            raise AttributeError(f"Modifying '{key}' is not allowed.")
        if key in self._constants:
            raise ValueError("Cannot override constants.")
        self._constants[key] = value

    def __getattr__(self, key):
        if key in self._constants:
            return self._constants[key]

        raise AttributeError(f"'{key}' not found")


if Settings.is_uninitilized():
    settings = Settings()
    settings.allow_unverified_code = (bool, False)
    settings.use_only_local_catalogs = (bool, False)
    settings.global_loader_limit = (int, None)
    settings.num_resamples_for_instance_metrics = (int, 1000)
    settings.num_resamples_for_global_metrics = (int, 100)
    settings.max_log_message_size = (int, 100000)
    settings.catalogs = None
    settings.artifactories = None
    settings.default_recipe = "dataset_recipe"
    settings.default_verbosity = "info"
    settings.use_eager_execution = False
    settings.remote_metrics = []
    settings.test_card_disable = (bool, False)
    settings.test_metric_disable = (bool, False)
    settings.metrics_master_key_token = None
    settings.seed = (int, 42)
    settings.skip_artifacts_prepare_and_verify = (bool, False)
    settings.data_classification_policy = None
    settings.mock_inference_mode = (bool, False)
    settings.disable_hf_datasets_cache = (bool, False)
    settings.stream_hf_datasets_by_default = (bool, False)
    settings.loader_cache_size = (int, 25)
    settings.loaders_max_retries = (int, 10)
    settings.task_data_as_text = (bool, True)
    settings.default_provider = "watsonx"
    settings.default_format = None
    settings.hf_load_from_offline = (bool, False)
    settings.hf_save_to_offline = (bool, False)
    settings.hf_offline_datasets_path = None
    settings.hf_offline_metrics_path = None
    settings.hf_offline_models_path = None
    settings.inference_engine_cache_path = "./inference_engine_cache/"
    settings.max_connection_retries = 3

if Constants.is_uninitilized():
    constants = Constants()
    constants.dataset_file = os.path.join(os.path.dirname(__file__), "dataset.py")
    constants.metric_file = os.path.join(os.path.dirname(__file__), "metric.py")
    constants.local_catalog_path = os.path.join(os.path.dirname(__file__), "catalog")
    unitxt_pkg = importlib.util.find_spec("unitxt")
    if unitxt_pkg and unitxt_pkg.origin:
        constants.package_dir = os.path.dirname(unitxt_pkg.origin)
        constants.default_catalog_path = os.path.join(constants.package_dir, "catalog")
    else:
        constants.default_catalog_path = constants.local_catalog_path
    constants.catalog_dir = constants.local_catalog_path
    constants.dataset_url = "unitxt/data"
    constants.metric_url = "unitxt/metric"
    constants.version = version
    constants.catalog_hierarchy_sep = "."
    constants.env_local_catalogs_paths_sep = ":"
    constants.non_registered_files = [
        "__init__.py",
        "artifact.py",
        "utils.py",
        "register.py",
        "metric.py",
        "dataset.py",
        "blocks.py",
    ]
    constants.codebase_url = "https://github.com/IBM/unitxt"
    constants.website_url = "https://www.unitxt.org"
    constants.inference_stream = "__INFERENCE_STREAM__"
    constants.instance_stream = "__INSTANCE_STREAM__"
    constants.image_tag = "unitxt-img"
    constants.demos_pool_field = "_demos_pool_"
    constants.demos_field = "demos"
    constants.instruction_field = "instruction"
    constants.system_prompt_field = "system_prompt"


def get_settings() -> Settings:
    return Settings()


def get_constants():
    return Constants()

1	import importlib.metadata	1✔
2	import importlib.util	1✔
3	import os	1✔
4	from contextlib import contextmanager	1✔
5
6	from .version import version	1✔
7
8
9	def cast_to_type(value, value_type):	1✔
10	if value_type is bool:	1✔
11	if value not in ["True", "False", True, False]:	1✔
12	raise ValueError(	1✔
13	f"Value must be in ['True', 'False', True, False] got {value}"
14	)
15	if value == "True":	1✔
16	return True	1✔
17	if value == "False":	1✔
18	return False	1✔
19	return value	1✔
20	if value_type is int:	1✔
21	return int(value)	1✔
22	if value_type is float:	1✔
23	return float(value)	1✔
24
25	raise ValueError("Unsupported type.")	×
26
27
28	class Settings:	1✔
29	_instance = None	1✔
30	_settings = {}	1✔
31	_types = {}	1✔
32	_logger = None	1✔
33
34	@classmethod	1✔
35	def is_uninitilized(cls):	1✔
36	return cls._instance is None	1✔
37
38	def __new__(cls):	1✔
39	if cls.is_uninitilized():	1✔
40	cls._instance = super().__new__(cls)	1✔
41	return cls._instance	1✔
42
43	def __setattr__(self, key, value):	1✔
44	if key.endswith("_key") or key in {"_instance", "_settings"}:	1✔
45	raise AttributeError(f"Modifying '{key}' is not allowed.")	1✔
46
47	if isinstance(value, tuple) and len(value) == 2:	1✔
48	value_type, value = value	1✔
49	if value_type not in [int, float, bool]:	1✔
50	raise ValueError(	1✔
51	f"Setting settings with tuple requires the first element to be either [int, float, bool], got {value_type}"
52	)
53	self._types[key] = value_type	1✔
54
55	if key in self._types and value is not None:	1✔
56	value_type = self._types[key]	1✔
57	value = cast_to_type(value, value_type)	1✔
58
59	if key in self._settings:	1✔
60	if self._logger is not None:	1✔
61	self._logger.info(	×
62	f"unitxt.settings.{key} changed: {self._settings[key]} -> {value}"
63	)
64	self._settings[key] = value	1✔
65
66	def __getattr__(self, key):	1✔
67	if key.endswith("_key"):	1✔
68	actual_key = key[:-4] # Remove the "_key" suffix	1✔
69	return self.environment_variable_key_name(actual_key)	1✔
70
71	key_name = self.environment_variable_key_name(key)	1✔
72	env_value = os.getenv(key_name)	1✔
73
74	if env_value is not None:	1✔
75	if key in self._types:	1✔
76	env_value = cast_to_type(env_value, self._types[key])	1✔
77	return env_value	1✔
78
79	if key in self._settings:	1✔
80	return self._settings[key]	1✔
81
82	raise AttributeError(f"'{key}' not found")	1✔
83
84	def environment_variable_key_name(self, key):	1✔
85	return "UNITXT_" + key.upper()	1✔
86
87	def get_all_environment_variables(self):	1✔
88	return [	×
89	self.environment_variable_key_name(key) for key in self._settings.keys()
90	]
91
92	@contextmanager	1✔
93	def context(self, **kwargs):	1✔
94	old_values = {key: self._settings.get(key, None) for key in kwargs}	1✔
95	try:	1✔
96	for key, value in kwargs.items():	1✔
97	self.__setattr__(key, value)	1✔
98	yield	1✔
99	finally:
100	for key, value in old_values.items():	1✔
101	self.__setattr__(key, value)	1✔
102
103
104	class Constants:	1✔
105	_instance = None	1✔
106	_constants = {}	1✔
107
108	@classmethod	1✔
109	def is_uninitilized(cls):	1✔
110	return cls._instance is None	1✔
111
112	def __new__(cls):	1✔
113	if cls.is_uninitilized():	1✔
114	cls._instance = super().__new__(cls)	1✔
115	return cls._instance	1✔
116
117	def __setattr__(self, key, value):	1✔
118	if key.endswith("_key") or key in {"_instance", "_constants"}:	1✔
119	raise AttributeError(f"Modifying '{key}' is not allowed.")	×
120	if key in self._constants:	1✔
121	raise ValueError("Cannot override constants.")	×
122	self._constants[key] = value	1✔
123
124	def __getattr__(self, key):	1✔
125	if key in self._constants:	1✔
126	return self._constants[key]	1✔
127
128	raise AttributeError(f"'{key}' not found")	×
129
130
131	if Settings.is_uninitilized():	1✔
132	settings = Settings()	1✔
133	settings.allow_unverified_code = (bool, False)	1✔
134	settings.use_only_local_catalogs = (bool, False)	1✔
135	settings.global_loader_limit = (int, None)	1✔
136	settings.num_resamples_for_instance_metrics = (int, 1000)	1✔
137	settings.num_resamples_for_global_metrics = (int, 100)	1✔
138	settings.max_log_message_size = (int, 100000)	1✔
139	settings.catalogs = None	1✔
140	settings.artifactories = None	1✔
141	settings.default_recipe = "dataset_recipe"	1✔
142	settings.default_verbosity = "info"	1✔
143	settings.use_eager_execution = False	1✔
144	settings.remote_metrics = []	1✔
145	settings.test_card_disable = (bool, False)	1✔
146	settings.test_metric_disable = (bool, False)	1✔
147	settings.metrics_master_key_token = None	1✔
148	settings.seed = (int, 42)	1✔
149	settings.skip_artifacts_prepare_and_verify = (bool, False)	1✔
150	settings.data_classification_policy = None	1✔
151	settings.mock_inference_mode = (bool, False)	1✔
152	settings.disable_hf_datasets_cache = (bool, False)	1✔
153	settings.stream_hf_datasets_by_default = (bool, False)	1✔
154	settings.loader_cache_size = (int, 25)	1✔
155	settings.loaders_max_retries = (int, 10)	1✔
156	settings.task_data_as_text = (bool, True)	1✔
157	settings.default_provider = "watsonx"	1✔
158	settings.default_format = None	1✔
159	settings.hf_load_from_offline = (bool, False)	1✔
160	settings.hf_save_to_offline = (bool, False)	1✔
161	settings.hf_offline_datasets_path = None	1✔
162	settings.hf_offline_metrics_path = None	1✔
163	settings.hf_offline_models_path = None	1✔
164	settings.inference_engine_cache_path = "./inference_engine_cache/"	1✔
165	settings.max_connection_retries = 3	1✔
166
167	if Constants.is_uninitilized():	1✔
168	constants = Constants()	1✔
169	constants.dataset_file = os.path.join(os.path.dirname(__file__), "dataset.py")	1✔
170	constants.metric_file = os.path.join(os.path.dirname(__file__), "metric.py")	1✔
171	constants.local_catalog_path = os.path.join(os.path.dirname(__file__), "catalog")	1✔
172	unitxt_pkg = importlib.util.find_spec("unitxt")	1✔
173	if unitxt_pkg and unitxt_pkg.origin:	1✔
174	constants.package_dir = os.path.dirname(unitxt_pkg.origin)	1✔
175	constants.default_catalog_path = os.path.join(constants.package_dir, "catalog")	1✔
176	else:
177	constants.default_catalog_path = constants.local_catalog_path	×
178	constants.catalog_dir = constants.local_catalog_path	1✔
179	constants.dataset_url = "unitxt/data"	1✔
180	constants.metric_url = "unitxt/metric"	1✔
181	constants.version = version	1✔
182	constants.catalog_hierarchy_sep = "."	1✔
183	constants.env_local_catalogs_paths_sep = ":"	1✔
184	constants.non_registered_files = [	1✔
185	"__init__.py",
186	"artifact.py",
187	"utils.py",
188	"register.py",
189	"metric.py",
190	"dataset.py",
191	"blocks.py",
192	]
193	constants.codebase_url = "https://github.com/IBM/unitxt"	1✔
194	constants.website_url = "https://www.unitxt.org"	1✔
195	constants.inference_stream = "__INFERENCE_STREAM__"	1✔
196	constants.instance_stream = "__INSTANCE_STREAM__"	1✔
197	constants.image_tag = "unitxt-img"	1✔
198	constants.demos_pool_field = "_demos_pool_"	1✔
199	constants.demos_field = "demos"	1✔
200	constants.instruction_field = "instruction"	1✔
201	constants.system_prompt_field = "system_prompt"	1✔
202
203
204	def get_settings() -> Settings:	1✔
205	return Settings()	1✔
206
207
208	def get_constants():	1✔
209	return Constants()	1✔

IBM / unitxt / 14841109746

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous