12999897484

Committed 27 Jan 2025 10:53PM UTC coverage: 78.988% (-0.2%) from 79.163%

Build # 12999897484

Build Type

Pull #1560

github

Committed by

web-flow

Commit Message

Merge 4c97c29e4 into fddf5e362

Pull Request Pull Request #1560: remove IterableDataset, use only Dataset

Run Details

1417 of 1792 branches covered (79.07%)

Branch coverage included in aggregate %.

9037 of 11443 relevant lines covered (78.97%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.85

src/unitxt/api.py

import json
from datetime import datetime
from functools import lru_cache
from typing import Any, Dict, List, Optional, Union

from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict

from .artifact import fetch_artifact
from .card import TaskCard
from .dataset_utils import get_dataset_artifact
from .error_utils import UnitxtError
from .inference import (
    InferenceEngine,
    LogProbInferenceEngine,
    OptionSelectingByLogProbsInferenceEngine,
)
from .loaders import LoadFromDictionary
from .logging_utils import get_logger
from .metric_utils import EvaluationResults, _compute, _inference_post_process
from .operator import SourceOperator
from .schema import loads_instance
from .settings_utils import get_constants, get_settings
from .standard import DatasetRecipe
from .task import Task

logger = get_logger()
constants = get_constants()
settings = get_settings()


def load(source: Union[SourceOperator, str]):
    assert isinstance(
        source, (SourceOperator, str)
    ), "source must be a SourceOperator or a string"
    if isinstance(source, str):
        source, _ = fetch_artifact(source)
    return source().to_dataset()


def _get_recipe_from_query(dataset_query: str) -> DatasetRecipe:
    dataset_query = dataset_query.replace("sys_prompt", "instruction")
    try:
        dataset_stream, _ = fetch_artifact(dataset_query)
    except:
        dataset_stream = get_dataset_artifact(dataset_query)
    return dataset_stream


def _get_recipe_from_dict(dataset_params: Dict[str, Any]) -> DatasetRecipe:
    recipe_attributes = list(DatasetRecipe.__dict__["__fields__"].keys())
    for param in dataset_params.keys():
        assert param in recipe_attributes, (
            f"The parameter '{param}' is not an attribute of the 'DatasetRecipe' class. "
            f"Please check if the name is correct. The available attributes are: '{recipe_attributes}'."
        )
    return DatasetRecipe(**dataset_params)


def _verify_dataset_args(dataset_query: Optional[str] = None, dataset_args=None):
    if dataset_query and dataset_args:
        raise ValueError(
            "Cannot provide 'dataset_query' and key-worded arguments at the same time. "
            "If you want to load dataset from a card in local catalog, use query only. "
            "Otherwise, use key-worded arguments only to specify properties of dataset."
        )

    if dataset_query:
        if not isinstance(dataset_query, str):
            raise ValueError(
                f"If specified, 'dataset_query' must be a string, however, "
                f"'{dataset_query}' was provided instead, which is of type "
                f"'{type(dataset_query)}'."
            )

    if not dataset_query and not dataset_args:
        raise ValueError(
            "Either 'dataset_query' or key-worded arguments must be provided."
        )


def load_recipe(dataset_query: Optional[str] = None, **kwargs) -> DatasetRecipe:
    if isinstance(dataset_query, DatasetRecipe):
        return dataset_query

    _verify_dataset_args(dataset_query, kwargs)

    if dataset_query:
        recipe = _get_recipe_from_query(dataset_query)

    if kwargs:
        recipe = _get_recipe_from_dict(kwargs)

    return recipe


def create_dataset(
    task: Union[str, Task],
    test_set: List[Dict[Any, Any]],
    train_set: Optional[List[Dict[Any, Any]]] = None,
    validation_set: Optional[List[Dict[Any, Any]]] = None,
    split: Optional[str] = None,
    **kwargs,
) -> Union[DatasetDict, IterableDatasetDict, Dataset, IterableDataset]:
    """Creates dataset from input data based on a specific task.

    Args:
        task:  The name of the task from the Unitxt Catalog (https://www.unitxt.ai/en/latest/catalog/catalog.tasks.__dir__.html)
        test_set : required list of instances
        train_set : optional train_set
        validation_set: optional validation set
        split: optional one split to choose
        **kwargs: Arguments used to load dataset from provided datasets (see load_dataset())

    Returns:
        DatasetDict

    Example:
        template = Template(...)
        dataset = create_dataset(task="tasks.qa.open", template=template, format="formats.chatapi")
    """
    data = {"test": test_set}
    if train_set is not None:
        data["train"] = train_set
    if validation_set is not None:
        data["validation"] = validation_set
    task, _ = fetch_artifact(task)

    if "template" not in kwargs and task.default_template is None:
        raise Exception(
            f"No 'template' was passed to the create_dataset() and the given task ('{task.__id__}') has no 'default_template' field."
        )

    card = TaskCard(loader=LoadFromDictionary(data=data), task=task)
    return load_dataset(card=card, split=split, **kwargs)


def load_dataset(
    dataset_query: Optional[str] = None,
    split: Optional[str] = None,
    streaming: bool = False,
    disable_cache: Optional[bool] = None,
    **kwargs,
) -> Union[DatasetDict, IterableDatasetDict, Dataset, IterableDataset]:
    """Loads dataset.

    If the 'dataset_query' argument is provided, then dataset is loaded from a card
    in local catalog based on parameters specified in the query.

    Alternatively, dataset is loaded from a provided card based on explicitly
    given parameters.

    Args:
        dataset_query (str, optional):
            A string query which specifies a dataset to load from
            local catalog or name of specific recipe or benchmark in the catalog. For
            example, ``"card=cards.wnli,template=templates.classification.multi_class.relation.default"``.
        streaming (bool, False):
            When True yields the data as Unitxt streams dictionary
        split (str, optional):
            The split of the data to load
        disable_cache (str, optional):
            Disable caching process of the data
        **kwargs:
            Arguments used to load dataset from provided card, which is not present in local catalog.

    Returns:
        DatasetDict

    :Example:

        .. code-block:: python

            dataset = load_dataset(
                dataset_query="card=cards.stsb,template=templates.regression.two_texts.simple,max_train_instances=5"
            )  # card and template must be present in local catalog

            # or built programmatically
            card = TaskCard(...)
            template = Template(...)
            loader_limit = 10
            dataset = load_dataset(card=card, template=template, loader_limit=loader_limit)

    """
    recipe = load_recipe(dataset_query, **kwargs)
    ms = recipe()
    return {stream_name: list(ms[stream_name]) for stream_name in ms}


def fill_metadata(**kwargs):
    metadata = kwargs.copy()
    metadata["unitxt_version"] = get_constants().version
    metadata["creation_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
    return metadata


def evaluate(
    predictions, dataset: Union[Dataset, IterableDataset] = None, data=None
) -> EvaluationResults:
    if dataset is None and data is None:
        raise UnitxtError(message="Specify 'dataset' in evaluate")
    if data is not None:
        dataset = data  # for backward compatibility
    evaluation_result = _compute(predictions=predictions, references=dataset)
    if hasattr(dataset, "info") and hasattr(dataset.info, "description"):
        evaluation_result.metadata["dataset"] = dataset.info.description
    if hasattr(predictions, "metadata"):
        evaluation_result.metadata["predictions"] = predictions.metadata
    evaluation_result.metadata["creation_time"] = datetime.now().strftime(
        "%Y-%m-%d %H:%M:%S.%f"
    )[:-3]
    return evaluation_result


def post_process(predictions, data) -> List[Dict[str, Any]]:
    return _inference_post_process(predictions=predictions, references=data)


@lru_cache
def _get_produce_with_cache(dataset_query: Optional[str] = None, **kwargs):
    return load_recipe(dataset_query, **kwargs).produce


def produce(
    instance_or_instances, dataset_query: Optional[str] = None, **kwargs
) -> Union[Dataset, Dict[str, Any]]:
    is_list = isinstance(instance_or_instances, list)
    if not is_list:
        instance_or_instances = [instance_or_instances]
    result = _get_produce_with_cache(dataset_query, **kwargs)(instance_or_instances)
    if not is_list:
        return result[0]
    return Dataset.from_list(result).with_transform(loads_instance)


def infer(
    instance_or_instances,
    engine: InferenceEngine,
    dataset_query: Optional[str] = None,
    return_data: bool = False,
    return_log_probs: bool = False,
    return_meta_data: bool = False,
    previous_messages: Optional[List[Dict[str, str]]] = None,
    **kwargs,
):
    dataset = produce(instance_or_instances, dataset_query, **kwargs)
    if previous_messages is not None:

        def add_previous_messages(example, index):
            example["source"] = previous_messages[index] + example["source"]
            return example

        dataset = dataset.map(add_previous_messages, with_indices=True)
    engine, _ = fetch_artifact(engine)
    if return_log_probs:
        if not isinstance(engine, LogProbInferenceEngine):
            raise NotImplementedError(
                f"Error in infer: return_log_probs set to True but supplied engine "
                f"{engine.__class__.__name__} does not support logprobs."
            )
        infer_outputs = engine.infer_log_probs(dataset, return_meta_data)
        raw_predictions = (
            [output.prediction for output in infer_outputs]
            if return_meta_data
            else infer_outputs
        )
        raw_predictions = [
            json.dumps(raw_prediction) for raw_prediction in raw_predictions
        ]
    else:
        infer_outputs = engine.infer(dataset, return_meta_data)
        raw_predictions = (
            [output.prediction for output in infer_outputs]
            if return_meta_data
            else infer_outputs
        )
    predictions = post_process(raw_predictions, dataset)
    if return_data:
        if return_meta_data:
            infer_output_list = [
                infer_output.__dict__ for infer_output in infer_outputs
            ]
            for infer_output in infer_output_list:
                del infer_output["prediction"]
            dataset = dataset.add_column("infer_meta_data", infer_output_list)
        dataset = dataset.add_column("prediction", predictions)
        return dataset.add_column("raw_prediction", raw_predictions)
    return predictions


def select(
    instance_or_instances,
    engine: OptionSelectingByLogProbsInferenceEngine,
    dataset_query: Optional[str] = None,
    return_data: bool = False,
    previous_messages: Optional[List[Dict[str, str]]] = None,
    **kwargs,
):
    dataset = produce(instance_or_instances, dataset_query, **kwargs)
    if previous_messages is not None:

        def add_previous_messages(example, index):
            example["source"] = previous_messages[index] + example["source"]
            return example

        dataset = dataset.map(add_previous_messages, with_indices=True)
    engine, _ = fetch_artifact(engine)
    predictions = engine.select(dataset)
    # predictions = post_process(raw_predictions, dataset)
    if return_data:
        return dataset.add_column("prediction", predictions)
    return predictions

1	import json	1✔
2	from datetime import datetime	1✔
3	from functools import lru_cache	1✔
4	from typing import Any, Dict, List, Optional, Union	1✔
5
6	from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict	1✔
7
8	from .artifact import fetch_artifact	1✔
9	from .card import TaskCard	1✔
10	from .dataset_utils import get_dataset_artifact	1✔
11	from .error_utils import UnitxtError	1✔
12	from .inference import (	1✔
13	InferenceEngine,
14	LogProbInferenceEngine,
15	OptionSelectingByLogProbsInferenceEngine,
16	)
17	from .loaders import LoadFromDictionary	1✔
18	from .logging_utils import get_logger	1✔
19	from .metric_utils import EvaluationResults, _compute, _inference_post_process	1✔
20	from .operator import SourceOperator	1✔
21	from .schema import loads_instance	1✔
22	from .settings_utils import get_constants, get_settings	1✔
23	from .standard import DatasetRecipe	1✔
24	from .task import Task	1✔
25
26	logger = get_logger()	1✔
27	constants = get_constants()	1✔
28	settings = get_settings()	1✔
29
30
31	def load(source: Union[SourceOperator, str]):	1✔
32	assert isinstance(	×
33	source, (SourceOperator, str)
34	), "source must be a SourceOperator or a string"
35	if isinstance(source, str):	×
36	source, _ = fetch_artifact(source)	×
37	return source().to_dataset()	×
38
39
40	def _get_recipe_from_query(dataset_query: str) -> DatasetRecipe:	1✔
41	dataset_query = dataset_query.replace("sys_prompt", "instruction")	1✔
42	try:	1✔
43	dataset_stream, _ = fetch_artifact(dataset_query)	1✔
44	except:	1✔
45	dataset_stream = get_dataset_artifact(dataset_query)	1✔
46	return dataset_stream	1✔
47
48
49	def _get_recipe_from_dict(dataset_params: Dict[str, Any]) -> DatasetRecipe:	1✔
50	recipe_attributes = list(DatasetRecipe.__dict__["__fields__"].keys())	1✔
51	for param in dataset_params.keys():	1✔
52	assert param in recipe_attributes, (	1✔
53	f"The parameter '{param}' is not an attribute of the 'DatasetRecipe' class. "
54	f"Please check if the name is correct. The available attributes are: '{recipe_attributes}'."
55	)
56	return DatasetRecipe(**dataset_params)	1✔
57
58
59	def _verify_dataset_args(dataset_query: Optional[str] = None, dataset_args=None):	1✔
60	if dataset_query and dataset_args:	1✔
61	raise ValueError(	×
62	"Cannot provide 'dataset_query' and key-worded arguments at the same time. "
63	"If you want to load dataset from a card in local catalog, use query only. "
64	"Otherwise, use key-worded arguments only to specify properties of dataset."
65	)
66
67	if dataset_query:	1✔
68	if not isinstance(dataset_query, str):	1✔
69	raise ValueError(	×
70	f"If specified, 'dataset_query' must be a string, however, "
71	f"'{dataset_query}' was provided instead, which is of type "
72	f"'{type(dataset_query)}'."
73	)
74
75	if not dataset_query and not dataset_args:	1✔
76	raise ValueError(	×
77	"Either 'dataset_query' or key-worded arguments must be provided."
78	)
79
80
81	def load_recipe(dataset_query: Optional[str] = None, **kwargs) -> DatasetRecipe:	1✔
82	if isinstance(dataset_query, DatasetRecipe):	1✔
83	return dataset_query	×
84
85	_verify_dataset_args(dataset_query, kwargs)	1✔
86
87	if dataset_query:	1✔
88	recipe = _get_recipe_from_query(dataset_query)	1✔
89
90	if kwargs:	1✔
91	recipe = _get_recipe_from_dict(kwargs)	1✔
92
93	return recipe	1✔
94
95
96	def create_dataset(	1✔
97	task: Union[str, Task],
98	test_set: List[Dict[Any, Any]],
99	train_set: Optional[List[Dict[Any, Any]]] = None,
100	validation_set: Optional[List[Dict[Any, Any]]] = None,
101	split: Optional[str] = None,
102	**kwargs,
103	) -> Union[DatasetDict, IterableDatasetDict, Dataset, IterableDataset]:
104	"""Creates dataset from input data based on a specific task.
105
106	Args:
107	task: The name of the task from the Unitxt Catalog (https://www.unitxt.ai/en/latest/catalog/catalog.tasks.__dir__.html)
108	test_set : required list of instances
109	train_set : optional train_set
110	validation_set: optional validation set
111	split: optional one split to choose
112	**kwargs: Arguments used to load dataset from provided datasets (see load_dataset())
113
114	Returns:
115	DatasetDict
116
117	Example:
118	template = Template(...)
119	dataset = create_dataset(task="tasks.qa.open", template=template, format="formats.chatapi")
120	"""
121	data = {"test": test_set}	×
122	if train_set is not None:	×
123	data["train"] = train_set	×
124	if validation_set is not None:	×
125	data["validation"] = validation_set	×
126	task, _ = fetch_artifact(task)	×
127
128	if "template" not in kwargs and task.default_template is None:	×
129	raise Exception(	×
130	f"No 'template' was passed to the create_dataset() and the given task ('{task.__id__}') has no 'default_template' field."
131	)
132
133	card = TaskCard(loader=LoadFromDictionary(data=data), task=task)	×
134	return load_dataset(card=card, split=split, **kwargs)	×
135
136
137	def load_dataset(	1✔
138	dataset_query: Optional[str] = None,
139	split: Optional[str] = None,
140	streaming: bool = False,
141	disable_cache: Optional[bool] = None,
142	**kwargs,
143	) -> Union[DatasetDict, IterableDatasetDict, Dataset, IterableDataset]:
144	"""Loads dataset.
145
146	If the 'dataset_query' argument is provided, then dataset is loaded from a card
147	in local catalog based on parameters specified in the query.
148
149	Alternatively, dataset is loaded from a provided card based on explicitly
150	given parameters.
151
152	Args:
153	dataset_query (str, optional):
154	A string query which specifies a dataset to load from
155	local catalog or name of specific recipe or benchmark in the catalog. For
156	example, ``"card=cards.wnli,template=templates.classification.multi_class.relation.default"``.
157	streaming (bool, False):
158	When True yields the data as Unitxt streams dictionary
159	split (str, optional):
160	The split of the data to load
161	disable_cache (str, optional):
162	Disable caching process of the data
163	**kwargs:
164	Arguments used to load dataset from provided card, which is not present in local catalog.
165
166	Returns:
167	DatasetDict
168
169	:Example:
170
171	.. code-block:: python
172
173	dataset = load_dataset(
174	dataset_query="card=cards.stsb,template=templates.regression.two_texts.simple,max_train_instances=5"
175	) # card and template must be present in local catalog
176
177	# or built programmatically
178	card = TaskCard(...)
179	template = Template(...)
180	loader_limit = 10
181	dataset = load_dataset(card=card, template=template, loader_limit=loader_limit)
182
183	"""
184	recipe = load_recipe(dataset_query, **kwargs)	1✔
185	ms = recipe()	1✔
186	return {stream_name: list(ms[stream_name]) for stream_name in ms}	1✔
187
188
189	def fill_metadata(**kwargs):	1✔
190	metadata = kwargs.copy()	×
191	metadata["unitxt_version"] = get_constants().version	×
192	metadata["creation_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]	×
193	return metadata	×
194
195
196	def evaluate(	1✔
197	predictions, dataset: Union[Dataset, IterableDataset] = None, data=None
198	) -> EvaluationResults:
199	if dataset is None and data is None:	1✔
200	raise UnitxtError(message="Specify 'dataset' in evaluate")	×
201	if data is not None:	1✔
202	dataset = data # for backward compatibility	1✔
203	evaluation_result = _compute(predictions=predictions, references=dataset)	1✔
204	if hasattr(dataset, "info") and hasattr(dataset.info, "description"):	1✔
205	evaluation_result.metadata["dataset"] = dataset.info.description	1✔
206	if hasattr(predictions, "metadata"):	1✔
207	evaluation_result.metadata["predictions"] = predictions.metadata	×
208	evaluation_result.metadata["creation_time"] = datetime.now().strftime(	1✔
209	"%Y-%m-%d %H:%M:%S.%f"
210	)[:-3]
211	return evaluation_result	1✔
212
213
214	def post_process(predictions, data) -> List[Dict[str, Any]]:	1✔
215	return _inference_post_process(predictions=predictions, references=data)	1✔
216
217
218	@lru_cache	1✔
219	def _get_produce_with_cache(dataset_query: Optional[str] = None, **kwargs):	1✔
220	return load_recipe(dataset_query, **kwargs).produce	1✔
221
222
223	def produce(	1✔
224	instance_or_instances, dataset_query: Optional[str] = None, **kwargs
225	) -> Union[Dataset, Dict[str, Any]]:
226	is_list = isinstance(instance_or_instances, list)	1✔
227	if not is_list:	1✔
228	instance_or_instances = [instance_or_instances]	1✔
229	result = _get_produce_with_cache(dataset_query, **kwargs)(instance_or_instances)	1✔
230	if not is_list:	1✔
231	return result[0]	1✔
232	return Dataset.from_list(result).with_transform(loads_instance)	1✔
233
234
235	def infer(	1✔
236	instance_or_instances,
237	engine: InferenceEngine,
238	dataset_query: Optional[str] = None,
239	return_data: bool = False,
240	return_log_probs: bool = False,
241	return_meta_data: bool = False,
242	previous_messages: Optional[List[Dict[str, str]]] = None,
243	**kwargs,
244	):
245	dataset = produce(instance_or_instances, dataset_query, **kwargs)	1✔
246	if previous_messages is not None:	1✔
247
248	def add_previous_messages(example, index):	×
249	example["source"] = previous_messages[index] + example["source"]	×
250	return example	×
251
252	dataset = dataset.map(add_previous_messages, with_indices=True)	×
253	engine, _ = fetch_artifact(engine)	1✔
254	if return_log_probs:	1✔
255	if not isinstance(engine, LogProbInferenceEngine):	×
256	raise NotImplementedError(	×
257	f"Error in infer: return_log_probs set to True but supplied engine "
258	f"{engine.__class__.__name__} does not support logprobs."
259	)
260	infer_outputs = engine.infer_log_probs(dataset, return_meta_data)	×
261	raw_predictions = (	×
262	[output.prediction for output in infer_outputs]
263	if return_meta_data
264	else infer_outputs
265	)
266	raw_predictions = [	×
267	json.dumps(raw_prediction) for raw_prediction in raw_predictions
268	]
269	else:
270	infer_outputs = engine.infer(dataset, return_meta_data)	1✔
271	raw_predictions = (	1✔
272	[output.prediction for output in infer_outputs]
273	if return_meta_data
274	else infer_outputs
275	)
276	predictions = post_process(raw_predictions, dataset)	1✔
277	if return_data:	1✔
278	if return_meta_data:	1✔
279	infer_output_list = [	×
280	infer_output.__dict__ for infer_output in infer_outputs
281	]
282	for infer_output in infer_output_list:	×
283	del infer_output["prediction"]	×
284	dataset = dataset.add_column("infer_meta_data", infer_output_list)	×
285	dataset = dataset.add_column("prediction", predictions)	1✔
286	return dataset.add_column("raw_prediction", raw_predictions)	1✔
287	return predictions	1✔
288
289
290	def select(	1✔
291	instance_or_instances,
292	engine: OptionSelectingByLogProbsInferenceEngine,
293	dataset_query: Optional[str] = None,
294	return_data: bool = False,
295	previous_messages: Optional[List[Dict[str, str]]] = None,
296	**kwargs,
297	):
298	dataset = produce(instance_or_instances, dataset_query, **kwargs)	×
299	if previous_messages is not None:	×
300
301	def add_previous_messages(example, index):	×
302	example["source"] = previous_messages[index] + example["source"]	×
303	return example	×
304
305	dataset = dataset.map(add_previous_messages, with_indices=True)	×
306	engine, _ = fetch_artifact(engine)	×
307	predictions = engine.select(dataset)	×
308	# predictions = post_process(raw_predictions, dataset)
309	if return_data:	×
310	return dataset.add_column("prediction", predictions)	×
311	return predictions	×

IBM / unitxt / 12999897484

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous