11766741741

Committed 10 Nov 2024 04:26PM UTC coverage: 94.813% (-0.3%) from 95.089%

Build # 11766741741

Build Type

Pull #491

github

Committed by

web-flow

Commit Message

Merge e34b5de8a into 78062fd9b

Pull Request Pull Request #491: Empty cache

Run Details

9 of 12 new or added lines in 2 files covered. (75.0%)

914 of 964 relevant lines covered (94.81%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/executorlib/standalone/serialize.py

import hashlib
import inspect
import re
from typing import Any, Tuple

import cloudpickle


def cloudpickle_register(ind: int = 2):
    """
    Cloudpickle can either pickle by value or pickle by reference. The functions which are communicated have to
    be pickled by value rather than by reference, so the module which calls the map function is pickled by value.
    https://github.com/cloudpipe/cloudpickle#overriding-pickles-serialization-mechanism-for-importable-constructs
    inspect can help to find the module which is calling executorlib
    https://docs.python.org/3/library/inspect.html
    to learn more about inspect another good read is:
    http://pymotw.com/2/inspect/index.html#module-inspect
    1 refers to 1 level higher than the map function

    Args:
        ind (int): index of the level at which pickle by value starts while for the rest pickle by reference is used
    """
    try:  # When executed in a jupyter notebook this can cause a ValueError - in this case we just ignore it.
        cloudpickle.register_pickle_by_value(inspect.getmodule(inspect.stack()[ind][0]))
    except IndexError:
        cloudpickle_register(ind=ind - 1)
    except ValueError:
        pass


def serialize_funct_h5(
    fn: callable, fn_args: list = [], fn_kwargs: dict = {}, resource_dict: dict = {}
) -> Tuple[str, dict]:
    """
    Serialize a function and its arguments and keyword arguments into an HDF5 file.

    Args:
        fn (callable): The function to be serialized.
        fn_args (list): The arguments of the function.
        fn_kwargs (dict): The keyword arguments of the function.
        resource_dict (dict): resource dictionary, which defines the resources used for the execution of the function.
                              Example resource dictionary: {
                                  cores: 1,
                                  threads_per_core: 1,
                                  gpus_per_worker: 0,
                                  oversubscribe: False,
                                  cwd: None,
                                  executor: None,
                                  hostname_localhost: False,
                              }

    Returns:
        Tuple[str, dict]: A tuple containing the task key and the serialized data.

    """
    binary_all = cloudpickle.dumps(
        {"fn": fn, "args": fn_args, "kwargs": fn_kwargs, "resource_dict": resource_dict}
    )
    task_key = fn.__name__ + _get_hash(binary=binary_all)
    data = {
        "fn": fn,
        "args": fn_args,
        "kwargs": fn_kwargs,
        "resource_dict": resource_dict,
    }
    return task_key, data


def _get_hash(binary: bytes) -> str:
    """
    Get the hash of a binary.

    Args:
        binary (bytes): The binary to be hashed.

    Returns:
        str: The hash of the binary.

    """
    # Remove specification of jupyter kernel from hash to be deterministic
    binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)
    return str(hashlib.md5(binary_no_ipykernel).hexdigest())

1	import hashlib	1✔
2	import inspect	1✔
3	import re	1✔
4	from typing import Any, Tuple	1✔
5
6	import cloudpickle	1✔
7
8
9	def cloudpickle_register(ind: int = 2):	1✔
10	"""
11	Cloudpickle can either pickle by value or pickle by reference. The functions which are communicated have to
12	be pickled by value rather than by reference, so the module which calls the map function is pickled by value.
13	https://github.com/cloudpipe/cloudpickle#overriding-pickles-serialization-mechanism-for-importable-constructs
14	inspect can help to find the module which is calling executorlib
15	https://docs.python.org/3/library/inspect.html
16	to learn more about inspect another good read is:
17	http://pymotw.com/2/inspect/index.html#module-inspect
18	1 refers to 1 level higher than the map function
19
20	Args:
21	ind (int): index of the level at which pickle by value starts while for the rest pickle by reference is used
22	"""
23	try: # When executed in a jupyter notebook this can cause a ValueError - in this case we just ignore it.	1✔
24	cloudpickle.register_pickle_by_value(inspect.getmodule(inspect.stack()[ind][0]))	1✔
25	except IndexError:	×
26	cloudpickle_register(ind=ind - 1)	×
27	except ValueError:	×
28	pass	×
29
30
31	def serialize_funct_h5(	1✔
32	fn: callable, fn_args: list = [], fn_kwargs: dict = {}, resource_dict: dict = {}
33	) -> Tuple[str, dict]:
34	"""
35	Serialize a function and its arguments and keyword arguments into an HDF5 file.
36
37	Args:
38	fn (callable): The function to be serialized.
39	fn_args (list): The arguments of the function.
40	fn_kwargs (dict): The keyword arguments of the function.
41	resource_dict (dict): resource dictionary, which defines the resources used for the execution of the function.
42	Example resource dictionary: {
43	cores: 1,
44	threads_per_core: 1,
45	gpus_per_worker: 0,
46	oversubscribe: False,
47	cwd: None,
48	executor: None,
49	hostname_localhost: False,
50	}
51
52	Returns:
53	Tuple[str, dict]: A tuple containing the task key and the serialized data.
54
55	"""
56	binary_all = cloudpickle.dumps(	1✔
57	{"fn": fn, "args": fn_args, "kwargs": fn_kwargs, "resource_dict": resource_dict}
58	)
59	task_key = fn.__name__ + _get_hash(binary=binary_all)	1✔
60	data = {	1✔
61	"fn": fn,
62	"args": fn_args,
63	"kwargs": fn_kwargs,
64	"resource_dict": resource_dict,
65	}
66	return task_key, data	1✔
67
68
69	def _get_hash(binary: bytes) -> str:	1✔
70	"""
71	Get the hash of a binary.
72
73	Args:
74	binary (bytes): The binary to be hashed.
75
76	Returns:
77	str: The hash of the binary.
78
79	"""
80	# Remove specification of jupyter kernel from hash to be deterministic
81	binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)	1✔
82	return str(hashlib.md5(binary_no_ipykernel).hexdigest())	1✔

pyiron / executorlib / 11766741741

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous