13118649262

Committed 03 Feb 2025 04:56PM UTC coverage: 95.996% (-0.5%) from 96.536%

Build # 13118649262

Build Type

Pull #555

github

Committed by

web-flow

Commit Message

Merge 2f06a3fcd into 5ec2a2015

Pull Request Pull Request #555: Add linting

Run Details

76 of 83 new or added lines in 16 files covered. (91.57%)

4 existing lines in 2 files now uncovered.

1079 of 1124 relevant lines covered (96.0%)

0.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.92

/executorlib/standalone/serialize.py

import hashlib
import inspect
import re
from typing import Callable, Optional

import cloudpickle


def cloudpickle_register(ind: int = 2):
    """
    Cloudpickle can either pickle by value or pickle by reference. The functions which are communicated have to
    be pickled by value rather than by reference, so the module which calls the map function is pickled by value.
    https://github.com/cloudpipe/cloudpickle#overriding-pickles-serialization-mechanism-for-importable-constructs
    inspect can help to find the module which is calling executorlib
    https://docs.python.org/3/library/inspect.html
    to learn more about inspect another good read is:
    http://pymotw.com/2/inspect/index.html#module-inspect
    1 refers to 1 level higher than the map function

    Args:
        ind (int): index of the level at which pickle by value starts while for the rest pickle by reference is used
    """
    try:  # When executed in a jupyter notebook this can cause a ValueError - in this case we just ignore it.
        cloudpickle.register_pickle_by_value(inspect.getmodule(inspect.stack()[ind][0]))
    except IndexError:
        cloudpickle_register(ind=ind - 1)
    except ValueError:
        pass


def serialize_funct_h5(
    fn: Callable, fn_args: Optional[list] = None, fn_kwargs: Optional[dict] = None, resource_dict: Optional[dict] = None
) -> tuple[str, dict]:
    """
    Serialize a function and its arguments and keyword arguments into an HDF5 file.

    Args:
        fn (Callable): The function to be serialized.
        fn_args (list): The arguments of the function.
        fn_kwargs (dict): The keyword arguments of the function.
        resource_dict (dict): resource dictionary, which defines the resources used for the execution of the function.
                              Example resource dictionary: {
                                  cores: 1,
                                  threads_per_core: 1,
                                  gpus_per_worker: 0,
                                  oversubscribe: False,
                                  cwd: None,
                                  executor: None,
                                  hostname_localhost: False,
                              }

    Returns:
        Tuple[str, dict]: A tuple containing the task key and the serialized data.

    """
    if fn_args is None:
        fn_args = []
    if fn_kwargs is None:
        fn_kwargs = {}
    if resource_dict is None:
        resource_dict = {}
    binary_all = cloudpickle.dumps(
        {"fn": fn, "args": fn_args, "kwargs": fn_kwargs, "resource_dict": resource_dict}
    )
    task_key = fn.__name__ + _get_hash(binary=binary_all)
    data = {
        "fn": fn,
        "args": fn_args,
        "kwargs": fn_kwargs,
        "resource_dict": resource_dict,
    }
    return task_key, data


def _get_hash(binary: bytes) -> str:
    """
    Get the hash of a binary.

    Args:
        binary (bytes): The binary to be hashed.

    Returns:
        str: The hash of the binary.

    """
    # Remove specification of jupyter kernel from hash to be deterministic
    binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)
    return str(hashlib.md5(binary_no_ipykernel).hexdigest())

1	import hashlib	1✔
2	import inspect	1✔
3	import re	1✔
4	from typing import Callable, Optional	1✔
5
6	import cloudpickle	1✔
7
8
9	def cloudpickle_register(ind: int = 2):	1✔
10	"""
11	Cloudpickle can either pickle by value or pickle by reference. The functions which are communicated have to
12	be pickled by value rather than by reference, so the module which calls the map function is pickled by value.
13	https://github.com/cloudpipe/cloudpickle#overriding-pickles-serialization-mechanism-for-importable-constructs
14	inspect can help to find the module which is calling executorlib
15	https://docs.python.org/3/library/inspect.html
16	to learn more about inspect another good read is:
17	http://pymotw.com/2/inspect/index.html#module-inspect
18	1 refers to 1 level higher than the map function
19
20	Args:
21	ind (int): index of the level at which pickle by value starts while for the rest pickle by reference is used
22	"""
23	try: # When executed in a jupyter notebook this can cause a ValueError - in this case we just ignore it.	1✔
24	cloudpickle.register_pickle_by_value(inspect.getmodule(inspect.stack()[ind][0]))	1✔
25	except IndexError:	×
26	cloudpickle_register(ind=ind - 1)	×
27	except ValueError:	×
28	pass	×
29
30
31	def serialize_funct_h5(	1✔
32	fn: Callable, fn_args: Optional[list] = None, fn_kwargs: Optional[dict] = None, resource_dict: Optional[dict] = None
33	) -> tuple[str, dict]:
34	"""
35	Serialize a function and its arguments and keyword arguments into an HDF5 file.
36
37	Args:
38	fn (Callable): The function to be serialized.
39	fn_args (list): The arguments of the function.
40	fn_kwargs (dict): The keyword arguments of the function.
41	resource_dict (dict): resource dictionary, which defines the resources used for the execution of the function.
42	Example resource dictionary: {
43	cores: 1,
44	threads_per_core: 1,
45	gpus_per_worker: 0,
46	oversubscribe: False,
47	cwd: None,
48	executor: None,
49	hostname_localhost: False,
50	}
51
52	Returns:
53	Tuple[str, dict]: A tuple containing the task key and the serialized data.
54
55	"""
56	if fn_args is None:	1✔
UNCOV 57	fn_args = []	×
58	if fn_kwargs is None:	1✔
NEW 59	fn_kwargs = {}	×
60	if resource_dict is None:	1✔
61	resource_dict = {}	1✔
62	binary_all = cloudpickle.dumps(	1✔
63	{"fn": fn, "args": fn_args, "kwargs": fn_kwargs, "resource_dict": resource_dict}
64	)
65	task_key = fn.__name__ + _get_hash(binary=binary_all)	1✔
66	data = {	1✔
67	"fn": fn,
68	"args": fn_args,
69	"kwargs": fn_kwargs,
70	"resource_dict": resource_dict,
71	}
72	return task_key, data	1✔
73
74
75	def _get_hash(binary: bytes) -> str:	1✔
76	"""
77	Get the hash of a binary.
78
79	Args:
80	binary (bytes): The binary to be hashed.
81
82	Returns:
83	str: The hash of the binary.
84
85	"""
86	# Remove specification of jupyter kernel from hash to be deterministic
87	binary_no_ipykernel = re.sub(b"(?<=/ipykernel_)(.*)(?=/)", b"", binary)	1✔
88	return str(hashlib.md5(binary_no_ipykernel).hexdigest())	1✔

pyiron / executorlib / 13118649262

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous