8238257922

Committed 11 Mar 2024 06:56PM UTC coverage: 80.273% (-0.03%) from 80.3%

Build # 8238257922

Build Type

push

github

Committed by

quaquel

Commit Message

bugfix to finalizer

fixes a bug in the finalizer in case of working directory models

Run Details

1 of 13 new or added lines in 2 files covered. (7.69%)

4757 of 5926 relevant lines covered (80.27%)

0.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/ema_workbench/em_framework/futures_util.py

import os
import random
import shutil
import string
import time


from collections import defaultdict

from ..util import get_module_logger


__all__ = ["finalizer", "setup_working_directories", "determine_rootdir"]

_logger = get_module_logger(__name__)


def determine_rootdir(msis):
    for model in msis:
        try:
            model.working_directory
        except AttributeError:
            root_dir = None
            break
    else:
        random_part = [random.choice(string.ascii_letters + string.digits) for _ in range(5)]
        random_part = "".join(random_part)
        root_dir = os.path.abspath("tmp" + random_part)
        os.makedirs(root_dir)
    return root_dir


def finalizer(experiment_runner):
    """cleanup"""

    def finalizer(tmpdir):
        _logger.info("finalizing")

        experiment_runner.cleanup()
        del experiment_runner

        time.sleep(1)

        if tmpdir:
            try:
                shutil.rmtree(tmpdir)
            except OSError:
                pass
    return finalizer


def setup_working_directories(models, root_dir):
    """copies the working directory of each model to a process specific
    temporary directory and update the working directory of the model

    Parameters
    ----------
    models : list
    root_dir : str

    """

    # group models by working directory to avoid copying the same directory
    # multiple times
    wd_by_model = defaultdict(list)
    for model in models:
        try:
            wd = model.working_directory
        except AttributeError:
            pass
        else:
            wd_by_model[wd].append(model)

    # if the dict is not empty
    if wd_by_model:
        # make a directory with the process id as identifier
        tmpdir_name = f"tmp{os.getpid()}"
        tmpdir = os.path.join(root_dir, tmpdir_name)
        os.mkdir(tmpdir)

        _logger.debug(f"setting up working directory: {tmpdir}")

        for key, value in wd_by_model.items():
            # we need a sub directory in the process working directory
            # for each unique model working directory
            subdir = os.path.basename(os.path.normpath(key))
            new_wd = os.path.join(tmpdir, subdir)

            # the copy operation
            shutil.copytree(key, new_wd)

            for model in value:
                model.working_directory = new_wd
        return tmpdir
    else:
        return None

1	import os	1✔
2	import random	1✔
3	import shutil	1✔
4	import string	1✔
5	import time	1✔
6
7
8	from collections import defaultdict	1✔
9
10	from ..util import get_module_logger	1✔
11
12
13	__all__ = ["finalizer", "setup_working_directories", "determine_rootdir"]	1✔
14
15	_logger = get_module_logger(__name__)	1✔
16
17
18	def determine_rootdir(msis):	1✔
19	for model in msis:	1✔
20	try:	1✔
21	model.working_directory	1✔
22	except AttributeError:	1✔
23	root_dir = None	1✔
24	break	1✔
25	else:
26	random_part = [random.choice(string.ascii_letters + string.digits) for _ in range(5)]	×
27	random_part = "".join(random_part)	×
28	root_dir = os.path.abspath("tmp" + random_part)	×
29	os.makedirs(root_dir)	×
30	return root_dir	1✔
31
32
33	def finalizer(experiment_runner):	1✔
34	"""cleanup"""
35
NEW 36	def finalizer(tmpdir):	×
NEW 37	_logger.info("finalizing")	×
38
NEW 39	experiment_runner.cleanup()	×
NEW 40	del experiment_runner	×
41
NEW 42	time.sleep(1)	×
43
NEW 44	if tmpdir:	×
NEW 45	try:	×
NEW 46	shutil.rmtree(tmpdir)	×
NEW 47	except OSError:	×
NEW 48	pass	×
NEW 49	return finalizer	×
50
51
52	def setup_working_directories(models, root_dir):	1✔
53	"""copies the working directory of each model to a process specific
54	temporary directory and update the working directory of the model
55
56	Parameters
57	----------
58	models : list
59	root_dir : str
60
61	"""
62
63	# group models by working directory to avoid copying the same directory
64	# multiple times
65	wd_by_model = defaultdict(list)	1✔
66	for model in models:	1✔
67	try:	1✔
68	wd = model.working_directory	1✔
69	except AttributeError:	1✔
70	pass	1✔
71	else:
72	wd_by_model[wd].append(model)	×
73
74	# if the dict is not empty
75	if wd_by_model:	1✔
76	# make a directory with the process id as identifier
77	tmpdir_name = f"tmp{os.getpid()}"	×
78	tmpdir = os.path.join(root_dir, tmpdir_name)	×
79	os.mkdir(tmpdir)	×
80
81	_logger.debug(f"setting up working directory: {tmpdir}")	×
82
83	for key, value in wd_by_model.items():	×
84	# we need a sub directory in the process working directory
85	# for each unique model working directory
86	subdir = os.path.basename(os.path.normpath(key))	×
87	new_wd = os.path.join(tmpdir, subdir)	×
88
89	# the copy operation
90	shutil.copytree(key, new_wd)	×
91
92	for model in value:	×
93	model.working_directory = new_wd	×
94	return tmpdir	×
95	else:
96	return None	1✔

quaquel / EMAworkbench / 8238257922

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous