• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pyiron / executorlib / 11766741741

10 Nov 2024 04:26PM CUT coverage: 94.813% (-0.3%) from 95.089%
11766741741

Pull #491

github

web-flow
Merge e34b5de8a into 78062fd9b
Pull Request #491: Empty cache

9 of 12 new or added lines in 2 files covered. (75.0%)

914 of 964 relevant lines covered (94.81%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.1
/executorlib/standalone/cache/queue.py
1
import os
1✔
2
import subprocess
1✔
3
from typing import List, Optional, Union
1✔
4

5
from pysqa import QueueAdapter
1✔
6

7

8
def execute_with_pysqa(
1✔
9
    command: str,
10
    resource_dict: dict,
11
    task_dependent_lst: List[int] = [],
12
    config_directory: Optional[str] = None,
13
    backend: Optional[str] = None,
14
    cache_directory: Optional[str] = None,
15
) -> int:
16
    """
17
    Execute a command by submitting it to the queuing system
18

19
    Args:
20
        command (list): The command to be executed.
21
        task_dependent_lst (list): A list of subprocesses that the current subprocess depends on. Defaults to [].
22
        resource_dict (dict): resource dictionary, which defines the resources used for the execution of the function.
23
                              Example resource dictionary: {
24
                                  cwd: None,
25
                              }
26
        config_directory (str, optional): path to the config directory.
27
        backend (str, optional): name of the backend used to spawn tasks.
28
        cache_directory (str): The directory to store the HDF5 files.
29

30
    Returns:
31
        int: queuing system ID
32
    """
33
    if resource_dict is None:
1✔
34
        resource_dict = {}
×
35
    if "cwd" in resource_dict and resource_dict["cwd"] is not None:
1✔
36
        cwd = resource_dict["cwd"]
1✔
37
    else:
38
        cwd = cache_directory
×
39
    qa = QueueAdapter(
1✔
40
        directory=config_directory,
41
        queue_type=backend,
42
        execute_command=_pysqa_execute_command,
43
    )
44
    submit_kwargs = {
1✔
45
        "command": " ".join(command),
46
        "dependency_list": [str(qid) for qid in task_dependent_lst],
47
        "working_directory": os.path.abspath(cwd),
48
    }
49
    if "cwd" in resource_dict:
1✔
50
        del resource_dict["cwd"]
1✔
51
    unsupported_keys = [
1✔
52
        "threads_per_core",
53
        "gpus_per_core",
54
        "openmpi_oversubscribe",
55
        "slurm_cmd_args",
56
    ]
57
    for k in unsupported_keys:
1✔
58
        if k in resource_dict:
1✔
59
            del resource_dict[k]
1✔
60
    if "job_name" not in resource_dict:
1✔
61
        resource_dict["job_name"] = "pysqa"
1✔
62
    submit_kwargs.update(resource_dict)
1✔
63
    return qa.submit_job(**submit_kwargs)
1✔
64

65

66
def _pysqa_execute_command(
1✔
67
    commands: str,
68
    working_directory: Optional[str] = None,
69
    split_output: bool = True,
70
    shell: bool = False,
71
    error_filename: str = "pysqa.err",
72
) -> Union[str, List[str]]:
73
    """
74
    A wrapper around the subprocess.check_output function. Modified from pysqa to raise an exception if the subprocess
75
    fails to submit the job to the queue.
76

77
    Args:
78
        commands (str): The command(s) to be executed on the command line
79
        working_directory (str, optional): The directory where the command is executed. Defaults to None.
80
        split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True.
81
        shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False.
82
        error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err".
83

84
    Returns:
85
        Union[str, List[str]]: Output of the shell command either as a string or as a list of strings
86
    """
87
    if shell and isinstance(commands, list):
1✔
88
        commands = " ".join(commands)
1✔
89
    out = subprocess.check_output(
1✔
90
        commands,
91
        cwd=working_directory,
92
        stderr=subprocess.STDOUT,
93
        universal_newlines=True,
94
        shell=not isinstance(commands, list),
95
    )
96
    if out is not None and split_output:
1✔
97
        return out.split("\n")
1✔
98
    else:
99
        return out
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc