4119519683

Build # 4119519683

Build Type

push

github

Committed by GitHub

Commit Message

Lintify (#287)

Run Details

36 of 36 new or added lines in 9 files covered. (100.0%)

5321 of 6332 relevant lines covered (84.03%)

1.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.64

/winterdrp/utils/execute_cmd.py

"""
Module for executing bash commands
"""
import logging
import os
import shutil
import subprocess
from pathlib import Path

import docker

from winterdrp.utils.dockerutil import (
    docker_batch_put,
    docker_get_new_files,
    docker_path,
    new_container,
)

logger = logging.getLogger(__name__)


class ExecutionError(Exception):
    """Error relating to executing bash command"""


DEFAULT_TIMEOUT = 300.0


def run_local(cmd: str, output_dir: str = ".", timeout: float = DEFAULT_TIMEOUT):
    """
    Function to run on local machine using subprocess, with error handling.

    After the specified 'cmd' command has been run, any newly-generated files
    will be copied out of the current directory to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the command you want to use to run sextractor.
    An example would be:
        cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
    output_dir: A local directory to save the output files to.
    timeout: Time to timeout in seconds

    Returns
    -------

    """

    try:
        # See what files are in the directory beforehand

        ignore_files = (
            subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
        )

        # Run sextractor

        rval = subprocess.run(
            cmd, check=True, capture_output=True, shell=True, timeout=timeout
        )

        msg = "Successfully executed command. "

        if rval.stdout.decode() != "":
            msg += f"Found the following output: {rval.stdout.decode()}"
        logger.debug(msg)

        try:
            os.makedirs(output_dir)
        except OSError:
            pass

        # Move new files to output dir

        new_files = [
            x
            for x in subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
            if x not in ignore_files
        ]

        current_dir = (
            subprocess.run("pwd", check=True, capture_output=True)
            .stdout.decode()
            .strip()
        )

        if len(new_files) > 0:
            logger.debug(
                f"The following new files were created in the current directory: "
                f"{new_files}"
            )

        for file in new_files:
            current_path = os.path.join(current_dir, file)
            output_path = os.path.join(output_dir, file)

            logger.info(f"File saved to {output_path}")

            shutil.move(current_path, output_path)

    except subprocess.CalledProcessError as err:
        msg = (
            f"Error found when running with command: \n \n '{err.cmd}' \n \n"
            f"This yielded a return code of {err.returncode}. "
            f"The following traceback was found: \n {err.stderr.decode()}"
        )
        logger.error(msg)
        raise ExecutionError(msg) from err


def temp_config(config_path: str | Path, output_dir: str | Path) -> Path:
    """
    Get a

    :param config_path:
    :param output_dir:
    :return:
    """
    basename = f"temp_{Path(config_path).name}"
    return Path(output_dir).joinpath(basename)


def run_docker(cmd: str, output_dir: Path | str = "."):
    """Function to run a command via Docker.
    A container will be generated automatically,
    but a Docker server must be running first.
    You can start one via the Desktop application,
    or on the command line with `docker start'.

    After the specified 'cmd' command has been run, any newly-generated files
     will be copied out of the container to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the base arguments you want to use to run sextractor.
    An example would be:
        cmd = 'image01.fits -c sex.config'
    output_dir: A local directory to save the output files to.

    Returns
    -------

    """

    container = new_container()

    try:
        container.attach()

        container.start()

        split = cmd.split(" -")

        # Reorganise the commands so that each '-x' argument is grouped together
        # Basically still work even if someone puts the filename in a weird place

        sorted_split = []

        for i, arg in enumerate(split):
            sep = arg.split(" ")
            sorted_split.append(" ".join(sep[:2]))
            if len(sep) > 2:
                sorted_split[0] += " " + " ".join(sep[2:])

        new_split = []

        # Loop over sextractor command, and
        # copy everything that looks like a file into container
        # Go through everything that looks like a file with paths in it after

        copy_list = []
        temp_files = []

        files_of_files = []

        for i, arg in enumerate(sorted_split):
            sep = arg.split(" ")

            if sep[0] == "c":
                files_of_files.append(sep[1])

            new = list(sep)

            for j, x in enumerate(sep):
                if len(x) > 0:
                    if os.path.isfile(x):
                        new[j] = docker_path(sep[j])
                        copy_list.append(sep[j])
                    elif x[0] == "@":
                        files_of_files.append(x[1:])
                    elif os.path.isdir(os.path.dirname(x)):
                        new[j] = docker_path(sep[j])

            new_split.append(" ".join(new))

        cmd = " -".join(new_split)

        # Be extra clever: go through files and check there too!

        logger.debug(
            f"Found the following files which should contain paths: {files_of_files}"
        )

        for path in files_of_files:
            new_file = []

            with open(path, "rb", encoding="utf8") as local_file:
                for line in local_file.readlines():
                    args = [x for x in line.decode().split(" ") if x not in [""]]
                    new_args = list(args)
                    for i, arg in enumerate(args):
                        if os.path.isfile(arg):
                            copy_list.append(arg)
                            new_args[i] = docker_path(arg)
                        elif os.path.isfile(arg.strip("\n")):
                            copy_list.append(arg.strip("\n"))
                            new_args[i] = str(docker_path(arg.strip("\n"))) + "\n"
                    new_file.append(" ".join(new_args))

            temp_file_path = temp_config(path, output_dir)

            with open(temp_file_path, "w", encoding="utf8") as temp_file:
                temp_file.writelines(new_file)

            copy_list.append(temp_file_path)

            cmd = cmd.replace(path + " ", str(docker_path(temp_file_path)) + " ")

        # Copy in files, and see what files are already there

        copy_list = list(set(copy_list))

        logger.debug(f"Copying {copy_list} into container")

        ignore_files = docker_batch_put(container=container, local_paths=copy_list)

        # Run command

        log = container.exec_run(cmd, stderr=True, stdout=True)

        for temp_file_path in temp_files:
            logger.debug(f"Deleting temporary file {temp_file_path}")
            os.remove(temp_file_path)

        if not log.output == b"":
            logger.info(f"Output: {log.output.decode()}")

        if not log.exit_code == 0:
            err = (
                f"Error running command: \n '{cmd}'\n "
                f"which resulted in returncode '{log.exit_code}' and"
                f"the following error message: \n '{log.output.decode()}'"
            )
            logger.error(err)
            raise subprocess.CalledProcessError(
                returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
            )

        # Copy out any files which did not exist before running sextractor

        docker_get_new_files(
            container=container, output_dir=output_dir, ignore_files=ignore_files
        )

    except docker.errors.APIError as err:
        logger.error(err)
        raise ExecutionError(err) from err
    finally:
        # In any case, clean up by killing the container and removing files
        container.kill()
        container.remove()


def execute(
    cmd: str,
    output_dir: Path | str = ".",
    local: bool = True,
    timeout: float = DEFAULT_TIMEOUT,
):
    """
    Generically execute a command either via bash or a docker container

    :param cmd: command
    :param output_dir: output directory for command
    :param local: boolean whether use local or docker
    :param timeout: timeout for local execution
    :return: None
    """
    logger.debug(
        f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
    )
    if local:
        run_local(cmd, output_dir=output_dir, timeout=timeout)
    else:
        run_docker(cmd, output_dir=output_dir)

1	"""
2	Module for executing bash commands
3	"""
4	import logging	2✔
5	import os	2✔
6	import shutil	2✔
7	import subprocess	2✔
8	from pathlib import Path	2✔
9
10	import docker	2✔
11
12	from winterdrp.utils.dockerutil import (	2✔
13	docker_batch_put,
14	docker_get_new_files,
15	docker_path,
16	new_container,
17	)
18
19	logger = logging.getLogger(__name__)	2✔
20
21
22	class ExecutionError(Exception):	2✔
23	"""Error relating to executing bash command"""
24
25
26	DEFAULT_TIMEOUT = 300.0	2✔
27
28
29	def run_local(cmd: str, output_dir: str = ".", timeout: float = DEFAULT_TIMEOUT):	2✔
30	"""
31	Function to run on local machine using subprocess, with error handling.
32
33	After the specified 'cmd' command has been run, any newly-generated files
34	will be copied out of the current directory to 'output_dir'
35
36	Parameters
37	----------
38	cmd: A string containing the command you want to use to run sextractor.
39	An example would be:
40	cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
41	output_dir: A local directory to save the output files to.
42	timeout: Time to timeout in seconds
43
44	Returns
45	-------
46
47	"""
48
49	try:	2✔
50	# See what files are in the directory beforehand
51
52	ignore_files = (	2✔
53	subprocess.run("ls", check=True, capture_output=True)
54	.stdout.decode()
55	.split("\n")
56	)
57
58	# Run sextractor
59
60	rval = subprocess.run(	2✔
61	cmd, check=True, capture_output=True, shell=True, timeout=timeout
62	)
63
64	msg = "Successfully executed command. "	2✔
65
66	if rval.stdout.decode() != "":	2✔
67	msg += f"Found the following output: {rval.stdout.decode()}"	×
68	logger.debug(msg)	2✔
69
70	try:	2✔
71	os.makedirs(output_dir)	2✔
72	except OSError:	2✔
73	pass	2✔
74
75	# Move new files to output dir
76
77	new_files = [	2✔
78	x
79	for x in subprocess.run("ls", check=True, capture_output=True)
80	.stdout.decode()
81	.split("\n")
82	if x not in ignore_files
83	]
84
85	current_dir = (	2✔
86	subprocess.run("pwd", check=True, capture_output=True)
87	.stdout.decode()
88	.strip()
89	)
90
91	if len(new_files) > 0:	2✔
92	logger.debug(	2✔
93	f"The following new files were created in the current directory: "
94	f"{new_files}"
95	)
96
97	for file in new_files:	2✔
98	current_path = os.path.join(current_dir, file)	2✔
99	output_path = os.path.join(output_dir, file)	2✔
100
101	logger.info(f"File saved to {output_path}")	2✔
102
103	shutil.move(current_path, output_path)	2✔
104
105	except subprocess.CalledProcessError as err:	×
106	msg = (	×
107	f"Error found when running with command: \n \n '{err.cmd}' \n \n"
108	f"This yielded a return code of {err.returncode}. "
109	f"The following traceback was found: \n {err.stderr.decode()}"
110	)
111	logger.error(msg)
112	raise ExecutionError(msg) from err
113
114
115	def temp_config(config_path: str \| Path, output_dir: str \| Path) -> Path:	2✔
116	"""
117	Get a
118
119	:param config_path:
120	:param output_dir:
121	:return:
122	"""
123	basename = f"temp_{Path(config_path).name}"	×
124	return Path(output_dir).joinpath(basename)	×
125
126
127	def run_docker(cmd: str, output_dir: Path \| str = "."):	2✔
128	"""Function to run a command via Docker.
129	A container will be generated automatically,
130	but a Docker server must be running first.
131	You can start one via the Desktop application,
132	or on the command line with `docker start'.
133
134	After the specified 'cmd' command has been run, any newly-generated files
135	will be copied out of the container to 'output_dir'
136
137	Parameters
138	----------
139	cmd: A string containing the base arguments you want to use to run sextractor.
140	An example would be:
141	cmd = 'image01.fits -c sex.config'
142	output_dir: A local directory to save the output files to.
143
144	Returns
145	-------
146
147	"""
148
149	container = new_container()	×
150
151	try:	×
152	container.attach()	×
153
154	container.start()	×
155
156	split = cmd.split(" -")	×
157
158	# Reorganise the commands so that each '-x' argument is grouped together
159	# Basically still work even if someone puts the filename in a weird place
160
161	sorted_split = []	×
162
163	for i, arg in enumerate(split):	×
164	sep = arg.split(" ")	×
165	sorted_split.append(" ".join(sep[:2]))	×
166	if len(sep) > 2:	×
167	sorted_split[0] += " " + " ".join(sep[2:])	×
168
169	new_split = []	×
170
171	# Loop over sextractor command, and
172	# copy everything that looks like a file into container
173	# Go through everything that looks like a file with paths in it after
174
175	copy_list = []	×
176	temp_files = []	×
177
178	files_of_files = []	×
179
180	for i, arg in enumerate(sorted_split):	×
181	sep = arg.split(" ")	×
182
183	if sep[0] == "c":	×
184	files_of_files.append(sep[1])	×
185
186	new = list(sep)	×
187
188	for j, x in enumerate(sep):	×
189	if len(x) > 0:	×
190	if os.path.isfile(x):	×
191	new[j] = docker_path(sep[j])	×
192	copy_list.append(sep[j])	×
193	elif x[0] == "@":	×
194	files_of_files.append(x[1:])	×
195	elif os.path.isdir(os.path.dirname(x)):	×
196	new[j] = docker_path(sep[j])	×
197
198	new_split.append(" ".join(new))	×
199
200	cmd = " -".join(new_split)	×
201
202	# Be extra clever: go through files and check there too!
203
204	logger.debug(	×
205	f"Found the following files which should contain paths: {files_of_files}"
206	)
207
208	for path in files_of_files:	×
209	new_file = []	×
210
211	with open(path, "rb", encoding="utf8") as local_file:	×
212	for line in local_file.readlines():	×
213	args = [x for x in line.decode().split(" ") if x not in [""]]	×
214	new_args = list(args)	×
215	for i, arg in enumerate(args):	×
216	if os.path.isfile(arg):	×
217	copy_list.append(arg)	×
218	new_args[i] = docker_path(arg)	×
219	elif os.path.isfile(arg.strip("\n")):	×
220	copy_list.append(arg.strip("\n"))	×
221	new_args[i] = str(docker_path(arg.strip("\n"))) + "\n"	×
222	new_file.append(" ".join(new_args))	×
223
224	temp_file_path = temp_config(path, output_dir)	×
225
226	with open(temp_file_path, "w", encoding="utf8") as temp_file:	×
227	temp_file.writelines(new_file)	×
228
229	copy_list.append(temp_file_path)	×
230
231	cmd = cmd.replace(path + " ", str(docker_path(temp_file_path)) + " ")	×
232
233	# Copy in files, and see what files are already there
234
235	copy_list = list(set(copy_list))	×
236
237	logger.debug(f"Copying {copy_list} into container")	×
238
239	ignore_files = docker_batch_put(container=container, local_paths=copy_list)	×
240
241	# Run command
242
243	log = container.exec_run(cmd, stderr=True, stdout=True)	×
244
245	for temp_file_path in temp_files:	×
246	logger.debug(f"Deleting temporary file {temp_file_path}")	×
247	os.remove(temp_file_path)	×
248
249	if not log.output == b"":	×
250	logger.info(f"Output: {log.output.decode()}")	×
251
252	if not log.exit_code == 0:	×
253	err = (
254	f"Error running command: \n '{cmd}'\n "
255	f"which resulted in returncode '{log.exit_code}' and"
256	f"the following error message: \n '{log.output.decode()}'"
257	)
258	logger.error(err)
259	raise subprocess.CalledProcessError(
260	returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
261	)
262
263	# Copy out any files which did not exist before running sextractor
264
265	docker_get_new_files(	×
266	container=container, output_dir=output_dir, ignore_files=ignore_files
267	)
268
269	except docker.errors.APIError as err:	×
270	logger.error(err)
271	raise ExecutionError(err) from err
272	finally:
273	# In any case, clean up by killing the container and removing files
274	container.kill()	×
275	container.remove()	×
276
277
278	def execute(	2✔
279	cmd: str,
280	output_dir: Path \| str = ".",
281	local: bool = True,
282	timeout: float = DEFAULT_TIMEOUT,
283	):
284	"""
285	Generically execute a command either via bash or a docker container
286
287	:param cmd: command
288	:param output_dir: output directory for command
289	:param local: boolean whether use local or docker
290	:param timeout: timeout for local execution
291	:return: None
292	"""
293	logger.debug(	2✔
294	f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
295	)
296	if local:	2✔
297	run_local(cmd, output_dir=output_dir, timeout=timeout)	2✔
298	else:
299	run_docker(cmd, output_dir=output_dir)	×

winter-telescope / winterdrp / 4119519683

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous