3769534174

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/winter-telescope/winterdrp/commit/<a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">fb65b52df">&lt;a href=&quot;https://github.com/winter-telescope/winterdrp/commit/</a><a class="double-link" href="https://github.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">fb65b52df</a><a href="https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/winter-telescope/winterdrp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&quot;&gt;fb65b52df&lt;/a&gt;&lt;a href=&quot;https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&amp;quot;&amp;... (continued)

Pull Request Pull Request #256: Lintify

Run Details

90 of 90 new or added lines in 15 files covered. (100.0%)

4632 of 6121 relevant lines covered (75.67%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.64

/winterdrp/utils/execute_cmd.py

"""
Module for executing bash commands
"""
import logging
import os
import shutil
import subprocess
from pathlib import Path

import docker

from winterdrp.utils.dockerutil import (
    docker_batch_put,
    docker_get_new_files,
    docker_path,
    new_container,
)

logger = logging.getLogger(__name__)


class ExecutionError(Exception):
    """Error relating to executing bash command"""


DEFAULT_TIMEOUT = 300.0


def run_local(cmd: str, output_dir: str = ".", timeout: float = DEFAULT_TIMEOUT):
    """
    Function to run on local machine using subprocess, with error handling.

    After the specified 'cmd' command has been run, any newly-generated files
    will be copied out of the current directory to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the command you want to use to run sextractor.
    An example would be:
        cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
    output_dir: A local directory to save the output files to.
    timeout: Time to timeout in seconds

    Returns
    -------

    """

    try:

        # See what files are in the directory beforehand

        ignore_files = (
            subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
        )

        # Run sextractor

        rval = subprocess.run(
            cmd, check=True, capture_output=True, shell=True, timeout=timeout
        )

        msg = "Successfully executed command. "

        if rval.stdout.decode() != "":
            msg += f"Found the following output: {rval.stdout.decode()}"
        logger.debug(msg)

        try:
            os.makedirs(output_dir)
        except OSError:
            pass

        # Move new files to output dir

        new_files = [
            x
            for x in subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
            if x not in ignore_files
        ]

        current_dir = (
            subprocess.run("pwd", check=True, capture_output=True)
            .stdout.decode()
            .strip()
        )

        if len(new_files) > 0:

            logger.debug(
                f"The following new files were created in the current directory: "
                f"{new_files}"
            )

        for file in new_files:

            current_path = os.path.join(current_dir, file)
            output_path = os.path.join(output_dir, file)

            logger.info(f"File saved to {output_path}")

            shutil.move(current_path, output_path)

    except subprocess.CalledProcessError as err:
        msg = (
            f"Error found when running with command: \n \n '{err.cmd}' \n \n"
            f"This yielded a return code of {err.returncode}. "
            f"The following traceback was found: \n {err.stderr.decode()}"
        )
        logger.error(msg)
        raise ExecutionError(msg) from err


def temp_config(config_path: str | Path, output_dir: str | Path) -> Path:
    """
    Get a

    :param config_path:
    :param output_dir:
    :return:
    """
    basename = f"temp_{Path(config_path).name}"
    return Path(output_dir).joinpath(basename)


def run_docker(cmd: str, output_dir: Path | str = "."):
    """Function to run a command via Docker.
    A container will be generated automatically,
    but a Docker server must be running first.
    You can start one via the Desktop application,
    or on the command line with `docker start'.

    After the specified 'cmd' command has been run, any newly-generated files
     will be copied out of the container to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the base arguments you want to use to run sextractor.
    An example would be:
        cmd = 'image01.fits -c sex.config'
    output_dir: A local directory to save the output files to.

    Returns
    -------

    """

    container = new_container()

    try:

        container.attach()

        container.start()

        split = cmd.split(" -")

        # Reorganise the commands so that each '-x' argument is grouped together
        # Basically still work even if someone puts the filename in a weird place

        sorted_split = []

        for i, arg in enumerate(split):
            sep = arg.split(" ")
            sorted_split.append(" ".join(sep[:2]))
            if len(sep) > 2:
                sorted_split[0] += " " + " ".join(sep[2:])

        new_split = []

        # Loop over sextractor command, and
        # copy everything that looks like a file into container
        # Go through everything that looks like a file with paths in it after

        copy_list = []
        temp_files = []

        files_of_files = []

        for i, arg in enumerate(sorted_split):
            sep = arg.split(" ")

            if sep[0] == "c":
                files_of_files.append(sep[1])

            new = list(sep)

            for j, x in enumerate(sep):
                if len(x) > 0:
                    if os.path.isfile(x):
                        new[j] = docker_path(sep[j])
                        copy_list.append(sep[j])
                    elif x[0] == "@":
                        files_of_files.append(x[1:])
                    elif os.path.isdir(os.path.dirname(x)):
                        new[j] = docker_path(sep[j])

            new_split.append(" ".join(new))

        cmd = " -".join(new_split)

        # Be extra clever: go through files and check there too!

        logger.debug(
            f"Found the following files which should contain paths: {files_of_files}"
        )

        for path in files_of_files:

            new_file = []

            with open(path, "rb", encoding="utf8") as local_file:
                for line in local_file.readlines():
                    args = [x for x in line.decode().split(" ") if x not in [""]]
                    new_args = list(args)
                    for i, arg in enumerate(args):
                        if os.path.isfile(arg):
                            copy_list.append(arg)
                            new_args[i] = docker_path(arg)
                        elif os.path.isfile(arg.strip("\n")):
                            copy_list.append(arg.strip("\n"))
                            new_args[i] = str(docker_path(arg.strip("\n"))) + "\n"
                    new_file.append(" ".join(new_args))

            temp_file_path = temp_config(path, output_dir)

            with open(temp_file_path, "w", encoding="utf8") as temp_file:
                temp_file.writelines(new_file)

            copy_list.append(temp_file_path)

            cmd = cmd.replace(path + " ", str(docker_path(temp_file_path)) + " ")

        # Copy in files, and see what files are already there

        copy_list = list(set(copy_list))

        logger.debug(f"Copying {copy_list} into container")

        ignore_files = docker_batch_put(container=container, local_paths=copy_list)

        # Run command

        log = container.exec_run(cmd, stderr=True, stdout=True)

        for temp_file_path in temp_files:
            logger.debug(f"Deleting temporary file {temp_file_path}")
            os.remove(temp_file_path)

        if not log.output == b"":
            logger.info(f"Output: {log.output.decode()}")

        if not log.exit_code == 0:
            err = (
                f"Error running command: \n '{cmd}'\n "
                f"which resulted in returncode '{log.exit_code}' and"
                f"the following error message: \n '{log.output.decode()}'"
            )
            logger.error(err)
            raise subprocess.CalledProcessError(
                returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
            )

        # Copy out any files which did not exist before running sextractor

        docker_get_new_files(
            container=container, output_dir=output_dir, ignore_files=ignore_files
        )

    except docker.errors.APIError as err:
        logger.error(err)
        raise ExecutionError(err) from err
    finally:
        # In any case, clean up by killing the container and removing files
        container.kill()
        container.remove()


def execute(
    cmd: str,
    output_dir: Path | str = ".",
    local: bool = True,
    timeout: float = DEFAULT_TIMEOUT,
):
    """
    Generically execute a command either via bash or a docker container

    :param cmd: command
    :param output_dir: output directory for command
    :param local: boolean whether use local or docker
    :param timeout: timeout for local execution
    :return: None
    """
    logger.debug(
        f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
    )
    if local:
        run_local(cmd, output_dir=output_dir, timeout=timeout)
    else:
        run_docker(cmd, output_dir=output_dir)

1	"""
2	Module for executing bash commands
3	"""
4	import logging	1✔
5	import os	1✔
6	import shutil	1✔
7	import subprocess	1✔
8	from pathlib import Path	1✔
9
10	import docker	1✔
11
12	from winterdrp.utils.dockerutil import (	1✔
13	docker_batch_put,
14	docker_get_new_files,
15	docker_path,
16	new_container,
17	)
18
19	logger = logging.getLogger(__name__)	1✔
20
21
22	class ExecutionError(Exception):	1✔
23	"""Error relating to executing bash command"""
24
25
26	DEFAULT_TIMEOUT = 300.0	1✔
27
28
29	def run_local(cmd: str, output_dir: str = ".", timeout: float = DEFAULT_TIMEOUT):	1✔
30	"""
31	Function to run on local machine using subprocess, with error handling.
32
33	After the specified 'cmd' command has been run, any newly-generated files
34	will be copied out of the current directory to 'output_dir'
35
36	Parameters
37	----------
38	cmd: A string containing the command you want to use to run sextractor.
39	An example would be:
40	cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
41	output_dir: A local directory to save the output files to.
42	timeout: Time to timeout in seconds
43
44	Returns
45	-------
46
47	"""
48
49	try:	1✔
50
51	# See what files are in the directory beforehand
52
53	ignore_files = (	1✔
54	subprocess.run("ls", check=True, capture_output=True)
55	.stdout.decode()
56	.split("\n")
57	)
58
59	# Run sextractor
60
61	rval = subprocess.run(	1✔
62	cmd, check=True, capture_output=True, shell=True, timeout=timeout
63	)
64
65	msg = "Successfully executed command. "	1✔
66
67	if rval.stdout.decode() != "":	1✔
68	msg += f"Found the following output: {rval.stdout.decode()}"	×
69	logger.debug(msg)	1✔
70
71	try:	1✔
72	os.makedirs(output_dir)	1✔
73	except OSError:	1✔
74	pass	1✔
75
76	# Move new files to output dir
77
78	new_files = [	1✔
79	x
80	for x in subprocess.run("ls", check=True, capture_output=True)
81	.stdout.decode()
82	.split("\n")
83	if x not in ignore_files
84	]
85
86	current_dir = (	1✔
87	subprocess.run("pwd", check=True, capture_output=True)
88	.stdout.decode()
89	.strip()
90	)
91
92	if len(new_files) > 0:	1✔
93
94	logger.debug(	1✔
95	f"The following new files were created in the current directory: "
96	f"{new_files}"
97	)
98
99	for file in new_files:	1✔
100
101	current_path = os.path.join(current_dir, file)	1✔
102	output_path = os.path.join(output_dir, file)	1✔
103
104	logger.info(f"File saved to {output_path}")	1✔
105
106	shutil.move(current_path, output_path)	1✔
107
108	except subprocess.CalledProcessError as err:	×
109	msg = (	×
110	f"Error found when running with command: \n \n '{err.cmd}' \n \n"
111	f"This yielded a return code of {err.returncode}. "
112	f"The following traceback was found: \n {err.stderr.decode()}"
113	)
114	logger.error(msg)
115	raise ExecutionError(msg) from err
116
117
118	def temp_config(config_path: str \| Path, output_dir: str \| Path) -> Path:	1✔
119	"""
120	Get a
121
122	:param config_path:
123	:param output_dir:
124	:return:
125	"""
126	basename = f"temp_{Path(config_path).name}"	×
127	return Path(output_dir).joinpath(basename)	×
128
129
130	def run_docker(cmd: str, output_dir: Path \| str = "."):	1✔
131	"""Function to run a command via Docker.
132	A container will be generated automatically,
133	but a Docker server must be running first.
134	You can start one via the Desktop application,
135	or on the command line with `docker start'.
136
137	After the specified 'cmd' command has been run, any newly-generated files
138	will be copied out of the container to 'output_dir'
139
140	Parameters
141	----------
142	cmd: A string containing the base arguments you want to use to run sextractor.
143	An example would be:
144	cmd = 'image01.fits -c sex.config'
145	output_dir: A local directory to save the output files to.
146
147	Returns
148	-------
149
150	"""
151
152	container = new_container()	×
153
154	try:	×
155
156	container.attach()	×
157
158	container.start()	×
159
160	split = cmd.split(" -")	×
161
162	# Reorganise the commands so that each '-x' argument is grouped together
163	# Basically still work even if someone puts the filename in a weird place
164
165	sorted_split = []	×
166
167	for i, arg in enumerate(split):	×
168	sep = arg.split(" ")	×
169	sorted_split.append(" ".join(sep[:2]))	×
170	if len(sep) > 2:	×
171	sorted_split[0] += " " + " ".join(sep[2:])	×
172
173	new_split = []	×
174
175	# Loop over sextractor command, and
176	# copy everything that looks like a file into container
177	# Go through everything that looks like a file with paths in it after
178
179	copy_list = []	×
180	temp_files = []	×
181
182	files_of_files = []	×
183
184	for i, arg in enumerate(sorted_split):	×
185	sep = arg.split(" ")	×
186
187	if sep[0] == "c":	×
188	files_of_files.append(sep[1])	×
189
190	new = list(sep)	×
191
192	for j, x in enumerate(sep):	×
193	if len(x) > 0:	×
194	if os.path.isfile(x):	×
195	new[j] = docker_path(sep[j])	×
196	copy_list.append(sep[j])	×
197	elif x[0] == "@":	×
198	files_of_files.append(x[1:])	×
199	elif os.path.isdir(os.path.dirname(x)):	×
200	new[j] = docker_path(sep[j])	×
201
202	new_split.append(" ".join(new))	×
203
204	cmd = " -".join(new_split)	×
205
206	# Be extra clever: go through files and check there too!
207
208	logger.debug(	×
209	f"Found the following files which should contain paths: {files_of_files}"
210	)
211
212	for path in files_of_files:	×
213
214	new_file = []	×
215
216	with open(path, "rb", encoding="utf8") as local_file:	×
217	for line in local_file.readlines():	×
218	args = [x for x in line.decode().split(" ") if x not in [""]]	×
219	new_args = list(args)	×
220	for i, arg in enumerate(args):	×
221	if os.path.isfile(arg):	×
222	copy_list.append(arg)	×
223	new_args[i] = docker_path(arg)	×
224	elif os.path.isfile(arg.strip("\n")):	×
225	copy_list.append(arg.strip("\n"))	×
226	new_args[i] = str(docker_path(arg.strip("\n"))) + "\n"	×
227	new_file.append(" ".join(new_args))	×
228
229	temp_file_path = temp_config(path, output_dir)	×
230
231	with open(temp_file_path, "w", encoding="utf8") as temp_file:	×
232	temp_file.writelines(new_file)	×
233
234	copy_list.append(temp_file_path)	×
235
236	cmd = cmd.replace(path + " ", str(docker_path(temp_file_path)) + " ")	×
237
238	# Copy in files, and see what files are already there
239
240	copy_list = list(set(copy_list))	×
241
242	logger.debug(f"Copying {copy_list} into container")	×
243
244	ignore_files = docker_batch_put(container=container, local_paths=copy_list)	×
245
246	# Run command
247
248	log = container.exec_run(cmd, stderr=True, stdout=True)	×
249
250	for temp_file_path in temp_files:	×
251	logger.debug(f"Deleting temporary file {temp_file_path}")	×
252	os.remove(temp_file_path)	×
253
254	if not log.output == b"":	×
255	logger.info(f"Output: {log.output.decode()}")	×
256
257	if not log.exit_code == 0:	×
258	err = (
259	f"Error running command: \n '{cmd}'\n "
260	f"which resulted in returncode '{log.exit_code}' and"
261	f"the following error message: \n '{log.output.decode()}'"
262	)
263	logger.error(err)
264	raise subprocess.CalledProcessError(
265	returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
266	)
267
268	# Copy out any files which did not exist before running sextractor
269
270	docker_get_new_files(	×
271	container=container, output_dir=output_dir, ignore_files=ignore_files
272	)
273
274	except docker.errors.APIError as err:	×
275	logger.error(err)
276	raise ExecutionError(err) from err
277	finally:
278	# In any case, clean up by killing the container and removing files
279	container.kill()	×
280	container.remove()	×
281
282
283	def execute(	1✔
284	cmd: str,
285	output_dir: Path \| str = ".",
286	local: bool = True,
287	timeout: float = DEFAULT_TIMEOUT,
288	):
289	"""
290	Generically execute a command either via bash or a docker container
291
292	:param cmd: command
293	:param output_dir: output directory for command
294	:param local: boolean whether use local or docker
295	:param timeout: timeout for local execution
296	:return: None
297	"""
298	logger.debug(	1✔
299	f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
300	)
301	if local:	1✔
302	run_local(cmd, output_dir=output_dir, timeout=timeout)	1✔
303	else:
304	run_docker(cmd, output_dir=output_dir)	×

winter-telescope / winterdrp / 3769534174

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous