3597069237

Build Type

github

Committed by GitHub

Commit Message

Merge 7b7dc3e1e into 3f47e794f

Pull Request Pull Request #224: Code with Style

Run Details

1490 of 1490 new or added lines in 93 files covered. (100.0%)

4571 of 6109 relevant lines covered (74.82%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.26

/winterdrp/utils/execute_cmd.py

import logging
import os
import shutil
import subprocess

import docker
from docker.errors import DockerException

from winterdrp.utils.dockerutil import (
    docker_batch_put,
    docker_get_new_files,
    docker_path,
    new_container,
)

logger = logging.getLogger(__name__)


class ExecutionError(Exception):
    pass


default_timeout = 300.0


def run_local(cmd: str, output_dir: str = ".", timeout: float = default_timeout):
    """
    Function to run on local machine using subprocess, with error handling.

    After the specified 'cmd' command has been run, any newly-generated files
    will be copied out of the current directory to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the command you want to use to run sextractor. An example would be:
        cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
    output_dir: A local directory to save the output files to.
    timeout: Time to timeout in seconds

    Returns
    -------

    """

    try:

        # See what files are in the directory beforehand

        ignore_files = (
            subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
        )

        # Run sextractor

        rval = subprocess.run(
            cmd, check=True, capture_output=True, shell=True, timeout=timeout
        )

        msg = f"Successfully executed command. "

        if rval.stdout.decode() != "":
            msg += f"Found the following output: {rval.stdout.decode()}"
        logger.debug(msg)

        try:
            os.makedirs(output_dir)
        except OSError:
            pass

        # Move new files to output dir

        new_files = [
            x
            for x in subprocess.run("ls", check=True, capture_output=True)
            .stdout.decode()
            .split("\n")
            if x not in ignore_files
        ]

        current_dir = (
            subprocess.run("pwd", check=True, capture_output=True)
            .stdout.decode()
            .strip()
        )

        if len(new_files) > 0:

            logger.debug(
                f"The following new files were created in the current directory: {new_files}"
            )

        for file in new_files:

            current_path = os.path.join(current_dir, file)
            output_path = os.path.join(output_dir, file)

            logger.info(f"File saved to {output_path}")

            shutil.move(current_path, output_path)

    except subprocess.CalledProcessError as err:
        msg = (
            f"Error found when running with command: \n \n '{err.cmd}' \n \n"
            f"This yielded a return code of {err.returncode}. The following traceback was found: \n {err.stderr.decode()}"
        )
        logger.error(msg)
        raise ExecutionError(msg)


def temp_config(config_path: str, output_dir: str) -> str:
    basename = f"temp_{os.path.basename(config_path)}"
    return os.path.join(output_dir, basename)


def run_docker(cmd: str, output_dir: str = "."):
    """Function to run a command via Docker. A container will be generated automatically,
    but a Docker server must be running first. You can start one via the Desktop application,
    or on the command line with `docker start'.

    After the specified 'cmd' command has been run, any newly-generated files
     will be copied out of the container to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the base arguments you want to use to run sextractor. An example would be:
        cmd = 'image01.fits -c sex.config'
    output_dir: A local directory to save the output files to.

    Returns
    -------

    """

    container = new_container()

    try:

        container.attach()

        container.start()

        split = cmd.split(" -")

        # Reorganise the commands so that each '-x' argument is grouped together
        # Basically still work even if someone puts the filename in a weird place

        sorted_split = []

        for i, arg in enumerate(split):
            sep = arg.split(" ")
            sorted_split.append(" ".join(sep[:2]))
            if len(sep) > 2:
                sorted_split[0] += " " + " ".join(sep[2:])

        new_split = []

        # Loop over sextractor command, and
        # copy everything that looks like a file into container
        # Go through everything that looks like a file with paths in it after

        copy_list = []
        temp_files = []

        files_of_files = []

        for i, arg in enumerate(sorted_split):
            sep = arg.split(" ")

            if sep[0] == "c":
                files_of_files.append(sep[1])

            new = list(sep)

            for j, x in enumerate(sep):
                if len(x) > 0:
                    if os.path.isfile(x):
                        new[j] = docker_path(sep[j])
                        copy_list.append(sep[j])
                    elif x[0] == "@":
                        files_of_files.append(x[1:])
                    elif os.path.isdir(os.path.dirname(x)):
                        new[j] = docker_path(sep[j])

            new_split.append(" ".join(new))

        cmd = " -".join(new_split)

        # Be extra clever: go through files and check there too!

        logger.debug(
            f"Found the following files which should contain paths: {files_of_files}"
        )

        for path in files_of_files:

            new_file = []

            with open(path, "rb") as f:
                for line in f.readlines():
                    args = [x for x in line.decode().split(" ") if x not in [""]]
                    new_args = list(args)
                    for i, arg in enumerate(args):
                        if os.path.isfile(arg):
                            copy_list.append(arg)
                            new_args[i] = docker_path(arg)
                        elif os.path.isfile(arg.strip("\n")):
                            copy_list.append(arg.strip("\n"))
                            new_args[i] = docker_path(arg.strip("\n")) + "\n"
                    new_file.append(" ".join(new_args))

            temp_file = temp_config(path, output_dir)

            with open(temp_file, "w") as g:
                g.writelines(new_file)

            copy_list.append(temp_file)

            cmd = cmd.replace(path + " ", docker_path(temp_file) + " ")

        # Copy in files, and see what files are already there

        copy_list = list(set(copy_list))

        logger.debug(f"Copying {copy_list} into container")

        ignore_files = docker_batch_put(container=container, local_paths=copy_list)

        # Run command

        log = container.exec_run(cmd, stderr=True, stdout=True)

        for temp_file in temp_files:
            logger.debug(f"Deleting temporary file {temp_file}")
            os.remove(temp_file)

        if not log.output == b"":
            logger.info(f"Output: {log.output.decode()}")

        if not log.exit_code == 0:
            err = (
                f"Error running command: \n '{cmd}'\n which resulted in returncode '{log.exit_code}' and"
                f"the following error message: \n '{log.output.decode()}'"
            )
            logger.error(err)
            raise subprocess.CalledProcessError(
                returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
            )

        # Copy out any files which did not exist before running sextractor

        docker_get_new_files(
            container=container, output_dir=output_dir, ignore_files=ignore_files
        )

    except docker.errors.APIError as err:
        logger.error(err)
        raise ExecutionError(err)
    finally:
        # In any case, clean up by killing the container and removing files
        container.kill()
        container.remove()


def execute(cmd, output_dir=".", local=True, timeout: float = default_timeout):
    logger.debug(
        f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
    )
    if local:
        run_local(cmd, output_dir=output_dir, timeout=timeout)
    else:
        run_docker(cmd, output_dir=output_dir)

1	import logging	1✔
2	import os	1✔
3	import shutil	1✔
4	import subprocess	1✔
5
6	import docker	1✔
7	from docker.errors import DockerException	1✔
8
9	from winterdrp.utils.dockerutil import (	1✔
10	docker_batch_put,
11	docker_get_new_files,
12	docker_path,
13	new_container,
14	)
15
16	logger = logging.getLogger(__name__)	1✔
17
18
19	class ExecutionError(Exception):	1✔
20	pass	1✔
21
22
23	default_timeout = 300.0	1✔
24
25
26	def run_local(cmd: str, output_dir: str = ".", timeout: float = default_timeout):	1✔
27	"""
28	Function to run on local machine using subprocess, with error handling.
29
30	After the specified 'cmd' command has been run, any newly-generated files
31	will be copied out of the current directory to 'output_dir'
32
33	Parameters
34	----------
35	cmd: A string containing the command you want to use to run sextractor. An example would be:
36	cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
37	output_dir: A local directory to save the output files to.
38	timeout: Time to timeout in seconds
39
40	Returns
41	-------
42
43	"""
44
45	try:	1✔
46
47	# See what files are in the directory beforehand
48
49	ignore_files = (	1✔
50	subprocess.run("ls", check=True, capture_output=True)
51	.stdout.decode()
52	.split("\n")
53	)
54
55	# Run sextractor
56
57	rval = subprocess.run(	1✔
58	cmd, check=True, capture_output=True, shell=True, timeout=timeout
59	)
60
61	msg = f"Successfully executed command. "	1✔
62
63	if rval.stdout.decode() != "":	1✔
64	msg += f"Found the following output: {rval.stdout.decode()}"	×
65	logger.debug(msg)	1✔
66
67	try:	1✔
68	os.makedirs(output_dir)	1✔
69	except OSError:	1✔
70	pass	1✔
71
72	# Move new files to output dir
73
74	new_files = [	1✔
75	x
76	for x in subprocess.run("ls", check=True, capture_output=True)
77	.stdout.decode()
78	.split("\n")
79	if x not in ignore_files
80	]
81
82	current_dir = (	1✔
83	subprocess.run("pwd", check=True, capture_output=True)
84	.stdout.decode()
85	.strip()
86	)
87
88	if len(new_files) > 0:	1✔
89
90	logger.debug(	1✔
91	f"The following new files were created in the current directory: {new_files}"
92	)
93
94	for file in new_files:	1✔
95
96	current_path = os.path.join(current_dir, file)	1✔
97	output_path = os.path.join(output_dir, file)	1✔
98
99	logger.info(f"File saved to {output_path}")	1✔
100
101	shutil.move(current_path, output_path)	1✔
102
103	except subprocess.CalledProcessError as err:	×
104	msg = (	×
105	f"Error found when running with command: \n \n '{err.cmd}' \n \n"
106	f"This yielded a return code of {err.returncode}. The following traceback was found: \n {err.stderr.decode()}"
107	)
108	logger.error(msg)
109	raise ExecutionError(msg)
110
111
112	def temp_config(config_path: str, output_dir: str) -> str:	1✔
113	basename = f"temp_{os.path.basename(config_path)}"	×
114	return os.path.join(output_dir, basename)	×
115
116
117	def run_docker(cmd: str, output_dir: str = "."):	1✔
118	"""Function to run a command via Docker. A container will be generated automatically,
119	but a Docker server must be running first. You can start one via the Desktop application,
120	or on the command line with `docker start'.
121
122	After the specified 'cmd' command has been run, any newly-generated files
123	will be copied out of the container to 'output_dir'
124
125	Parameters
126	----------
127	cmd: A string containing the base arguments you want to use to run sextractor. An example would be:
128	cmd = 'image01.fits -c sex.config'
129	output_dir: A local directory to save the output files to.
130
131	Returns
132	-------
133
134	"""
135
136	container = new_container()	×
137
138	try:	×
139
140	container.attach()	×
141
142	container.start()	×
143
144	split = cmd.split(" -")	×
145
146	# Reorganise the commands so that each '-x' argument is grouped together
147	# Basically still work even if someone puts the filename in a weird place
148
149	sorted_split = []	×
150
151	for i, arg in enumerate(split):	×
152	sep = arg.split(" ")	×
153	sorted_split.append(" ".join(sep[:2]))	×
154	if len(sep) > 2:	×
155	sorted_split[0] += " " + " ".join(sep[2:])	×
156
157	new_split = []	×
158
159	# Loop over sextractor command, and
160	# copy everything that looks like a file into container
161	# Go through everything that looks like a file with paths in it after
162
163	copy_list = []	×
164	temp_files = []	×
165
166	files_of_files = []	×
167
168	for i, arg in enumerate(sorted_split):	×
169	sep = arg.split(" ")	×
170
171	if sep[0] == "c":	×
172	files_of_files.append(sep[1])	×
173
174	new = list(sep)	×
175
176	for j, x in enumerate(sep):	×
177	if len(x) > 0:	×
178	if os.path.isfile(x):	×
179	new[j] = docker_path(sep[j])	×
180	copy_list.append(sep[j])	×
181	elif x[0] == "@":	×
182	files_of_files.append(x[1:])	×
183	elif os.path.isdir(os.path.dirname(x)):	×
184	new[j] = docker_path(sep[j])	×
185
186	new_split.append(" ".join(new))	×
187
188	cmd = " -".join(new_split)	×
189
190	# Be extra clever: go through files and check there too!
191
192	logger.debug(	×
193	f"Found the following files which should contain paths: {files_of_files}"
194	)
195
196	for path in files_of_files:	×
197
198	new_file = []	×
199
200	with open(path, "rb") as f:	×
201	for line in f.readlines():	×
202	args = [x for x in line.decode().split(" ") if x not in [""]]	×
203	new_args = list(args)	×
204	for i, arg in enumerate(args):	×
205	if os.path.isfile(arg):	×
206	copy_list.append(arg)	×
207	new_args[i] = docker_path(arg)	×
208	elif os.path.isfile(arg.strip("\n")):	×
209	copy_list.append(arg.strip("\n"))	×
210	new_args[i] = docker_path(arg.strip("\n")) + "\n"	×
211	new_file.append(" ".join(new_args))	×
212
213	temp_file = temp_config(path, output_dir)	×
214
215	with open(temp_file, "w") as g:	×
216	g.writelines(new_file)	×
217
218	copy_list.append(temp_file)	×
219
220	cmd = cmd.replace(path + " ", docker_path(temp_file) + " ")	×
221
222	# Copy in files, and see what files are already there
223
224	copy_list = list(set(copy_list))	×
225
226	logger.debug(f"Copying {copy_list} into container")	×
227
228	ignore_files = docker_batch_put(container=container, local_paths=copy_list)	×
229
230	# Run command
231
232	log = container.exec_run(cmd, stderr=True, stdout=True)	×
233
234	for temp_file in temp_files:	×
235	logger.debug(f"Deleting temporary file {temp_file}")	×
236	os.remove(temp_file)	×
237
238	if not log.output == b"":	×
239	logger.info(f"Output: {log.output.decode()}")	×
240
241	if not log.exit_code == 0:	×
242	err = (
243	f"Error running command: \n '{cmd}'\n which resulted in returncode '{log.exit_code}' and"
244	f"the following error message: \n '{log.output.decode()}'"
245	)
246	logger.error(err)
247	raise subprocess.CalledProcessError(
248	returncode=log.exit_code, cmd=cmd, stderr=log.output.decode()
249	)
250
251	# Copy out any files which did not exist before running sextractor
252
253	docker_get_new_files(	×
254	container=container, output_dir=output_dir, ignore_files=ignore_files
255	)
256
257	except docker.errors.APIError as err:	×
258	logger.error(err)
259	raise ExecutionError(err)
260	finally:
261	# In any case, clean up by killing the container and removing files
262	container.kill()	×
263	container.remove()	×
264
265
266	def execute(cmd, output_dir=".", local=True, timeout: float = default_timeout):	1✔
267	logger.debug(	1✔
268	f"Using '{['docker', 'local'][local]}' " f" installation to run `{cmd}`"
269	)
270	if local:	1✔
271	run_local(cmd, output_dir=output_dir, timeout=timeout)	1✔
272	else:
273	run_docker(cmd, output_dir=output_dir)	×

winter-telescope / winterdrp / 3597069237

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous