3596687365

Build Type

push

github

Committed by Robert Stein

Commit Message

Add black

Run Details

4582 of 6121 relevant lines covered (74.86%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.26

/winterdrp/utils/execute_cmd.py

import logging
import subprocess
import os
import docker
import shutil
from docker.errors import DockerException
from winterdrp.utils.dockerutil import new_container, docker_path, docker_batch_put, docker_get_new_files

logger = logging.getLogger(__name__)


class ExecutionError(Exception):
    pass


default_timeout = 300.


def run_local(
        cmd: str,
        output_dir: str = ".",
        timeout: float = default_timeout
):
    """
    Function to run on local machine using subprocess, with error handling.

    After the specified 'cmd' command has been run, any newly-generated files
    will be copied out of the current directory to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the command you want to use to run sextractor. An example would be:
        cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
    output_dir: A local directory to save the output files to.
    timeout: Time to timeout in seconds

    Returns
    -------

    """

    try:

        # See what files are in the directory beforehand

        ignore_files = subprocess.run("ls", check=True, capture_output=True).stdout.decode().split("\n")

        # Run sextractor

        rval = subprocess.run(cmd, check=True, capture_output=True, shell=True, timeout=timeout)

        msg = f'Successfully executed command. '

        if rval.stdout.decode() != "":
            msg += f"Found the following output: {rval.stdout.decode()}"
        logger.debug(msg)

        try:
            os.makedirs(output_dir)
        except OSError:
            pass

        # Move new files to output dir

        new_files = [
            x for x in subprocess.run("ls", check=True, capture_output=True).stdout.decode().split("\n")
            if x not in ignore_files
        ]

        current_dir = subprocess.run("pwd", check=True, capture_output=True).stdout.decode().strip()

        if len(new_files) > 0:

            logger.debug(f"The following new files were created in the current directory: {new_files}")

        for file in new_files:

            current_path = os.path.join(current_dir, file)
            output_path = os.path.join(output_dir, file)

            logger.info(f"File saved to {output_path}")

            shutil.move(current_path, output_path)

    except subprocess.CalledProcessError as err:
        msg = f"Error found when running with command: \n \n '{err.cmd}' \n \n" \
              f"This yielded a return code of {err.returncode}. The following traceback was found: \n {err.stderr.decode()}"
        logger.error(msg)
        raise ExecutionError(msg)


def temp_config(
        config_path: str,
        output_dir: str
) -> str:
    basename = f"temp_{os.path.basename(config_path)}"
    return os.path.join(output_dir, basename)


def run_docker(
        cmd: str,
        output_dir: str = "."
):
    """Function to run a command via Docker. A container will be generated automatically,
    but a Docker server must be running first. You can start one via the Desktop application,
    or on the command line with `docker start'.

    After the specified 'cmd' command has been run, any newly-generated files
     will be copied out of the container to 'output_dir'

    Parameters
    ----------
    cmd: A string containing the base arguments you want to use to run sextractor. An example would be:
        cmd = 'image01.fits -c sex.config'
    output_dir: A local directory to save the output files to.

    Returns
    -------

    """

    container = new_container()

    try:

        container.attach()

        container.start()

        split = cmd.split(" -")

        # Reorganise the commands so that each '-x' argument is grouped together
        # Basically still work even if someone puts the filename in a weird place

        sorted_split = []

        for i, arg in enumerate(split):
            sep = arg.split(" ")
            sorted_split.append(" ".join(sep[:2]))
            if len(sep) > 2:
                sorted_split[0] += " " + " ".join(sep[2:])

        new_split = []

        # Loop over sextractor command, and
        # copy everything that looks like a file into container
        # Go through everything that looks like a file with paths in it after

        copy_list = []
        temp_files = []

        files_of_files = []

        for i, arg in enumerate(sorted_split):
            sep = arg.split(" ")

            if sep[0] == "c":
                files_of_files.append(sep[1])

            new = list(sep)

            for j, x in enumerate(sep):
                if len(x) > 0:
                    if os.path.isfile(x):
                        new[j] = docker_path(sep[j])
                        copy_list.append(sep[j])
                    elif x[0] == "@":
                        files_of_files.append(x[1:])
                    elif os.path.isdir(os.path.dirname(x)):
                        new[j] = docker_path(sep[j])

            new_split.append(" ".join(new))

        cmd = " -".join(new_split)

        # Be extra clever: go through files and check there too!

        logger.debug(f"Found the following files which should contain paths: {files_of_files}")

        for path in files_of_files:

            new_file = []

            with open(path, "rb") as f:
                for line in f.readlines():
                    args = [x for x in line.decode().split(" ") if x not in [""]]
                    new_args = list(args)
                    for i, arg in enumerate(args):
                        if os.path.isfile(arg):
                            copy_list.append(arg)
                            new_args[i] = docker_path(arg)
                        elif os.path.isfile(arg.strip("\n")):
                            copy_list.append(arg.strip("\n"))
                            new_args[i] = docker_path(arg.strip("\n")) + "\n"
                    new_file.append(" ".join(new_args))

            temp_file = temp_config(path, output_dir)

            with open(temp_file, "w") as g:
                g.writelines(new_file)

            copy_list.append(temp_file)

            cmd = cmd.replace(path + " ", docker_path(temp_file) + " ")

        # Copy in files, and see what files are already there

        copy_list = list(set(copy_list))

        logger.debug(f"Copying {copy_list} into container")

        ignore_files = docker_batch_put(
            container=container,
            local_paths=copy_list
        )

        # Run command

        log = container.exec_run(cmd, stderr=True, stdout=True)

        for temp_file in temp_files:
            logger.debug(f"Deleting temporary file {temp_file}")
            os.remove(temp_file)

        if not log.output == b"":
            logger.info(f"Output: {log.output.decode()}")

        if not log.exit_code == 0:
            err = f"Error running command: \n '{cmd}'\n which resulted in returncode '{log.exit_code}' and" \
                  f"the following error message: \n '{log.output.decode()}'"
            logger.error(err)
            raise subprocess.CalledProcessError(
                returncode=log.exit_code,
                cmd=cmd,
                stderr=log.output.decode()
            )

        # Copy out any files which did not exist before running sextractor

        docker_get_new_files(
            container=container,
            output_dir=output_dir,
            ignore_files=ignore_files
        )

    except docker.errors.APIError as err:
        logger.error(err)
        raise ExecutionError(err)
    finally:
        # In any case, clean up by killing the container and removing files
        container.kill()
        container.remove()


def execute(
        cmd,
        output_dir=".",
        local=True,
        timeout: float = default_timeout
):
    logger.debug(f"Using '{['docker', 'local'][local]}' "
                 f" installation to run `{cmd}`")
    if local:
        run_local(cmd, output_dir=output_dir, timeout=timeout)
    else:
        run_docker(cmd, output_dir=output_dir)

1	import logging	1✔
2	import subprocess	1✔
3	import os	1✔
4	import docker	1✔
5	import shutil	1✔
6	from docker.errors import DockerException	1✔
7	from winterdrp.utils.dockerutil import new_container, docker_path, docker_batch_put, docker_get_new_files	1✔
8
9	logger = logging.getLogger(__name__)	1✔
10
11
12	class ExecutionError(Exception):	1✔
13	pass	1✔
14
15
16	default_timeout = 300.	1✔
17
18
19	def run_local(	1✔
20	cmd: str,
21	output_dir: str = ".",
22	timeout: float = default_timeout
23	):
24	"""
25	Function to run on local machine using subprocess, with error handling.
26
27	After the specified 'cmd' command has been run, any newly-generated files
28	will be copied out of the current directory to 'output_dir'
29
30	Parameters
31	----------
32	cmd: A string containing the command you want to use to run sextractor. An example would be:
33	cmd = '/usr/bin/source-extractor image0001.fits -c sex.config'
34	output_dir: A local directory to save the output files to.
35	timeout: Time to timeout in seconds
36
37	Returns
38	-------
39
40	"""
41
42	try:	1✔
43
44	# See what files are in the directory beforehand
45
46	ignore_files = subprocess.run("ls", check=True, capture_output=True).stdout.decode().split("\n")	1✔
47
48	# Run sextractor
49
50	rval = subprocess.run(cmd, check=True, capture_output=True, shell=True, timeout=timeout)	1✔
51
52	msg = f'Successfully executed command. '	1✔
53
54	if rval.stdout.decode() != "":	1✔
55	msg += f"Found the following output: {rval.stdout.decode()}"	×
56	logger.debug(msg)	1✔
57
58	try:	1✔
59	os.makedirs(output_dir)	1✔
60	except OSError:	1✔
61	pass	1✔
62
63	# Move new files to output dir
64
65	new_files = [	1✔
66	x for x in subprocess.run("ls", check=True, capture_output=True).stdout.decode().split("\n")
67	if x not in ignore_files
68	]
69
70	current_dir = subprocess.run("pwd", check=True, capture_output=True).stdout.decode().strip()	1✔
71
72	if len(new_files) > 0:	1✔
73
74	logger.debug(f"The following new files were created in the current directory: {new_files}")	1✔
75
76	for file in new_files:	1✔
77
78	current_path = os.path.join(current_dir, file)	1✔
79	output_path = os.path.join(output_dir, file)	1✔
80
81	logger.info(f"File saved to {output_path}")	1✔
82
83	shutil.move(current_path, output_path)	1✔
84
85	except subprocess.CalledProcessError as err:	×
86	msg = f"Error found when running with command: \n \n '{err.cmd}' \n \n" \	×
87	f"This yielded a return code of {err.returncode}. The following traceback was found: \n {err.stderr.decode()}"
88	logger.error(msg)
89	raise ExecutionError(msg)
90
91
92	def temp_config(	1✔
93	config_path: str,
94	output_dir: str
95	) -> str:
96	basename = f"temp_{os.path.basename(config_path)}"	×
97	return os.path.join(output_dir, basename)	×
98
99
100	def run_docker(	1✔
101	cmd: str,
102	output_dir: str = "."
103	):
104	"""Function to run a command via Docker. A container will be generated automatically,
105	but a Docker server must be running first. You can start one via the Desktop application,
106	or on the command line with `docker start'.
107
108	After the specified 'cmd' command has been run, any newly-generated files
109	will be copied out of the container to 'output_dir'
110
111	Parameters
112	----------
113	cmd: A string containing the base arguments you want to use to run sextractor. An example would be:
114	cmd = 'image01.fits -c sex.config'
115	output_dir: A local directory to save the output files to.
116
117	Returns
118	-------
119
120	"""
121
122	container = new_container()	×
123
124	try:	×
125
126	container.attach()	×
127
128	container.start()	×
129
130	split = cmd.split(" -")	×
131
132	# Reorganise the commands so that each '-x' argument is grouped together
133	# Basically still work even if someone puts the filename in a weird place
134
135	sorted_split = []	×
136
137	for i, arg in enumerate(split):	×
138	sep = arg.split(" ")	×
139	sorted_split.append(" ".join(sep[:2]))	×
140	if len(sep) > 2:	×
141	sorted_split[0] += " " + " ".join(sep[2:])	×
142
143	new_split = []	×
144
145	# Loop over sextractor command, and
146	# copy everything that looks like a file into container
147	# Go through everything that looks like a file with paths in it after
148
149	copy_list = []	×
150	temp_files = []	×
151
152	files_of_files = []	×
153
154	for i, arg in enumerate(sorted_split):	×
155	sep = arg.split(" ")	×
156
157	if sep[0] == "c":	×
158	files_of_files.append(sep[1])	×
159
160	new = list(sep)	×
161
162	for j, x in enumerate(sep):	×
163	if len(x) > 0:	×
164	if os.path.isfile(x):	×
165	new[j] = docker_path(sep[j])	×
166	copy_list.append(sep[j])	×
167	elif x[0] == "@":	×
168	files_of_files.append(x[1:])	×
169	elif os.path.isdir(os.path.dirname(x)):	×
170	new[j] = docker_path(sep[j])	×
171
172	new_split.append(" ".join(new))	×
173
174	cmd = " -".join(new_split)	×
175
176	# Be extra clever: go through files and check there too!
177
178	logger.debug(f"Found the following files which should contain paths: {files_of_files}")	×
179
180	for path in files_of_files:	×
181
182	new_file = []	×
183
184	with open(path, "rb") as f:	×
185	for line in f.readlines():	×
186	args = [x for x in line.decode().split(" ") if x not in [""]]	×
187	new_args = list(args)	×
188	for i, arg in enumerate(args):	×
189	if os.path.isfile(arg):	×
190	copy_list.append(arg)	×
191	new_args[i] = docker_path(arg)	×
192	elif os.path.isfile(arg.strip("\n")):	×
193	copy_list.append(arg.strip("\n"))	×
194	new_args[i] = docker_path(arg.strip("\n")) + "\n"	×
195	new_file.append(" ".join(new_args))	×
196
197	temp_file = temp_config(path, output_dir)	×
198
199	with open(temp_file, "w") as g:	×
200	g.writelines(new_file)	×
201
202	copy_list.append(temp_file)	×
203
204	cmd = cmd.replace(path + " ", docker_path(temp_file) + " ")	×
205
206	# Copy in files, and see what files are already there
207
208	copy_list = list(set(copy_list))	×
209
210	logger.debug(f"Copying {copy_list} into container")	×
211
212	ignore_files = docker_batch_put(	×
213	container=container,
214	local_paths=copy_list
215	)
216
217	# Run command
218
219	log = container.exec_run(cmd, stderr=True, stdout=True)	×
220
221	for temp_file in temp_files:	×
222	logger.debug(f"Deleting temporary file {temp_file}")	×
223	os.remove(temp_file)	×
224
225	if not log.output == b"":	×
226	logger.info(f"Output: {log.output.decode()}")	×
227
228	if not log.exit_code == 0:	×
229	err = f"Error running command: \n '{cmd}'\n which resulted in returncode '{log.exit_code}' and" \
230	f"the following error message: \n '{log.output.decode()}'"
231	logger.error(err)
232	raise subprocess.CalledProcessError(
233	returncode=log.exit_code,
234	cmd=cmd,
235	stderr=log.output.decode()
236	)
237
238	# Copy out any files which did not exist before running sextractor
239
240	docker_get_new_files(	×
241	container=container,
242	output_dir=output_dir,
243	ignore_files=ignore_files
244	)
245
246	except docker.errors.APIError as err:	×
247	logger.error(err)
248	raise ExecutionError(err)
249	finally:
250	# In any case, clean up by killing the container and removing files
251	container.kill()	×
252	container.remove()	×
253
254
255	def execute(	1✔
256	cmd,
257	output_dir=".",
258	local=True,
259	timeout: float = default_timeout
260	):
261	logger.debug(f"Using '{['docker', 'local'][local]}' "	1✔
262	f" installation to run `{cmd}`")
263	if local:	1✔
264	run_local(cmd, output_dir=output_dir, timeout=timeout)	1✔
265	else:
266	run_docker(cmd, output_dir=output_dir)	×

winter-telescope / winterdrp / 3596687365

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous