3769534174

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/winter-telescope/winterdrp/commit/<a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/<a class="double-link" href="https://git"><a class=hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">fb65b52df">&lt;a href=&quot;https://github.com/winter-telescope/winterdrp/commit/</a><a class="double-link" href="https://github.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">fb65b52df</a><a href="https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/winter-telescope/winterdrp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&quot;&gt;fb65b52df&lt;/a&gt;&lt;a href=&quot;https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/winter-telescope/winterdrp/commit/fb65b52dfae2af597cf36513c5c72601721ab3b3&amp;quot;&amp;... (continued)

Pull Request Pull Request #256: Lintify

Run Details

90 of 90 new or added lines in 15 files covered. (100.0%)

4632 of 6121 relevant lines covered (75.67%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.69

/winterdrp/utils/dockerutil.py

"""
Module containing docker integration (beta-stage)
"""
import io
import logging
import os
import tarfile
from pathlib import Path

import docker
from docker.errors import DockerException
from docker.models.containers import Container

logger = logging.getLogger(__name__)

DOCKER_IMAGE_NAME = "robertdstein/astrodocker"
docker_dir = Path("/usr/src/astrodocker")


def new_container():
    """Generate a new docker.models.containers.Container object, using the default
    docker daemon and the docker image.
    If the image is not found locally,
    the image will first be pulled from DockerHub.

    This function requires a Docker daemon to first be running.

    Returns
    -------
    A docker container built with the {DOCKER_IMAGE_NAME} image
    """
    try:
        client = docker.from_env()
    except DockerException as exc:
        err = (
            "Unable to connect to Docker daemon. "
            "Have you installed Docker, and started a daemon? "
            "Find out more at https://www.docker.com "
        )
        logger.error(err)
        raise ConnectionError(err) from exc

    if len(client.images.list(DOCKER_IMAGE_NAME)) < 1:
        logger.info(f"Pulling docker image {DOCKER_IMAGE_NAME}")
        client.images.pull(DOCKER_IMAGE_NAME)

    return client.containers.run(DOCKER_IMAGE_NAME, tty=True, detach=True)


def docker_path(file_path: str | Path) -> Path:
    """
    Converts a local path to the corresponding path in the docker container

    :param file_path: file path
    :return:
    """
    return docker_dir.joinpath(Path(file_path).name)


def docker_get(container: Container, local_path: str | Path):
    """Function to cope one file from the Docker container 'container' to 'local_path'.
    The file in the container should have
    the same name as the base file in 'local_path'.

    Parameters
    ----------
    container: A docker.models.container.Container object
    local_path: Local path of file to copy to

    Returns
    -------
    """

    container_path = docker_path(local_path)

    with open(local_path, "wb") as local_file:
        bits, _ = container.get_archive(container_path.as_posix())
        for chunk in bits:
            local_file.write(chunk)


def docker_put(container: Container, local_path: str | Path):
    """Function to one file, at 'local_path' into the Docker container 'container'

    Parameters
    ----------
    container: A docker.models.container.Container object
    local_path: Local path of file to copy

    Returns
    -------
    """
    stream = io.BytesIO()

    with tarfile.open(fileobj=stream, mode="w|") as tar, open(
        local_path, "rb"
    ) as local_file:
        info = tar.gettarinfo(fileobj=local_file)
        info.name = os.path.basename(local_path)
        tar.addfile(info, local_file)

    return container.put_archive(docker_dir.as_posix(), stream.getvalue())


def docker_batch_put(container: Container, local_paths: str | list):
    """Function to copy multiple files into a Docker container

    Parameters
    ----------
    container: A docker.models.container.Container object
    local_paths: Local paths of each file to copy

    Returns
    -------
    Returns a list of files in the docker container after the copying is done
    """

    if isinstance(local_paths, str):
        local_paths = [local_paths]

    for local_path in local_paths:
        docker_put(container, local_path)

    return (
        container.exec_run("ls", stderr=True, stdout=True).output.decode().split("\n")
    )


def docker_get_new_files(
    container: Container, output_dir: str | Path, ignore_files: list[str | Path]
):
    """
    Function to copy new files out of a container.
    All files in the work directory of 'container'
    will be copied out to 'output_dir',
    unless they appear in the 'ignore_files' list.

    The normal procedure is to run this in tandem with docker_batch_put(),
    so that only new files
    are copied out of the container. For example:

        ignore_files = docker_batch_put(
            container=container,
            local_paths=list_of_files_to_copy_into_container
        )

        container.exec_run(some_docker_command)

        docker_get_new_files(
            container=container,
            output_dir=output_dir,
            ignore_files=ignore_files
        )

    Parameters
    ----------
    container: A docker.models.container.Container object
    output_dir: A local directory to save the output files to.
    ignore_files: List of files to ignore (i.e to not copy)

    Returns
    -------
    """

    new_files = [
        x
        for x in container.exec_run("ls", stderr=True, stdout=True)
        .output.decode()
        .split("\n")
        if x not in ignore_files
    ]

    # Make output directory if it doesn't exist

    output_dir = Path(output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    # Collect output files

    for output_file in new_files:

        output_path = output_dir.joinpath(output_file)

        docker_get(container, output_path)

        if output_path.exists():
            logger.debug(f"Saved to {output_path}")
        else:
            raise FileNotFoundError(f"Unable to find {output_path}")

1	"""
2	Module containing docker integration (beta-stage)
3	"""
4	import io	1✔
5	import logging	1✔
6	import os	1✔
7	import tarfile	1✔
8	from pathlib import Path	1✔
9
10	import docker	1✔
11	from docker.errors import DockerException	1✔
12	from docker.models.containers import Container	1✔
13
14	logger = logging.getLogger(__name__)	1✔
15
16	DOCKER_IMAGE_NAME = "robertdstein/astrodocker"	1✔
17	docker_dir = Path("/usr/src/astrodocker")	1✔
18
19
20	def new_container():	1✔
21	"""Generate a new docker.models.containers.Container object, using the default
22	docker daemon and the docker image.
23	If the image is not found locally,
24	the image will first be pulled from DockerHub.
25
26	This function requires a Docker daemon to first be running.
27
28	Returns
29	-------
30	A docker container built with the {DOCKER_IMAGE_NAME} image
31	"""
32	try:	×
33	client = docker.from_env()	×
34	except DockerException as exc:	×
35	err = (
36	"Unable to connect to Docker daemon. "
37	"Have you installed Docker, and started a daemon? "
38	"Find out more at https://www.docker.com "
39	)
40	logger.error(err)
41	raise ConnectionError(err) from exc
42
43	if len(client.images.list(DOCKER_IMAGE_NAME)) < 1:	×
44	logger.info(f"Pulling docker image {DOCKER_IMAGE_NAME}")	×
45	client.images.pull(DOCKER_IMAGE_NAME)	×
46
47	return client.containers.run(DOCKER_IMAGE_NAME, tty=True, detach=True)	×
48
49
50	def docker_path(file_path: str \| Path) -> Path:	1✔
51	"""
52	Converts a local path to the corresponding path in the docker container
53
54	:param file_path: file path
55	:return:
56	"""
57	return docker_dir.joinpath(Path(file_path).name)	×
58
59
60	def docker_get(container: Container, local_path: str \| Path):	1✔
61	"""Function to cope one file from the Docker container 'container' to 'local_path'.
62	The file in the container should have
63	the same name as the base file in 'local_path'.
64
65	Parameters
66	----------
67	container: A docker.models.container.Container object
68	local_path: Local path of file to copy to
69
70	Returns
71	-------
72	"""
73
74	container_path = docker_path(local_path)	×
75
76	with open(local_path, "wb") as local_file:	×
77	bits, _ = container.get_archive(container_path.as_posix())	×
78	for chunk in bits:	×
79	local_file.write(chunk)	×
80
81
82	def docker_put(container: Container, local_path: str \| Path):	1✔
83	"""Function to one file, at 'local_path' into the Docker container 'container'
84
85	Parameters
86	----------
87	container: A docker.models.container.Container object
88	local_path: Local path of file to copy
89
90	Returns
91	-------
92	"""
93	stream = io.BytesIO()	×
94
95	with tarfile.open(fileobj=stream, mode="w\|") as tar, open(	×
96	local_path, "rb"
97	) as local_file:
98	info = tar.gettarinfo(fileobj=local_file)	×
99	info.name = os.path.basename(local_path)	×
100	tar.addfile(info, local_file)	×
101
102	return container.put_archive(docker_dir.as_posix(), stream.getvalue())	×
103
104
105	def docker_batch_put(container: Container, local_paths: str \| list):	1✔
106	"""Function to copy multiple files into a Docker container
107
108	Parameters
109	----------
110	container: A docker.models.container.Container object
111	local_paths: Local paths of each file to copy
112
113	Returns
114	-------
115	Returns a list of files in the docker container after the copying is done
116	"""
117
118	if isinstance(local_paths, str):	×
119	local_paths = [local_paths]	×
120
121	for local_path in local_paths:	×
122	docker_put(container, local_path)	×
123
124	return (	×
125	container.exec_run("ls", stderr=True, stdout=True).output.decode().split("\n")
126	)
127
128
129	def docker_get_new_files(	1✔
130	container: Container, output_dir: str \| Path, ignore_files: list[str \| Path]
131	):
132	"""
133	Function to copy new files out of a container.
134	All files in the work directory of 'container'
135	will be copied out to 'output_dir',
136	unless they appear in the 'ignore_files' list.
137
138	The normal procedure is to run this in tandem with docker_batch_put(),
139	so that only new files
140	are copied out of the container. For example:
141
142	ignore_files = docker_batch_put(
143	container=container,
144	local_paths=list_of_files_to_copy_into_container
145	)
146
147	container.exec_run(some_docker_command)
148
149	docker_get_new_files(
150	container=container,
151	output_dir=output_dir,
152	ignore_files=ignore_files
153	)
154
155	Parameters
156	----------
157	container: A docker.models.container.Container object
158	output_dir: A local directory to save the output files to.
159	ignore_files: List of files to ignore (i.e to not copy)
160
161	Returns
162	-------
163	"""
164
165	new_files = [	×
166	x
167	for x in container.exec_run("ls", stderr=True, stdout=True)
168	.output.decode()
169	.split("\n")
170	if x not in ignore_files
171	]
172
173	# Make output directory if it doesn't exist
174
175	output_dir = Path(output_dir)	×
176	output_dir.mkdir(parents=True, exist_ok=True)	×
177
178	# Collect output files
179
180	for output_file in new_files:	×
181
182	output_path = output_dir.joinpath(output_file)	×
183
184	docker_get(container, output_path)	×
185
186	if output_path.exists():	×
187	logger.debug(f"Saved to {output_path}")	×
188	else:
189	raise FileNotFoundError(f"Unable to find {output_path}")

winter-telescope / winterdrp / 3769534174

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous