2221026373

Build Type

github

Committed by GitHub

Commit Message

Merge 597fae41e into d55f76503

Pull Request Pull Request #35: Simplify dependencies

Run Details

12 of 68 new or added lines in 14 files covered. (17.65%)

16 existing lines in 1 file now uncovered.

661 of 3293 relevant lines covered (20.07%)

0.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/miranda/remote/rsdtmf.py

"""
=================
rstdmf Operations
=================

Classes:

 * rstdmfError - the exception raised on failure.

Objects:

 * DiskSpaceEvent - threading event for when there is not enough space on disk.

Functions:

 * :func:`rstdmf_rename` - single rstdmf call with file renaming at the end.
 * :func:`local_storage_for_rstdmf` - storage object for rstdmf_divisions.
 * :func:`rstdmf_divisions` - rstdmf calls based on file numbers and size.

Notes
-----
The restored files are always renamed to their full path with '/' replaced
by '.' (leading '/' removed) to allow for restoring files with the same name
in different locations.
e.g. /server/path/sample.tar becomes server.path.sample.tar
The python trick to convert is::

   >>> path_string = Path(dmf1_file).absolute() # to get full path if necessary.
   >>> str(path_string).replace('/','.').lstrip('.')

"""
import logging.config
import os
import subprocess
import threading
import time
from pathlib import Path
from typing import List, Union

from miranda.scripting import LOGGING_CONFIG
from miranda.storage import (
    DiskSpaceError,
    FileMeta,
    StorageState,
    size_division,
    size_evaluation,
)
from miranda.units import GiB
from miranda.utils import find_filepaths, yesno_prompt

from .ops import transfer_file

DiskSpaceEvent = threading.Event()

logging.config.dictConfig(LOGGING_CONFIG)

__all__ = [
    "local_storage_for_rstdmf",
    "rstdmf_divisions",
    "rstdmf_rename",
    "RstdmfError",
]


def rstdmf_rename(file_list: Union[str, List[str]], restore_path: str):
    """Single rstdmf call with file renaming at the end.

    Parameters
    ----------
    file_list : List[str]
      list of files to restore.
    restore_path : str

    Notes
    -----
    The restored files are renamed to their full path with '/' replaced by '.'.
        e.g. /dmf1/scenario/sample.tar becomes dmf1.scenario.sample.tar

    """
    # Make sure we have the full path of each argument
    if not isinstance(file_list, list):
        file_list = [file_list]
    file_list = [Path(x).absolute().as_posix() for x in file_list]
    restore_path = Path(restore_path).absolute()

    # Create output directory, if necessary
    try:
        Path(restore_path).mkdir(parents=True, exist_ok=True)
    except OSError:
        raise RstdmfError(f"Cannot create restore path {restore_path}.")

    # Create string with list of files
    file_list_string = "' '".join(file_list)
    file_list_string = "'" + file_list_string + "'"
    file_list_string = file_list_string.replace("(", r"\(")
    file_list_string = file_list_string.replace(")", r"\)")

    # rstdmf call
    flag_system = subprocess.call(
        ["rstdmf", "-m", "-b", file_list_string, restore_path]
    )
    if flag_system:
        raise RstdmfError("rstdmf call failed.")

    # rename files
    for file_to_restore in file_list:
        file_name = file_to_restore.replace("/", ".").lstrip(".")
        restored_file = Path(restore_path).joinpath(Path(file_to_restore).name)
        try:
            transfer_file(restored_file, Path(restore_path).joinpath(file_name))
        except OSError:
            raise RstdmfError(f"Could not move {restored_file}.")


def local_storage_for_rstdmf(
    files_for_rstdmf: List[str], restore_path: str, max_size_on_disk: int
):
    """Create a local storage object for rstdmf_divisions function.

    Parameters
    ----------
    files_for_rstdmf : List[str]
        files that will be restored.
    restore_path : str
    max_size_on_disk : int
        maximum amount of space to be used on disk.

    Returns
    -------
    out : StorageState object

    """

    # Get files already restored on disk and their size
    files_on_disk = list()
    for file_on_disk in files_for_rstdmf:
        renamed_file = file_on_disk.replace("/", ".").lstrip(".")
        renamed_path = Path(restore_path).joinpath(renamed_file)
        if os.path.isfile(renamed_path):
            files_on_disk.append(renamed_path)
        elif os.path.isdir(renamed_path):
            files_within = find_filepaths(renamed_path)
            files_on_disk.append(
                [FileMeta(renamed_path.as_posix()), size_evaluation(files_within)]
            )
    size_on_disk = size_evaluation(files_on_disk)
    # create a local storage object
    return StorageState(
        restore_path, max_size_on_disk, size_on_disk, max_size_on_disk - size_on_disk
    )


#
def rstdmf_divisions(
    paths_to_restore,
    restore_path,
    disk_space_margin: int = 100 * GiB,
    disk_refresh_time: int = 60,
    rstdmf_size_limit: int = 10 * GiB,
    rstdmf_file_limit: int = 24,
    max_size_on_disk: int = 0,
    verbose: bool = False,
    preserve_order: bool = True,
):
    """Securely make call(s) to rstdmf.

    Parameters
    ----------
    paths_to_restore : list of strings
        list of files to restore (path or FileMeta objects), giving
        directories also works but they are considered like a single file
        (i.e. be careful of their size!).
    restore_path : str
        Path where files will be restored.
    disk_space_margin : int
        minimum space required on disk to proceed, set to 0 for no minimum
        requirement (default: 100 GiB).
    disk_refresh_time : int
        time to wait to lookup disk space again when full (default: 60 s).
    rstdmf_size_limit : int
        size limit of a single rstdmf call, set to 0 for no size limit
        (default: 10 GiB).
    rstdmf_file_limit : int
        limit in number of files of a single rstdmf call, set to 0 for no
        limit (default: 24).
    max_size_on_disk : int
        maximum size of restored files on disk, set to 0 for no maximum size
        (default: 0).
    verbose : bool
        verbose mode (default: off).
    preserve_order : bool
        flag to force files to be restored in the order they are given
        (default: True).

    Notes
    -----
    The restored files are renamed to their full path with '/' replaced by '.':
    for example, ``/server/path/sample.tar`` becomes ``server.path.sample.tar``
    This allows getting files with the same name in different locations.
    If this renamed file is already in the target location, it is not restored.

    """

    # Make a list of files that should be restored
    files_for_rstdmf = []
    for file_to_restore in paths_to_restore:
        # If a directory is given, create its associated FileMeta object
        if os.path.isdir(file_to_restore):
            files_within = find_filepaths(file_to_restore)
            file_to_restore = FileMeta(file_to_restore, size_evaluation(files_within))
        # If just a file name is given, convert it to a FileMeta object
        elif not isinstance(file_to_restore, FileMeta):
            file_to_restore = FileMeta(file_to_restore)
        # Skip files that do not exist or are already in the restoration path
        file_name = file_to_restore.path.replace("/", ".").lstrip(".")
        if (
            Path(restore_path).joinpath(file_name).exists()
            or not Path(file_to_restore).exists()
        ):
            continue
        # Make sure no files are repeated
        for file_for_rstdmf in files_for_rstdmf:
            if file_for_rstdmf.path == file_to_restore.path:
                break
        else:
            files_for_rstdmf.append(file_to_restore)
    if not files_for_rstdmf:
        logging.warning("All files are already on disk or nonexistent.")
        return

    # Divide files to satisfy rstdmf constraints
    if (max_size_on_disk != 0) and (max_size_on_disk < rstdmf_size_limit):
        division_size_limit = max_size_on_disk
    else:
        division_size_limit = rstdmf_size_limit
    divisions = size_division(
        files_for_rstdmf,
        division_size_limit,
        rstdmf_file_limit,
        check_name_repetition=True,
        preserve_order=preserve_order,
    )

    try:
        storage = StorageState(restore_path)
    except DiskSpaceError:
        storage = StorageState(restore_path, 0, 0, 0)
    if verbose:
        size_of_files = size_evaluation(files_for_rstdmf)
        logging.warning(
            f"Disk space: {float(storage.free_space) / GiB} GiB, Restoration size: {size_of_files / GiB} GiB, "
            + f"Subdivisions: {str(len(divisions))}"
        )
        if not yesno_prompt("Proceed with restoration?"):
            return

    for division in divisions:
        size_of_files = size_evaluation(division)
        list_of_path = [file_meta.path for file_meta in division]
        # If we can't get disk space, disregard it in the process
        try:
            storage = StorageState(restore_path)
        except DiskSpaceError:
            storage = StorageState(restore_path, 0, 0, 0)
            disk_space_margin = 0
        local_storage = local_storage_for_rstdmf(
            files_for_rstdmf, restore_path, max_size_on_disk
        )
        if (
            (disk_space_margin != 0)
            and (storage.free_space - disk_space_margin < size_of_files)
        ) or ((max_size_on_disk != 0) and (local_storage.free_space < size_of_files)):
            DiskSpaceEvent.set()
            if storage.free_space - disk_space_margin < size_of_files:
                logging.warning(
                    f"Disk space running low! Starting wait loop ({disk_refresh_time} seconds)."
                )
                logging.warning(
                    f"Disk space: {float(storage.free_space) / GiB} GB, "
                    f"Disk space to maintain: {disk_space_margin / GiB} GiB, "
                    f"Restore size: {str(size_of_files / GiB)} GiB."
                )
            elif local_storage.free_space < size_of_files:
                logging.warning(
                    f"Reached size limit of files on disk ({float(max_size_on_disk) / GiB} GiB)."
                    f" Starting wait loop ({disk_refresh_time} seconds)."
                )
            while (storage.free_space - disk_space_margin < size_of_files) or (
                local_storage.free_space < size_of_files
            ):
                time.sleep(disk_refresh_time)
                # If we can't get disk space, disregard it in the process
                try:
                    storage = StorageState(restore_path)
                except DiskSpaceError:
                    storage = StorageState(restore_path, 0, 0, 0)
                    disk_space_margin = 0
                local_storage = local_storage_for_rstdmf(
                    files_for_rstdmf, restore_path, max_size_on_disk
                )
        DiskSpaceEvent.clear()
        try:
            rstdmf_rename(list_of_path, restore_path)
        except RstdmfError:
            raise


class RstdmfError(Exception):
    pass

1	"""
2	=================
3	rstdmf Operations
4	=================
5
6	Classes:
7
8	* rstdmfError - the exception raised on failure.
9
10	Objects:
11
12	* DiskSpaceEvent - threading event for when there is not enough space on disk.
13
14	Functions:
15
16	* :func:`rstdmf_rename` - single rstdmf call with file renaming at the end.
17	* :func:`local_storage_for_rstdmf` - storage object for rstdmf_divisions.
18	* :func:`rstdmf_divisions` - rstdmf calls based on file numbers and size.
19
20	Notes
21	-----
22	The restored files are always renamed to their full path with '/' replaced
23	by '.' (leading '/' removed) to allow for restoring files with the same name
24	in different locations.
25	e.g. /server/path/sample.tar becomes server.path.sample.tar
26	The python trick to convert is::
27
28	>>> path_string = Path(dmf1_file).absolute() # to get full path if necessary.
29	>>> str(path_string).replace('/','.').lstrip('.')
30
31	"""
NEW 32	import logging.config	×
33	import os	×
34	import subprocess	×
35	import threading	×
36	import time	×
37	from pathlib import Path	×
38	from typing import List, Union	×
39
40	from miranda.scripting import LOGGING_CONFIG	×
41	from miranda.storage import (	×
42	DiskSpaceError,
43	FileMeta,
44	StorageState,
45	size_division,
46	size_evaluation,
47	)
48	from miranda.units import GiB	×
49	from miranda.utils import find_filepaths, yesno_prompt	×
50
NEW 51	from .ops import transfer_file	×
52
53	DiskSpaceEvent = threading.Event()	×
54
NEW 55	logging.config.dictConfig(LOGGING_CONFIG)	×
56
57	__all__ = [	×
58	"local_storage_for_rstdmf",
59	"rstdmf_divisions",
60	"rstdmf_rename",
61	"RstdmfError",
62	]
63
64
65	def rstdmf_rename(file_list: Union[str, List[str]], restore_path: str):	×
66	"""Single rstdmf call with file renaming at the end.
67
68	Parameters
69	----------
70	file_list : List[str]
71	list of files to restore.
72	restore_path : str
73
74	Notes
75	-----
76	The restored files are renamed to their full path with '/' replaced by '.'.
77	e.g. /dmf1/scenario/sample.tar becomes dmf1.scenario.sample.tar
78
79	"""
80	# Make sure we have the full path of each argument
81	if not isinstance(file_list, list):	×
82	file_list = [file_list]	×
83	file_list = [Path(x).absolute().as_posix() for x in file_list]	×
84	restore_path = Path(restore_path).absolute()	×
85
86	# Create output directory, if necessary
87	try:	×
88	Path(restore_path).mkdir(parents=True, exist_ok=True)	×
89	except OSError:	×
90	raise RstdmfError(f"Cannot create restore path {restore_path}.")	×
91
92	# Create string with list of files
93	file_list_string = "' '".join(file_list)	×
94	file_list_string = "'" + file_list_string + "'"	×
95	file_list_string = file_list_string.replace("(", r"\(")	×
96	file_list_string = file_list_string.replace(")", r"\)")	×
97
98	# rstdmf call
99	flag_system = subprocess.call(	×
100	["rstdmf", "-m", "-b", file_list_string, restore_path]
101	)
102	if flag_system:	×
103	raise RstdmfError("rstdmf call failed.")	×
104
105	# rename files
106	for file_to_restore in file_list:	×
107	file_name = file_to_restore.replace("/", ".").lstrip(".")	×
108	restored_file = Path(restore_path).joinpath(Path(file_to_restore).name)	×
109	try:	×
110	transfer_file(restored_file, Path(restore_path).joinpath(file_name))	×
111	except OSError:	×
112	raise RstdmfError(f"Could not move {restored_file}.")	×
113
114
115	def local_storage_for_rstdmf(	×
116	files_for_rstdmf: List[str], restore_path: str, max_size_on_disk: int
117	):
118	"""Create a local storage object for rstdmf_divisions function.
119
120	Parameters
121	----------
122	files_for_rstdmf : List[str]
123	files that will be restored.
124	restore_path : str
125	max_size_on_disk : int
126	maximum amount of space to be used on disk.
127
128	Returns
129	-------
130	out : StorageState object
131
132	"""
133
134	# Get files already restored on disk and their size
135	files_on_disk = list()	×
136	for file_on_disk in files_for_rstdmf:	×
137	renamed_file = file_on_disk.replace("/", ".").lstrip(".")	×
138	renamed_path = Path(restore_path).joinpath(renamed_file)	×
139	if os.path.isfile(renamed_path):	×
140	files_on_disk.append(renamed_path)	×
141	elif os.path.isdir(renamed_path):	×
142	files_within = find_filepaths(renamed_path)	×
143	files_on_disk.append(	×
144	[FileMeta(renamed_path.as_posix()), size_evaluation(files_within)]
145	)
146	size_on_disk = size_evaluation(files_on_disk)	×
147	# create a local storage object
148	return StorageState(	×
149	restore_path, max_size_on_disk, size_on_disk, max_size_on_disk - size_on_disk
150	)
151
152
153	#
154	def rstdmf_divisions(	×
155	paths_to_restore,
156	restore_path,
157	disk_space_margin: int = 100 * GiB,
158	disk_refresh_time: int = 60,
159	rstdmf_size_limit: int = 10 * GiB,
160	rstdmf_file_limit: int = 24,
161	max_size_on_disk: int = 0,
162	verbose: bool = False,
163	preserve_order: bool = True,
164	):
165	"""Securely make call(s) to rstdmf.
166
167	Parameters
168	----------
169	paths_to_restore : list of strings
170	list of files to restore (path or FileMeta objects), giving
171	directories also works but they are considered like a single file
172	(i.e. be careful of their size!).
173	restore_path : str
174	Path where files will be restored.
175	disk_space_margin : int
176	minimum space required on disk to proceed, set to 0 for no minimum
177	requirement (default: 100 GiB).
178	disk_refresh_time : int
179	time to wait to lookup disk space again when full (default: 60 s).
180	rstdmf_size_limit : int
181	size limit of a single rstdmf call, set to 0 for no size limit
182	(default: 10 GiB).
183	rstdmf_file_limit : int
184	limit in number of files of a single rstdmf call, set to 0 for no
185	limit (default: 24).
186	max_size_on_disk : int
187	maximum size of restored files on disk, set to 0 for no maximum size
188	(default: 0).
189	verbose : bool
190	verbose mode (default: off).
191	preserve_order : bool
192	flag to force files to be restored in the order they are given
193	(default: True).
194
195	Notes
196	-----
197	The restored files are renamed to their full path with '/' replaced by '.':
198	for example, ``/server/path/sample.tar`` becomes ``server.path.sample.tar``
199	This allows getting files with the same name in different locations.
200	If this renamed file is already in the target location, it is not restored.
201
202	"""
203
204	# Make a list of files that should be restored
205	files_for_rstdmf = []	×
206	for file_to_restore in paths_to_restore:	×
207	# If a directory is given, create its associated FileMeta object
208	if os.path.isdir(file_to_restore):	×
209	files_within = find_filepaths(file_to_restore)	×
210	file_to_restore = FileMeta(file_to_restore, size_evaluation(files_within))	×
211	# If just a file name is given, convert it to a FileMeta object
212	elif not isinstance(file_to_restore, FileMeta):	×
213	file_to_restore = FileMeta(file_to_restore)	×
214	# Skip files that do not exist or are already in the restoration path
215	file_name = file_to_restore.path.replace("/", ".").lstrip(".")	×
216	if (	×
217	Path(restore_path).joinpath(file_name).exists()
218	or not Path(file_to_restore).exists()
219	):
220	continue	×
221	# Make sure no files are repeated
222	for file_for_rstdmf in files_for_rstdmf:	×
223	if file_for_rstdmf.path == file_to_restore.path:	×
224	break	×
225	else:
226	files_for_rstdmf.append(file_to_restore)	×
227	if not files_for_rstdmf:	×
228	logging.warning("All files are already on disk or nonexistent.")	×
229	return	×
230
231	# Divide files to satisfy rstdmf constraints
232	if (max_size_on_disk != 0) and (max_size_on_disk < rstdmf_size_limit):	×
233	division_size_limit = max_size_on_disk	×
234	else:
235	division_size_limit = rstdmf_size_limit	×
236	divisions = size_division(	×
237	files_for_rstdmf,
238	division_size_limit,
239	rstdmf_file_limit,
240	check_name_repetition=True,
241	preserve_order=preserve_order,
242	)
243
244	try:	×
245	storage = StorageState(restore_path)	×
246	except DiskSpaceError:	×
247	storage = StorageState(restore_path, 0, 0, 0)	×
248	if verbose:	×
249	size_of_files = size_evaluation(files_for_rstdmf)	×
250	logging.warning(	×
251	f"Disk space: {float(storage.free_space) / GiB} GiB, Restoration size: {size_of_files / GiB} GiB, "
252	+ f"Subdivisions: {str(len(divisions))}"
253	)
254	if not yesno_prompt("Proceed with restoration?"):	×
255	return	×
256
257	for division in divisions:	×
258	size_of_files = size_evaluation(division)	×
259	list_of_path = [file_meta.path for file_meta in division]	×
260	# If we can't get disk space, disregard it in the process
261	try:	×
262	storage = StorageState(restore_path)	×
263	except DiskSpaceError:	×
264	storage = StorageState(restore_path, 0, 0, 0)	×
265	disk_space_margin = 0	×
266	local_storage = local_storage_for_rstdmf(	×
267	files_for_rstdmf, restore_path, max_size_on_disk
268	)
269	if (	×
270	(disk_space_margin != 0)
271	and (storage.free_space - disk_space_margin < size_of_files)
272	) or ((max_size_on_disk != 0) and (local_storage.free_space < size_of_files)):
273	DiskSpaceEvent.set()	×
274	if storage.free_space - disk_space_margin < size_of_files:	×
275	logging.warning(	×
276	f"Disk space running low! Starting wait loop ({disk_refresh_time} seconds)."
277	)
278	logging.warning(	×
279	f"Disk space: {float(storage.free_space) / GiB} GB, "
280	f"Disk space to maintain: {disk_space_margin / GiB} GiB, "
281	f"Restore size: {str(size_of_files / GiB)} GiB."
282	)
283	elif local_storage.free_space < size_of_files:	×
284	logging.warning(	×
285	f"Reached size limit of files on disk ({float(max_size_on_disk) / GiB} GiB)."
286	f" Starting wait loop ({disk_refresh_time} seconds)."
287	)
288	while (storage.free_space - disk_space_margin < size_of_files) or (	×
289	local_storage.free_space < size_of_files
290	):
291	time.sleep(disk_refresh_time)	×
292	# If we can't get disk space, disregard it in the process
293	try:	×
294	storage = StorageState(restore_path)	×
295	except DiskSpaceError:	×
296	storage = StorageState(restore_path, 0, 0, 0)	×
297	disk_space_margin = 0	×
298	local_storage = local_storage_for_rstdmf(	×
299	files_for_rstdmf, restore_path, max_size_on_disk
300	)
301	DiskSpaceEvent.clear()	×
302	try:	×
303	rstdmf_rename(list_of_path, restore_path)	×
304	except RstdmfError:	×
305	raise	×
306
307
308	class RstdmfError(Exception):	×
309	pass	×

Ouranosinc / miranda / 2221026373

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous