2157914261

Build Type

github

Committed by GitHub

Commit Message

Merge 5282ae2c2 into 9ac032fc5

Pull Request Pull Request #24: Add CMIP file structure, use pyessv controlled vocabularies, and major refactoring

Run Details

242 of 1112 new or added lines in 35 files covered. (21.76%)

12 existing lines in 4 files now uncovered.

735 of 3250 relevant lines covered (22.62%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.51

/miranda/archive/archiver.py

#!/bin/env python3
import logging
import logging.config
from collections import defaultdict
from datetime import datetime as dt
from pathlib import Path
from typing import List, Union

from miranda.archive.ops import create_archive, create_remote_directory, transfer_file
from miranda.connect import Connection
from miranda.scripting import LOGGING_CONFIG
from miranda.storage import report_file_size
from miranda.utils import find_filepaths, single_item_list, working_directory

from .groupings import group_by_deciphered_date, group_by_size, group_by_subdirectories

logging.config.dictConfig(LOGGING_CONFIG)

__all__ = ["archive_database"]


def archive_database(
    source: Union[Path, str, List],
    common_path: Union[Path, str],
    destination: Union[Path, str],
    file_suffixes: str = ".nc",
    server: str = None,
    username: str = None,
    project_name: str = None,
    overwrite: bool = False,
    compression: bool = False,
    recursive: bool = False,
    use_grouping: bool = True,
    use_subdirectories: bool = True,
    dry_run: bool = False,
) -> None:
    """
    Given a source, destination, and dependent on file size limit, create tarfile archives and transfer
     files to another server for backup purposes
    """
    project = "{}_{}{}.{}"

    if not project_name:
        project_name = destination.name

    if compression:
        suffix = "tar.gz"
    elif not compression:
        suffix = "tar"
    else:
        raise ValueError(f"Compression: {compression}")

    file_list, source_path = find_filepaths(
        source=source, recursive=recursive, file_suffixes=file_suffixes
    )

    if use_subdirectories:
        file_groups = group_by_subdirectories(file_list, within=common_path)

    else:
        file_groups = defaultdict(list)
        for f in file_list:
            file_groups["."].append(f)

    connection = Connection(username=username, host=server, protocol="sftp")

    if dry_run:
        logging.info(
            "Running archival functions in `dry_run` mode. No files will be transferred."
        )

    try:
        successful_transfers = list()
        with connection as ctx:
            for group_name, members in file_groups.items():
                remote_path = Path(destination, group_name)

                if not dry_run:
                    if not remote_path.exists():
                        create_remote_directory(remote_path, transport=ctx)

                if use_grouping:
                    dated_groups = group_by_deciphered_date(members)
                else:
                    dated_groups = dict(group_name=members)

                for common_date, files in dated_groups.items():
                    if not use_grouping or single_item_list(files):
                        for archive_file in files:
                            transfer = Path(remote_path, archive_file.name)

                            if transfer.is_file():
                                if not overwrite:
                                    logging.info(f"{transfer} exists. Skipping file.")
                                    continue
                                logging.info(f"{transfer} exists. Overwriting.")
                            if not dry_run:
                                if transfer_file(archive_file, transfer, transport=ctx):
                                    successful_transfers.append(archive_file)
                            else:
                                successful_transfers.append(archive_file)

                    elif use_grouping or not single_item_list(files):
                        sized_groups = group_by_size(files)

                        for i, sized_group in enumerate(sized_groups):
                            if len(sized_groups) > 1:
                                part = f"_{str(i + 1).zfill(3)}"
                            else:
                                part = ""

                            archive_file = project.format(
                                project_name, group_name, common_date, part, suffix
                            )
                            transfer = Path(remote_path, archive_file)

                            if transfer.is_file():
                                if not overwrite:
                                    logging.info(
                                        f'File "{transfer}" exists. Skipping file.'
                                    )
                                    continue
                                logging.info(f'File "{transfer}" exists. Overwriting.')

                            with working_directory(source_path):
                                if not dry_run:
                                    if create_archive(
                                        sized_group,
                                        transfer,
                                        transport=ctx,
                                        compression=compression,
                                        recursive=recursive,
                                    ):
                                        successful_transfers.extend(sized_group)
                                else:
                                    successful_transfers.extend(sized_group)
                    else:
                        raise FileNotFoundError("No files found in grouping.")

        logging.info(
            "Transferred {} of {} files totalling {}.".format(
                len(successful_transfers),
                len([f for f in file_list]),
                report_file_size(successful_transfers),
            )
        )

    except Exception as e:
        msg = f"{e}: Failed to transfer files."
        logging.error(msg)
        raise RuntimeError(msg) from e


if __name__ == "__main__":
    logging.basicConfig(
        filename=f"{dt.strftime(dt.now(), '%Y%m%d')}_{Path(__name__).stem}.log",
        level=logging.INFO,
    )

1	#!/bin/env python3
2	import logging	3✔
3	import logging.config	3✔
4	from collections import defaultdict	3✔
5	from datetime import datetime as dt	3✔
6	from pathlib import Path	3✔
7	from typing import List, Union	3✔
8
9	from miranda.archive.ops import create_archive, create_remote_directory, transfer_file	3✔
10	from miranda.connect import Connection	3✔
11	from miranda.scripting import LOGGING_CONFIG	3✔
12	from miranda.storage import report_file_size	3✔
13	from miranda.utils import find_filepaths, single_item_list, working_directory	3✔
14
15	from .groupings import group_by_deciphered_date, group_by_size, group_by_subdirectories	3✔
16
17	logging.config.dictConfig(LOGGING_CONFIG)	3✔
18
19	__all__ = ["archive_database"]	3✔
20
21
22	def archive_database(	3✔
23	source: Union[Path, str, List],
24	common_path: Union[Path, str],
25	destination: Union[Path, str],
26	file_suffixes: str = ".nc",
27	server: str = None,
28	username: str = None,
29	project_name: str = None,
30	overwrite: bool = False,
31	compression: bool = False,
32	recursive: bool = False,
33	use_grouping: bool = True,
34	use_subdirectories: bool = True,
35	dry_run: bool = False,
36	) -> None:
37	"""
38	Given a source, destination, and dependent on file size limit, create tarfile archives and transfer
39	files to another server for backup purposes
40	"""
41	project = "{}_{}{}.{}"	×
42
43	if not project_name:	×
44	project_name = destination.name	×
45
46	if compression:	×
47	suffix = "tar.gz"	×
48	elif not compression:	×
49	suffix = "tar"	×
50	else:
NEW 51	raise ValueError(f"Compression: {compression}")	×
52
53	file_list, source_path = find_filepaths(	×
54	source=source, recursive=recursive, file_suffixes=file_suffixes
55	)
56
57	if use_subdirectories:	×
58	file_groups = group_by_subdirectories(file_list, within=common_path)	×
59
60	else:
61	file_groups = defaultdict(list)	×
62	for f in file_list:	×
63	file_groups["."].append(f)	×
64
65	connection = Connection(username=username, host=server, protocol="sftp")	×
66
67	if dry_run:	×
68	logging.info(	×
69	"Running archival functions in `dry_run` mode. No files will be transferred."
70	)
71
72	try:	×
73	successful_transfers = list()	×
74	with connection as ctx:	×
75	for group_name, members in file_groups.items():	×
76	remote_path = Path(destination, group_name)	×
77
78	if not dry_run:	×
79	if not remote_path.exists():	×
80	create_remote_directory(remote_path, transport=ctx)	×
81
82	if use_grouping:	×
83	dated_groups = group_by_deciphered_date(members)	×
84	else:
85	dated_groups = dict(group_name=members)	×
86
87	for common_date, files in dated_groups.items():	×
88	if not use_grouping or single_item_list(files):	×
89	for archive_file in files:	×
90	transfer = Path(remote_path, archive_file.name)	×
91
92	if transfer.is_file():	×
93	if not overwrite:	×
NEW 94	logging.info(f"{transfer} exists. Skipping file.")	×
95	continue	×
NEW 96	logging.info(f"{transfer} exists. Overwriting.")	×
97	if not dry_run:	×
98	if transfer_file(archive_file, transfer, transport=ctx):	×
99	successful_transfers.append(archive_file)	×
100	else:
101	successful_transfers.append(archive_file)	×
102
103	elif use_grouping or not single_item_list(files):	×
104	sized_groups = group_by_size(files)	×
105
106	for i, sized_group in enumerate(sized_groups):	×
107	if len(sized_groups) > 1:	×
108	part = f"_{str(i + 1).zfill(3)}"	×
109	else:
110	part = ""	×
111
112	archive_file = project.format(	×
113	project_name, group_name, common_date, part, suffix
114	)
115	transfer = Path(remote_path, archive_file)	×
116
117	if transfer.is_file():	×
118	if not overwrite:	×
119	logging.info(	×
120	f'File "{transfer}" exists. Skipping file.'
121	)
122	continue	×
123	logging.info(f'File "{transfer}" exists. Overwriting.')	×
124
125	with working_directory(source_path):	×
126	if not dry_run:	×
127	if create_archive(	×
128	sized_group,
129	transfer,
130	transport=ctx,
131	compression=compression,
132	recursive=recursive,
133	):
134	successful_transfers.extend(sized_group)	×
135	else:
136	successful_transfers.extend(sized_group)	×
137	else:
138	raise FileNotFoundError("No files found in grouping.")	×
139
140	logging.info(	×
141	"Transferred {} of {} files totalling {}.".format(
142	len(successful_transfers),
143	len([f for f in file_list]),
144	report_file_size(successful_transfers),
145	)
146	)
147
148	except Exception as e:	×
149	msg = f"{e}: Failed to transfer files."	×
150	logging.error(msg)	×
151	raise RuntimeError(msg) from e	×
152
153
154	if __name__ == "__main__":	3✔
155	logging.basicConfig(	×
156	filename=f"{dt.strftime(dt.now(), '%Y%m%d')}_{Path(__name__).stem}.log",
157	level=logging.INFO,
158	)

Ouranosinc / miranda / 2157914261

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous