2198161128

Build # 2198161128

Build Type

github

Committed by GitHub

Commit Message

Merge 7896c44d6 into d55f76503

Pull Request Pull Request #33: Support CORDEX and CMIP5/6

Run Details

2 of 28 new or added lines in 5 files covered. (7.14%)

574 existing lines in 15 files now uncovered.

737 of 3288 relevant lines covered (22.41%)

0.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

30.49

/miranda/convert/_data_definitions.py

import json
import logging.config
import os
from pathlib import Path
from typing import Dict, List, Union

from miranda.scripting import LOGGING_CONFIG
from miranda.storage import report_file_size

logging.config.dictConfig(LOGGING_CONFIG)

__all__ = [
    "gather_agcfsr",
    "gather_agmerra",
    "gather_era5_land",
    "gather_era5_single_levels",
    "gather_nrcan_gridded_obs",
    "gather_sc_earth",
    "gather_wfdei_gem_capa",
    "era5_variables",
    "nrcan_variables",
    "nasa_ag_variables",
    "sc_earth_variables",
    "wfdei_gem_capa_variables",
    "reanalysis_project_institutes",
    "xarray_frequencies_to_cmip6like",
]

data_folder = Path(__file__).parent / "data"
era5_variables = json.load(open(data_folder / "ecmwf_cf_attrs.json"))[
    "variable_entry"
].keys()
nrcan_variables = ["tasmin", "tasmax", "pr"]
nasa_ag_variables = json.load(open(data_folder / "nasa_cf_attrs.json"))[
    "variable_entry"
].keys()
sc_earth_variables = ["prcp", "tdew", "tmean", "trange", "wind"]
wfdei_gem_capa_variables = json.load(open(data_folder / "usask_cf_attrs.json"))[
    "variable_entry"
].keys()

reanalysis_project_institutes = {
    "cfsr": "ncar",
    "era5": "ecmwf",
    "era5-single-levels-preliminary-back-extension": "ecmwf",
    "era5-single-levels": "ecmwf",
    "era5-land": "ecmwf",
    "merra2": "nasa",
    "nrcan-gridded-10km": "nrcan",
    "wfdei-gem-capa": "usask",
}


# Manually map xarray frequencies to CMIP6/CMIP5 controlled vocabulary.
# see: https://github.com/ES-DOC/pyessv-archive
xarray_frequencies_to_cmip6like = {
    "H": "hr",
    "D": "day",
    "W": "sem",
    "M": "mon",
    "Q": "qtr",  # TODO does this make sense? does not exist in cmip6 CV
    "A": "yr",
    "Y": "yr",
}


def gather_era5_single_levels(
    path: Union[str, os.PathLike], back_extension: bool = False
) -> Dict[str, List[Path]]:
    # ERA5 source data
    source_era5 = Path(path)
    logging.info("Gathering ERA5 from %s" % source_era5.as_posix())
    infiles_era5 = list()
    for v in era5_variables:
        infiles_era5.extend(list(sorted(source_era5.rglob(f"{v}_*.nc"))))
    logging.info(
        f"Found {len(infiles_era5)} files, totalling {report_file_size(infiles_era5)}."
    )
    if not back_extension:
        return {"era5-single-levels": infiles_era5}
    return {"era5-single-levels-preliminary-back-extension": infiles_era5}


def gather_era5_land_sea_mask(path: Union[str, os.PathLike]) -> Dict:
    try:
        land_sea_mask = dict(lsm=next(Path(path).glob("sftlf*.nc")))
    except StopIteration:
        logging.error("No land_sea_mask found for ERA5.")
        raise FileNotFoundError()
    return land_sea_mask


def gather_era5_land(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # ERA5-Land source data
    source_era5l = Path(path)
    logging.info("Gathering ERA5-Land from %s" % source_era5l.as_posix())
    infiles_era5l = list()
    for v in era5_variables:
        infiles_era5l.extend(list(sorted(source_era5l.rglob(f"{v}_*.nc"))))
    logging.info(
        f"Found {len(infiles_era5l)} files, totalling {report_file_size(infiles_era5l)}."
    )
    return {"era5-land": infiles_era5l}


def gather_agmerra(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # agMERRA source data
    source_agmerra = Path(path)
    logging.info("Gathering agMERRA from %s" % source_agmerra.as_posix())
    infiles_agmerra = list()
    for v in nasa_ag_variables:
        infiles_agmerra.extend(list(sorted(source_agmerra.rglob(f"AgMERRA_*_{v}.nc4"))))
    logging.info(
        f"Found {len(infiles_agmerra)} files, totalling {report_file_size(infiles_agmerra)}."
    )
    return dict(cfsr=infiles_agmerra)


def gather_agcfsr(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # agCFSR source data
    source_agcfsr = Path(path)
    logging.info("Gathering CFSR from %s" % source_agcfsr.as_posix())
    infiles_agcfsr = list()
    for v in nasa_ag_variables:
        infiles_agcfsr.extend(list(sorted(source_agcfsr.rglob(f"AgCFSR_*_{v}.nc4"))))
    logging.info(
        f"Found {len(infiles_agcfsr)} files, totalling {report_file_size(infiles_agcfsr)}."
    )
    return dict(cfsr=infiles_agcfsr)


def gather_nrcan_gridded_obs(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # NRCan Gridded Obs source data
    source_nrcan = Path(path)
    logging.info("Gathering NRCAN Gridded Obs from %s" % source_nrcan.as_posix())
    infiles_nrcan = list()
    for v in nrcan_variables:
        infiles_nrcan.extend(list(sorted(source_nrcan.joinpath(v).glob(f"*{v}_*.nc"))))
    logging.info(
        f"Found {len(infiles_nrcan)} files, totalling {report_file_size(infiles_nrcan)}."
    )
    return dict(nrcan=infiles_nrcan)


def gather_wfdei_gem_capa(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # WFDEI-GEM-CaPa source data
    source_wfdei = Path(path)
    logging.info("Gathering WFDEI-GEM_CaPa from %s" % source_wfdei.as_posix())
    infiles_wfdei = list()
    for v in wfdei_gem_capa_variables:
        infiles_wfdei.extend(list(sorted(source_wfdei.rglob(f"{v}_*.nc"))))
    logging.info(
        f"Found {len(infiles_wfdei)} files, totalling {report_file_size(infiles_wfdei)}."
    )
    return {"wfdei-gem-capa": infiles_wfdei}


def gather_sc_earth(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:
    # SC_Earth source data
    source_sc_earth = Path(path)
    logging.info("Gathering SC-Earth from %s" % source_sc_earth.as_posix())
    infiles_sc_earth = list()
    for v in sc_earth_variables:
        infiles_sc_earth.extend(
            list(sorted(source_sc_earth.rglob(f"SC-Earth_{v}_*.nc")))
        )
    logging.info(
        f"Found {len(infiles_sc_earth)} files, totalling {report_file_size(infiles_sc_earth)}."
    )
    return {"wfdei-gem-capa": infiles_sc_earth}

1	import json	3✔
2	import logging.config	3✔
3	import os	3✔
4	from pathlib import Path	3✔
5	from typing import Dict, List, Union	3✔
6
7	from miranda.scripting import LOGGING_CONFIG	3✔
8	from miranda.storage import report_file_size	3✔
9
10	logging.config.dictConfig(LOGGING_CONFIG)	3✔
11
12	__all__ = [	3✔
13	"gather_agcfsr",
14	"gather_agmerra",
15	"gather_era5_land",
16	"gather_era5_single_levels",
17	"gather_nrcan_gridded_obs",
18	"gather_sc_earth",
19	"gather_wfdei_gem_capa",
20	"era5_variables",
21	"nrcan_variables",
22	"nasa_ag_variables",
23	"sc_earth_variables",
24	"wfdei_gem_capa_variables",
25	"reanalysis_project_institutes",
26	"xarray_frequencies_to_cmip6like",
27	]
28
29	data_folder = Path(__file__).parent / "data"	3✔
30	era5_variables = json.load(open(data_folder / "ecmwf_cf_attrs.json"))[	3✔
31	"variable_entry"
32	].keys()
33	nrcan_variables = ["tasmin", "tasmax", "pr"]	3✔
34	nasa_ag_variables = json.load(open(data_folder / "nasa_cf_attrs.json"))[	3✔
35	"variable_entry"
36	].keys()
37	sc_earth_variables = ["prcp", "tdew", "tmean", "trange", "wind"]	3✔
38	wfdei_gem_capa_variables = json.load(open(data_folder / "usask_cf_attrs.json"))[	3✔
39	"variable_entry"
40	].keys()
41
42	reanalysis_project_institutes = {	3✔
43	"cfsr": "ncar",
44	"era5": "ecmwf",
45	"era5-single-levels-preliminary-back-extension": "ecmwf",
46	"era5-single-levels": "ecmwf",
47	"era5-land": "ecmwf",
48	"merra2": "nasa",
49	"nrcan-gridded-10km": "nrcan",
50	"wfdei-gem-capa": "usask",
51	}
52
53
54	# Manually map xarray frequencies to CMIP6/CMIP5 controlled vocabulary.
55	# see: https://github.com/ES-DOC/pyessv-archive
56	xarray_frequencies_to_cmip6like = {	3✔
57	"H": "hr",
58	"D": "day",
59	"W": "sem",
60	"M": "mon",
61	"Q": "qtr", # TODO does this make sense? does not exist in cmip6 CV
62	"A": "yr",
63	"Y": "yr",
64	}
65
66
67	def gather_era5_single_levels(	3✔
68	path: Union[str, os.PathLike], back_extension: bool = False
69	) -> Dict[str, List[Path]]:
70	# ERA5 source data
UNCOV 71	source_era5 = Path(path)	×
UNCOV 72	logging.info("Gathering ERA5 from %s" % source_era5.as_posix())	×
73	infiles_era5 = list()	×
74	for v in era5_variables:	×
75	infiles_era5.extend(list(sorted(source_era5.rglob(f"{v}_*.nc"))))	×
76	logging.info(	×
77	f"Found {len(infiles_era5)} files, totalling {report_file_size(infiles_era5)}."
78	)
79	if not back_extension:	×
80	return {"era5-single-levels": infiles_era5}	×
UNCOV 81	return {"era5-single-levels-preliminary-back-extension": infiles_era5}	×
82
83
84	def gather_era5_land_sea_mask(path: Union[str, os.PathLike]) -> Dict:	3✔
85	try:	×
86	land_sea_mask = dict(lsm=next(Path(path).glob("sftlf*.nc")))	×
UNCOV 87	except StopIteration:	×
UNCOV 88	logging.error("No land_sea_mask found for ERA5.")	×
89	raise FileNotFoundError()	×
90	return land_sea_mask	×
91
92
93	def gather_era5_land(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
94	# ERA5-Land source data
95	source_era5l = Path(path)	×
96	logging.info("Gathering ERA5-Land from %s" % source_era5l.as_posix())	×
97	infiles_era5l = list()	×
98	for v in era5_variables:	×
99	infiles_era5l.extend(list(sorted(source_era5l.rglob(f"{v}_*.nc"))))	×
100	logging.info(	×
101	f"Found {len(infiles_era5l)} files, totalling {report_file_size(infiles_era5l)}."
102	)
103	return {"era5-land": infiles_era5l}	×
104
105
106	def gather_agmerra(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
107	# agMERRA source data
108	source_agmerra = Path(path)	×
109	logging.info("Gathering agMERRA from %s" % source_agmerra.as_posix())	×
110	infiles_agmerra = list()	×
111	for v in nasa_ag_variables:	×
112	infiles_agmerra.extend(list(sorted(source_agmerra.rglob(f"AgMERRA_*_{v}.nc4"))))	×
113	logging.info(	×
114	f"Found {len(infiles_agmerra)} files, totalling {report_file_size(infiles_agmerra)}."
115	)
UNCOV 116	return dict(cfsr=infiles_agmerra)	×
117
118
119	def gather_agcfsr(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
120	# agCFSR source data
121	source_agcfsr = Path(path)	×
122	logging.info("Gathering CFSR from %s" % source_agcfsr.as_posix())	×
123	infiles_agcfsr = list()	×
124	for v in nasa_ag_variables:	×
UNCOV 125	infiles_agcfsr.extend(list(sorted(source_agcfsr.rglob(f"AgCFSR_*_{v}.nc4"))))	×
UNCOV 126	logging.info(	×
127	f"Found {len(infiles_agcfsr)} files, totalling {report_file_size(infiles_agcfsr)}."
128	)
UNCOV 129	return dict(cfsr=infiles_agcfsr)	×
130
131
132	def gather_nrcan_gridded_obs(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
133	# NRCan Gridded Obs source data
134	source_nrcan = Path(path)	×
135	logging.info("Gathering NRCAN Gridded Obs from %s" % source_nrcan.as_posix())	×
136	infiles_nrcan = list()	×
137	for v in nrcan_variables:	×
UNCOV 138	infiles_nrcan.extend(list(sorted(source_nrcan.joinpath(v).glob(f"{v}_.nc"))))	×
UNCOV 139	logging.info(	×
140	f"Found {len(infiles_nrcan)} files, totalling {report_file_size(infiles_nrcan)}."
141	)
UNCOV 142	return dict(nrcan=infiles_nrcan)	×
143
144
145	def gather_wfdei_gem_capa(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
146	# WFDEI-GEM-CaPa source data
147	source_wfdei = Path(path)	×
148	logging.info("Gathering WFDEI-GEM_CaPa from %s" % source_wfdei.as_posix())	×
149	infiles_wfdei = list()	×
150	for v in wfdei_gem_capa_variables:	×
UNCOV 151	infiles_wfdei.extend(list(sorted(source_wfdei.rglob(f"{v}_*.nc"))))	×
UNCOV 152	logging.info(	×
153	f"Found {len(infiles_wfdei)} files, totalling {report_file_size(infiles_wfdei)}."
154	)
UNCOV 155	return {"wfdei-gem-capa": infiles_wfdei}	×
156
157
158	def gather_sc_earth(path: Union[str, os.PathLike]) -> Dict[str, List[Path]]:	3✔
159	# SC_Earth source data
160	source_sc_earth = Path(path)	×
161	logging.info("Gathering SC-Earth from %s" % source_sc_earth.as_posix())	×
162	infiles_sc_earth = list()	×
163	for v in sc_earth_variables:	×
UNCOV 164	infiles_sc_earth.extend(	×
165	list(sorted(source_sc_earth.rglob(f"SC-Earth_{v}_*.nc")))
166	)
UNCOV 167	logging.info(	×
168	f"Found {len(infiles_sc_earth)} files, totalling {report_file_size(infiles_sc_earth)}."
169	)
UNCOV 170	return {"wfdei-gem-capa": infiles_sc_earth}	×

Ouranosinc / miranda / 2198161128

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous