1715021976

Build # 1715021976

Build Type

github

Committed by GitHub

Commit Message

Merge 3cd491177 into 5c54767a4

Pull Request Pull Request #21: Update ECMWF requests calls to newest API conventions

Run Details

16 of 301 new or added lines in 6 files covered. (5.32%)

95 existing lines in 3 files now uncovered.

494 of 2418 relevant lines covered (20.43%)

0.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.9

/miranda/ecmwf/era5.py

import functools
import logging.config
import multiprocessing
import os
from datetime import date
from datetime import datetime as dt
from pathlib import Path
from typing import List, Mapping, Optional, Tuple

from cdsapi import Client

from miranda.scripting import LOGGING_CONFIG

logging.config.dictConfig(LOGGING_CONFIG)

__all__ = ["request_era5"]


def request_era5(
    variables: Optional[Mapping[str, str]] = None,
    projects: List[str] = None,
    domain: str = "AMNO",
    year_start: int = 1981,
    year_end: Optional[int] = None,
    processes: int = 4,
) -> None:
    """Request ERA5/ERA5-Land from Copernicus Data Store in NetCDF4 format.

    Parameters
    ----------
    variables: Mapping[str, str], optional
    projects : List[{"era5", "era5-land"}]
    domain : {"GLOBAL", "AMNO", "CAN", "QC"}
    year_start : int
    year_end : int, optional
    processes : int

    Returns
    -------
    None
    """
    # Variables of interest
    variable_reference = dict(
        pr="total_precipitation",
        vas="10m_v_component_of_wind",
        uas="10m_u_component_of_wind",
        td="2m_dewpoint_temperature",
        tas="2m_temperature",
        potevap="potential evaporation",
        snd="snow_depth",
        prsn="snowfall",
    )

    v_requested = dict()
    if variables:
        for v in variables:
            if v in variable_reference:
                v_requested[v] = variable_reference[v]
    else:
        v_requested = variable_reference

    if year_end is None:
        year_end = date.today().year
    years = range(year_start, year_end)

    months = [str(d).zfill(2) for d in range(13)]
    yearmonth = list()
    for y in years:
        for m in months:
            yearmonth.append((y, m))

    project_names = list()
    if "era5" in projects:
        project_names.append("reanalysis-era5-single-levels")
    if "era5-land" in projects:
        project_names.append("reanalysis-era5-land")
    product = project_names[0].split("-")[0]

    target = Path().cwd().joinpath("downloaded")

    Path(target).mkdir(exist_ok=True)
    os.chdir(target)

    for p in projects:
        proc = multiprocessing.Pool(processes=processes)
        func = functools.partial(_request_direct_era, v_requested, p, domain, product)

        logging.info([func, dt.now().strftime("%Y-%m-%d %X")])

        proc.map(func, yearmonth)
        proc.close()
        proc.join()


def _request_direct_era(
    variables: Mapping[str, str],
    project: str,
    domain: str,
    product: str,
    yearmonth: Tuple[int, str],
):
    """Launch formatted request."""
    year, month = yearmonth
    days = [str(d).zfill(2) for d in range(32)]
    times = ["{}:00".format(str(t).zfill(2)) for t in range(24)]

    if domain.upper() == "GLOBAL":
        region = [90, -180, -90, 180]
    elif domain.upper() == "AMNO":
        domain = "NAM"
        region = [90, -180, 10, -10]
    elif domain.upper() == "CAN":
        region = [83.5, -141, 41.5, -52.5]
    elif domain.upper() == "QC":
        region = [63, -80, 44.5, -57]
    else:
        raise ValueError()

    c = Client()

    for var in variables.keys():
        netcdf_name = f"{var}_{'-'.join(project.split('-')[1:])}_{product}_hourly_{year}{month}_{domain.upper()}.nc"

        if Path(netcdf_name).exists():
            continue

        request_kwargs = dict(
            variable=variables[var],
            year=year,
            month=month,
            day=days,
            time=times,
            area=region,
            format="netcdf",
        )

        if project == "reanalysis-era5-single-levels":
            request_kwargs.update(dict(product_type=product))

        c.retrieve(
            project,
            request_kwargs,
            netcdf_name,
        )

1	import functools	3✔
2	import logging.config	3✔
3	import multiprocessing	3✔
4	import os	3✔
5	from datetime import date	3✔
6	from datetime import datetime as dt	3✔
7	from pathlib import Path	3✔
8	from typing import List, Mapping, Optional, Tuple	3✔
9
10	from cdsapi import Client	3✔
11
12	from miranda.scripting import LOGGING_CONFIG	3✔
13
14	logging.config.dictConfig(LOGGING_CONFIG)	3✔
15
16	__all__ = ["request_era5"]	3✔
17
18
19	def request_era5(	3✔
20	variables: Optional[Mapping[str, str]] = None,
21	projects: List[str] = None,
22	domain: str = "AMNO",
23	year_start: int = 1981,
24	year_end: Optional[int] = None,
25	processes: int = 4,
26	) -> None:
27	"""Request ERA5/ERA5-Land from Copernicus Data Store in NetCDF4 format.
28
29	Parameters
30	----------
31	variables: Mapping[str, str], optional
32	projects : List[{"era5", "era5-land"}]
33	domain : {"GLOBAL", "AMNO", "CAN", "QC"}
34	year_start : int
35	year_end : int, optional
36	processes : int
37
38	Returns
39	-------
40	None
41	"""
42	# Variables of interest
43	variable_reference = dict(	×
44	pr="total_precipitation",
45	vas="10m_v_component_of_wind",
46	uas="10m_u_component_of_wind",
47	td="2m_dewpoint_temperature",
48	tas="2m_temperature",
49	potevap="potential evaporation",
50	snd="snow_depth",
51	prsn="snowfall",
52	)
53
54	v_requested = dict()	×
55	if variables:	×
56	for v in variables:	×
57	if v in variable_reference:	×
58	v_requested[v] = variable_reference[v]	×
59	else:
60	v_requested = variable_reference	×
61
62	if year_end is None:	×
63	year_end = date.today().year	×
64	years = range(year_start, year_end)	×
65
66	months = [str(d).zfill(2) for d in range(13)]	×
67	yearmonth = list()	×
68	for y in years:	×
69	for m in months:	×
70	yearmonth.append((y, m))	×
71
72	project_names = list()	×
73	if "era5" in projects:	×
74	project_names.append("reanalysis-era5-single-levels")	×
75	if "era5-land" in projects:	×
76	project_names.append("reanalysis-era5-land")	×
77	product = project_names[0].split("-")[0]	×
78
79	target = Path().cwd().joinpath("downloaded")	×
80
81	Path(target).mkdir(exist_ok=True)	×
82	os.chdir(target)	×
83
84	for p in projects:	×
85	proc = multiprocessing.Pool(processes=processes)	×
86	func = functools.partial(_request_direct_era, v_requested, p, domain, product)	×
87
88	logging.info([func, dt.now().strftime("%Y-%m-%d %X")])	×
89
90	proc.map(func, yearmonth)	×
91	proc.close()	×
92	proc.join()	×
93
94
95	def _request_direct_era(	3✔
96	variables: Mapping[str, str],
97	project: str,
98	domain: str,
99	product: str,
100	yearmonth: Tuple[int, str],
101	):
102	"""Launch formatted request."""
103	year, month = yearmonth	×
104	days = [str(d).zfill(2) for d in range(32)]	×
105	times = ["{}:00".format(str(t).zfill(2)) for t in range(24)]	×
106
107	if domain.upper() == "GLOBAL":	×
NEW 108	region = [90, -180, -90, 180]	×
109	elif domain.upper() == "AMNO":	×
NEW 110	domain = "NAM"	×
NEW 111	region = [90, -180, 10, -10]	×
112	elif domain.upper() == "CAN":	×
NEW 113	region = [83.5, -141, 41.5, -52.5]	×
114	elif domain.upper() == "QC":	×
NEW UNCOV 115	region = [63, -80, 44.5, -57]	×
116	else:
UNCOV 117	raise ValueError()	×
118
UNCOV 119	c = Client()	×
120
121	for var in variables.keys():	×
NEW UNCOV 122	netcdf_name = f"{var}_{'-'.join(project.split('-')[1:])}_{product}_hourly_{year}{month}_{domain.upper()}.nc"	×
123
124	if Path(netcdf_name).exists():	×
UNCOV 125	continue	×
126
UNCOV 127	request_kwargs = dict(	×
128	variable=variables[var],
129	year=year,
130	month=month,
131	day=days,
132	time=times,
133	area=region,
134	format="netcdf",
135	)
136
137	if project == "reanalysis-era5-single-levels":	×
UNCOV 138	request_kwargs.update(dict(product_type=product))	×
139
UNCOV 140	c.retrieve(	×
141	project,
142	request_kwargs,
143	netcdf_name,
144	)

Ouranosinc / miranda / 1715021976

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous