Build has been canceled!

2228419799

Build # 2228419799

Build Type

github

Committed by GitHub

Commit Message

Merge e25774d57 into d076d8475

Pull Request Pull Request #33: Support CORDEX and CMIP5/6

Run Details

32 of 162 new or added lines in 15 files covered. (19.75%)

6 existing lines in 5 files now uncovered.

659 of 3225 relevant lines covered (20.43%)

0.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.56

/miranda/convert/ecmwf.py

import itertools as it
import logging.config
import multiprocessing
import os
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Optional

import xarray
from dask.diagnostics import ProgressBar

from miranda.scripting import LOGGING_CONFIG

logging.config.dictConfig(LOGGING_CONFIG)


def tigge_convert(
    source: Optional[os.PathLike] = None,
    target: Optional[os.PathLike] = None,
    processes: int = 8,
) -> None:
    """Convert grib2 file to netCDF format.

    Parameters
    ----------
    source : os.PathLike, optional
    target : os.PathLike, optional
    processes : int

    Returns
    -------
    None
    """

    def _tigge_convert(fn):
        """Launch reformatting function."""
        infile, output_folder = fn
        try:
            for f in Path(infile.parent).glob(infile.name.replace(".grib", "*.idx")):
                f.unlink(missing_ok=True)

            ds = xarray.open_dataset(
                infile,
                engine="cfgrib",
                chunks="auto",
            )

            encoding = {var: dict(zlib=True) for var in ds.data_vars}
            encoding["time"] = {"dtype": "single"}
            tf = tempfile.NamedTemporaryFile(suffix=".nc", delete=False)

            with ProgressBar():
                logging.info(f"converting: {infile.name}")
                ds.to_netcdf(
                    tf.name, format="NETCDF4", engine="netcdf4", encoding=encoding
                )

            shutil.move(
                tf.name,
                output_folder.joinpath(infile.name.replace(".grib", ".nc")).as_posix(),
            )

        except ValueError:
            logging.error(f"error converting {infile.name} : File may be corrupted.")

    if source is None:
        source = Path().cwd().joinpath("downloaded")

    all_files = list(Path(source).glob("*.grib2"))
    if len(all_files) == 0:
        raise FileNotFoundError("TIGGE files not found.")

    if target is None:
        target = Path().cwd().joinpath("converted")
    else:
        target = Path(target)
    target.mkdir(exist_ok=True)

    with multiprocessing.Pool(processes=processes) as p:
        combs = list(it.product(*[all_files, [target]]))
        p.map(_tigge_convert, combs)
        p.close()
        p.join()

1	import itertools as it	3✔
2	import logging.config	3✔
3	import multiprocessing	3✔
4	import os	3✔
5	import shutil	3✔
6	import sys	3✔
7	import tempfile	3✔
8	from pathlib import Path	3✔
9	from typing import Optional	3✔
10
11	import xarray	3✔
12	from dask.diagnostics import ProgressBar	3✔
13
14	from miranda.scripting import LOGGING_CONFIG	3✔
15
16	logging.config.dictConfig(LOGGING_CONFIG)	3✔
17
18
19	def tigge_convert(	3✔
20	source: Optional[os.PathLike] = None,
21	target: Optional[os.PathLike] = None,
22	processes: int = 8,
23	) -> None:
24	"""Convert grib2 file to netCDF format.
25
26	Parameters
27	----------
28	source : os.PathLike, optional
29	target : os.PathLike, optional
30	processes : int
31
32	Returns
33	-------
34	None
35	"""
36
37	def _tigge_convert(fn):	×
38	"""Launch reformatting function."""
39	infile, output_folder = fn	×
40	try:	×
41	for f in Path(infile.parent).glob(infile.name.replace(".grib", "*.idx")):	×
42	f.unlink(missing_ok=True)	×
43
44	ds = xarray.open_dataset(	×
45	infile,
46	engine="cfgrib",
47	chunks="auto",
48	)
49
50	encoding = {var: dict(zlib=True) for var in ds.data_vars}	×
51	encoding["time"] = {"dtype": "single"}	×
NEW 52	tf = tempfile.NamedTemporaryFile(suffix=".nc", delete=False)	×
53
54	with ProgressBar():	×
NEW 55	logging.info(f"converting: {infile.name}")	×
56	ds.to_netcdf(	×
57	tf.name, format="NETCDF4", engine="netcdf4", encoding=encoding
58	)
59
60	shutil.move(	×
61	tf.name,
62	output_folder.joinpath(infile.name.replace(".grib", ".nc")).as_posix(),
63	)
64
65	except ValueError:	×
NEW 66	logging.error(f"error converting {infile.name} : File may be corrupted.")	×
67
68	if source is None:	×
NEW 69	source = Path().cwd().joinpath("downloaded")	×
70
NEW 71	all_files = list(Path(source).glob("*.grib2"))	×
NEW 72	if len(all_files) == 0:	×
NEW 73	raise FileNotFoundError("TIGGE files not found.")	×
74
NEW 75	if target is None:	×
NEW 76	target = Path().cwd().joinpath("converted")	×
77	else:
NEW 78	target = Path(target)	×
UNCOV 79	target.mkdir(exist_ok=True)	×
80
NEW 81	with multiprocessing.Pool(processes=processes) as p:	×
NEW 82	combs = list(it.product(*[all_files, [target]]))	×
NEW 83	p.map(_tigge_convert, combs)	×
NEW 84	p.close()	×
NEW 85	p.join()	×

Ouranosinc / miranda / 2228419799

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous