2117681512

Build Type

github

Committed by GitHub

Commit Message

Merge 9bbb68671 into bf78f91b7

Pull Request Pull Request #24: Add CMIP file structure, use pyessv controlled vocabularies, and major refactoring

Run Details

241 of 1086 new or added lines in 35 files covered. (22.19%)

13 existing lines in 4 files now uncovered.

735 of 3230 relevant lines covered (22.76%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.03

/miranda/convert/ecmwf.py

import itertools as it
import multiprocessing
import os
import shutil
import tempfile
from pathlib import Path
from typing import Optional

import xarray
from dask.diagnostics import ProgressBar


def tigge_convert(
    source: Optional[os.PathLike] = None,
    target: Optional[os.PathLike] = None,
    processes: int = 8,
) -> None:
    """Convert grib2 file to netCDF format.

    Parameters
    ----------
    source : os.PathLike, optional
    target : os.PathLike, optional
    processes : int

    Returns
    -------
    None
    """

    def _tigge_convert(fn):
        """Launch reformatting function."""
        infile, output_folder = fn
        try:
            for f in Path(infile.parent).glob(infile.name.replace(".grib", "*.idx")):
                f.unlink(missing_ok=True)

            ds = xarray.open_dataset(
                infile,
                engine="cfgrib",
                chunks="auto",
            )

            encoding = {var: dict(zlib=True) for var in ds.data_vars}
            encoding["time"] = {"dtype": "single"}
            tmpfile = tempfile.NamedTemporaryFile(suffix=".nc", delete=False)

            with ProgressBar():
                print("converting ", infile.name)
                ds.to_netcdf(
                    tmpfile.name, format="NETCDF4", engine="netcdf4", encoding=encoding
                )

            shutil.move(
                tmpfile.name,
                output_folder.joinpath(infile.name.replace(".grib", ".nc")).as_posix(),
            )

        except ValueError:
            print(f"error converting {infile.name} : File may be corrupted")

    if source is None:
        source = Path().cwd().joinpath("download")
    if target is None:
        target = Path().cwd().joinpath("converted")

    all_files = Path(source).glob("*.grib2")

    target = Path(target)
    target.mkdir(exist_ok=True)

    p = multiprocessing.Pool(processes=processes)

    combs = list(it.product(*[all_files, [target]]))
    p.map(_tigge_convert, combs)
    p.close()
    p.join()

1	import itertools as it	3✔
2	import multiprocessing	3✔
3	import os	3✔
4	import shutil	3✔
5	import tempfile	3✔
6	from pathlib import Path	3✔
7	from typing import Optional	3✔
8
9	import xarray	3✔
10	from dask.diagnostics import ProgressBar	3✔
11
12
13	def tigge_convert(	3✔
14	source: Optional[os.PathLike] = None,
15	target: Optional[os.PathLike] = None,
16	processes: int = 8,
17	) -> None:
18	"""Convert grib2 file to netCDF format.
19
20	Parameters
21	----------
22	source : os.PathLike, optional
23	target : os.PathLike, optional
24	processes : int
25
26	Returns
27	-------
28	None
29	"""
30
NEW 31	def _tigge_convert(fn):	×
32	"""Launch reformatting function."""
NEW 33	infile, output_folder = fn	×
NEW 34	try:	×
NEW 35	for f in Path(infile.parent).glob(infile.name.replace(".grib", "*.idx")):	×
NEW 36	f.unlink(missing_ok=True)	×
37
NEW 38	ds = xarray.open_dataset(	×
39	infile,
40	engine="cfgrib",
41	chunks="auto",
42	)
43
NEW 44	encoding = {var: dict(zlib=True) for var in ds.data_vars}	×
NEW 45	encoding["time"] = {"dtype": "single"}	×
NEW 46	tmpfile = tempfile.NamedTemporaryFile(suffix=".nc", delete=False)	×
47
NEW 48	with ProgressBar():	×
NEW 49	print("converting ", infile.name)	×
NEW 50	ds.to_netcdf(	×
51	tmpfile.name, format="NETCDF4", engine="netcdf4", encoding=encoding
52	)
53
NEW 54	shutil.move(	×
55	tmpfile.name,
56	output_folder.joinpath(infile.name.replace(".grib", ".nc")).as_posix(),
57	)
58
NEW 59	except ValueError:	×
NEW 60	print(f"error converting {infile.name} : File may be corrupted")	×
61
NEW 62	if source is None:	×
NEW 63	source = Path().cwd().joinpath("download")	×
NEW 64	if target is None:	×
NEW 65	target = Path().cwd().joinpath("converted")	×
66
NEW 67	all_files = Path(source).glob("*.grib2")	×
68
NEW 69	target = Path(target)	×
NEW 70	target.mkdir(exist_ok=True)	×
71
NEW 72	p = multiprocessing.Pool(processes=processes)	×
73
NEW 74	combs = list(it.product(*[all_files, [target]]))	×
NEW 75	p.map(_tigge_convert, combs)	×
NEW 76	p.close()	×
NEW 77	p.join()	×

Ouranosinc / miranda / 2117681512

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous