1954258533

Build # 1954258533

Build Type

github

Committed by GitHub

Commit Message

Merge 81f1fb372 into 250bc4dd4

Pull Request Pull Request #24: Add CMIP file structure - WIP

Run Details

137 of 670 new or added lines in 33 files covered. (20.45%)

10 existing lines in 3 files now uncovered.

679 of 3022 relevant lines covered (22.47%)

0.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33

/miranda/eccc/_utils.py

import logging.config
from datetime import datetime as dt
from typing import Dict, List, Tuple, Union

from miranda.scripting import LOGGING_CONFIG

__all__ = ["cf_hourly_metadata", "cf_daily_metadata", "ahccd_metadata"]

logging.config.dictConfig(LOGGING_CONFIG)


def cf_hourly_metadata(variable_code: Union[int, str]) -> Dict[str, Union[int, float]]:
    """

    Parameters
    ----------
    variable_code: Union[int, str]

    Returns
    -------
    dict
    """
    ec_hourly_variables = {
        "061": {
            "nc_units": "W m-2 h-1",
            "scale_factor": 1e6 / (60 * 60),
            "add_offset": 0,
            "long_name": "RF1 Global Solar Radiation",
            "standard_name": "solar_radiation_flux",
            "nc_name": "rf1_radiation",
        },
        "071": {
            "nc_units": "m",
            "scale_factor": 30,
            "add_offset": 0,
            "long_name": "Ceiling height of lowest layer of clouds",
            "standard_name": "ceiling_cloud_height",
            "nc_name": "ceiling_hgt",
        },
        "072": {
            "nc_units": "m",
            "scale_factor": 100,
            "add_offset": 0,
            "long_name": "Visibility",
            "standard_name": "visibility_in_air",
            "nc_name": "visibility",
        },
        "073": {
            "nc_units": "Pa",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Sea Level Pressure",
            "standard_name": "air_pressure_at_mean_sea_level",
            "nc_name": "psl",
        },
        "074": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Dew Point Temperature",
            "standard_name": "dew_point_temperature",
            "nc_name": "tds",
        },
        "075": {
            "nc_units": "degree",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Wind Direction at 2 m (U2A Anemometer) (16 pts)",
            "standard_name": "wind_direction_u2a",
            "nc_name": "wind_dir_u2a_16",
        },
        "076": {
            "nc_units": "m s-1",
            "scale_factor": 0.277777778,
            "add_offset": 0,
            "long_name": "Wind Speed (U2A Anemometer)",
            "standard_name": "wind_speed_u2a",
            "nc_name": "wind_speed_u2a",
        },
        "077": {
            "nc_units": "Pa",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Station Pressure",
            "standard_name": "atmospheric_pressure",
            "nc_name": "pressure",
        },
        "078": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Dry Bulb Temperature",
            "standard_name": "dry_bulb_temperature",
            "nc_name": "tas_dry",
        },
        "079": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Wet Bulb temperature",
            "standard_name": "wet_bulb_temperature",
            "nc_name": "tas_wet",
        },
        "080": {
            "nc_units": "%",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Relative Humidity",
            "standard_name": "relative_humidity",
            "nc_name": "hur",
        },
        "081": {
            "nc_units": "%",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Total Cloud Opacity",
            "standard_name": "cloud_albedo",
            "nc_name": "clo",
        },
        "082": {
            "nc_units": "%",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Total Cloud Amount",
            "standard_name": "cloud_area_fraction",
            "nc_name": "clt",
        },
        "089": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Freezing Rain",
            "standard_name": "freezing_rain",
            "nc_name": "freeze_rain",
        },
        "094": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Ice Pellets",
            "standard_name": "ice_pellet_presence",
            "nc_name": "ice_pellets",
        },
        "107": {
            "nc_units": "%",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Lowest cloud layer opacity",
            "standard_name": "low_type_cloud_opacity_fraction",
            "nc_name": "cloud_opac",
        },
        "108": {
            "nc_units": "%",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Lowest cloud layer amount or condition",
            "standard_name": "low_type_cloud_area_fraction",
            "nc_name": "cloud_frac",
        },
        "109": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Lowest cloud layer type",
            "standard_name": "low_type_cloud_type",
            "nc_name": "low_cloud_type",
        },
        "110": {
            "nc_units": "m",
            "scale_factor": 30,
            "add_offset": 0,
            "long_name": "Lowest cloud layer height",
            "standard_name": "low_type_cloud_height",
            "nc_name": "low_cloud_hgt",
        },
        "123": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Rainfall",
            "standard_name": "rainfall_flux",
            "nc_name": "rainfall",
        },
        "133": {
            "nc_units": "s",
            "scale_factor": 3600,
            "add_offset": 0,
            "long_name": "Sunshine",
            "standard_name": "duration_of_sunshine",
            "nc_name": "sun",
        },
        "156": {
            "nc_units": "degree",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Wind Direction at 2 m (U2A Anemometer) (36 pts)",
            "standard_name": "wind_direction_u2a",
            "nc_name": "wind_dir_u2a_36",
        },
        "262": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Precipitation (minutes 00-60)",
            "standard_name": "precipitation_flux",
            "nc_name": "precipitation",
        },
        "263": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Precipitation (minutes 00-15)",
            "standard_name": "precipitation_flux",
            "nc_name": "precipitation_q1",
        },
        "264": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Precipitation (minutes 15-30)",
            "standard_name": "precipitation_flux",
            "nc_name": "precipitation_q2",
        },
        "265": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Precipitation (minutes 30-45)",
            "standard_name": "precipitation_flux",
            "nc_name": "precipitation_q3",
        },
        "266": {
            "nc_units": "kg m-2 s-1",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Total Precipitation (minutes 45-60)",
            "standard_name": "precipitation_flux",
            "nc_name": "precipitation_q4",
        },
        "267": {
            "nc_units": "kg m-2",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Precipitation Gauge Weight per Unit Area (at minute 15)",
            "standard_name": "precipitation_amount",
            "nc_name": "precipitation_weight_q1",
        },
        "268": {
            "nc_units": "kg m-2",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Precipitation Gauge Weight per Unit Area (at minute 30)",
            "standard_name": "precipitation_amount",
            "nc_name": "precipitation_weight_q2",
        },
        "269": {
            "nc_units": "kg m-2",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Precipitation Gauge Weight per Unit Area (at minute 45)",
            "standard_name": "precipitation_amount",
            "nc_name": "precipitation_weight_q3",
        },
        "270": {
            "nc_units": "kg m-2",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Precipitation Gauge Weight per Unit Area (at minute 60)",
            "standard_name": "precipitation_amount",
            "nc_name": "precipitation_weight_q4",
        },
        "271": {
            "nc_units": "m s-1",
            "scale_factor": 0.02777778,
            "add_offset": 0,
            "long_name": "Wind Speed at 2 m (minutes 00-15)",
            "standard_name": "wind_speed",
            "nc_name": "wind_speed_q1",
        },
        "272": {
            "nc_units": "m s-1",
            "scale_factor": 0.02777778,
            "add_offset": 0,
            "long_name": "Wind Speed at 2 m (minutes 15-30)",
            "standard_name": "wind_speed",
            "nc_name": "wind_speed_q2",
        },
        "273": {
            "nc_units": "m s-1",
            "scale_factor": 0.02777778,
            "add_offset": 0,
            "long_name": "Wind Speed at 2 m (minutes 30-45)",
            "standard_name": "wind_speed",
            "nc_name": "wind_speed_q3",
        },
        "274": {
            "nc_units": "m s-1",
            "scale_factor": 0.02777778,
            "add_offset": 0,
            "long_name": "Wind Speed at 2 m (minutes 45-60)",
            "standard_name": "wind_speed",
            "nc_name": "wind_speed_q4",
        },
        "275": {
            "nc_units": "m",
            "scale_factor": 0.01,
            "add_offset": 0,
            "long_name": "Snow Depth (at minute 60)",
            "standard_name": "surface_snow_thickness",
            "nc_name": "snd_q4",
        },
        "276": {
            "nc_units": "m",
            "scale_factor": 0.01,
            "add_offset": 0,
            "long_name": "Snow Depth (at minute 15)",
            "standard_name": "surface_snow_thickness",
            "nc_name": "snd_q1",
        },
        "277": {
            "nc_units": "m",
            "scale_factor": 0.01,
            "add_offset": 0,
            "long_name": "Snow Depth (at minute 30)",
            "standard_name": "surface_snow_thickness",
            "nc_name": "snd_q2",
        },
        "278": {
            "nc_units": "m",
            "scale_factor": 0.01,
            "add_offset": 0,
            "long_name": "Snow Depth (at minute 45)",
            "standard_name": "surface_snow_thickness",
            "nc_name": "snd_q3",
        },
        "279": {
            "nc_units": "degree",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Wind Direction at 2 m (minutes 50-60)",
            "standard_name": "wind_direction",
            "nc_name": "wind_dir",
        },
        "280": {
            "nc_units": "m s-1",
            "scale_factor": 0.02777778,
            "add_offset": 0,
            "long_name": "Wind Speed at 2 m (minutes 50-60)",
            "standard_name": "wind_speed",
            "nc_name": "wind_speed",
        },
    }
    code = str(variable_code).zfill(3)
    if code in ["061"]:
        raise NotImplementedError()
    try:
        variable = ec_hourly_variables[code]
        variable["missing_flags"] = "M"
        variable["least_significant_digit"] = ""
    except KeyError:
        logging.error(f"Hourly variable `{code}` not supported.")
        raise
    return variable


def cf_daily_metadata(variable_code: Union[int, str]) -> Dict[str, Union[int, float]]:
    """

    Parameters
    ----------
    variable_code: Union[int, str]

    Returns
    -------
    dict
    """
    ec_daily_variables = {
        "001": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Daily Maximum Temperature",
            "standard_name": "air_temperature_maximum",
            "nc_name": "tasmax",
        },
        "002": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Daily Minimum Temperature",
            "standard_name": "air_temperature_minimum",
            "nc_name": "tasmin",
        },
        "003": {
            "nc_units": "K",
            "scale_factor": 0.1,
            "add_offset": 273.15,
            "long_name": "Daily Mean Temperature",
            "standard_name": "air_temperature",
            "nc_name": "tas",
        },
        "010": {
            "nc_units": "mm",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Daily Total Rainfall",
            "standard_name": "liquid_precipitation_amount",
            "nc_name": "prlptot",
        },
        "011": {
            "nc_units": "cm",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Daily Total Snowfall",
            "standard_name": "solid_precipitation_amount",
            "nc_name": "prsntot",
        },
        "012": {
            "nc_units": "mm",
            "scale_factor": 0.1,
            "add_offset": 0,
            "long_name": "Daily Total Precipitation",
            "standard_name": "precipitation_amount",
            "nc_name": "prcptot",
        },
        "013": {
            "nc_units": "m",
            "scale_factor": 0.01,
            "add_offset": 0,
            "long_name": "Snow on the Ground",
            "standard_name": "surface_snow_thickness",
            "nc_name": "sndtot",
        },
        "014": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Thunderstorms",
            "standard_name": "thunderstorm_presence",
            "nc_name": "thunder",
        },
        "015": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Freezing rain or drizzle",
            "standard_name": "freeze_rain_drizzle_presence",
            "nc_name": "freezing_rain_drizzle",
        },
        "016": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Hail",
            "standard_name": "hail_presence",
            "nc_name": "hail",
        },
        "017": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Fog or Ice Fog",
            "standard_name": "fog_ice_fog_presence",
            "nc_name": "fog_ice_fog",
        },
        "018": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Smoke or Haze",
            "standard_name": "smoke_haze_presence",
            "nc_name": "smoke_haze",
        },
        "019": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Blowing Dust or Sand",
            "standard_name": "blowing_dust_sand_presence",
            "nc_name": "blowing_dust_sand",
        },
        "020": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Blowing snow",
            "standard_name": "blowing_snow_presence",
            "nc_name": "blow_snow",
        },
        "021": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Wind speed >= 28 Knots",
            "standard_name": "wind_exceeding_28_knots",
            "nc_name": "wind_gt_28kt",
        },
        "022": {
            "nc_units": "1",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "Wind speed >= 34 Knots",
            "standard_name": "wind_exceeding_34_knots",
            "nc_name": "wind_gt_34kt",
        },
        "023": {
            "nc_units": "degree",
            "scale_factor": 10,
            "add_offset": 0,
            "long_name": "Direction of extreme gust (16 pts) to December 1976",
            "standard_name": "wind_to_direction",
            "nc_name": "gust_dir",
        },
        "024": {
            "nc_units": "m s-1",
            "scale_factor": 0.2777778,
            "add_offset": 0,
            "long_name": "Speed of extreme gust",
            "standard_name": "wind_speed_of_gust",
            "nc_name": "gust_speed",
        },
        "025": {
            "nc_units": "h",
            "scale_factor": 1,
            "add_offset": 0,
            "long_name": "UTC hour of extreme gust",
            "standard_name": "hour_of_extreme_gust",
            "nc_name": "gust_hour",
        },
    }
    code = str(variable_code).zfill(3)
    try:
        variable = ec_daily_variables[code]
        variable["missing_flags"] = "M"
        variable["least_significant_digit"] = ""
    except KeyError:
        logging.error(f"Daily variable `{code}` not supported.")
        raise
    return variable


def ahccd_metadata(
    code: str, gen: int
) -> (Dict[str, Union[str, float]], Dict, List[Tuple[int, int]], int):
    """

    Parameters
    ----------
    code: {"dx", "dn", "dm", "dt", "ds", "dr"}
    gen: {1, 2, 3}

    Returns
    -------
    Dict[str, Union[str, float]], Dict, List[Tuple[int, int]], int
    """
    generation = {1: "First", 2: "Second", 3: "Third"}.get(gen)

    ec_ahccd_attrs = dict(
        dx=dict(
            variable="tasmax",
            units="deg C",
            standard_name="air_temperature",
            long_name="Near-Surface Maximum Daily Air Temperature",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
        ),
        dn=dict(
            variable="tasmin",
            units="deg C",
            standard_name="air_temperature",
            long_name="Near-Surface Minimum Daily Air Temperature",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
        ),
        dm=dict(
            variable="tas",
            units="deg C",
            standard_name="air_temperature",
            long_name="Near-Surface Daily Mean Air Temperature",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
        ),
        dt=dict(
            variable="pr",
            units="mm day-1",
            standard_name="precipitation_flux",
            long_name="Daily Total Precipitation",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
        ),
        ds=dict(
            variable="prsn",
            units="mm day-1",
            standard_name="snowfall_flux",
            long_name="Daily Snowfall",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
        ),
        dr=dict(
            variable="prlp",
            units="mm day-1",
            standard_name="rainfall_flux",
            long_name="Daily Rainfall",
            comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
        ),
    )
    try:
        variable = ec_ahccd_attrs[code]
        variable["missing_flags"] = "M"
        if variable["variable"].startswith("tas"):
            variable["NaN_value"] = -9999.9
            column_names = [
                "No",
                "StnId",
                "Station name",
                "Prov",
                "FromYear",
                "FromMonth",
                "ToYear",
                "ToMonth",
                "%Miss",
                "Lat(deg)",
                "Long(deg)",
                "Elev(m)",
                "Joined",
                "RCS",
            ]
            column_spaces = [(0, 5), (5, 6), (6, 8), (8, 9)]
            ii = 9
            for i in range(1, 32):
                column_spaces.append((ii, ii + 7))
                ii += 7
                column_spaces.append((ii, ii + 1))
                ii += 1
            header_row = 3

        elif variable["variable"].startswith("pr"):
            variable["NaN_value"] = -9999.99
            column_names = [
                "Prov",
                "Station name",
                "stnid",
                "beg yr",
                "beg mon",
                "end yr",
                "end mon",
                "lat (deg)",
                "long (deg)",
                "elev (m)",
                "stns joined",
            ]
            column_spaces = [(0, 4), (4, 5), (5, 7), (7, 8)]
            ii = 8
            for i in range(1, 32):
                column_spaces.append((ii, ii + 8))
                ii += 8
                column_spaces.append((ii, ii + 1))
                ii += 1
            header_row = 0

        else:
            raise KeyError

        column_names = {
            col.lower()
            .split("(")[0]
            .replace("%", "pct_")
            .strip()
            .replace(" ", "_"): col
            for col in list(column_names)
        }

        if gen == 3:
            _citation = (
                "Vincent, L.A., M.M. Hartwell and X.L. Wang, 2020: A Third Generation of Homogenized "
                "Temperature for Trend Analysis and Monitoring Changes in Canada’s Climate. "
                "Atmosphere-Ocean. https://doi.org/10.1080/07055900.2020.1765728"
            )
        elif gen == 2:
            _citation = (
                "Mekis, É and L.A. Vincent, 2011: An overview of the second generation adjusted daily "
                "precipitation dataset for trend analysis in Canada. Atmosphere-Ocean 49(2), "
                "163-177 doi:10.1080/07055900.2011.583910"
            )
        else:
            msg = f"Generation '{gen}' not supported."
            raise NotImplementedError(msg)

        global_attrs = dict(
            title=f"{generation} Generation of Homogenized Daily {variable['variable']} "
            "for Canada (Updated to December 2019)",
            history=f"{dt.today().strftime('%Y-%m-%d')}: Convert from original format to NetCDF",
            type="station_obs",
            institute="Environment and Climate Change Canada",
            institute_id="ECCC",
            dataset_id=f"AHCCD_gen{gen}_day_{variable['variable']}",
            frequency="day",
            licence_type="permissive",
            licence="https:/open.canada.ca/en/open-government-licence-canada",
            citation=_citation,
        )

    except KeyError as e:
        msg = f"AHCCD variable '{code}' or generation '{gen}' not supported."
        logging.error(msg)
        raise NotImplementedError(msg) from e

    return variable, column_names, column_spaces, header_row, global_attrs

1	import logging.config	3✔
2	from datetime import datetime as dt	3✔
3	from typing import Dict, List, Tuple, Union	3✔
4
5	from miranda.scripting import LOGGING_CONFIG	3✔
6
7	__all__ = ["cf_hourly_metadata", "cf_daily_metadata", "ahccd_metadata"]	3✔
8
9	logging.config.dictConfig(LOGGING_CONFIG)	3✔
10
11
12	def cf_hourly_metadata(variable_code: Union[int, str]) -> Dict[str, Union[int, float]]:	3✔
13	"""
14
15	Parameters
16	----------
17	variable_code: Union[int, str]
18
19	Returns
20	-------
21	dict
22	"""
23	ec_hourly_variables = {	3✔
24	"061": {
25	"nc_units": "W m-2 h-1",
26	"scale_factor": 1e6 / (60 * 60),
27	"add_offset": 0,
28	"long_name": "RF1 Global Solar Radiation",
29	"standard_name": "solar_radiation_flux",
30	"nc_name": "rf1_radiation",
31	},
32	"071": {
33	"nc_units": "m",
34	"scale_factor": 30,
35	"add_offset": 0,
36	"long_name": "Ceiling height of lowest layer of clouds",
37	"standard_name": "ceiling_cloud_height",
38	"nc_name": "ceiling_hgt",
39	},
40	"072": {
41	"nc_units": "m",
42	"scale_factor": 100,
43	"add_offset": 0,
44	"long_name": "Visibility",
45	"standard_name": "visibility_in_air",
46	"nc_name": "visibility",
47	},
48	"073": {
49	"nc_units": "Pa",
50	"scale_factor": 10,
51	"add_offset": 0,
52	"long_name": "Sea Level Pressure",
53	"standard_name": "air_pressure_at_mean_sea_level",
54	"nc_name": "psl",
55	},
56	"074": {
57	"nc_units": "K",
58	"scale_factor": 0.1,
59	"add_offset": 273.15,
60	"long_name": "Dew Point Temperature",
61	"standard_name": "dew_point_temperature",
62	"nc_name": "tds",
63	},
64	"075": {
65	"nc_units": "degree",
66	"scale_factor": 10,
67	"add_offset": 0,
68	"long_name": "Wind Direction at 2 m (U2A Anemometer) (16 pts)",
69	"standard_name": "wind_direction_u2a",
70	"nc_name": "wind_dir_u2a_16",
71	},
72	"076": {
73	"nc_units": "m s-1",
74	"scale_factor": 0.277777778,
75	"add_offset": 0,
76	"long_name": "Wind Speed (U2A Anemometer)",
77	"standard_name": "wind_speed_u2a",
78	"nc_name": "wind_speed_u2a",
79	},
80	"077": {
81	"nc_units": "Pa",
82	"scale_factor": 10,
83	"add_offset": 0,
84	"long_name": "Station Pressure",
85	"standard_name": "atmospheric_pressure",
86	"nc_name": "pressure",
87	},
88	"078": {
89	"nc_units": "K",
90	"scale_factor": 0.1,
91	"add_offset": 273.15,
92	"long_name": "Dry Bulb Temperature",
93	"standard_name": "dry_bulb_temperature",
94	"nc_name": "tas_dry",
95	},
96	"079": {
97	"nc_units": "K",
98	"scale_factor": 0.1,
99	"add_offset": 273.15,
100	"long_name": "Wet Bulb temperature",
101	"standard_name": "wet_bulb_temperature",
102	"nc_name": "tas_wet",
103	},
104	"080": {
105	"nc_units": "%",
106	"scale_factor": 1,
107	"add_offset": 0,
108	"long_name": "Relative Humidity",
109	"standard_name": "relative_humidity",
110	"nc_name": "hur",
111	},
112	"081": {
113	"nc_units": "%",
114	"scale_factor": 10,
115	"add_offset": 0,
116	"long_name": "Total Cloud Opacity",
117	"standard_name": "cloud_albedo",
118	"nc_name": "clo",
119	},
120	"082": {
121	"nc_units": "%",
122	"scale_factor": 10,
123	"add_offset": 0,
124	"long_name": "Total Cloud Amount",
125	"standard_name": "cloud_area_fraction",
126	"nc_name": "clt",
127	},
128	"089": {
129	"nc_units": "1",
130	"scale_factor": 1,
131	"add_offset": 0,
132	"long_name": "Freezing Rain",
133	"standard_name": "freezing_rain",
134	"nc_name": "freeze_rain",
135	},
136	"094": {
137	"nc_units": "1",
138	"scale_factor": 1,
139	"add_offset": 0,
140	"long_name": "Ice Pellets",
141	"standard_name": "ice_pellet_presence",
142	"nc_name": "ice_pellets",
143	},
144	"107": {
145	"nc_units": "%",
146	"scale_factor": 10,
147	"add_offset": 0,
148	"long_name": "Lowest cloud layer opacity",
149	"standard_name": "low_type_cloud_opacity_fraction",
150	"nc_name": "cloud_opac",
151	},
152	"108": {
153	"nc_units": "%",
154	"scale_factor": 10,
155	"add_offset": 0,
156	"long_name": "Lowest cloud layer amount or condition",
157	"standard_name": "low_type_cloud_area_fraction",
158	"nc_name": "cloud_frac",
159	},
160	"109": {
161	"nc_units": "1",
162	"scale_factor": 1,
163	"add_offset": 0,
164	"long_name": "Lowest cloud layer type",
165	"standard_name": "low_type_cloud_type",
166	"nc_name": "low_cloud_type",
167	},
168	"110": {
169	"nc_units": "m",
170	"scale_factor": 30,
171	"add_offset": 0,
172	"long_name": "Lowest cloud layer height",
173	"standard_name": "low_type_cloud_height",
174	"nc_name": "low_cloud_hgt",
175	},
176	"123": {
177	"nc_units": "kg m-2 s-1",
178	"scale_factor": 0.1,
179	"add_offset": 0,
180	"long_name": "Total Rainfall",
181	"standard_name": "rainfall_flux",
182	"nc_name": "rainfall",
183	},
184	"133": {
185	"nc_units": "s",
186	"scale_factor": 3600,
187	"add_offset": 0,
188	"long_name": "Sunshine",
189	"standard_name": "duration_of_sunshine",
190	"nc_name": "sun",
191	},
192	"156": {
193	"nc_units": "degree",
194	"scale_factor": 10,
195	"add_offset": 0,
196	"long_name": "Wind Direction at 2 m (U2A Anemometer) (36 pts)",
197	"standard_name": "wind_direction_u2a",
198	"nc_name": "wind_dir_u2a_36",
199	},
200	"262": {
201	"nc_units": "kg m-2 s-1",
202	"scale_factor": 0.1,
203	"add_offset": 0,
204	"long_name": "Total Precipitation (minutes 00-60)",
205	"standard_name": "precipitation_flux",
206	"nc_name": "precipitation",
207	},
208	"263": {
209	"nc_units": "kg m-2 s-1",
210	"scale_factor": 0.1,
211	"add_offset": 0,
212	"long_name": "Total Precipitation (minutes 00-15)",
213	"standard_name": "precipitation_flux",
214	"nc_name": "precipitation_q1",
215	},
216	"264": {
217	"nc_units": "kg m-2 s-1",
218	"scale_factor": 0.1,
219	"add_offset": 0,
220	"long_name": "Total Precipitation (minutes 15-30)",
221	"standard_name": "precipitation_flux",
222	"nc_name": "precipitation_q2",
223	},
224	"265": {
225	"nc_units": "kg m-2 s-1",
226	"scale_factor": 0.1,
227	"add_offset": 0,
228	"long_name": "Total Precipitation (minutes 30-45)",
229	"standard_name": "precipitation_flux",
230	"nc_name": "precipitation_q3",
231	},
232	"266": {
233	"nc_units": "kg m-2 s-1",
234	"scale_factor": 0.1,
235	"add_offset": 0,
236	"long_name": "Total Precipitation (minutes 45-60)",
237	"standard_name": "precipitation_flux",
238	"nc_name": "precipitation_q4",
239	},
240	"267": {
241	"nc_units": "kg m-2",
242	"scale_factor": 0.1,
243	"add_offset": 0,
244	"long_name": "Precipitation Gauge Weight per Unit Area (at minute 15)",
245	"standard_name": "precipitation_amount",
246	"nc_name": "precipitation_weight_q1",
247	},
248	"268": {
249	"nc_units": "kg m-2",
250	"scale_factor": 0.1,
251	"add_offset": 0,
252	"long_name": "Precipitation Gauge Weight per Unit Area (at minute 30)",
253	"standard_name": "precipitation_amount",
254	"nc_name": "precipitation_weight_q2",
255	},
256	"269": {
257	"nc_units": "kg m-2",
258	"scale_factor": 0.1,
259	"add_offset": 0,
260	"long_name": "Precipitation Gauge Weight per Unit Area (at minute 45)",
261	"standard_name": "precipitation_amount",
262	"nc_name": "precipitation_weight_q3",
263	},
264	"270": {
265	"nc_units": "kg m-2",
266	"scale_factor": 0.1,
267	"add_offset": 0,
268	"long_name": "Precipitation Gauge Weight per Unit Area (at minute 60)",
269	"standard_name": "precipitation_amount",
270	"nc_name": "precipitation_weight_q4",
271	},
272	"271": {
273	"nc_units": "m s-1",
274	"scale_factor": 0.02777778,
275	"add_offset": 0,
276	"long_name": "Wind Speed at 2 m (minutes 00-15)",
277	"standard_name": "wind_speed",
278	"nc_name": "wind_speed_q1",
279	},
280	"272": {
281	"nc_units": "m s-1",
282	"scale_factor": 0.02777778,
283	"add_offset": 0,
284	"long_name": "Wind Speed at 2 m (minutes 15-30)",
285	"standard_name": "wind_speed",
286	"nc_name": "wind_speed_q2",
287	},
288	"273": {
289	"nc_units": "m s-1",
290	"scale_factor": 0.02777778,
291	"add_offset": 0,
292	"long_name": "Wind Speed at 2 m (minutes 30-45)",
293	"standard_name": "wind_speed",
294	"nc_name": "wind_speed_q3",
295	},
296	"274": {
297	"nc_units": "m s-1",
298	"scale_factor": 0.02777778,
299	"add_offset": 0,
300	"long_name": "Wind Speed at 2 m (minutes 45-60)",
301	"standard_name": "wind_speed",
302	"nc_name": "wind_speed_q4",
303	},
304	"275": {
305	"nc_units": "m",
306	"scale_factor": 0.01,
307	"add_offset": 0,
308	"long_name": "Snow Depth (at minute 60)",
309	"standard_name": "surface_snow_thickness",
310	"nc_name": "snd_q4",
311	},
312	"276": {
313	"nc_units": "m",
314	"scale_factor": 0.01,
315	"add_offset": 0,
316	"long_name": "Snow Depth (at minute 15)",
317	"standard_name": "surface_snow_thickness",
318	"nc_name": "snd_q1",
319	},
320	"277": {
321	"nc_units": "m",
322	"scale_factor": 0.01,
323	"add_offset": 0,
324	"long_name": "Snow Depth (at minute 30)",
325	"standard_name": "surface_snow_thickness",
326	"nc_name": "snd_q2",
327	},
328	"278": {
329	"nc_units": "m",
330	"scale_factor": 0.01,
331	"add_offset": 0,
332	"long_name": "Snow Depth (at minute 45)",
333	"standard_name": "surface_snow_thickness",
334	"nc_name": "snd_q3",
335	},
336	"279": {
337	"nc_units": "degree",
338	"scale_factor": 1,
339	"add_offset": 0,
340	"long_name": "Wind Direction at 2 m (minutes 50-60)",
341	"standard_name": "wind_direction",
342	"nc_name": "wind_dir",
343	},
344	"280": {
345	"nc_units": "m s-1",
346	"scale_factor": 0.02777778,
347	"add_offset": 0,
348	"long_name": "Wind Speed at 2 m (minutes 50-60)",
349	"standard_name": "wind_speed",
350	"nc_name": "wind_speed",
351	},
352	}
353	code = str(variable_code).zfill(3)	3✔
354	if code in ["061"]:	3✔
355	raise NotImplementedError()	×
356	try:	3✔
357	variable = ec_hourly_variables[code]	3✔
358	variable["missing_flags"] = "M"	3✔
359	variable["least_significant_digit"] = ""	3✔
360	except KeyError:	×
NEW 361	logging.error(f"Hourly variable `{code}` not supported.")	×
362	raise	×
363	return variable	3✔
364
365
366	def cf_daily_metadata(variable_code: Union[int, str]) -> Dict[str, Union[int, float]]:	3✔
367	"""
368
369	Parameters
370	----------
371	variable_code: Union[int, str]
372
373	Returns
374	-------
375	dict
376	"""
377	ec_daily_variables = {	3✔
378	"001": {
379	"nc_units": "K",
380	"scale_factor": 0.1,
381	"add_offset": 273.15,
382	"long_name": "Daily Maximum Temperature",
383	"standard_name": "air_temperature_maximum",
384	"nc_name": "tasmax",
385	},
386	"002": {
387	"nc_units": "K",
388	"scale_factor": 0.1,
389	"add_offset": 273.15,
390	"long_name": "Daily Minimum Temperature",
391	"standard_name": "air_temperature_minimum",
392	"nc_name": "tasmin",
393	},
394	"003": {
395	"nc_units": "K",
396	"scale_factor": 0.1,
397	"add_offset": 273.15,
398	"long_name": "Daily Mean Temperature",
399	"standard_name": "air_temperature",
400	"nc_name": "tas",
401	},
402	"010": {
403	"nc_units": "mm",
404	"scale_factor": 0.1,
405	"add_offset": 0,
406	"long_name": "Daily Total Rainfall",
407	"standard_name": "liquid_precipitation_amount",
408	"nc_name": "prlptot",
409	},
410	"011": {
411	"nc_units": "cm",
412	"scale_factor": 0.1,
413	"add_offset": 0,
414	"long_name": "Daily Total Snowfall",
415	"standard_name": "solid_precipitation_amount",
416	"nc_name": "prsntot",
417	},
418	"012": {
419	"nc_units": "mm",
420	"scale_factor": 0.1,
421	"add_offset": 0,
422	"long_name": "Daily Total Precipitation",
423	"standard_name": "precipitation_amount",
424	"nc_name": "prcptot",
425	},
426	"013": {
427	"nc_units": "m",
428	"scale_factor": 0.01,
429	"add_offset": 0,
430	"long_name": "Snow on the Ground",
431	"standard_name": "surface_snow_thickness",
432	"nc_name": "sndtot",
433	},
434	"014": {
435	"nc_units": "1",
436	"scale_factor": 1,
437	"add_offset": 0,
438	"long_name": "Thunderstorms",
439	"standard_name": "thunderstorm_presence",
440	"nc_name": "thunder",
441	},
442	"015": {
443	"nc_units": "1",
444	"scale_factor": 1,
445	"add_offset": 0,
446	"long_name": "Freezing rain or drizzle",
447	"standard_name": "freeze_rain_drizzle_presence",
448	"nc_name": "freezing_rain_drizzle",
449	},
450	"016": {
451	"nc_units": "1",
452	"scale_factor": 1,
453	"add_offset": 0,
454	"long_name": "Hail",
455	"standard_name": "hail_presence",
456	"nc_name": "hail",
457	},
458	"017": {
459	"nc_units": "1",
460	"scale_factor": 1,
461	"add_offset": 0,
462	"long_name": "Fog or Ice Fog",
463	"standard_name": "fog_ice_fog_presence",
464	"nc_name": "fog_ice_fog",
465	},
466	"018": {
467	"nc_units": "1",
468	"scale_factor": 1,
469	"add_offset": 0,
470	"long_name": "Smoke or Haze",
471	"standard_name": "smoke_haze_presence",
472	"nc_name": "smoke_haze",
473	},
474	"019": {
475	"nc_units": "1",
476	"scale_factor": 1,
477	"add_offset": 0,
478	"long_name": "Blowing Dust or Sand",
479	"standard_name": "blowing_dust_sand_presence",
480	"nc_name": "blowing_dust_sand",
481	},
482	"020": {
483	"nc_units": "1",
484	"scale_factor": 1,
485	"add_offset": 0,
486	"long_name": "Blowing snow",
487	"standard_name": "blowing_snow_presence",
488	"nc_name": "blow_snow",
489	},
490	"021": {
491	"nc_units": "1",
492	"scale_factor": 1,
493	"add_offset": 0,
494	"long_name": "Wind speed >= 28 Knots",
495	"standard_name": "wind_exceeding_28_knots",
496	"nc_name": "wind_gt_28kt",
497	},
498	"022": {
499	"nc_units": "1",
500	"scale_factor": 1,
501	"add_offset": 0,
502	"long_name": "Wind speed >= 34 Knots",
503	"standard_name": "wind_exceeding_34_knots",
504	"nc_name": "wind_gt_34kt",
505	},
506	"023": {
507	"nc_units": "degree",
508	"scale_factor": 10,
509	"add_offset": 0,
510	"long_name": "Direction of extreme gust (16 pts) to December 1976",
511	"standard_name": "wind_to_direction",
512	"nc_name": "gust_dir",
513	},
514	"024": {
515	"nc_units": "m s-1",
516	"scale_factor": 0.2777778,
517	"add_offset": 0,
518	"long_name": "Speed of extreme gust",
519	"standard_name": "wind_speed_of_gust",
520	"nc_name": "gust_speed",
521	},
522	"025": {
523	"nc_units": "h",
524	"scale_factor": 1,
525	"add_offset": 0,
526	"long_name": "UTC hour of extreme gust",
527	"standard_name": "hour_of_extreme_gust",
528	"nc_name": "gust_hour",
529	},
530	}
531	code = str(variable_code).zfill(3)	3✔
532	try:	3✔
533	variable = ec_daily_variables[code]	3✔
534	variable["missing_flags"] = "M"	3✔
535	variable["least_significant_digit"] = ""	3✔
536	except KeyError:	×
NEW 537	logging.error(f"Daily variable `{code}` not supported.")	×
538	raise	×
539	return variable	3✔
540
541
542	def ahccd_metadata(	3✔
543	code: str, gen: int
544	) -> (Dict[str, Union[str, float]], Dict, List[Tuple[int, int]], int):
545	"""
546
547	Parameters
548	----------
549	code: {"dx", "dn", "dm", "dt", "ds", "dr"}
550	gen: {1, 2, 3}
551
552	Returns
553	-------
554	Dict[str, Union[str, float]], Dict, List[Tuple[int, int]], int
555	"""
556	generation = {1: "First", 2: "Second", 3: "Third"}.get(gen)	×
557
558	ec_ahccd_attrs = dict(	×
559	dx=dict(
560	variable="tasmax",
561	units="deg C",
562	standard_name="air_temperature",
563	long_name="Near-Surface Maximum Daily Air Temperature",
564	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
565	),
566	dn=dict(
567	variable="tasmin",
568	units="deg C",
569	standard_name="air_temperature",
570	long_name="Near-Surface Minimum Daily Air Temperature",
571	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
572	),
573	dm=dict(
574	variable="tas",
575	units="deg C",
576	standard_name="air_temperature",
577	long_name="Near-Surface Daily Mean Air Temperature",
578	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Temperature Data",
579	),
580	dt=dict(
581	variable="pr",
582	units="mm day-1",
583	standard_name="precipitation_flux",
584	long_name="Daily Total Precipitation",
585	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
586	),
587	ds=dict(
588	variable="prsn",
589	units="mm day-1",
590	standard_name="snowfall_flux",
591	long_name="Daily Snowfall",
592	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
593	),
594	dr=dict(
595	variable="prlp",
596	units="mm day-1",
597	standard_name="rainfall_flux",
598	long_name="Daily Rainfall",
599	comment=f"ECCC {generation} Generation of Adjusted and Homogenized Precipitation Data",
600	),
601	)
602	try:	×
603	variable = ec_ahccd_attrs[code]	×
604	variable["missing_flags"] = "M"	×
605	if variable["variable"].startswith("tas"):	×
606	variable["NaN_value"] = -9999.9	×
607	column_names = [	×
608	"No",
609	"StnId",
610	"Station name",
611	"Prov",
612	"FromYear",
613	"FromMonth",
614	"ToYear",
615	"ToMonth",
616	"%Miss",
617	"Lat(deg)",
618	"Long(deg)",
619	"Elev(m)",
620	"Joined",
621	"RCS",
622	]
623	column_spaces = [(0, 5), (5, 6), (6, 8), (8, 9)]	×
624	ii = 9	×
625	for i in range(1, 32):	×
626	column_spaces.append((ii, ii + 7))	×
627	ii += 7	×
628	column_spaces.append((ii, ii + 1))	×
629	ii += 1	×
630	header_row = 3	×
631
632	elif variable["variable"].startswith("pr"):	×
633	variable["NaN_value"] = -9999.99	×
634	column_names = [	×
635	"Prov",
636	"Station name",
637	"stnid",
638	"beg yr",
639	"beg mon",
640	"end yr",
641	"end mon",
642	"lat (deg)",
643	"long (deg)",
644	"elev (m)",
645	"stns joined",
646	]
647	column_spaces = [(0, 4), (4, 5), (5, 7), (7, 8)]	×
648	ii = 8	×
649	for i in range(1, 32):	×
650	column_spaces.append((ii, ii + 8))	×
651	ii += 8	×
652	column_spaces.append((ii, ii + 1))	×
653	ii += 1	×
654	header_row = 0	×
655
656	else:
657	raise KeyError	×
658
659	column_names = {	×
660	col.lower()
661	.split("(")[0]
662	.replace("%", "pct_")
663	.strip()
664	.replace(" ", "_"): col
665	for col in list(column_names)
666	}
667
668	if gen == 3:	×
669	_citation = (	×
670	"Vincent, L.A., M.M. Hartwell and X.L. Wang, 2020: A Third Generation of Homogenized "
671	"Temperature for Trend Analysis and Monitoring Changes in Canada’s Climate. "
672	"Atmosphere-Ocean. https://doi.org/10.1080/07055900.2020.1765728"
673	)
674	elif gen == 2:	×
675	_citation = (	×
676	"Mekis, É and L.A. Vincent, 2011: An overview of the second generation adjusted daily "
677	"precipitation dataset for trend analysis in Canada. Atmosphere-Ocean 49(2), "
678	"163-177 doi:10.1080/07055900.2011.583910"
679	)
680	else:
681	msg = f"Generation '{gen}' not supported."	×
682	raise NotImplementedError(msg)	×
683
684	global_attrs = dict(	×
685	title=f"{generation} Generation of Homogenized Daily {variable['variable']} "
686	"for Canada (Updated to December 2019)",
687	history=f"{dt.today().strftime('%Y-%m-%d')}: Convert from original format to NetCDF",
688	type="station_obs",
689	institute="Environment and Climate Change Canada",
690	institute_id="ECCC",
691	dataset_id=f"AHCCD_gen{gen}_day_{variable['variable']}",
692	frequency="day",
693	licence_type="permissive",
694	licence="https:/open.canada.ca/en/open-government-licence-canada",
695	citation=_citation,
696	)
697
698	except KeyError as e:	×
699	msg = f"AHCCD variable '{code}' or generation '{gen}' not supported."	×
700	logging.error(msg)	×
701	raise NotImplementedError(msg) from e	×
702
703	return variable, column_names, column_spaces, header_row, global_attrs	×

Ouranosinc / miranda / 1954258533

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous