1947394058

Build # 1947394058

Build Type

github

Committed by GitHub

Commit Message

Merge f8951d551 into 250bc4dd4

Pull Request Pull Request #24: Add CMIP file structure - WIP

Run Details

115 of 660 new or added lines in 28 files covered. (17.42%)

8 existing lines in 2 files now uncovered.

660 of 3021 relevant lines covered (21.85%)

0.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.12

/miranda/decode/_time.py

import logging
from logging import config
from typing import Union

import cftime
import pandas as pd
from pandas._libs.tslibs import NaTType  # noqa

from miranda.scripting import LOGGING_CONFIG

logging.config.dictConfig(LOGGING_CONFIG)

__all__ = [
    "date_parser",
    "DecoderError",
    "BASIC_DT_VALIDATION",
    "DATE_VALIDATION",
    "TIME_UNITS_TO_FREQUENCY",
    "TIME_UNITS_TO_TIMEDELTA",
]


BASIC_DT_VALIDATION = r"\s*(?=\d{2}(?:\d{2})?)"
DATE_VALIDATION = r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$"
TIME_UNITS_TO_FREQUENCY = {
    "subhrPt": "sub-hr",
    "hourly": "1hr",
    "hours": "1hr",
    "hr": "1hr",
    "6-hourly": "6hr",
    "daily": "day",
    "days": "day",
    "day": "day",
    "weekly": "sem",
    "weeks": "sem",
    "sem": "sem",
    "monthly": "mon",
    "months": "mon",
    "mon": "mon",
    "monC": "monC",
    "Amon": "mon",
    "yearly": "yr",
    "years": "yr",
    "annual": "yr",
    "yr": "yr",
    "yrPt": "yrPt",
    "decadal": "dec",
    "decades": "dec",
    "dec": "dec",
    "fixed": "fx",
    "fx": "fx",
}
TIME_UNITS_TO_TIMEDELTA = {
    "hourly": "1h",
    "hours": "1h",
    "1hr": "1h",
    "1hrCM": "1h",
    "1hrPt": "1h",
    "3hr": "3h",
    "3hrPt": "3h",
    "6-hourly": "6h",
    "6hr": "6h",
    "6hrPt": "6h",
    "daily": "1d",
    "day": "1d",
    "days": "1d",
    "weekly": "7d",
    "weeks": "7d",
    "sem": "7d",
    "mon": "30d",
    "monC": "30d",
    "monPt": "30d",
    "Amon": "30d",
    "QS": "90d",
    "qtr": "90d",
    "yearly": "365d",
    "years": "365d",
    "year": "365d",
    "yr": "365d",
    "yrPt": "365d",
}


class DecoderError(Exception):
    pass


def date_parser(
    date: str,
    *,
    end_of_period: bool = False,
    out_dtype: str = "str",
    strtime_format: str = "%Y-%m-%d",
) -> Union[str, pd.Timestamp, NaTType]:
    """Returns a datetime from a string.

    Parameters
    ----------
    date : str
      Date to be converted.
    end_of_period : bool
      If True, the date will be the end of month or year depending on what's most appropriate.
    out_dtype: {"datetime", "str"}
      Returned object type.
    strtime_format: str
      If out_dtype=='str', this sets the strftime format.

    Returns
    -------
    pd.Timestamp or str or pd.NaT
      Parsed date.

    Notes
    -----
    Adapted from code written by Gabriel Rondeau-Genesse (@RondeauG)
    """

    # Formats, ordered depending on string length
    formats = {
        4: ["%Y"],
        6: ["%Y%m"],
        7: ["%Y-%m"],
        8: ["%Y%m%d"],
        10: ["%Y%m%d%H", "%Y-%m-%d"],
        12: ["%Y%m%d%H%M"],
        13: ["%Y%m-%Y%m"],
        17: ["%Y%m%d-%Y%m%d"],
        19: ["%Y-%m-%dT%H:%M:%S"],
        21: ["%Y%m%d%H-%Y%m%d%H"],
    }
    end_date_found = False

    def _parse_date(d, fmts):
        for fmt in fmts:
            try:
                s = pd.to_datetime(d, format=fmt)
                match = fmt
                break
            except ValueError:
                pass
        else:
            raise DecoderError("Can't parse date {d} with supported formats {fmts}.")
        return s, match

    date_format = None
    if isinstance(date, str):
        if len(date) in [13, 17]:
            dates = date.split("-")
            if not end_of_period:
                date = dates[0]
            else:
                date = dates[1]
                end_date_found = True

        try:
            possible_formats = formats[len(date)]
            date, date_format = _parse_date(date, possible_formats)
        except KeyError:
            # Return NaT for fixed/missing/ill-formatted date strings
            return pd.NaT

    elif isinstance(date, cftime.datetime):
        for n in range(3):
            try:
                date = pd.Timestamp.fromisoformat((date - pd.Timedelta(n)).isoformat())
            except ValueError:  # We are NOT catching OutOfBoundsDatetime.
                pass
            else:
                break
        else:
            raise DecoderError(
                "Unable to parse cftime date {date}, even when moving back 2 days."
            )
    elif not isinstance(date, pd.Timestamp):
        date = pd.Timestamp(date)  # noqa

    if end_of_period and date_format and not end_date_found:
        if "m" not in date_format:
            date = date + pd.tseries.offsets.YearEnd(1)
        elif "d" not in date_format:
            date = date + pd.tseries.offsets.MonthEnd(1)
        # TODO: Implement sub-daily?

    if out_dtype == "str":
        return date.strftime(strtime_format)

    return date

1	import logging	3✔
2	from logging import config	3✔
3	from typing import Union	3✔
4
5	import cftime	3✔
6	import pandas as pd	3✔
7	from pandas._libs.tslibs import NaTType # noqa	3✔
8
9	from miranda.scripting import LOGGING_CONFIG	3✔
10
11	logging.config.dictConfig(LOGGING_CONFIG)	3✔
12
13	__all__ = [	3✔
14	"date_parser",
15	"DecoderError",
16	"BASIC_DT_VALIDATION",
17	"DATE_VALIDATION",
18	"TIME_UNITS_TO_FREQUENCY",
19	"TIME_UNITS_TO_TIMEDELTA",
20	]
21
22
23	BASIC_DT_VALIDATION = r"\s*(?=\d{2}(?:\d{2})?)"	3✔
24	DATE_VALIDATION = r"^\d{4}-(0[1-9]\|1[0-2])-(0[1-9]\|[12][0-9]\|3[01])$"	3✔
25	TIME_UNITS_TO_FREQUENCY = {	3✔
26	"subhrPt": "sub-hr",
27	"hourly": "1hr",
28	"hours": "1hr",
29	"hr": "1hr",
30	"6-hourly": "6hr",
31	"daily": "day",
32	"days": "day",
33	"day": "day",
34	"weekly": "sem",
35	"weeks": "sem",
36	"sem": "sem",
37	"monthly": "mon",
38	"months": "mon",
39	"mon": "mon",
40	"monC": "monC",
41	"Amon": "mon",
42	"yearly": "yr",
43	"years": "yr",
44	"annual": "yr",
45	"yr": "yr",
46	"yrPt": "yrPt",
47	"decadal": "dec",
48	"decades": "dec",
49	"dec": "dec",
50	"fixed": "fx",
51	"fx": "fx",
52	}
53	TIME_UNITS_TO_TIMEDELTA = {	3✔
54	"hourly": "1h",
55	"hours": "1h",
56	"1hr": "1h",
57	"1hrCM": "1h",
58	"1hrPt": "1h",
59	"3hr": "3h",
60	"3hrPt": "3h",
61	"6-hourly": "6h",
62	"6hr": "6h",
63	"6hrPt": "6h",
64	"daily": "1d",
65	"day": "1d",
66	"days": "1d",
67	"weekly": "7d",
68	"weeks": "7d",
69	"sem": "7d",
70	"mon": "30d",
71	"monC": "30d",
72	"monPt": "30d",
73	"Amon": "30d",
74	"QS": "90d",
75	"qtr": "90d",
76	"yearly": "365d",
77	"years": "365d",
78	"year": "365d",
79	"yr": "365d",
80	"yrPt": "365d",
81	}
82
83
84	class DecoderError(Exception):	3✔
85	pass	3✔
86
87
88	def date_parser(	3✔
89	date: str,
90	*,
91	end_of_period: bool = False,
92	out_dtype: str = "str",
93	strtime_format: str = "%Y-%m-%d",
94	) -> Union[str, pd.Timestamp, NaTType]:
95	"""Returns a datetime from a string.
96
97	Parameters
98	----------
99	date : str
100	Date to be converted.
101	end_of_period : bool
102	If True, the date will be the end of month or year depending on what's most appropriate.
103	out_dtype: {"datetime", "str"}
104	Returned object type.
105	strtime_format: str
106	If out_dtype=='str', this sets the strftime format.
107
108	Returns
109	-------
110	pd.Timestamp or str or pd.NaT
111	Parsed date.
112
113	Notes
114	-----
115	Adapted from code written by Gabriel Rondeau-Genesse (@RondeauG)
116	"""
117
118	# Formats, ordered depending on string length
NEW 119	formats = {	×
120	4: ["%Y"],
121	6: ["%Y%m"],
122	7: ["%Y-%m"],
123	8: ["%Y%m%d"],
124	10: ["%Y%m%d%H", "%Y-%m-%d"],
125	12: ["%Y%m%d%H%M"],
126	13: ["%Y%m-%Y%m"],
127	17: ["%Y%m%d-%Y%m%d"],
128	19: ["%Y-%m-%dT%H:%M:%S"],
129	21: ["%Y%m%d%H-%Y%m%d%H"],
130	}
NEW 131	end_date_found = False	×
132
NEW 133	def _parse_date(d, fmts):	×
NEW 134	for fmt in fmts:	×
NEW 135	try:	×
NEW 136	s = pd.to_datetime(d, format=fmt)	×
NEW 137	match = fmt	×
NEW 138	break	×
NEW 139	except ValueError:	×
NEW 140	pass	×
141	else:
NEW 142	raise DecoderError("Can't parse date {d} with supported formats {fmts}.")	×
NEW 143	return s, match	×
144
NEW 145	date_format = None	×
NEW 146	if isinstance(date, str):	×
NEW 147	if len(date) in [13, 17]:	×
NEW 148	dates = date.split("-")	×
NEW 149	if not end_of_period:	×
NEW 150	date = dates[0]	×
151	else:
NEW 152	date = dates[1]	×
NEW 153	end_date_found = True	×
154
NEW 155	try:	×
NEW 156	possible_formats = formats[len(date)]	×
NEW 157	date, date_format = _parse_date(date, possible_formats)	×
NEW 158	except KeyError:	×
159	# Return NaT for fixed/missing/ill-formatted date strings
NEW 160	return pd.NaT	×
161
NEW 162	elif isinstance(date, cftime.datetime):	×
NEW 163	for n in range(3):	×
NEW 164	try:	×
NEW 165	date = pd.Timestamp.fromisoformat((date - pd.Timedelta(n)).isoformat())	×
NEW 166	except ValueError: # We are NOT catching OutOfBoundsDatetime.	×
NEW 167	pass	×
168	else:
NEW 169	break	×
170	else:
NEW 171	raise DecoderError(	×
172	"Unable to parse cftime date {date}, even when moving back 2 days."
173	)
NEW 174	elif not isinstance(date, pd.Timestamp):	×
NEW 175	date = pd.Timestamp(date) # noqa	×
176
NEW 177	if end_of_period and date_format and not end_date_found:	×
NEW 178	if "m" not in date_format:	×
NEW 179	date = date + pd.tseries.offsets.YearEnd(1)	×
NEW 180	elif "d" not in date_format:	×
NEW 181	date = date + pd.tseries.offsets.MonthEnd(1)	×
182	# TODO: Implement sub-daily?
183
NEW 184	if out_dtype == "str":	×
NEW 185	return date.strftime(strtime_format)	×
186
NEW 187	return date	×

Ouranosinc / miranda / 1947394058

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous