9347528555

Committed 03 Jun 2024 09:16AM UTC coverage: 92.866% (+5.0%) from 87.87%

Build # 9347528555

Build Type

push

github

Committed by

ghiggi

Commit Message

Generalize checks defaults

Run Details

10 of 10 new or added lines in 1 file covered. (100.0%)

266 existing lines in 25 files now uncovered.

13199 of 14213 relevant lines covered (92.87%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.34

/gpm/dataset/decoding/dataarray_attrs.py

# -----------------------------------------------------------------------------.
# MIT License

# Copyright (c) 2024 GPM-API developers
#
# This file is part of GPM-API.

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# -----------------------------------------------------------------------------.
"""This module contains functions to standardize GPM-API Dataset attributes."""
import re

import numpy as np


def convert_string_to_number(string):
    if string.isdigit():
        return int(string)
    return float(string)


def ensure_dtype_name(dtype):
    """Ensure the dtype is a string name.

    This function convert numpy.dtype to the string name.
    """
    if isinstance(dtype, np.dtype):
        dtype = dtype.name
    return dtype


def _check_fillvalue_format(attrs):
    # Ensure fill values are numbers
    if "CodeMissingValue" in attrs and isinstance(attrs["CodeMissingValue"], str):
        attrs["CodeMissingValue"] = convert_string_to_number(attrs["CodeMissingValue"])
    if "_FillValue" in attrs and isinstance(attrs["_FillValue"], str):
        attrs["_FillValue"] = convert_string_to_number(attrs["_FillValue"])

    # Check _FillValue and CodeMissingValue agrees
    # - Do not since _FillValue often badly defined !
    # if "_FillValue" in attrs  and "CodeMissingValue" in attrs:
    #     if attrs["_FillValue"] != attrs["CodeMissingValue"]:
    #         name = da.name
    #         fillvalue = attrs["_FillValue"]
    #         codevalue = attrs["CodeMissingValue"]
    #         raise ValueError(f"In {name}, _FillValue is {fillvalue} and CodeMissingValue is {codevalue}")

    # Convert CodeMissingValue' to _FillValue if available
    if "CodeMissingValue" in attrs:
        attrs["_FillValue"] = attrs["CodeMissingValue"]

    # Remove 'CodeMissingValue'
    _ = attrs.pop("CodeMissingValue", None)

    return attrs


def _sanitize_attributes(attrs):
    # Convert 'Units' to 'units'
    if not attrs.get("units", False) and attrs.get("Units", False):
        attrs["units"] = attrs.pop("Units")

    # Remove 'Units'
    attrs.pop("Units", None)

    # Remove 'DimensionNames'
    attrs.pop("DimensionNames", None)

    # Sanitize LongName if present
    if "LongName" in attrs:
        attrs["description"] = re.sub(
            " +",
            " ",
            attrs["LongName"].replace("\n", " ").replace("\t", " "),
        ).strip()
        attrs.pop("LongName")
    return attrs


def _format_dataarray_attrs(da, product=None):
    attrs = da.attrs

    # Ensure fill values are numbers
    # - If CodeMissingValue is present, it is used as _FillValue
    # - _FillValue are moved to encoding by xr.decode_cf !
    attrs = _check_fillvalue_format(attrs)

    # Remove Units, DimensionNames and sanitize LongName
    attrs = _sanitize_attributes(attrs)

    # Ensure encoding and source_dtype is a dtype string name
    if "dtype" in da.encoding:
        da.encoding["dtype"] = ensure_dtype_name(da.encoding["dtype"])

    if "source_dtype" in attrs:
        attrs["source_dtype"] = ensure_dtype_name(attrs["source_dtype"])

    # Add source dtype from encoding if not present
    if "source_dtype" not in attrs and "dtype" in da.encoding:
        attrs["source_dtype"] = da.encoding["dtype"]

    # Add gpm_api product name
    if product is not None:
        attrs["gpm_api_product"] = product

    # Attach attributes
    da.attrs = attrs

    return da


def standardize_dataarrays_attrs(ds, product):
    # Sanitize variable attributes
    for var, da in ds.items():
        ds[var] = _format_dataarray_attrs(da, product)

    # Drop attributes from bounds coordinates
    # - https://github.com/pydata/xarray/issues/8368
    # - Attribute is lost when writing to netcdf
    bounds_coords = ["time_bnds", "lon_bnds", "lat_bnds"]
    for bnds in bounds_coords:
        if bnds in ds:
            ds[bnds].attrs = {}

    # Sanitize coordinates attributes
    for coord in list(ds.coords):
        ds[coord].attrs = _sanitize_attributes(ds[coord].attrs)

    return ds

1	# -----------------------------------------------------------------------------.
2	# MIT License
3
4	# Copyright (c) 2024 GPM-API developers
5	#
6	# This file is part of GPM-API.
7
8	# Permission is hereby granted, free of charge, to any person obtaining a copy
9	# of this software and associated documentation files (the "Software"), to deal
10	# in the Software without restriction, including without limitation the rights
11	# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12	# copies of the Software, and to permit persons to whom the Software is
13	# furnished to do so, subject to the following conditions:
14	#
15	# The above copyright notice and this permission notice shall be included in all
16	# copies or substantial portions of the Software.
17	#
18	# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19	# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20	# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21	# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22	# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23	# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24	# SOFTWARE.
25
26	# -----------------------------------------------------------------------------.
27	"""This module contains functions to standardize GPM-API Dataset attributes."""	1✔
28	import re	1✔
29
30	import numpy as np	1✔
31
32
33	def convert_string_to_number(string):	1✔
34	if string.isdigit():	1✔
35	return int(string)	1✔
36	return float(string)	1✔
37
38
39	def ensure_dtype_name(dtype):	1✔
40	"""Ensure the dtype is a string name.
41
42	This function convert numpy.dtype to the string name.
43	"""
44	if isinstance(dtype, np.dtype):	1✔
45	dtype = dtype.name	1✔
46	return dtype	1✔
47
48
49	def _check_fillvalue_format(attrs):	1✔
50	# Ensure fill values are numbers
51	if "CodeMissingValue" in attrs and isinstance(attrs["CodeMissingValue"], str):	1✔
52	attrs["CodeMissingValue"] = convert_string_to_number(attrs["CodeMissingValue"])	1✔
53	if "_FillValue" in attrs and isinstance(attrs["_FillValue"], str):	1✔
54	attrs["_FillValue"] = convert_string_to_number(attrs["_FillValue"])	×
55
56	# Check _FillValue and CodeMissingValue agrees
57	# - Do not since _FillValue often badly defined !
58	# if "_FillValue" in attrs and "CodeMissingValue" in attrs:
59	# if attrs["_FillValue"] != attrs["CodeMissingValue"]:
60	# name = da.name
61	# fillvalue = attrs["_FillValue"]
62	# codevalue = attrs["CodeMissingValue"]
63	# raise ValueError(f"In {name}, _FillValue is {fillvalue} and CodeMissingValue is {codevalue}")
64
65	# Convert CodeMissingValue' to _FillValue if available
66	if "CodeMissingValue" in attrs:	1✔
67	attrs["_FillValue"] = attrs["CodeMissingValue"]	1✔
68
69	# Remove 'CodeMissingValue'
70	_ = attrs.pop("CodeMissingValue", None)	1✔
71
72	return attrs	1✔
73
74
75	def _sanitize_attributes(attrs):	1✔
76	# Convert 'Units' to 'units'
77	if not attrs.get("units", False) and attrs.get("Units", False):	1✔
UNCOV 78	attrs["units"] = attrs.pop("Units")	×
79
80	# Remove 'Units'
81	attrs.pop("Units", None)	1✔
82
83	# Remove 'DimensionNames'
84	attrs.pop("DimensionNames", None)	1✔
85
86	# Sanitize LongName if present
87	if "LongName" in attrs:	1✔
88	attrs["description"] = re.sub(	1✔
89	" +",
90	" ",
91	attrs["LongName"].replace("\n", " ").replace("\t", " "),
92	).strip()
93	attrs.pop("LongName")	1✔
94	return attrs	1✔
95
96
97	def _format_dataarray_attrs(da, product=None):	1✔
98	attrs = da.attrs	1✔
99
100	# Ensure fill values are numbers
101	# - If CodeMissingValue is present, it is used as _FillValue
102	# - _FillValue are moved to encoding by xr.decode_cf !
103	attrs = _check_fillvalue_format(attrs)	1✔
104
105	# Remove Units, DimensionNames and sanitize LongName
106	attrs = _sanitize_attributes(attrs)	1✔
107
108	# Ensure encoding and source_dtype is a dtype string name
109	if "dtype" in da.encoding:	1✔
110	da.encoding["dtype"] = ensure_dtype_name(da.encoding["dtype"])	1✔
111
112	if "source_dtype" in attrs:	1✔
UNCOV 113	attrs["source_dtype"] = ensure_dtype_name(attrs["source_dtype"])	×
114
115	# Add source dtype from encoding if not present
116	if "source_dtype" not in attrs and "dtype" in da.encoding:	1✔
117	attrs["source_dtype"] = da.encoding["dtype"]	1✔
118
119	# Add gpm_api product name
120	if product is not None:	1✔
121	attrs["gpm_api_product"] = product	1✔
122
123	# Attach attributes
124	da.attrs = attrs	1✔
125
126	return da	1✔
127
128
129	def standardize_dataarrays_attrs(ds, product):	1✔
130	# Sanitize variable attributes
131	for var, da in ds.items():	1✔
132	ds[var] = _format_dataarray_attrs(da, product)	1✔
133
134	# Drop attributes from bounds coordinates
135	# - https://github.com/pydata/xarray/issues/8368
136	# - Attribute is lost when writing to netcdf
137	bounds_coords = ["time_bnds", "lon_bnds", "lat_bnds"]	1✔
138	for bnds in bounds_coords:	1✔
139	if bnds in ds:	1✔
140	ds[bnds].attrs = {}	1✔
141
142	# Sanitize coordinates attributes
143	for coord in list(ds.coords):	1✔
144	ds[coord].attrs = _sanitize_attributes(ds[coord].attrs)	1✔
145
146	return ds	1✔

ghiggi / gpm_api / 9347528555

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous