1091846040

Build Type

github-actions

Committed by GitHub

Commit Message

<a href="https://github.com/USDA-ARS-NWRC/smrf/commit/<a class=hub.com/USDA-ARS-NWRC/smrf/commit/57fd90a636ce840407a82bfb641e93cce52b93cf">57fd90a63<a href="https://github.com/USDA-ARS-NWRC/smrf/commit/57fd90a636ce840407a82bfb641e93cce52b93cf">">Merge </a><a class="double-link" href="https://github.com/USDA-ARS-NWRC/smrf/commit/<a class="double-link" href="https://github.com/USDA-ARS-NWRC/smrf/commit/02cbd33e57b836758e727d167ae71dc478643d5b">02cbd33e5</a>">02cbd33e5</a><a href="https://github.com/USDA-ARS-NWRC/smrf/commit/57fd90a636ce840407a82bfb641e93cce52b93cf"> into 465d42cca">465d42cca</a>

Pull Request Pull Request #213: Precip - input and output data scalars and testing

Run Details

140 of 140 new or added lines in 9 files covered. (100.0%)

3928 of 4679 relevant lines covered (83.95%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.76

/smrf/tests/smrf_test_case.py

import logging
import os
import shutil
import unittest
from copy import deepcopy
from pathlib import Path

import netCDF4 as nc
import numpy as np
from inicheck.tools import cast_all_variables, get_user_config

import smrf
from smrf.framework.model_framework import run_smrf
from smrf.tests import non_distributed_variables


class SMRFTestCase(unittest.TestCase):
    """
    The base test case for SMRF that will load in the configuration file
    and store as the base config. Also will remove the output
    directory upon tear down.

    Runs the short simulation over reynolds mountain east
    """
    DIST_VARIABLES = frozenset([
        'air_temp',
        'cloud_factor',
        'precip',
        'thermal',
        'vapor_pressure',
        'wind',
    ])

    THREAD_CONFIG = {
        'threading': True,
        'max_queue': 1,
        'time_out': 5
    }

    BASE_INI_FILE_NAME = 'config.ini'

    test_dir = Path(smrf.__file__).parent.joinpath('tests')
    basin_dir = test_dir.joinpath('basins', 'RME')
    config_file = os.path.join(basin_dir, BASE_INI_FILE_NAME)

    @property
    def dist_variables(self):
        if self._dist_variables is None:
            self._dist_variables = list(self.DIST_VARIABLES)
        return self._dist_variables

    @property
    def base_config(self):
        return self.base_config_copy()

    @classmethod
    def configure(cls):
        cls.run_config = cls.base_config_copy()

    @property
    def smrf_instance(self):
        return smrf.framework.SMRF(self.config_file)

    @classmethod
    def base_config_copy(cls):
        return deepcopy(cls._base_config)

    @classmethod
    def load_base_config(cls):
        cls._base_config = get_user_config(cls.config_file, modules='smrf')

    @classmethod
    def setUpClass(cls):
        cls.load_base_config()
        cls.create_output_dir()
        cls.configure()

    @classmethod
    def tearDownClass(cls):
        cls.remove_output_dir()
        delattr(cls, 'output_dir')

    @classmethod
    def create_output_dir(cls):
        folder = os.path.join(cls._base_config.cfg['output']['out_location'])

        # Remove any potential files to ensure fresh run
        if os.path.isdir(folder):
            shutil.rmtree(folder)

        os.makedirs(folder)
        cls.output_dir = Path(folder)

    @classmethod
    def remove_output_dir(cls):
        if hasattr(cls, 'output_dir') and \
                os.path.exists(cls.output_dir):
            shutil.rmtree(cls.output_dir)

    @classmethod
    def thread_config(cls):
        config = cls.base_config_copy()
        config.raw_cfg['system'].update(cls.THREAD_CONFIG)

        config.apply_recipes()
        config = cast_all_variables(config, config.mcfg)

        return config

    @staticmethod
    def can_i_run_smrf(config):
        """
        Test whether a config is possible to run
        """
        try:
            run_smrf(config)
            return True

        except Exception as e:
            print(e)
            return False

    @staticmethod
    def assert_gold_equal(gold, not_gold, error_msg):
        """Compare two arrays

        Arguments:
            gold {array} -- gold array
            not_gold {array} -- not gold array
            error_msg {str} -- error message to display
        """

        if os.getenv('NOT_ON_GOLD_HOST') is None:
            try:
                np.testing.assert_array_equal(
                    not_gold,
                    gold,
                    err_msg=error_msg
                )
            except AssertionError:
                for rtol in [1e-5, 1e-4]:
                    status = np.allclose(
                        not_gold,
                        gold,
                        atol=0,
                        rtol=rtol
                    )
                    if status:
                        break
                if not status:
                    raise AssertionError(error_msg)
                print('Arrays are not exact match but close with rtol={}'.format(rtol))  # noqa
        else:
            np.testing.assert_almost_equal(
                not_gold,
                gold,
                decimal=3,
                err_msg=error_msg
            )

    def setUp(self):
        self._dist_variables = None

        # clear the logger
        for handler in logging.root.handlers:
            logging.root.removeHandler(handler)

    def tearDown(self):
        logging.shutdown()

    def compare_hrrr_gold(self):
        """
        Compare the model results with the gold standard
        """
        [
            self.compare_netcdf_files(file_name.name)
            for file_name in self.gold_dir.glob('*.nc')
        ]

    @staticmethod
    def _validate_time(gold, test):
        np.testing.assert_equal(
            gold.variables['time'][:],
            test.variables['time'][:],
            err_msg="Time steps did not match: \nGOLD {0} \n TEST {1}".format(
                gold.variables['time'], test.variables['time']
            )
        )

    def compare_netcdf_files(self, output_file):
        """
        Compare two netcdf files to ensure that they are identical. The
        tests will compare the attributes of each variable and ensure that
        the values are exact
        """

        gold = nc.Dataset(self.gold_dir.joinpath(output_file))
        test = nc.Dataset(self.output_dir.joinpath(output_file))

        self._validate_time(gold, test)

        # go through all variables and compare everything including
        # the attributes and data
        for var_name, v in gold.variables.items():

            # compare the dimensions
            for att in gold.variables[var_name].ncattrs():
                self.assertEqual(
                    getattr(gold.variables[var_name], att),
                    getattr(test.variables[var_name], att))

            # only compare those that are floats
            if gold.variables[var_name].datatype != np.dtype('S1'):
                error_msg = "Variable: {0} did not match gold standard". \
                    format(var_name)
                self.assert_gold_equal(
                    gold.variables[var_name][:],
                    test.variables[var_name][:],
                    error_msg
                )

        gold.close()
        test.close()

    def evaluate_netcdf_ratio(self, output_file):
        """
        Find a ratio between the primary variable for 2 datasets.
        Args:
            output_file: path to the test output file

        Returns:
            a tuple of the ratio ndarray and an ndarry of indices that should
            be ignored
        """

        with nc.Dataset(self.gold_dir.joinpath(output_file)) as gold, \
                nc.Dataset(self.output_dir.joinpath(output_file)) as test:

            self._validate_time(gold, test)

            for var_name, v in gold.variables.items():
                if var_name not in non_distributed_variables:
                    # ratio is meaningless if both cells are zero or nan
                    ignore_indexes = (
                        (test.variables[var_name][:] == 0.0)
                        & (gold.variables[var_name][:] == 0.0)) \
                        | (np.isnan(test.variables[var_name][:])
                           & np.isnan(gold.variables[var_name][:]))
                    return (test.variables[var_name][:] / gold.variables[
                                                              var_name][:],
                            ignore_indexes)

        raise ValueError(f'{output_file} did not return a valid ratio')

1	import logging	1✔
2	import os	1✔
3	import shutil	1✔
4	import unittest	1✔
5	from copy import deepcopy	1✔
6	from pathlib import Path	1✔
7
8	import netCDF4 as nc	1✔
9	import numpy as np	1✔
10	from inicheck.tools import cast_all_variables, get_user_config	1✔
11
12	import smrf	1✔
13	from smrf.framework.model_framework import run_smrf	1✔
14	from smrf.tests import non_distributed_variables	1✔
15
16
17	class SMRFTestCase(unittest.TestCase):	1✔
18	"""
19	The base test case for SMRF that will load in the configuration file
20	and store as the base config. Also will remove the output
21	directory upon tear down.
22
23	Runs the short simulation over reynolds mountain east
24	"""
25	DIST_VARIABLES = frozenset([	1✔
26	'air_temp',
27	'cloud_factor',
28	'precip',
29	'thermal',
30	'vapor_pressure',
31	'wind',
32	])
33
34	THREAD_CONFIG = {	1✔
35	'threading': True,
36	'max_queue': 1,
37	'time_out': 5
38	}
39
40	BASE_INI_FILE_NAME = 'config.ini'	1✔
41
42	test_dir = Path(smrf.__file__).parent.joinpath('tests')	1✔
43	basin_dir = test_dir.joinpath('basins', 'RME')	1✔
44	config_file = os.path.join(basin_dir, BASE_INI_FILE_NAME)	1✔
45
46	@property	1✔
47	def dist_variables(self):
48	if self._dist_variables is None:	1✔
49	self._dist_variables = list(self.DIST_VARIABLES)	1✔
50	return self._dist_variables	1✔
51
52	@property	1✔
53	def base_config(self):
54	return self.base_config_copy()	1✔
55
56	@classmethod	1✔
57	def configure(cls):
58	cls.run_config = cls.base_config_copy()	1✔
59
60	@property	1✔
61	def smrf_instance(self):
62	return smrf.framework.SMRF(self.config_file)	1✔
63
64	@classmethod	1✔
65	def base_config_copy(cls):
66	return deepcopy(cls._base_config)	1✔
67
68	@classmethod	1✔
69	def load_base_config(cls):
70	cls._base_config = get_user_config(cls.config_file, modules='smrf')	1✔
71
72	@classmethod	1✔
73	def setUpClass(cls):
74	cls.load_base_config()	1✔
75	cls.create_output_dir()	1✔
76	cls.configure()	1✔
77
78	@classmethod	1✔
79	def tearDownClass(cls):
80	cls.remove_output_dir()	1✔
81	delattr(cls, 'output_dir')	1✔
82
83	@classmethod	1✔
84	def create_output_dir(cls):
85	folder = os.path.join(cls._base_config.cfg['output']['out_location'])	1✔
86
87	# Remove any potential files to ensure fresh run
88	if os.path.isdir(folder):	1✔
89	shutil.rmtree(folder)	1✔
90
91	os.makedirs(folder)	1✔
92	cls.output_dir = Path(folder)	1✔
93
94	@classmethod	1✔
95	def remove_output_dir(cls):
96	if hasattr(cls, 'output_dir') and \	1✔
97	os.path.exists(cls.output_dir):
98	shutil.rmtree(cls.output_dir)	1✔
99
100	@classmethod	1✔
101	def thread_config(cls):
102	config = cls.base_config_copy()	1✔
103	config.raw_cfg['system'].update(cls.THREAD_CONFIG)	1✔
104
105	config.apply_recipes()	1✔
106	config = cast_all_variables(config, config.mcfg)	1✔
107
108	return config	1✔
109
110	@staticmethod	1✔
111	def can_i_run_smrf(config):
112	"""
113	Test whether a config is possible to run
114	"""
115	try:	×
116	run_smrf(config)	×
117	return True	×
118
119	except Exception as e:	×
120	print(e)	×
121	return False	×
122
123	@staticmethod	1✔
124	def assert_gold_equal(gold, not_gold, error_msg):
125	"""Compare two arrays
126
127	Arguments:
128	gold {array} -- gold array
129	not_gold {array} -- not gold array
130	error_msg {str} -- error message to display
131	"""
132
133	if os.getenv('NOT_ON_GOLD_HOST') is None:	1✔
134	try:	1✔
135	np.testing.assert_array_equal(	1✔
136	not_gold,
137	gold,
138	err_msg=error_msg
139	)
140	except AssertionError:	×
141	for rtol in [1e-5, 1e-4]:	×
142	status = np.allclose(	×
143	not_gold,
144	gold,
145	atol=0,
146	rtol=rtol
147	)
148	if status:	×
149	break	×
150	if not status:	×
151	raise AssertionError(error_msg)	×
152	print('Arrays are not exact match but close with rtol={}'.format(rtol)) # noqa	×
153	else:
154	np.testing.assert_almost_equal(	×
155	not_gold,
156	gold,
157	decimal=3,
158	err_msg=error_msg
159	)
160
161	def setUp(self):	1✔
162	self._dist_variables = None	1✔
163
164	# clear the logger
165	for handler in logging.root.handlers:	1✔
166	logging.root.removeHandler(handler)	1✔
167
168	def tearDown(self):	1✔
169	logging.shutdown()	1✔
170
171	def compare_hrrr_gold(self):	1✔
172	"""
173	Compare the model results with the gold standard
174	"""
175	[	1✔
176	self.compare_netcdf_files(file_name.name)
177	for file_name in self.gold_dir.glob('*.nc')
178	]
179
180	@staticmethod	1✔
181	def _validate_time(gold, test):
182	np.testing.assert_equal(	1✔
183	gold.variables['time'][:],
184	test.variables['time'][:],
185	err_msg="Time steps did not match: \nGOLD {0} \n TEST {1}".format(
186	gold.variables['time'], test.variables['time']
187	)
188	)
189
190	def compare_netcdf_files(self, output_file):	1✔
191	"""
192	Compare two netcdf files to ensure that they are identical. The
193	tests will compare the attributes of each variable and ensure that
194	the values are exact
195	"""
196
197	gold = nc.Dataset(self.gold_dir.joinpath(output_file))	1✔
198	test = nc.Dataset(self.output_dir.joinpath(output_file))	1✔
199
200	self._validate_time(gold, test)	1✔
201
202	# go through all variables and compare everything including
203	# the attributes and data
204	for var_name, v in gold.variables.items():	1✔
205
206	# compare the dimensions
207	for att in gold.variables[var_name].ncattrs():	1✔
208	self.assertEqual(	1✔
209	getattr(gold.variables[var_name], att),
210	getattr(test.variables[var_name], att))
211
212	# only compare those that are floats
213	if gold.variables[var_name].datatype != np.dtype('S1'):	1✔
214	error_msg = "Variable: {0} did not match gold standard". \	1✔
215	format(var_name)
216	self.assert_gold_equal(	1✔
217	gold.variables[var_name][:],
218	test.variables[var_name][:],
219	error_msg
220	)
221
222	gold.close()	1✔
223	test.close()	1✔
224
225	def evaluate_netcdf_ratio(self, output_file):	1✔
226	"""
227	Find a ratio between the primary variable for 2 datasets.
228	Args:
229	output_file: path to the test output file
230
231	Returns:
232	a tuple of the ratio ndarray and an ndarry of indices that should
233	be ignored
234	"""
235
236	with nc.Dataset(self.gold_dir.joinpath(output_file)) as gold, \	1✔
237	nc.Dataset(self.output_dir.joinpath(output_file)) as test:
238
239	self._validate_time(gold, test)	1✔
240
241	for var_name, v in gold.variables.items():	1✔
242	if var_name not in non_distributed_variables:	1✔
243	# ratio is meaningless if both cells are zero or nan
244	ignore_indexes = (	1✔
245	(test.variables[var_name][:] == 0.0)
246	& (gold.variables[var_name][:] == 0.0)) \
247	\| (np.isnan(test.variables[var_name][:])
248	& np.isnan(gold.variables[var_name][:]))
249	return (test.variables[var_name][:] / gold.variables[	1✔
250	var_name][:],
251	ignore_indexes)
252
253	raise ValueError(f'{output_file} did not return a valid ratio')	×

USDA-ARS-NWRC / smrf / 1091846040

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous