305eb64a-9a20-4e4e-9c1e-8cc67e861cd6

Committed 09 Sep 2025 04:56PM UTC coverage: 51.086% (-43.4%) from 94.444%

Build # 305eb64a-9a20-4e4e-9c1e-8cc67e861cd6

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #766 from dbekaert/dev

v0.6.0

Run Details

133 of 261 new or added lines in 18 files covered. (50.96%)

3199 of 6262 relevant lines covered (51.09%)

0.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.51

/tools/RAiDER/processWM.py

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
#  Author: Jeremy Maurer, Raymond Hogenson & David Bekaert
#  Copyright 2019, by the California Institute of Technology. ALL RIGHTS
#  RESERVED. United States Government Sponsorship acknowledged.
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
import os
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np

from RAiDER.logger import logger
from RAiDER.models.customExceptions import (
    CriticalError,
    DatetimeOutsideRange,
    ExistingWeatherModelTooSmall,
    TryToKeepGoingError,
)
from RAiDER.models.weatherModel import checkContainment_raw, make_raw_weather_data_filename, make_weather_model_filename


def prepareWeatherModel(
    weather_model,
    time,
    ll_bounds,
    download_only: bool = False,
    makePlots: bool = False,
    force_download: bool = False,
) -> str:
    """Parse inputs to download and prepare a weather model grid for interpolation.

    Args:
        weather_model: WeatherModel   - instantiated weather model object
        time: datetime                - Python datetime to request. Will be rounded to nearest available time
        ll_bounds: list/array         - SNWE bounds target area to ensure weather model contains them
        download_only: bool           - False if preprocessing weather model data
        makePlots: bool               - whether to write debug plots
        force_download: bool          - True if you want to download even when the weather model exists

    Returns:
        str: filename of the netcdf file to which the weather model has been written
    """
    # set the bounding box from the in the case that it hasn't been set
    if weather_model.get_latlon_bounds() is None:
        weather_model.set_latlon_bounds(ll_bounds)

    # Ensure the file output location exists
    wmLoc = weather_model.get_wmLoc()
    weather_model.setTime(time)

    # get the path to the less processed weather model file
    path_wm_raw = make_raw_weather_data_filename(wmLoc, weather_model.Model(), time)

    # get the path to the more processed (cropped) weather model file
    path_wm_crop = weather_model.out_file(wmLoc)

    # check whether weather model files exists and/or or should be downloaded
    if os.path.exists(path_wm_crop) and not force_download:
        logger.warning(
            'Processed weather model already exists, please remove it ("%s") if you want to download a new one.',
            path_wm_crop,
        )

    # check whether the raw weather model covers this area
    elif os.path.exists(path_wm_raw) and checkContainment_raw(path_wm_raw, ll_bounds) and not force_download:
        logger.warning(
            'Raw weather model already exists, please remove it ("%s") if you want to download a new one.',
            path_wm_raw,
        )

    # if no weather model files supplied, check the standard location
    else:
        os.makedirs(os.path.dirname(path_wm_raw), exist_ok=True)
        try:
            weather_model.fetch(Path(path_wm_raw), time)
        except DatetimeOutsideRange:
            raise TryToKeepGoingError

    # If only downloading, exit now
    if download_only:
        logger.warning('download_only flag selected. No further processing will happen.')
        return None

    # Otherwise, load the weather model data
    f = weather_model.load()

    if f is not None:
        logger.warning('The processed weather model file already exists, so I will use that.')

        containment = weather_model.checkContainment(ll_bounds)
        if not containment and weather_model.Model() not in 'HRRR HRRRAK'.split():
            raise ExistingWeatherModelTooSmall

        return f

    # Logging some basic info
    logger.debug('Number of weather model nodes: %s', np.prod(weather_model.getWetRefractivity().shape))
    shape = weather_model.getWetRefractivity().shape
    logger.debug(f'Shape of weather model: {shape}')
    logger.debug(
        'Bounds of the weather model: %.2f/%.2f/%.2f/%.2f (SNWE)',
        np.nanmin(weather_model._ys),
        np.nanmax(weather_model._ys),
        np.nanmin(weather_model._xs),
        np.nanmax(weather_model._xs),
    )
    logger.debug('Weather model: %s', weather_model.Model())
    logger.debug('Mean value of the wet refractivity: %f', np.nanmean(weather_model.getWetRefractivity()))
    logger.debug('Mean value of the hydrostatic refractivity: %f', np.nanmean(weather_model.getHydroRefractivity()))
    logger.debug(weather_model)

    if makePlots:
        weather_model.plot('wh', True)
        weather_model.plot('pqt', True)
        plt.close('all')

    try:
        f = weather_model.write()
        containment = weather_model.checkContainment(ll_bounds)

    except Exception as e:
        logger.exception('Unable to save weathermodel to file')
        logger.exception(e)
        raise CriticalError

    finally:
        wm = weather_model.Model()
        del weather_model

    if not containment and wm not in 'HRRR'.split():
        raise ExistingWeatherModelTooSmall
    else:
        return f


def _weather_model_debug(los, lats, lons, ll_bounds, weather_model, wmLoc, time, out, download_only) -> None:
    """RaiderWeatherModelDebug main function."""
    logger.debug('Starting to run the weather model calculation with debugging plots')
    logger.debug('Time type: %s', type(time))
    logger.debug('Time: %s', time.strftime('%Y%m%d'))

    # location of the weather model files
    logger.debug('Beginning weather model pre-processing')
    logger.debug('Download-only is %s', download_only)
    if wmLoc is None:
        wmLoc = os.path.join(out, 'weather_files')

    # weather model calculation
    wm_filename = make_weather_model_filename(weather_model['name'], time, ll_bounds)
    weather_model_file = os.path.join(wmLoc, wm_filename)

    if not os.path.exists(weather_model_file):
        prepareWeatherModel(
            weather_model,
            time,
            wmLoc=wmLoc,
            lats=lats,
            lons=lons,
            ll_bounds=ll_bounds,
            download_only=download_only,
            makePlots=True,
        )
        try:
            weather_model.write2NETCDF4(weather_model_file)
        except:
            logger.exception('Unable to save weathermodel to file')

1	# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2	#
3	# Author: Jeremy Maurer, Raymond Hogenson & David Bekaert
4	# Copyright 2019, by the California Institute of Technology. ALL RIGHTS
5	# RESERVED. United States Government Sponsorship acknowledged.
6	#
7	# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
8	import os	1✔
9	from pathlib import Path	1✔
10
11	import matplotlib.pyplot as plt	1✔
12	import numpy as np	1✔
13
14	from RAiDER.logger import logger	1✔
15	from RAiDER.models.customExceptions import (	1✔
16	CriticalError,
17	DatetimeOutsideRange,
18	ExistingWeatherModelTooSmall,
19	TryToKeepGoingError,
20	)
21	from RAiDER.models.weatherModel import checkContainment_raw, make_raw_weather_data_filename, make_weather_model_filename	1✔
22
23
24	def prepareWeatherModel(	1✔
25	weather_model,
26	time,
27	ll_bounds,
28	download_only: bool = False,
29	makePlots: bool = False,
30	force_download: bool = False,
31	) -> str:
32	"""Parse inputs to download and prepare a weather model grid for interpolation.
33
34	Args:
35	weather_model: WeatherModel - instantiated weather model object
36	time: datetime - Python datetime to request. Will be rounded to nearest available time
37	ll_bounds: list/array - SNWE bounds target area to ensure weather model contains them
38	download_only: bool - False if preprocessing weather model data
39	makePlots: bool - whether to write debug plots
40	force_download: bool - True if you want to download even when the weather model exists
41
42	Returns:
43	str: filename of the netcdf file to which the weather model has been written
44	"""
45	# set the bounding box from the in the case that it hasn't been set
46	if weather_model.get_latlon_bounds() is None:	1✔
47	weather_model.set_latlon_bounds(ll_bounds)	×
48
49	# Ensure the file output location exists
50	wmLoc = weather_model.get_wmLoc()	1✔
51	weather_model.setTime(time)	1✔
52
53	# get the path to the less processed weather model file
54	path_wm_raw = make_raw_weather_data_filename(wmLoc, weather_model.Model(), time)	1✔
55
56	# get the path to the more processed (cropped) weather model file
57	path_wm_crop = weather_model.out_file(wmLoc)	1✔
58
59	# check whether weather model files exists and/or or should be downloaded
60	if os.path.exists(path_wm_crop) and not force_download:	1✔
61	logger.warning(	1✔
62	'Processed weather model already exists, please remove it ("%s") if you want to download a new one.',
63	path_wm_crop,
64	)
65
66	# check whether the raw weather model covers this area
67	elif os.path.exists(path_wm_raw) and checkContainment_raw(path_wm_raw, ll_bounds) and not force_download:	×
68	logger.warning(	×
69	'Raw weather model already exists, please remove it ("%s") if you want to download a new one.',
70	path_wm_raw,
71	)
72
73	# if no weather model files supplied, check the standard location
74	else:
75	os.makedirs(os.path.dirname(path_wm_raw), exist_ok=True)	×
76	try:	×
NEW 77	weather_model.fetch(Path(path_wm_raw), time)	×
78	except DatetimeOutsideRange:	×
79	raise TryToKeepGoingError	×
80
81	# If only downloading, exit now
82	if download_only:	1✔
83	logger.warning('download_only flag selected. No further processing will happen.')	×
84	return None	×
85
86	# Otherwise, load the weather model data
87	f = weather_model.load()	1✔
88
89	if f is not None:	1✔
90	logger.warning('The processed weather model file already exists, so I will use that.')	1✔
91
92	containment = weather_model.checkContainment(ll_bounds)	1✔
93	if not containment and weather_model.Model() not in 'HRRR HRRRAK'.split():	1✔
94	raise ExistingWeatherModelTooSmall	×
95
96	return f	1✔
97
98	# Logging some basic info
99	logger.debug('Number of weather model nodes: %s', np.prod(weather_model.getWetRefractivity().shape))	×
100	shape = weather_model.getWetRefractivity().shape	×
101	logger.debug(f'Shape of weather model: {shape}')	×
102	logger.debug(	×
103	'Bounds of the weather model: %.2f/%.2f/%.2f/%.2f (SNWE)',
104	np.nanmin(weather_model._ys),
105	np.nanmax(weather_model._ys),
106	np.nanmin(weather_model._xs),
107	np.nanmax(weather_model._xs),
108	)
109	logger.debug('Weather model: %s', weather_model.Model())	×
110	logger.debug('Mean value of the wet refractivity: %f', np.nanmean(weather_model.getWetRefractivity()))	×
111	logger.debug('Mean value of the hydrostatic refractivity: %f', np.nanmean(weather_model.getHydroRefractivity()))	×
112	logger.debug(weather_model)	×
113
114	if makePlots:	×
115	weather_model.plot('wh', True)	×
116	weather_model.plot('pqt', True)	×
117	plt.close('all')	×
118
119	try:	×
120	f = weather_model.write()	×
121	containment = weather_model.checkContainment(ll_bounds)	×
122
123	except Exception as e:	×
124	logger.exception('Unable to save weathermodel to file')	×
125	logger.exception(e)	×
126	raise CriticalError	×
127
128	finally:
129	wm = weather_model.Model()	×
130	del weather_model	×
131
132	if not containment and wm not in 'HRRR'.split():	×
133	raise ExistingWeatherModelTooSmall	×
134	else:
135	return f	×
136
137
138	def _weather_model_debug(los, lats, lons, ll_bounds, weather_model, wmLoc, time, out, download_only) -> None:	1✔
139	"""RaiderWeatherModelDebug main function."""
140	logger.debug('Starting to run the weather model calculation with debugging plots')	×
141	logger.debug('Time type: %s', type(time))	×
142	logger.debug('Time: %s', time.strftime('%Y%m%d'))	×
143
144	# location of the weather model files
145	logger.debug('Beginning weather model pre-processing')	×
146	logger.debug('Download-only is %s', download_only)	×
147	if wmLoc is None:	×
148	wmLoc = os.path.join(out, 'weather_files')	×
149
150	# weather model calculation
151	wm_filename = make_weather_model_filename(weather_model['name'], time, ll_bounds)	×
152	weather_model_file = os.path.join(wmLoc, wm_filename)	×
153
154	if not os.path.exists(weather_model_file):	×
155	prepareWeatherModel(	×
156	weather_model,
157	time,
158	wmLoc=wmLoc,
159	lats=lats,
160	lons=lons,
161	ll_bounds=ll_bounds,
162	download_only=download_only,
163	makePlots=True,
164	)
165	try:	×
166	weather_model.write2NETCDF4(weather_model_file)	×
167	except:	×
168	logger.exception('Unable to save weathermodel to file')	×

dbekaert / RAiDER / 305eb64a-9a20-4e4e-9c1e-8cc67e861cd6

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous