4f753398-e5bc-44cd-96d0-6e7eb936a270

Committed 29 Jun 2025 03:46AM UTC coverage: 94.756% (+0.5%) from 94.207%

Build # 4f753398-e5bc-44cd-96d0-6e7eb936a270

Build Type

Pull #745

circleci

Committed by

garlic-os

Commit Message

Update CHANGELOG.md for #745

Pull Request Pull Request #745: Run tests in parallel, use temp directories for all test artifacts

Run Details

425 of 516 new or added lines in 24 files covered. (82.36%)

2 existing lines in 2 files now uncovered.

3198 of 3375 relevant lines covered (94.76%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

18.06

/test/test_temporal_interpolate.py

from pathlib import Path
import pytest
import subprocess
import numpy as np
import xarray as xr


from test import WM

from RAiDER.logger import logger
from RAiDER.utilFcns import write_yaml

wm = 'ERA5' if WM == 'ERA-5' else WM


@pytest.mark.long
def test_cube_timemean(tmp_path: Path):
    """Test the mean interpolation by computing cube delays at 1:30PM vs mean of 12 PM / 3PM for GMAO."""
    ## make the lat lon grid
    S, N, W, E = 34, 35, -117, -116
    date       = 20200130
    dct_hrs    = {'GMAO': [12, 15, '13:30:00'], 'MERRA2': [12, 15, '13:30:00'], 'ERA5': [13, 14, '13:30:00']}
    hr1, hr2, ti = dct_hrs[wm]

    grp = {
            'date_group': {'date_start': date},
            'weather_model': WM,
            'aoi_group': {'bounding_box': [S, N, W, E]},
            'time_group': {'interpolate_time': 'none'},
            'runtime_group': {'output_directory': tmp_path},
        }

    ## run raider without interpolation for two exact weather model times
    for hr in [hr1, hr2]:
        grp['time_group'].update({'time': f'{hr}:00:00'})
        ## generate the default run config file and overwrite it with new parms
        cfg  = write_yaml(grp, tmp_path / 'temp1.yaml')

        ## run raider for the default date
        cmd  = f'raider.py {cfg}'
        proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
        assert proc.returncode == 0

    ## run interpolation in the middle of the two
    grp['time_group'] =  {'time': ti, 'interpolate_time': 'center_time'}
    cfg  = write_yaml(grp, tmp_path / 'temp2.yaml')

    cmd  = f'raider.py {cfg}'
    proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
    assert proc.returncode == 0

    with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr1}0000_ztd.nc') as ds:
        da1_tot = ds['wet'] + ds['hydro']

    with xr.open_dataset(tmp_path/ f'{WM}_tropo_{date}T{hr2}0000_ztd.nc') as ds:
        da2_tot = ds['wet'] + ds['hydro']

    with xr.open_dataset(tmp_path/ f'{WM}_tropo_{date}T{ti.replace(":", "")}_ztd.nc') as ds:
        da_interp_tot = ds['wet'] + ds['hydro']

    da_mu = (da1_tot + da2_tot) / 2
    assert np.allclose(da_mu, da_interp_tot)


@pytest.mark.long
def test_cube_weighting(tmp_path: Path):
    """Test the weighting by comparing a small crop with numpy directly."""
    import datetime as dt
    ## make the lat lon grid
    S, N, W, E = 34, 35, -117, -116
    date       = 20200130
    dct_hrs    = {'GMAO': [12, 15, '12:05:00'], 'MERRA2': [12, 15, '13:30:00'], 'ERA5': [13, 14, '13:05:00']}
    hr1, hr2, ti = dct_hrs[WM]

    grp = {
            'date_group': {'date_start': date},
            'weather_model': WM,
            'aoi_group': {'bounding_box': [S, N, W, E]},
            'time_group': {'interpolate_time': 'none'},
            'runtime_group': {'output_directory': tmp_path},
        }


    ## run raider without interpolation for two exact weather model times
    for hr in [hr1, hr2]:
        grp['time_group'].update({'time': f'{hr}:00:00'})
        ## generate the default run config file and overwrite it with new parms
        cfg  = write_yaml(grp, tmp_path / 'temp1.yaml')

        ## run raider for the default date
        cmd  = f'raider.py {cfg}'
        proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
        assert proc.returncode == 0

    ## run interpolation very near the first
    grp['time_group'] =  {'time': ti, 'interpolate_time': 'center_time'}
    cfg  = write_yaml(grp, tmp_path / 'temp2.yaml')

    cmd  = f'raider.py {cfg}'
    proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)

    ## double check on weighting

    with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr1}0000_ztd.nc') as ds:
        da1_tot = ds['wet'] + ds['hydro']

    with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr2}0000_ztd.nc') as ds:
        da2_tot = ds['wet'] + ds['hydro']

    with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{ti.replace(":", "")}_ztd.nc') as ds:
        da_interp_tot = ds['wet'] + ds['hydro']

    dt1 = dt.datetime.strptime(f'{date}{hr1}', '%Y%m%d%H')
    dt2 = dt.datetime.strptime(f'{date}{hr2}', '%Y%m%d%H')
    dt_ref = dt.datetime.strptime(f'{date}{ti}', '%Y%m%d%H:%M:%S')

    wgts  = np.array([(dt_ref-dt1).seconds, (dt2-dt_ref).seconds])
    da1_crop = da1_tot.isel(z=0, y=slice(0,1), x=slice(0, 2))
    da2_crop = da2_tot.isel(z=0, y=slice(0,1), x=slice(0, 2))
    da_out_crop = da_interp_tot.isel(z=0, y=slice(0,1), x=slice(0,2))

    dat = np.vstack([da1_crop.data, da2_crop.data])

    logger.info ('Tstart: %s, Tend: %s, Tref: %s', dt1, dt2, dt_ref)
    logger.info ('Weights: %s', wgts)
    logger.info ('Data from two dates: %s', dat)
    logger.info ('Weighted mean: %s', da_out_crop.data)
    assert np.allclose(da_out_crop, np.average(dat, weights=1/wgts, axis=0))

1	from pathlib import Path	1✔
2	import pytest	1✔
3	import subprocess	1✔
4	import numpy as np	1✔
5	import xarray as xr	1✔
6
7
8	from test import WM	1✔
9
10	from RAiDER.logger import logger	1✔
11	from RAiDER.utilFcns import write_yaml	1✔
12
13	wm = 'ERA5' if WM == 'ERA-5' else WM	1✔
14
15
16	@pytest.mark.long	1✔
17	def test_cube_timemean(tmp_path: Path):	1✔
18	"""Test the mean interpolation by computing cube delays at 1:30PM vs mean of 12 PM / 3PM for GMAO."""
19	## make the lat lon grid
20	S, N, W, E = 34, 35, -117, -116	×
21	date = 20200130	×
22	dct_hrs = {'GMAO': [12, 15, '13:30:00'], 'MERRA2': [12, 15, '13:30:00'], 'ERA5': [13, 14, '13:30:00']}	×
23	hr1, hr2, ti = dct_hrs[wm]	×
24
25	grp = {	×
26	'date_group': {'date_start': date},
27	'weather_model': WM,
28	'aoi_group': {'bounding_box': [S, N, W, E]},
29	'time_group': {'interpolate_time': 'none'},
30	'runtime_group': {'output_directory': tmp_path},
31	}
32
33	## run raider without interpolation for two exact weather model times
UNCOV 34	for hr in [hr1, hr2]:	×
35	grp['time_group'].update({'time': f'{hr}:00:00'})	×
36	## generate the default run config file and overwrite it with new parms
NEW 37	cfg = write_yaml(grp, tmp_path / 'temp1.yaml')	×
38
39	## run raider for the default date
40	cmd = f'raider.py {cfg}'	×
41	proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)	×
NEW 42	assert proc.returncode == 0	×
43
44	## run interpolation in the middle of the two
45	grp['time_group'] = {'time': ti, 'interpolate_time': 'center_time'}	×
NEW 46	cfg = write_yaml(grp, tmp_path / 'temp2.yaml')	×
47
48	cmd = f'raider.py {cfg}'	×
49	proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)	×
NEW 50	assert proc.returncode == 0	×
51
NEW 52	with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr1}0000_ztd.nc') as ds:	×
53	da1_tot = ds['wet'] + ds['hydro']	×
54
NEW 55	with xr.open_dataset(tmp_path/ f'{WM}_tropo_{date}T{hr2}0000_ztd.nc') as ds:	×
56	da2_tot = ds['wet'] + ds['hydro']	×
57
NEW 58	with xr.open_dataset(tmp_path/ f'{WM}_tropo_{date}T{ti.replace(":", "")}_ztd.nc') as ds:	×
59	da_interp_tot = ds['wet'] + ds['hydro']	×
60
61	da_mu = (da1_tot + da2_tot) / 2	×
62	assert np.allclose(da_mu, da_interp_tot)	×
63
64
65	@pytest.mark.long	1✔
66	def test_cube_weighting(tmp_path: Path):	1✔
67	"""Test the weighting by comparing a small crop with numpy directly."""
NEW 68	import datetime as dt	×
69	## make the lat lon grid
70	S, N, W, E = 34, 35, -117, -116	×
71	date = 20200130	×
72	dct_hrs = {'GMAO': [12, 15, '12:05:00'], 'MERRA2': [12, 15, '13:30:00'], 'ERA5': [13, 14, '13:05:00']}	×
73	hr1, hr2, ti = dct_hrs[WM]	×
74
75	grp = {	×
76	'date_group': {'date_start': date},
77	'weather_model': WM,
78	'aoi_group': {'bounding_box': [S, N, W, E]},
79	'time_group': {'interpolate_time': 'none'},
80	'runtime_group': {'output_directory': tmp_path},
81	}
82
83
84	## run raider without interpolation for two exact weather model times
85	for hr in [hr1, hr2]:	×
86	grp['time_group'].update({'time': f'{hr}:00:00'})	×
87	## generate the default run config file and overwrite it with new parms
NEW 88	cfg = write_yaml(grp, tmp_path / 'temp1.yaml')	×
89
90	## run raider for the default date
91	cmd = f'raider.py {cfg}'	×
92	proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)	×
NEW 93	assert proc.returncode == 0	×
94
95	## run interpolation very near the first
96	grp['time_group'] = {'time': ti, 'interpolate_time': 'center_time'}	×
NEW 97	cfg = write_yaml(grp, tmp_path / 'temp2.yaml')	×
98
99	cmd = f'raider.py {cfg}'	×
100	proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)	×
101
102	## double check on weighting
103
NEW 104	with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr1}0000_ztd.nc') as ds:	×
105	da1_tot = ds['wet'] + ds['hydro']	×
106
NEW 107	with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{hr2}0000_ztd.nc') as ds:	×
108	da2_tot = ds['wet'] + ds['hydro']	×
109
NEW 110	with xr.open_dataset(tmp_path / f'{WM}_tropo_{date}T{ti.replace(":", "")}_ztd.nc') as ds:	×
111	da_interp_tot = ds['wet'] + ds['hydro']	×
112
NEW 113	dt1 = dt.datetime.strptime(f'{date}{hr1}', '%Y%m%d%H')	×
NEW 114	dt2 = dt.datetime.strptime(f'{date}{hr2}', '%Y%m%d%H')	×
NEW 115	dt_ref = dt.datetime.strptime(f'{date}{ti}', '%Y%m%d%H:%M:%S')	×
116
117	wgts = np.array([(dt_ref-dt1).seconds, (dt2-dt_ref).seconds])	×
118	da1_crop = da1_tot.isel(z=0, y=slice(0,1), x=slice(0, 2))	×
119	da2_crop = da2_tot.isel(z=0, y=slice(0,1), x=slice(0, 2))	×
120	da_out_crop = da_interp_tot.isel(z=0, y=slice(0,1), x=slice(0,2))	×
121
122	dat = np.vstack([da1_crop.data, da2_crop.data])	×
123
124	logger.info ('Tstart: %s, Tend: %s, Tref: %s', dt1, dt2, dt_ref)	×
125	logger.info ('Weights: %s', wgts)	×
126	logger.info ('Data from two dates: %s', dat)	×
127	logger.info ('Weighted mean: %s', da_out_crop.data)	×
128	assert np.allclose(da_out_crop, np.average(dat, weights=1/wgts, axis=0))	×

dbekaert / RAiDER / 4f753398-e5bc-44cd-96d0-6e7eb936a270

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous