23369877357

Committed 21 Mar 2026 02:13AM UTC coverage: 90.512% (+0.02%) from 90.497%

Build # 23369877357

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1342 from spedas/t_1341

Adds support to load USGS Variometer data, adds test cases for loading variometer data.

Coverage Stats

13 of 48 new or added lines in 3 files covered. (27.08%)

10491 existing lines in 260 files now uncovered.

44303 of 48947 relevant lines covered (90.51%)

1.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/pyspedas/tplot_tools/tplot_math/degap.py

import pyspedas
from pyspedas.tplot_tools import store_data, get_data, tnames
import numpy as np
import copy
import logging


def degap(
    tvar,
    dt=None,
    margin=0.25,
    maxgap=None,
    func="nan",
    newname=None,
    onenanpergap=False,
    twonanpergap=False,
):
    """
    Fills gaps in the data either with NaNs or the last number.

    Parameters
    ----------
        tvar : str or list[str]
            Names of tplot variables to degap (wildcards accepted)
        dt : int/float
            Step size of the data in seconds, default is to use the median time interval
        margin : int/float, optional, default is 0.25 seconds
            The maximum deviation from the step size allowed before degapping occurs.  In other words, if you'd like to fill in data every 4 seconds
            but occasionally the data is 4.1 seconds apart, set the margin to .1 so that a data point is not inserted there.
        maxgap : int|float, optional
            Maximum gap length (in seconds) that will be filled.  If None, defaults to entire time range (i.e.
            all gaps with length > (dt+margin) will be filled)
        func : str, optional
            Either 'nan' or 'ffill', which overrides normal interpolation with NaN
            substitution or forward-filled values.
        newname : str, optional
            The new tplot variable name to store the data into.  If None, then the data is overwritten.
            This is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
        onenanpergap : bool
            if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
        twonanpergap : bool
            if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
    Returns
    -------
        None
            Creates a new tplot variable with the degap data

    Examples
    --------

        >>> import pyspedas
        >>> time = [pyspedas.time_float("2020-01-01") + i for i in [1, 2, 3, 4, 5, 6, 9, 10, 11]]
        >>> y = [1, 2, 3, 4, 5, 6, 9, 10, 11]
        >>> pyspedas.store_data("a", data={"x": time, "y": y})
        >>> degap("a", newname="b")
        >>> b = pyspedas.get("b")
        >>> print(b)
    """

    # check for globbed or array input, and call recursively
    tn = tnames(tvar)
    if len(tn) == 0:
        return
    elif len(tn) > 1:
        for j in range(len(tn)):
            degap(
                tn[j],
                dt=dt,
                margin=margin,
                func=func,
                onenanpergap=onenanpergap,
                twonanpergap=twonanpergap,
            )
        return

    # here we have 1 variable

    # fix from T.Hori, 2023-04-10, jimm02
    #    gap_size = np.diff(pyspedas.tplot_tools.data_quants[tvar].coords['time']) This is in Nanoseconds, and causes a type mismatch with dt+margin
    #    new_tvar_index = pyspedas.tplot_tools.data_quants[tvar].coords['time']
    new_tvar_index = get_data(tvar)[0]  # Unix time float64
    gap_size = np.diff(new_tvar_index)
    if maxgap is None:
        maxgap = np.nanmax(new_tvar_index)-np.nanmin(new_tvar_index)

    # Default for dt is the median value of gap_size, the time interval differences
    if dt == None:
        dt = np.median(gap_size)

    gap_index_locations = np.where((gap_size > dt + margin) & (gap_size < maxgap))
    values_to_add = np.array([])
    if onenanpergap == True:
        for i in gap_index_locations[0]:
            values_to_add = np.append(values_to_add, new_tvar_index[i] + dt)
    elif twonanpergap == True:
        # add two NaN values between the two values, either at margin if it's nonzero, or at dt/2
        # since the gap is greater than dt, this will work
        if margin > 0.0:
            if margin < dt / 2.0:
                dt_nan = margin
            else:
                dt_nan = dt / 2.0
        else:
            dt_nan = dt / 2.0
        for i in gap_index_locations[0]:
            values_to_add = np.append(values_to_add, new_tvar_index[i] + dt_nan)
            values_to_add = np.append(values_to_add, new_tvar_index[i + 1] - dt_nan)
    else:
        for i in gap_index_locations[0]:
            values_to_add = np.append(
                values_to_add, np.arange(new_tvar_index[i], new_tvar_index[i + 1], dt)
            )

    # new_index = np.sort(np.unique(np.concatenate((values_to_add, new_tvar_index))))
    new_index_float64 = np.sort(
        np.unique(np.concatenate((values_to_add, new_tvar_index)))
    )

    if func == "nan":
        method = None
    if func == "ffill":
        method = "ffill"

    # Replace any NaN or inf time values with 0.0  (Is this needed? It seems like it could result in non-monotonic times)
    cond=np.logical_not(np.isfinite(new_index_float64))
    new_index_float64[cond]=0.0

    # Convert back to datetime64 (nanoseconds since epoch)
    new_index=np.array(new_index_float64*1e9,dtype='datetime64[ns]')

    # This can fail if the stored quantities are np.datetime64 and new_index is something else, like datetime.datetime
    a = pyspedas.tplot_tools.data_quants[tvar].reindex({"time": new_index}, method=method)

    if newname is None:
        a.name = tvar
        a.attrs = copy.deepcopy(pyspedas.tplot_tools.data_quants[tvar].attrs)
        pyspedas.tplot_tools.data_quants[tvar] = copy.deepcopy(a)
    else:
        if "spec_bins" in a.coords:
            store_data(
                newname,
                data={"x": a.coords["time"], "y": a.values, "v": a.coords["spec_bins"]},
            )
            pyspedas.tplot_tools.data_quants[newname].attrs = copy.deepcopy(
                pyspedas.tplot_tools.data_quants[tvar].attrs
            )
        else:
            store_data(newname, data={"x": a.coords["time"], "y": a.values})
            pyspedas.tplot_tools.data_quants[newname].attrs = copy.deepcopy(
                pyspedas.tplot_tools.data_quants[tvar].attrs
            )

    return

1	import pyspedas	2✔
2	from pyspedas.tplot_tools import store_data, get_data, tnames	2✔
3	import numpy as np	2✔
4	import copy	2✔
5	import logging	2✔
6
7
8	def degap(	2✔
9	tvar,
10	dt=None,
11	margin=0.25,
12	maxgap=None,
13	func="nan",
14	newname=None,
15	onenanpergap=False,
16	twonanpergap=False,
17	):
18	"""
19	Fills gaps in the data either with NaNs or the last number.
20
21	Parameters
22	----------
23	tvar : str or list[str]
24	Names of tplot variables to degap (wildcards accepted)
25	dt : int/float
26	Step size of the data in seconds, default is to use the median time interval
27	margin : int/float, optional, default is 0.25 seconds
28	The maximum deviation from the step size allowed before degapping occurs. In other words, if you'd like to fill in data every 4 seconds
29	but occasionally the data is 4.1 seconds apart, set the margin to .1 so that a data point is not inserted there.
30	maxgap : int\|float, optional
31	Maximum gap length (in seconds) that will be filled. If None, defaults to entire time range (i.e.
32	all gaps with length > (dt+margin) will be filled)
33	func : str, optional
34	Either 'nan' or 'ffill', which overrides normal interpolation with NaN
35	substitution or forward-filled values.
36	newname : str, optional
37	The new tplot variable name to store the data into. If None, then the data is overwritten.
38	This is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
39	onenanpergap : bool
40	if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
41	twonanpergap : bool
42	if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
43	Returns
44	-------
45	None
46	Creates a new tplot variable with the degap data
47
48	Examples
49	--------
50
51	>>> import pyspedas
52	>>> time = [pyspedas.time_float("2020-01-01") + i for i in [1, 2, 3, 4, 5, 6, 9, 10, 11]]
53	>>> y = [1, 2, 3, 4, 5, 6, 9, 10, 11]
54	>>> pyspedas.store_data("a", data={"x": time, "y": y})
55	>>> degap("a", newname="b")
56	>>> b = pyspedas.get("b")
57	>>> print(b)
58	"""
59
60	# check for globbed or array input, and call recursively
61	tn = tnames(tvar)	2✔
62	if len(tn) == 0:	2✔
63	return	×
64	elif len(tn) > 1:	2✔
65	for j in range(len(tn)):	×
66	degap(	×
67	tn[j],
68	dt=dt,
69	margin=margin,
70	func=func,
71	onenanpergap=onenanpergap,
72	twonanpergap=twonanpergap,
73	)
74	return	×
75
76	# here we have 1 variable
77
78	# fix from T.Hori, 2023-04-10, jimm02
79	# gap_size = np.diff(pyspedas.tplot_tools.data_quants[tvar].coords['time']) This is in Nanoseconds, and causes a type mismatch with dt+margin
80	# new_tvar_index = pyspedas.tplot_tools.data_quants[tvar].coords['time']
81	new_tvar_index = get_data(tvar)[0] # Unix time float64	2✔
82	gap_size = np.diff(new_tvar_index)	2✔
83	if maxgap is None:	2✔
84	maxgap = np.nanmax(new_tvar_index)-np.nanmin(new_tvar_index)	2✔
85
86	# Default for dt is the median value of gap_size, the time interval differences
87	if dt == None:	2✔
UNCOV 88	dt = np.median(gap_size)	1✔
89
90	gap_index_locations = np.where((gap_size > dt + margin) & (gap_size < maxgap))	2✔
91	values_to_add = np.array([])	2✔
92	if onenanpergap == True:	2✔
UNCOV 93	for i in gap_index_locations[0]:	1✔
UNCOV 94	values_to_add = np.append(values_to_add, new_tvar_index[i] + dt)	1✔
95	elif twonanpergap == True:	2✔
96	# add two NaN values between the two values, either at margin if it's nonzero, or at dt/2
97	# since the gap is greater than dt, this will work
UNCOV 98	if margin > 0.0:	1✔
UNCOV 99	if margin < dt / 2.0:	1✔
UNCOV 100	dt_nan = margin	1✔
101	else:
102	dt_nan = dt / 2.0	×
103	else:
104	dt_nan = dt / 2.0	×
UNCOV 105	for i in gap_index_locations[0]:	1✔
UNCOV 106	values_to_add = np.append(values_to_add, new_tvar_index[i] + dt_nan)	1✔
UNCOV 107	values_to_add = np.append(values_to_add, new_tvar_index[i + 1] - dt_nan)	1✔
108	else:
109	for i in gap_index_locations[0]:	2✔
UNCOV 110	values_to_add = np.append(	1✔
111	values_to_add, np.arange(new_tvar_index[i], new_tvar_index[i + 1], dt)
112	)
113
114	# new_index = np.sort(np.unique(np.concatenate((values_to_add, new_tvar_index))))
115	new_index_float64 = np.sort(	2✔
116	np.unique(np.concatenate((values_to_add, new_tvar_index)))
117	)
118
119	if func == "nan":	2✔
120	method = None	2✔
121	if func == "ffill":	2✔
122	method = "ffill"	×
123
124	# Replace any NaN or inf time values with 0.0 (Is this needed? It seems like it could result in non-monotonic times)
125	cond=np.logical_not(np.isfinite(new_index_float64))	2✔
126	new_index_float64[cond]=0.0	2✔
127
128	# Convert back to datetime64 (nanoseconds since epoch)
129	new_index=np.array(new_index_float64*1e9,dtype='datetime64[ns]')	2✔
130
131	# This can fail if the stored quantities are np.datetime64 and new_index is something else, like datetime.datetime
132	a = pyspedas.tplot_tools.data_quants[tvar].reindex({"time": new_index}, method=method)	2✔
133
134	if newname is None:	2✔
135	a.name = tvar	2✔
136	a.attrs = copy.deepcopy(pyspedas.tplot_tools.data_quants[tvar].attrs)	2✔
137	pyspedas.tplot_tools.data_quants[tvar] = copy.deepcopy(a)	2✔
138	else:
UNCOV 139	if "spec_bins" in a.coords:	1✔
140	store_data(	×
141	newname,
142	data={"x": a.coords["time"], "y": a.values, "v": a.coords["spec_bins"]},
143	)
144	pyspedas.tplot_tools.data_quants[newname].attrs = copy.deepcopy(	×
145	pyspedas.tplot_tools.data_quants[tvar].attrs
146	)
147	else:
UNCOV 148	store_data(newname, data={"x": a.coords["time"], "y": a.values})	1✔
UNCOV 149	pyspedas.tplot_tools.data_quants[newname].attrs = copy.deepcopy(	1✔
150	pyspedas.tplot_tools.data_quants[tvar].attrs
151	)
152
153	return	2✔

spedas / pyspedas / 23369877357

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous