16972251084

Committed 14 Aug 2025 05:28PM UTC coverage: 89.527% (+0.01%) from 89.515%

Build # 16972251084

Build Type

push

github

Committed by

jameswilburlewis

Commit Message

Tweaked wording and formatting for readthedocs

Run Details

40100 of 44791 relevant lines covered (89.53%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.7

/pyspedas/pytplot/tplot_math/degap.py

import pyspedas
from pyspedas.pytplot import store_data, get_data, tnames
import numpy as np
import copy
import logging


def degap(
    tvar,
    dt=None,
    margin=0.25,
    maxgap=None,
    func="nan",
    newname=None,
    new_tvar=None,
    onenanpergap=False,
    twonanpergap=False,
):
    """
    Fills gaps in the data either with NaNs or the last number.

    Parameters
    ----------
        tvar : str or list[str]
            Names of tplot variables to degap (wildcards accepted)
        dt : int/float
            Step size of the data in seconds, default is to use the median time interval
        margin : int/float, optional, default is 0.25 seconds
            The maximum deviation from the step size allowed before degapping occurs.  In other words, if you'd like to fill in data every 4 seconds
            but occasionally the data is 4.1 seconds apart, set the margin to .1 so that a data point is not inserted there.
        maxgap : int|float, optional
            Maximum gap length (in seconds) that will be filled.  If None, defaults to entire time range (i.e.
            all gaps with length > (dt+margin) will be filled)
        func : str, optional
            Either 'nan' or 'ffill', which overrides normal interpolation with NaN
            substitution or forward-filled values.
        newname : str, optional
            The new tplot variable name to store the data into.  If None, then the data is overwritten.
            THIS is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
        new_tvar : str, optional (Deprecated)
            The new tplot variable name to store the data into.  If None, then the data is overwritten.
            THIS is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
        onenanpergap : bool
            if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
        twonanpergap : bool
            if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
    Returns
    -------
        None
            Creates a new tplot variable with the degap data

    Examples
    --------

        >>> import pyspedas
        >>> time = [pyspedas.time_float("2020-01-01") + i for i in [1, 2, 3, 4, 5, 6, 9, 10, 11]]
        >>> y = [1, 2, 3, 4, 5, 6, 9, 10, 11]
        >>> pyspedas.store_data("a", data={"x": time, "y": y})
        >>> degap("a", newname="b")
        >>> b = pyspedas.get("b")
        >>> print(b)
    """

    # new_tvar is deprecated in favor of newname
    if new_tvar is not None:
        logging.info(
            "degap: The new_tvar parameter is deprecated. Please use newname instead."
        )
        newname = new_tvar

    # check for globbed or array input, and call recursively
    tn = tnames(tvar)
    if len(tn) == 0:
        return
    elif len(tn) > 1:
        for j in range(len(tn)):
            degap(
                tn[j],
                dt=dt,
                margin=margin,
                func=func,
                onenanpergap=onenanpergap,
                twonanpergap=twonanpergap,
            )
        return

    # here we have 1 variable

    # fix from T.Hori, 2023-04-10, jimm02
    #    gap_size = np.diff(pyspedas.pytplot.data_quants[tvar].coords['time']) This is in Nanoseconds, and causes a type mismatch with dt+margin
    #    new_tvar_index = pyspedas.pytplot.data_quants[tvar].coords['time']
    new_tvar_index = get_data(tvar)[0]  # Unix time float64
    gap_size = np.diff(new_tvar_index)
    if maxgap is None:
        maxgap = np.nanmax(new_tvar_index)-np.nanmin(new_tvar_index)

    # Default for dt is the median value of gap_size, the time interval differences
    if dt == None:
        dt = np.median(gap_size)

    gap_index_locations = np.where((gap_size > dt + margin) & (gap_size < maxgap))
    values_to_add = np.array([])
    if onenanpergap == True:
        for i in gap_index_locations[0]:
            values_to_add = np.append(values_to_add, new_tvar_index[i] + dt)
    elif twonanpergap == True:
        # add two NaN values between the two values, either at margin if it's nonzero, or at dt/2
        # since the gap is greater than dt, this will work
        if margin > 0.0:
            if margin < dt / 2.0:
                dt_nan = margin
            else:
                dt_nan = dt / 2.0
        else:
            dt_nan = dt / 2.0
        for i in gap_index_locations[0]:
            values_to_add = np.append(values_to_add, new_tvar_index[i] + dt_nan)
            values_to_add = np.append(values_to_add, new_tvar_index[i + 1] - dt_nan)
    else:
        for i in gap_index_locations[0]:
            values_to_add = np.append(
                values_to_add, np.arange(new_tvar_index[i], new_tvar_index[i + 1], dt)
            )

    # new_index = np.sort(np.unique(np.concatenate((values_to_add, new_tvar_index))))
    new_index_float64 = np.sort(
        np.unique(np.concatenate((values_to_add, new_tvar_index)))
    )

    if func == "nan":
        method = None
    if func == "ffill":
        method = "ffill"

    # Replace any NaN or inf time values with 0.0  (Is this needed? It seems like it could result in non-monotonic times)
    cond=np.logical_not(np.isfinite(new_index_float64))
    new_index_float64[cond]=0.0

    # Convert back to datetime64 (nanoseconds since epoch)
    new_index=np.array(new_index_float64*1e9,dtype='datetime64[ns]')

    # This can fail if the stored quantities are np.datetime64 and new_index is something else, like datetime.datetime
    a = pyspedas.pytplot.data_quants[tvar].reindex({"time": new_index}, method=method)

    if newname is None:
        a.name = tvar
        a.attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)
        pyspedas.pytplot.data_quants[tvar] = copy.deepcopy(a)
    else:
        if "spec_bins" in a.coords:
            store_data(
                newname,
                data={"x": a.coords["time"], "y": a.values, "v": a.coords["spec_bins"]},
            )
            pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(
                pyspedas.pytplot.data_quants[tvar].attrs
            )
        else:
            store_data(newname, data={"x": a.coords["time"], "y": a.values})
            pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(
                pyspedas.pytplot.data_quants[tvar].attrs
            )

    return

1	import pyspedas	1✔
2	from pyspedas.pytplot import store_data, get_data, tnames	1✔
3	import numpy as np	1✔
4	import copy	1✔
5	import logging	1✔
6
7
8	def degap(	1✔
9	tvar,
10	dt=None,
11	margin=0.25,
12	maxgap=None,
13	func="nan",
14	newname=None,
15	new_tvar=None,
16	onenanpergap=False,
17	twonanpergap=False,
18	):
19	"""
20	Fills gaps in the data either with NaNs or the last number.
21
22	Parameters
23	----------
24	tvar : str or list[str]
25	Names of tplot variables to degap (wildcards accepted)
26	dt : int/float
27	Step size of the data in seconds, default is to use the median time interval
28	margin : int/float, optional, default is 0.25 seconds
29	The maximum deviation from the step size allowed before degapping occurs. In other words, if you'd like to fill in data every 4 seconds
30	but occasionally the data is 4.1 seconds apart, set the margin to .1 so that a data point is not inserted there.
31	maxgap : int\|float, optional
32	Maximum gap length (in seconds) that will be filled. If None, defaults to entire time range (i.e.
33	all gaps with length > (dt+margin) will be filled)
34	func : str, optional
35	Either 'nan' or 'ffill', which overrides normal interpolation with NaN
36	substitution or forward-filled values.
37	newname : str, optional
38	The new tplot variable name to store the data into. If None, then the data is overwritten.
39	THIS is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
40	new_tvar : str, optional (Deprecated)
41	The new tplot variable name to store the data into. If None, then the data is overwritten.
42	THIS is not an option for multiple variable input, for multiple or pseudo variables, the data is overwritten.
43	onenanpergap : bool
44	if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
45	twonanpergap : bool
46	if set to True, then only insert one NaN value, rather than adding NaN values at dt resolution
47	Returns
48	-------
49	None
50	Creates a new tplot variable with the degap data
51
52	Examples
53	--------
54
55	>>> import pyspedas
56	>>> time = [pyspedas.time_float("2020-01-01") + i for i in [1, 2, 3, 4, 5, 6, 9, 10, 11]]
57	>>> y = [1, 2, 3, 4, 5, 6, 9, 10, 11]
58	>>> pyspedas.store_data("a", data={"x": time, "y": y})
59	>>> degap("a", newname="b")
60	>>> b = pyspedas.get("b")
61	>>> print(b)
62	"""
63
64	# new_tvar is deprecated in favor of newname
65	if new_tvar is not None:	1✔
66	logging.info(	×
67	"degap: The new_tvar parameter is deprecated. Please use newname instead."
68	)
69	newname = new_tvar	×
70
71	# check for globbed or array input, and call recursively
72	tn = tnames(tvar)	1✔
73	if len(tn) == 0:	1✔
74	return	×
75	elif len(tn) > 1:	1✔
76	for j in range(len(tn)):	×
77	degap(	×
78	tn[j],
79	dt=dt,
80	margin=margin,
81	func=func,
82	onenanpergap=onenanpergap,
83	twonanpergap=twonanpergap,
84	)
85	return	×
86
87	# here we have 1 variable
88
89	# fix from T.Hori, 2023-04-10, jimm02
90	# gap_size = np.diff(pyspedas.pytplot.data_quants[tvar].coords['time']) This is in Nanoseconds, and causes a type mismatch with dt+margin
91	# new_tvar_index = pyspedas.pytplot.data_quants[tvar].coords['time']
92	new_tvar_index = get_data(tvar)[0] # Unix time float64	1✔
93	gap_size = np.diff(new_tvar_index)	1✔
94	if maxgap is None:	1✔
95	maxgap = np.nanmax(new_tvar_index)-np.nanmin(new_tvar_index)	1✔
96
97	# Default for dt is the median value of gap_size, the time interval differences
98	if dt == None:	1✔
99	dt = np.median(gap_size)	1✔
100
101	gap_index_locations = np.where((gap_size > dt + margin) & (gap_size < maxgap))	1✔
102	values_to_add = np.array([])	1✔
103	if onenanpergap == True:	1✔
104	for i in gap_index_locations[0]:	1✔
105	values_to_add = np.append(values_to_add, new_tvar_index[i] + dt)	1✔
106	elif twonanpergap == True:	1✔
107	# add two NaN values between the two values, either at margin if it's nonzero, or at dt/2
108	# since the gap is greater than dt, this will work
109	if margin > 0.0:	1✔
110	if margin < dt / 2.0:	1✔
111	dt_nan = margin	1✔
112	else:
113	dt_nan = dt / 2.0	×
114	else:
115	dt_nan = dt / 2.0	×
116	for i in gap_index_locations[0]:	1✔
117	values_to_add = np.append(values_to_add, new_tvar_index[i] + dt_nan)	1✔
118	values_to_add = np.append(values_to_add, new_tvar_index[i + 1] - dt_nan)	1✔
119	else:
120	for i in gap_index_locations[0]:	1✔
121	values_to_add = np.append(	1✔
122	values_to_add, np.arange(new_tvar_index[i], new_tvar_index[i + 1], dt)
123	)
124
125	# new_index = np.sort(np.unique(np.concatenate((values_to_add, new_tvar_index))))
126	new_index_float64 = np.sort(	1✔
127	np.unique(np.concatenate((values_to_add, new_tvar_index)))
128	)
129
130	if func == "nan":	1✔
131	method = None	1✔
132	if func == "ffill":	1✔
133	method = "ffill"	×
134
135	# Replace any NaN or inf time values with 0.0 (Is this needed? It seems like it could result in non-monotonic times)
136	cond=np.logical_not(np.isfinite(new_index_float64))	1✔
137	new_index_float64[cond]=0.0	1✔
138
139	# Convert back to datetime64 (nanoseconds since epoch)
140	new_index=np.array(new_index_float64*1e9,dtype='datetime64[ns]')	1✔
141
142	# This can fail if the stored quantities are np.datetime64 and new_index is something else, like datetime.datetime
143	a = pyspedas.pytplot.data_quants[tvar].reindex({"time": new_index}, method=method)	1✔
144
145	if newname is None:	1✔
146	a.name = tvar	1✔
147	a.attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)	1✔
148	pyspedas.pytplot.data_quants[tvar] = copy.deepcopy(a)	1✔
149	else:
150	if "spec_bins" in a.coords:	1✔
151	store_data(	×
152	newname,
153	data={"x": a.coords["time"], "y": a.values, "v": a.coords["spec_bins"]},
154	)
155	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(	×
156	pyspedas.pytplot.data_quants[tvar].attrs
157	)
158	else:
159	store_data(newname, data={"x": a.coords["time"], "y": a.values})	1✔
160	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(	1✔
161	pyspedas.pytplot.data_quants[tvar].attrs
162	)
163
164	return	1✔

spedas / pyspedas / 16972251084

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous