16972251084

Committed 14 Aug 2025 05:28PM UTC coverage: 89.527% (+0.01%) from 89.515%

Build # 16972251084

Build Type

push

github

Committed by

jameswilburlewis

Commit Message

Tweaked wording and formatting for readthedocs

Run Details

40100 of 44791 relevant lines covered (89.53%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.99

/pyspedas/pytplot/tplot_math/deflag.py

import pyspedas
from pyspedas.pytplot import store_data, get_data, tnames
import copy
import numpy as np
import logging


def deflag(tvar, flag=None, newname=None, new_tvar=None, method=None, fillval=None):
    """
    Replace NaN or other 'flag' values in arrays with interpolated or other values.

    Parameters
    ----------

        tvar: str or list[str]
            Names of tplot variables to deflag (wildcards accepted)
        flag : int,list
            Flagged data will be converted to NaNs.
        method : str, optional
            Method to apply. Valid options::

                'repeat': Repeat last good value
                'linear': Interpolate linearly over gap
                'replace': Replace flagged values with fillval, or NaN if fillval not specified
                'remove_nan': Remove timestamps and values with a NaN in any dimension

        newname : str
            Name of new tvar for deflagged data storage.
            If not specified, then the data in tvar1 will be replaced.
            This is not an option for multiple variable input, for
            multiple or pseudo variables, the data is overwritten.
        new_tvar : str (Deprecated)
            Name of new tvar for deflagged data storage.  
            If not specified, then the data in tvar1 will be replaced.
            THIS is not an option for multiple variable input, for 
            multiple or pseudo variables, the data is overwritten.
        fillval: int, float (optional)
            Value to use as replacement when method='replace'

    Notes
    -----

       deflag only works for 1 or 2-d data arrays; ntimes or (ntimes, nspectral_bins)

    Returns
    -------

        None

    Examples
    --------

        >>> pyspedas.store_data('d', data={'x':[2,5,8,11,14,17,21], 'y':[[1,1],[2,2],[100,4],[4,90],[5,5],[6,6],[7,7]]})
        >>> # Remove any instances of [100,90,7,2,57] from 'd', store in 'e'.
        >>> pyspedas.deflag('d',[100,90,7,2,57],newname='e')

    """

    # new_tvar is deprecated in favor of newname
    if new_tvar is not None:
        logging.info("deflag: The new_tvar parameter is deprecated. Please use newname instead.")
        newname = new_tvar

    # for linear method, and flag of NaN, or none, interp_nan an be called
    #    if (flag == None or np.isnan(flag)) and method == 'linear':
    #        interp_nan(tvar, newname=newname)
    #        return
    
    # check for globbed or array input, and call recursively
    tn = tnames(tvar)
    if len(tn) == 0:
        return
    elif len(tn) > 1:
        for j in range(len(tn)):
            deflag(tn[j], flag, method=method, fillval=fillval)
        return

    # Now flag needs to be an array
    if flag is None:
        flag = np.zeros(1)+np.nan
    else:
        if not isinstance(flag, np.ndarray):
            flag_array = np.array(flag)
        else:
            flag_array = flag
        # Ok, now if flag was a scalar, the array will have ndim = 0
        if flag_array.ndim == 0:
            flag = np.array([flag])
        else:
            flag = flag_array
    
    nf = len(flag)
    if method == 'remove_nan':  # this is different from the other methods, which retain all time intervals
        a = copy.deepcopy(get_data(tvar))
        alen = len(a)
        # Ignore more than 2d Y input
        if alen > 3:
            logging.info('deflag is not used for more than 2-d input')
            return

        time = a[0]
        data = a[1]
        # Added this to prevent error on !-d arrays (see below)
        data_dim = data.ndim
        new_time = []
        new_data = []
        
        if alen == 3:  # v variable
            v = a[2]
            if v.ndim == 1:
                new_v = v
                append_v = False
            else:
                new_v = []
                append_v = True

        # Fill the new variable
        non_nan_seen = False
        for j in range(len(time)):
            # This used to be "if len(data[j]) > 1", which failed if data is 1-D so that data[j] is scalar
            # Instead we go by the dimensions of the data array itself
            if data_dim > 1:
                tj = np.sum(data[j])
            else:
                tj = data[j]
            if not np.isnan(tj):
                non_nan_seen = True
                new_time.append(time[j])
                new_data.append(data[j])
                if alen == 3 and append_v:
                    new_v.append(v[j])
        if non_nan_seen is False:
            logging.warning('No unflagged data in %s, returning.', tvar)
            return
        if newname is None:
            if alen == 2:
                store_data(tvar, data={'x': new_time, 'y': new_data})
            else:
                store_data(tvar, data={'x': new_time, 'y': new_data, 'v': new_v})
        else:
            if alen == 2:
                store_data(newname, data={'x': new_time, 'y': new_data})
            else:
                store_data(newname, data={'x': new_time, 'y': new_data, 'v': new_v})
            pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)
    elif method == 'repeat' or method == 'linear' or method == 'replace':
        a = copy.deepcopy(get_data(tvar))
        time = a[0]
        data = a[1]
        # Force the data into a 2-d view, retaining the original shape so we can restore it later
        original_data_shape = data.shape
        data = data.reshape(len(time), -1)
        alen = len(a)
        if alen > 3:
            logging.info('deflag is not used for more than 2-d input')
            return
        if alen == 3:
            v = a[2]
        ntimes = time.shape[0]
        nydim = data.ndim
        if nydim == 1:
            ny = 1
        else:
            ny = data.shape[1]
        for i in range(nf):
            if np.isnan(flag[i]):  # NaN flags need special handling
                flag_is_nan = True
            else:
                flag_is_nan = False
            for k in range(ny):
                #print(data[:, k])
                if (flag_is_nan):
                    flagged_data = np.argwhere(np.isnan(data[:, k])).flatten()
                else:
                    flagged_data = np.argwhere(data[:, k] == flag[i]).flatten()
                if len(flagged_data) > 0:
                    if flag_is_nan:
                        okval = np.argwhere(np.isfinite(data[:, k])).flatten()
                    else:
                        okval = np.argwhere(data[:, k] != flag[i]).flatten()

                    if len(okval) == 0 and method in ['linear', 'repeat']:
                        logging.info('No unflagged data in %s, returning', tvar)
                        return
                    if method == 'repeat':  # flagged data repeats the previous unflagged value
                        for j in range(ntimes):
                            if (flag_is_nan):
                                if np.isnan(data[j, k]):
                                    if j == 0:
                                        data[j, k] = data[okval[0], k]
                                    else:
                                        data[j, k] = data[j-1, k]
                            else:
                                if data[j, k] == flag[i]:
                                    if j == 0:
                                        data[j, k] = data[okval[0], k]
                                    else:
                                        data[j, k] = data[j-1, k]
                        #print("After repeat: ", data[:, k])
                    elif method == 'replace':
                        if fillval is None:
                            fv = np.nan
                        else:
                            fv = fillval
                        data[flagged_data, k] = fv
                        #print("After replace: ", data[:,k])
                    else:  # method = 'linear'
                        # interpolate flagged data, using np.interp
                        dataj = data[okval, k]
                        timej = time[okval]
                        timej_flagged = time[flagged_data]
                        dataj_interpd = np.interp(timej_flagged, timej, dataj)
                        data[flagged_data, k] = dataj_interpd
                        #print("After linear: ", data[:,k])
                else:
                    logging.info("No flagged data in %s", tvar)

        if newname is None:
            if alen == 2:
                store_data(tvar, data={'x': time, 'y': data.reshape(original_data_shape)})
            else:
                store_data(tvar, data={'x': time, 'y': data.reshape(original_data_shape), 'v': v})
        else:
            if alen == 2:
                store_data(newname, data={'x': time, 'y': data.reshape(original_data_shape)})
            else:
                store_data(newname, data={'x': time, 'y': data.reshape(original_data_shape), 'v': v})
                pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)
    else:  # any other option includes method=None, replace flags with NaN
        nf = len(flag)
        a = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].where(pyspedas.pytplot.data_quants[tvar] != flag[0]))
        if nf > 1:
            for j in range(nf):
                a = copy.deepcopy(a.where(a != flag[j]))
        if newname is None:
            a.name = tvar
            pyspedas.pytplot.data_quants[tvar] = a
        else:
            if 'spec_bins' in a.coords:
                store_data(newname, data={'x': a.coords['time'], 'y': a.values, 'v': a.coords['spec_bins']})
                pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)
            else:
                store_data(newname, data={'x': a.coords['time'], 'y': a.values})
                pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)

    return

1	import pyspedas	1✔
2	from pyspedas.pytplot import store_data, get_data, tnames	1✔
3	import copy	1✔
4	import numpy as np	1✔
5	import logging	1✔
6
7
8	def deflag(tvar, flag=None, newname=None, new_tvar=None, method=None, fillval=None):	1✔
9	"""
10	Replace NaN or other 'flag' values in arrays with interpolated or other values.
11
12	Parameters
13	----------
14
15	tvar: str or list[str]
16	Names of tplot variables to deflag (wildcards accepted)
17	flag : int,list
18	Flagged data will be converted to NaNs.
19	method : str, optional
20	Method to apply. Valid options::
21
22	'repeat': Repeat last good value
23	'linear': Interpolate linearly over gap
24	'replace': Replace flagged values with fillval, or NaN if fillval not specified
25	'remove_nan': Remove timestamps and values with a NaN in any dimension
26
27	newname : str
28	Name of new tvar for deflagged data storage.
29	If not specified, then the data in tvar1 will be replaced.
30	This is not an option for multiple variable input, for
31	multiple or pseudo variables, the data is overwritten.
32	new_tvar : str (Deprecated)
33	Name of new tvar for deflagged data storage.
34	If not specified, then the data in tvar1 will be replaced.
35	THIS is not an option for multiple variable input, for
36	multiple or pseudo variables, the data is overwritten.
37	fillval: int, float (optional)
38	Value to use as replacement when method='replace'
39
40	Notes
41	-----
42
43	deflag only works for 1 or 2-d data arrays; ntimes or (ntimes, nspectral_bins)
44
45	Returns
46	-------
47
48	None
49
50	Examples
51	--------
52
53	>>> pyspedas.store_data('d', data={'x':[2,5,8,11,14,17,21], 'y':[[1,1],[2,2],[100,4],[4,90],[5,5],[6,6],[7,7]]})
54	>>> # Remove any instances of [100,90,7,2,57] from 'd', store in 'e'.
55	>>> pyspedas.deflag('d',[100,90,7,2,57],newname='e')
56
57	"""
58
59	# new_tvar is deprecated in favor of newname
60	if new_tvar is not None:	1✔
61	logging.info("deflag: The new_tvar parameter is deprecated. Please use newname instead.")	×
62	newname = new_tvar	×
63
64	# for linear method, and flag of NaN, or none, interp_nan an be called
65	# if (flag == None or np.isnan(flag)) and method == 'linear':
66	# interp_nan(tvar, newname=newname)
67	# return
68
69	# check for globbed or array input, and call recursively
70	tn = tnames(tvar)	1✔
71	if len(tn) == 0:	1✔
72	return	×
73	elif len(tn) > 1:	1✔
74	for j in range(len(tn)):	×
75	deflag(tn[j], flag, method=method, fillval=fillval)	×
76	return	×
77
78	# Now flag needs to be an array
79	if flag is None:	1✔
80	flag = np.zeros(1)+np.nan	1✔
81	else:
82	if not isinstance(flag, np.ndarray):	×
83	flag_array = np.array(flag)	×
84	else:
85	flag_array = flag	×
86	# Ok, now if flag was a scalar, the array will have ndim = 0
87	if flag_array.ndim == 0:	×
88	flag = np.array([flag])	×
89	else:
90	flag = flag_array	×
91
92	nf = len(flag)	1✔
93	if method == 'remove_nan': # this is different from the other methods, which retain all time intervals	1✔
94	a = copy.deepcopy(get_data(tvar))	1✔
95	alen = len(a)	1✔
96	# Ignore more than 2d Y input
97	if alen > 3:	1✔
98	logging.info('deflag is not used for more than 2-d input')	×
99	return	×
100
101	time = a[0]	1✔
102	data = a[1]	1✔
103	# Added this to prevent error on !-d arrays (see below)
104	data_dim = data.ndim	1✔
105	new_time = []	1✔
106	new_data = []	1✔
107
108	if alen == 3: # v variable	1✔
109	v = a[2]	×
110	if v.ndim == 1:	×
111	new_v = v	×
112	append_v = False	×
113	else:
114	new_v = []	×
115	append_v = True	×
116
117	# Fill the new variable
118	non_nan_seen = False	1✔
119	for j in range(len(time)):	1✔
120	# This used to be "if len(data[j]) > 1", which failed if data is 1-D so that data[j] is scalar
121	# Instead we go by the dimensions of the data array itself
122	if data_dim > 1:	1✔
123	tj = np.sum(data[j])	1✔
124	else:
125	tj = data[j]	1✔
126	if not np.isnan(tj):	1✔
127	non_nan_seen = True	1✔
128	new_time.append(time[j])	1✔
129	new_data.append(data[j])	1✔
130	if alen == 3 and append_v:	1✔
131	new_v.append(v[j])	×
132	if non_nan_seen is False:	1✔
133	logging.warning('No unflagged data in %s, returning.', tvar)	1✔
134	return	1✔
135	if newname is None:	1✔
136	if alen == 2:	×
137	store_data(tvar, data={'x': new_time, 'y': new_data})	×
138	else:
139	store_data(tvar, data={'x': new_time, 'y': new_data, 'v': new_v})	×
140	else:
141	if alen == 2:	1✔
142	store_data(newname, data={'x': new_time, 'y': new_data})	1✔
143	else:
144	store_data(newname, data={'x': new_time, 'y': new_data, 'v': new_v})	×
145	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)	1✔
146	elif method == 'repeat' or method == 'linear' or method == 'replace':	1✔
147	a = copy.deepcopy(get_data(tvar))	1✔
148	time = a[0]	1✔
149	data = a[1]	1✔
150	# Force the data into a 2-d view, retaining the original shape so we can restore it later
151	original_data_shape = data.shape	1✔
152	data = data.reshape(len(time), -1)	1✔
153	alen = len(a)	1✔
154	if alen > 3:	1✔
155	logging.info('deflag is not used for more than 2-d input')	×
156	return	×
157	if alen == 3:	1✔
158	v = a[2]	×
159	ntimes = time.shape[0]	1✔
160	nydim = data.ndim	1✔
161	if nydim == 1:	1✔
162	ny = 1	×
163	else:
164	ny = data.shape[1]	1✔
165	for i in range(nf):	1✔
166	if np.isnan(flag[i]): # NaN flags need special handling	1✔
167	flag_is_nan = True	1✔
168	else:
169	flag_is_nan = False	×
170	for k in range(ny):	1✔
171	#print(data[:, k])
172	if (flag_is_nan):	1✔
173	flagged_data = np.argwhere(np.isnan(data[:, k])).flatten()	1✔
174	else:
175	flagged_data = np.argwhere(data[:, k] == flag[i]).flatten()	×
176	if len(flagged_data) > 0:	1✔
177	if flag_is_nan:	1✔
178	okval = np.argwhere(np.isfinite(data[:, k])).flatten()	1✔
179	else:
180	okval = np.argwhere(data[:, k] != flag[i]).flatten()	×
181
182	if len(okval) == 0 and method in ['linear', 'repeat']:	1✔
183	logging.info('No unflagged data in %s, returning', tvar)	1✔
184	return	1✔
185	if method == 'repeat': # flagged data repeats the previous unflagged value	1✔
186	for j in range(ntimes):	1✔
187	if (flag_is_nan):	1✔
188	if np.isnan(data[j, k]):	1✔
189	if j == 0:	1✔
190	data[j, k] = data[okval[0], k]	1✔
191	else:
192	data[j, k] = data[j-1, k]	1✔
193	else:
194	if data[j, k] == flag[i]:	×
195	if j == 0:	×
196	data[j, k] = data[okval[0], k]	×
197	else:
198	data[j, k] = data[j-1, k]	×
199	#print("After repeat: ", data[:, k])
200	elif method == 'replace':	1✔
201	if fillval is None:	1✔
202	fv = np.nan	×
203	else:
204	fv = fillval	1✔
205	data[flagged_data, k] = fv	1✔
206	#print("After replace: ", data[:,k])
207	else: # method = 'linear'
208	# interpolate flagged data, using np.interp
209	dataj = data[okval, k]	1✔
210	timej = time[okval]	1✔
211	timej_flagged = time[flagged_data]	1✔
212	dataj_interpd = np.interp(timej_flagged, timej, dataj)	1✔
213	data[flagged_data, k] = dataj_interpd	1✔
214	#print("After linear: ", data[:,k])
215	else:
216	logging.info("No flagged data in %s", tvar)	1✔
217
218	if newname is None:	1✔
219	if alen == 2:	×
220	store_data(tvar, data={'x': time, 'y': data.reshape(original_data_shape)})	×
221	else:
222	store_data(tvar, data={'x': time, 'y': data.reshape(original_data_shape), 'v': v})	×
223	else:
224	if alen == 2:	1✔
225	store_data(newname, data={'x': time, 'y': data.reshape(original_data_shape)})	1✔
226	else:
227	store_data(newname, data={'x': time, 'y': data.reshape(original_data_shape), 'v': v})	×
228	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)	×
229	else: # any other option includes method=None, replace flags with NaN
230	nf = len(flag)	×
231	a = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].where(pyspedas.pytplot.data_quants[tvar] != flag[0]))	×
232	if nf > 1:	×
233	for j in range(nf):	×
234	a = copy.deepcopy(a.where(a != flag[j]))	×
235	if newname is None:	×
236	a.name = tvar	×
237	pyspedas.pytplot.data_quants[tvar] = a	×
238	else:
239	if 'spec_bins' in a.coords:	×
240	store_data(newname, data={'x': a.coords['time'], 'y': a.values, 'v': a.coords['spec_bins']})	×
241	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)	×
242	else:
243	store_data(newname, data={'x': a.coords['time'], 'y': a.values})	×
244	pyspedas.pytplot.data_quants[newname].attrs = copy.deepcopy(pyspedas.pytplot.data_quants[tvar].attrs)	×
245
246	return	1✔

spedas / pyspedas / 16972251084

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous