23096825121

Committed 14 Mar 2026 09:34PM UTC coverage: 10.65% (-79.8%) from 90.486%

Build # 23096825121

Build Type

push

github

Committed by

jameswilburlewis

Commit Message

Allow uploading of hidden files

Coverage Stats

5208 of 48900 relevant lines covered (10.65%)

0.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.33

/pyspedas/tplot_tools/tplot_math/join_vec.py

import pyspedas
from pyspedas.tplot_tools import store_data, tplot_wildcard_expand
import pandas as pd
import copy
import xarray as xr
import logging
import numpy as np

from pyspedas.tplot_tools import convert_tplotxarray_to_pandas_dataframe


# JOIN TVARS
# join TVars into single TVar with multiple columns
def join_vec(tvars, newname=None, merge=False):
    """
    Joins 1D tplot variables into one tplot variable.

    .. note::
        This analysis routine assumes the data is no more than 2 dimensions. If there are more, they may become flattened!

    Parameters
    ----------
    tvars : list of str
        Name of tplot variables to join together.
    newname : str, optional
        The name of the new tplot variable. If not specified (the default), a name will be assigned.
    merge : bool, optional
        Whether or not to merge the created variable into an older variable.
        Default is False.

    Returns
    -------
    None

    Examples
    --------
    >>> import pyspedas
    >>> import numpy as np
    >>> pyspedas.store_data('d', data={'x':[2,5,8,11,14,17,21], 'y':[[1,1,50],[2,2,3],[100,4,47],[4,90,5],[5,5,99],[6,6,25],[7,7,-5]]})
    >>> pyspedas.store_data('e', data={'x':[2,5,8,11,14,17,21], 'y':[[np.nan,1,1],[np.nan,2,3],[4,np.nan,47],[4,np.nan,5],[5,5,99],[6,6,25],[7,np.nan,-5]]})
    >>> pyspedas.store_data('g', data={'x':[0,4,8,12,16,19,21], 'y':[[8,1,1],[100,2,3],[4,2,47],[4,39,5],[5,5,99],[6,6,25],[7,-2,-5]]})
    >>> pyspedas.join_vec(['d','e','g'],newname='deg')

    """

    to_merge = False
    if newname in pyspedas.tplot_tools.data_quants.keys() and merge:
        prev_data_quant = pyspedas.tplot_tools.data_quants[newname]
        to_merge = True

    # Allow wildcard specification
    tvars=tplot_wildcard_expand(tvars)

    if newname is None:
        newname = "-".join(tvars) + "_joined"

    has_extra_v_values = False
    v_shape = None
    data_shape = None
    combined_v = None

    # Make sure all the components exist and are compatible to be joined
    for i, val in enumerate(tvars):
        dq = pyspedas.tplot_tools.data_quants[val]
        data_shape = dq.values.shape
        if len(data_shape) != 1:
            logging.error(f"join_vec: variable {val} must be 1-dimensional, but has shape {data_shape}.")
            return None
        if i == 0:
            if 'extra_v_values' in dq.attrs.keys():
                has_extra_v_values = True
                extra_v_values = dq.attrs['extra_v_values']
                v_shape = extra_v_values.shape
                # If extra_v_values has multiple elements, maybe it's time varying....it should match the number of timestamps.
                # We'll create the array here, and populate each row as we go.
                if has_extra_v_values:
                    if len(extra_v_values) == 1:
                        combined_v = np.empty((len(tvars),),dtype=extra_v_values.dtype)
                        combined_v[0] = extra_v_values[0]
                    elif len(extra_v_values) == data_shape[0]:
                        combined_v = np.empty((data_shape[0],len(tvars)), dtype=extra_v_values.dtype)
                        combined_v[:,0] = extra_v_values
                    else:
                        logging.error(f"join_vec: variable {val} has shape {data_shape}, but has incompatible extra_v_values shape {v_shape}.")
                        return None
            df = convert_tplotxarray_to_pandas_dataframe(tvars[i], no_spec_bins=True)

        else:
            if 'extra_v_values' in dq.attrs.keys():
                current_has_extra_v_values = True
                current_extra_v_values = dq.attrs['extra_v_values']
                current_v_shape = current_extra_v_values.shape
            else:
                current_has_extra_v_values = False
                current_extra_v_values = None
                current_v_shape = None
            current_data_shape = dq.values.shape

            if current_has_extra_v_values != has_extra_v_values:
                logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched extra_v_values attributes.")
                return None
            elif has_extra_v_values and v_shape != current_v_shape:
                logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched extra_v_values shapes {v_shape} and {current_v_shape}.")
                return None
            elif data_shape != current_data_shape:
                logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched data shapes {data_shape} and {current_data_shape}")
                return None

            if has_extra_v_values and v_shape[0] == 1:
                combined_v[i] = current_extra_v_values[0]
            elif has_extra_v_values and v_shape[0] > 1:
                combined_v[:,i] = current_extra_v_values
            d = convert_tplotxarray_to_pandas_dataframe(tvars[i], no_spec_bins=True)
            df = pd.concat([df, d], axis=1)

    if combined_v is None:
        store_data(newname, data={"x": df.index, "y": df.values})
    else:
        store_data(newname, data={"x": df.index, "y": df.values, "v": combined_v})

    if to_merge is True:
        cur_data_quant = pyspedas.tplot_tools.data_quants[newname]
        plot_options = copy.deepcopy(pyspedas.tplot_tools.data_quants[newname].attrs)
        pyspedas.tplot_tools.data_quants[newname] = xr.concat(
            [prev_data_quant, cur_data_quant], dim="time"
        ).sortby("time")
        pyspedas.tplot_tools.data_quants[newname].attrs = plot_options

    return newname

1	import pyspedas	1✔
2	from pyspedas.tplot_tools import store_data, tplot_wildcard_expand	1✔
3	import pandas as pd	1✔
4	import copy	1✔
5	import xarray as xr	1✔
6	import logging	1✔
7	import numpy as np	1✔
8
9	from pyspedas.tplot_tools import convert_tplotxarray_to_pandas_dataframe	1✔
10
11
12	# JOIN TVARS
13	# join TVars into single TVar with multiple columns
14	def join_vec(tvars, newname=None, merge=False):	1✔
15	"""
16	Joins 1D tplot variables into one tplot variable.
17
18	.. note::
19	This analysis routine assumes the data is no more than 2 dimensions. If there are more, they may become flattened!
20
21	Parameters
22	----------
23	tvars : list of str
24	Name of tplot variables to join together.
25	newname : str, optional
26	The name of the new tplot variable. If not specified (the default), a name will be assigned.
27	merge : bool, optional
28	Whether or not to merge the created variable into an older variable.
29	Default is False.
30
31	Returns
32	-------
33	None
34
35	Examples
36	--------
37	>>> import pyspedas
38	>>> import numpy as np
39	>>> pyspedas.store_data('d', data={'x':[2,5,8,11,14,17,21], 'y':[[1,1,50],[2,2,3],[100,4,47],[4,90,5],[5,5,99],[6,6,25],[7,7,-5]]})
40	>>> pyspedas.store_data('e', data={'x':[2,5,8,11,14,17,21], 'y':[[np.nan,1,1],[np.nan,2,3],[4,np.nan,47],[4,np.nan,5],[5,5,99],[6,6,25],[7,np.nan,-5]]})
41	>>> pyspedas.store_data('g', data={'x':[0,4,8,12,16,19,21], 'y':[[8,1,1],[100,2,3],[4,2,47],[4,39,5],[5,5,99],[6,6,25],[7,-2,-5]]})
42	>>> pyspedas.join_vec(['d','e','g'],newname='deg')
43
44	"""
45
46	to_merge = False	×
47	if newname in pyspedas.tplot_tools.data_quants.keys() and merge:	×
48	prev_data_quant = pyspedas.tplot_tools.data_quants[newname]	×
49	to_merge = True	×
50
51	# Allow wildcard specification
52	tvars=tplot_wildcard_expand(tvars)	×
53
54	if newname is None:	×
55	newname = "-".join(tvars) + "_joined"	×
56
57	has_extra_v_values = False	×
58	v_shape = None	×
59	data_shape = None	×
60	combined_v = None	×
61
62	# Make sure all the components exist and are compatible to be joined
63	for i, val in enumerate(tvars):	×
64	dq = pyspedas.tplot_tools.data_quants[val]	×
65	data_shape = dq.values.shape	×
66	if len(data_shape) != 1:	×
67	logging.error(f"join_vec: variable {val} must be 1-dimensional, but has shape {data_shape}.")	×
68	return None	×
69	if i == 0:	×
70	if 'extra_v_values' in dq.attrs.keys():	×
71	has_extra_v_values = True	×
72	extra_v_values = dq.attrs['extra_v_values']	×
73	v_shape = extra_v_values.shape	×
74	# If extra_v_values has multiple elements, maybe it's time varying....it should match the number of timestamps.
75	# We'll create the array here, and populate each row as we go.
76	if has_extra_v_values:	×
77	if len(extra_v_values) == 1:	×
78	combined_v = np.empty((len(tvars),),dtype=extra_v_values.dtype)	×
79	combined_v[0] = extra_v_values[0]	×
80	elif len(extra_v_values) == data_shape[0]:	×
81	combined_v = np.empty((data_shape[0],len(tvars)), dtype=extra_v_values.dtype)	×
82	combined_v[:,0] = extra_v_values	×
83	else:
84	logging.error(f"join_vec: variable {val} has shape {data_shape}, but has incompatible extra_v_values shape {v_shape}.")	×
85	return None	×
86	df = convert_tplotxarray_to_pandas_dataframe(tvars[i], no_spec_bins=True)	×
87
88	else:
89	if 'extra_v_values' in dq.attrs.keys():	×
90	current_has_extra_v_values = True	×
91	current_extra_v_values = dq.attrs['extra_v_values']	×
92	current_v_shape = current_extra_v_values.shape	×
93	else:
94	current_has_extra_v_values = False	×
95	current_extra_v_values = None	×
96	current_v_shape = None	×
97	current_data_shape = dq.values.shape	×
98
99	if current_has_extra_v_values != has_extra_v_values:	×
100	logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched extra_v_values attributes.")	×
101	return None	×
102	elif has_extra_v_values and v_shape != current_v_shape:	×
103	logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched extra_v_values shapes {v_shape} and {current_v_shape}.")	×
104	return None	×
105	elif data_shape != current_data_shape:	×
106	logging.error(f"join_vec: Unable to combine {tvars[0]} and {tvars[i]}, mismatched data shapes {data_shape} and {current_data_shape}")	×
107	return None	×
108
109	if has_extra_v_values and v_shape[0] == 1:	×
110	combined_v[i] = current_extra_v_values[0]	×
111	elif has_extra_v_values and v_shape[0] > 1:	×
112	combined_v[:,i] = current_extra_v_values	×
113	d = convert_tplotxarray_to_pandas_dataframe(tvars[i], no_spec_bins=True)	×
114	df = pd.concat([df, d], axis=1)	×
115
116	if combined_v is None:	×
117	store_data(newname, data={"x": df.index, "y": df.values})	×
118	else:
119	store_data(newname, data={"x": df.index, "y": df.values, "v": combined_v})	×
120
121	if to_merge is True:	×
122	cur_data_quant = pyspedas.tplot_tools.data_quants[newname]	×
123	plot_options = copy.deepcopy(pyspedas.tplot_tools.data_quants[newname].attrs)	×
124	pyspedas.tplot_tools.data_quants[newname] = xr.concat(	×
125	[prev_data_quant, cur_data_quant], dim="time"
126	).sortby("time")
127	pyspedas.tplot_tools.data_quants[newname].attrs = plot_options	×
128
129	return newname	×

spedas / pyspedas / 23096825121

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous