1478

Committed 06 Mar 2025 01:07AM UTC coverage: 88.136%. Remained the same

Build # 1478

Build Type

push

gitlab-ci

Committed by

Commit Message

Merge branch 'fix_issue' into 'master'

Update packages and also test for python 3.13

See merge request obob/pymatreader!49

Coverage Stats

1 of 3 new or added lines in 1 file covered. (33.33%)

156 of 177 relevant lines covered (88.14%)

7.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/pymatreader/utils.py

# Copyright (c) 2018, Dirk Gütlin & Thomas Hartmann
# All rights reserved.
#
# This file is part of the pymatreader Project, see:
# https://gitlab.com/obob/pymatreader
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Utility functions for pymatreader."""

from __future__ import annotations

import types
from typing import TYPE_CHECKING
from warnings import warn

import numpy as np

if TYPE_CHECKING:
    from collections.abc import Iterable

    import h5py

try:
    from scipy.io.matlab import MatlabFunction, MatlabOpaque
except ImportError:  # scipy < 1.8
    from scipy.io.matlab.mio5 import MatlabFunction
    from scipy.io.matlab.mio5_params import MatlabOpaque


standard_matlab_classes = (
    'char',
    'cell',
    'float',
    'double',
    'int',
    'int8',
    'int16',
    'int32',
    'int64',
    'uint',
    'uint8',
    'uint16',
    'logical',
    'uint32',
    'uint64',
    'struct',
    'unknown',
)


def _import_h5py() -> h5py:
    try:
        import h5py
    except Exception as exc:
        raise ImportError(f'h5py is required to read MATLAB files >= v7.3 ({exc})')
    return h5py


def _hdf5todict(
    hdf5_object: h5py.Dataset | h5py.Group,
    variable_names: Iterable | None = None,
    ignore_fields: Iterable | None = None,
) -> dict:
    """
    Recursively converts a hdf5 object to a python dictionary, converting all types as well.

    Parameters
    ----------
    hdf5_object: Union[h5py.Group, h5py.Dataset]
        Object to convert. Can be a h5py File, Group or Dataset
    variable_names: iterable, optional
        Tuple or list of variables to include. If set to none, all
        variable are read.
    ignore_fields: iterable, optional
        Tuple or list of fields to ignore. If set to none, all fields will
        be read.

    Returns
    -------
    dict
        Python dictionary
    """
    h5py = _import_h5py()

    if isinstance(hdf5_object, h5py.Group):
        return _handle_hdf5_group(hdf5_object, variable_names=variable_names, ignore_fields=ignore_fields)

    elif isinstance(hdf5_object, h5py.Dataset):
        result = _handle_hdf5_dataset(hdf5_object)
        if not isinstance(result, dict):
            raise TypeError('Unknown type in hdf5 file')
        return result

    raise TypeError('Unknown type in hdf5 file')


def _handle_hdf5_list(
    hdf5_object: list | types.GeneratorType | h5py.Dataset | h5py.Group,
    variable_names: Iterable | None = None,
    ignore_fields: Iterable | None = None,
) -> np.ndarray | list | str | int | float | complex | dict | None:
    h5py = _import_h5py()

    if isinstance(hdf5_object, h5py.Group):
        return _handle_hdf5_group(hdf5_object, variable_names=variable_names, ignore_fields=ignore_fields)

    elif isinstance(hdf5_object, h5py.Dataset):
        return _handle_hdf5_dataset(hdf5_object)
    elif isinstance(hdf5_object, (list, types.GeneratorType)):
        return [_handle_hdf5_list(item) for item in hdf5_object]

    raise TypeError('Unknown type in hdf5 file')


def _handle_hdf5_group(
    hdf5_object: h5py.Dataset, variable_names: Iterable | None = None, ignore_fields: Iterable | None = None
) -> dict:
    all_keys = set(hdf5_object.keys())
    if ignore_fields:
        all_keys = all_keys - set(ignore_fields)

    if variable_names:
        all_keys = all_keys & set(variable_names)

    return_dict = {}

    for key in all_keys:
        return_dict[key] = _handle_hdf5_list(hdf5_object[key], variable_names=None, ignore_fields=ignore_fields)

    return return_dict


def _handle_hdf5_dataset(hdf5_object: h5py.Dataset) -> np.ndarray | int | float | complex | str | list | dict | None:
    h5py = _import_h5py()

    data: np.ndarray

    if 'MATLAB_empty' in hdf5_object.attrs:  # noqa SIM108
        data = np.empty((0,))
    else:
        # this used to be just hdf5_object.value, but this is deprecated
        data = hdf5_object[()]

    matlab_class = hdf5_object.attrs.get('MATLAB_class', b'unknown').decode()

    if matlab_class not in standard_matlab_classes:
        warn(
            'Complex objects (like classes) are not supported. '
            'They are imported on a best effort base '
            'but your mileage will vary.'
        )

    if matlab_class == 'string':
        warn(
            'pymatreader cannot import Matlab string variables. '
            'Please convert these variables to char arrays in Matlab.'
        )
        return None

    if data.dtype == np.dtype('object'):
        data_list = [hdf5_object.file[cur_data] for cur_data in data.flatten()]

        if len(data_list) == 1 and matlab_class == 'cell':
            if isinstance(data_list[0], h5py.Group):
                return _handle_hdf5_group(data_list[0])

            matlab_class = data_list[0].attrs.get('MATLAB_class', matlab_class).decode()

            return _assign_types(data_list[0][()], matlab_class)

        return _assign_types(_handle_hdf5_list(data_list), matlab_class)

    return _assign_types(data, matlab_class)


def _convert_string_hdf5(values: np.ndarray) -> str | np.ndarray:
    if values.size > 1:
        return ''.join(chr(c) for c in values.flatten())
    else:
        try:
            return chr(int(values))
        except TypeError:
            return np.array([])


def _assign_types(
    values: np.ndarray | np.float64 | dict | list | str | int | float | complex | None, matlab_class: str
) -> np.ndarray | int | float | complex | str | list | dict | None:
    """Private function, which assigns correct types to h5py extracted values from _browse_dataset()."""
    if matlab_class == 'char' and isinstance(values, np.ndarray):
        values = np.squeeze(values).T
        return _handle_hdf5_strings(values)
    elif isinstance(values, np.ndarray):
        return _handle_ndarray(values)
    elif isinstance(values, np.float64):
        return float(values)
    else:
        return values


def _handle_ndarray(values: np.ndarray) -> np.ndarray | int | float | complex:
    """Handle conversion of ndarrays."""
    values = np.squeeze(values).T
    if values.dtype.names == ('real', 'imag'):
        values = np.array(values.view(complex))

    if values.size == 1:
        return values.item()
    else:
        return values


def _handle_hdf5_strings(values: np.ndarray) -> str | np.ndarray | list[str | np.ndarray]:
    if values.ndim in (0, 1):
        return _convert_string_hdf5(values)
    elif values.ndim == 2:  # noqa PLR2004
        return [_convert_string_hdf5(cur_val) for cur_val in values]
    else:
        raise RuntimeError('String arrays with more than 2 dimensionsare not supported at the moment.')


def _parse_scipy_mat_dict(data: dict) -> dict:
    """
    Parse a scipy.io.matlab.mio5_params.mat_struct dictionary.

    Parameters
    ----------
    data: dict
        data to be parsed

    Returns
    -------
    dict
        parsed data
    """
    for key in data:  # noqa PLC0206
        data[key] = _check_for_scipy_mat_struct(data[key])

    return data


def _check_for_scipy_mat_struct(data: dict | np.ndarray | MatlabOpaque) -> dict | np.ndarray | list | None:
    """
    Check all entries of data for occurrences of scipy.io.matlab.mio5_params.mat_struct and convert them.

    Parameters
    ----------
    data: any
        data to be checked

    Returns
    -------
    object
        checked and converted data
    """
    if isinstance(data, dict):
        return _parse_scipy_mat_dict(data)

    if isinstance(data, MatlabOpaque):
        try:
            if data[0][2] == b'string':
                warn(
                    'pymatreader cannot import Matlab string variables. '
                    'Please convert these variables to char arrays '
                    'in Matlab.'
                )
                return None
        except IndexError:
            pass
        warn(
            'Complex objects (like classes) are not supported. '
            'They are imported on a best effort base '
            'but your mileage will vary.'
        )

    if isinstance(data, np.ndarray):
        return _handle_scipy_ndarray(data)

    return data


def _handle_scipy_ndarray(data: np.ndarray | MatlabFunction) -> np.ndarray | list:
    if data.dtype == np.dtype('object') and not isinstance(data, MatlabFunction):
        as_list = []
        for element in data:
            as_list.append(_check_for_scipy_mat_struct(element))
        data = as_list
    elif isinstance(data.dtype.names, tuple):
        data = _todict_from_np_struct(data)
        data = _check_for_scipy_mat_struct(data)

    if isinstance(data, np.ndarray):
        data = np.array(data)

    return data


def _todict_from_np_struct(data: np.ndarray) -> dict[str, np.ndarray | int | float | str | list]:
    data_dict: dict[str, np.ndarray | int | float | str | list] = {}

    for cur_field_name in data.dtype.names:
        try:
            n_items = len(data[cur_field_name])
            cur_list = []

            for idx in np.arange(n_items):
                cur_value = data[cur_field_name].item(idx)
                cur_value = _check_for_scipy_mat_struct(cur_value)
                cur_list.append(cur_value)

            data_dict[cur_field_name] = cur_list
        except TypeError:
            cur_value = data[cur_field_name].item(0)
            cur_value = _check_for_scipy_mat_struct(cur_value)
            data_dict[cur_field_name] = cur_value

    return data_dict

1	# Copyright (c) 2018, Dirk Gütlin & Thomas Hartmann
2	# All rights reserved.
3	#
4	# This file is part of the pymatreader Project, see:
5	# https://gitlab.com/obob/pymatreader
6	#
7	# Redistribution and use in source and binary forms, with or without
8	# modification, are permitted provided that the following conditions are met:
9	#
10	# * Redistributions of source code must retain the above copyright notice, this
11	# list of conditions and the following disclaimer.
12	#
13	# * Redistributions in binary form must reproduce the above copyright notice,
14	# this list of conditions and the following disclaimer in the documentation
15	# and/or other materials provided with the distribution.
16	#
17	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27	# POSSIBILITY OF SUCH DAMAGE.
28
29	"""Utility functions for pymatreader."""
30
31	from __future__ import annotations	9✔
32
33	import types	9✔
34	from typing import TYPE_CHECKING	9✔
35	from warnings import warn	9✔
36
37	import numpy as np	9✔
38
39	if TYPE_CHECKING:	9✔
40	from collections.abc import Iterable	×
41
42	import h5py	×
43
44	try:	9✔
45	from scipy.io.matlab import MatlabFunction, MatlabOpaque	9✔
46	except ImportError: # scipy < 1.8	×
47	from scipy.io.matlab.mio5 import MatlabFunction	×
48	from scipy.io.matlab.mio5_params import MatlabOpaque	×
49
50
51	standard_matlab_classes = (	9✔
52	'char',
53	'cell',
54	'float',
55	'double',
56	'int',
57	'int8',
58	'int16',
59	'int32',
60	'int64',
61	'uint',
62	'uint8',
63	'uint16',
64	'logical',
65	'uint32',
66	'uint64',
67	'struct',
68	'unknown',
69	)
70
71
72	def _import_h5py() -> h5py:	9✔
73	try:	9✔
74	import h5py	9✔
75	except Exception as exc:	×
NEW 76	raise ImportError(f'h5py is required to read MATLAB files >= v7.3 ({exc})')	×
77	return h5py	9✔
78
79
80	def _hdf5todict(	9✔
81	hdf5_object: h5py.Dataset \| h5py.Group,
82	variable_names: Iterable \| None = None,
83	ignore_fields: Iterable \| None = None,
84	) -> dict:
85	"""
86	Recursively converts a hdf5 object to a python dictionary, converting all types as well.
87
88	Parameters
89	----------
90	hdf5_object: Union[h5py.Group, h5py.Dataset]
91	Object to convert. Can be a h5py File, Group or Dataset
92	variable_names: iterable, optional
93	Tuple or list of variables to include. If set to none, all
94	variable are read.
95	ignore_fields: iterable, optional
96	Tuple or list of fields to ignore. If set to none, all fields will
97	be read.
98
99	Returns
100	-------
101	dict
102	Python dictionary
103	"""
104	h5py = _import_h5py()	9✔
105
106	if isinstance(hdf5_object, h5py.Group):	9✔
107	return _handle_hdf5_group(hdf5_object, variable_names=variable_names, ignore_fields=ignore_fields)	9✔
108
109	elif isinstance(hdf5_object, h5py.Dataset):	×
110	result = _handle_hdf5_dataset(hdf5_object)	×
111	if not isinstance(result, dict):	×
112	raise TypeError('Unknown type in hdf5 file')	×
113	return result	×
114
115	raise TypeError('Unknown type in hdf5 file')	×
116
117
118	def _handle_hdf5_list(	9✔
119	hdf5_object: list \| types.GeneratorType \| h5py.Dataset \| h5py.Group,
120	variable_names: Iterable \| None = None,
121	ignore_fields: Iterable \| None = None,
122	) -> np.ndarray \| list \| str \| int \| float \| complex \| dict \| None:
123	h5py = _import_h5py()	9✔
124
125	if isinstance(hdf5_object, h5py.Group):	9✔
126	return _handle_hdf5_group(hdf5_object, variable_names=variable_names, ignore_fields=ignore_fields)	9✔
127
128	elif isinstance(hdf5_object, h5py.Dataset):	9✔
129	return _handle_hdf5_dataset(hdf5_object)	9✔
130	elif isinstance(hdf5_object, (list, types.GeneratorType)):	9✔
131	return [_handle_hdf5_list(item) for item in hdf5_object]	9✔
132
133	raise TypeError('Unknown type in hdf5 file')	×
134
135
136	def _handle_hdf5_group(	9✔
137	hdf5_object: h5py.Dataset, variable_names: Iterable \| None = None, ignore_fields: Iterable \| None = None
138	) -> dict:
139	all_keys = set(hdf5_object.keys())	9✔
140	if ignore_fields:	9✔
141	all_keys = all_keys - set(ignore_fields)	9✔
142
143	if variable_names:	9✔
144	all_keys = all_keys & set(variable_names)	9✔
145
146	return_dict = {}	9✔
147
148	for key in all_keys:	9✔
149	return_dict[key] = _handle_hdf5_list(hdf5_object[key], variable_names=None, ignore_fields=ignore_fields)	9✔
150
151	return return_dict	9✔
152
153
154	def _handle_hdf5_dataset(hdf5_object: h5py.Dataset) -> np.ndarray \| int \| float \| complex \| str \| list \| dict \| None:	9✔
155	h5py = _import_h5py()	9✔
156
157	data: np.ndarray
158
159	if 'MATLAB_empty' in hdf5_object.attrs: # noqa SIM108	9✔
160	data = np.empty((0,))	9✔
161	else:
162	# this used to be just hdf5_object.value, but this is deprecated
163	data = hdf5_object[()]	9✔
164
165	matlab_class = hdf5_object.attrs.get('MATLAB_class', b'unknown').decode()	9✔
166
167	if matlab_class not in standard_matlab_classes:	9✔
168	warn(	9✔
169	'Complex objects (like classes) are not supported. '
170	'They are imported on a best effort base '
171	'but your mileage will vary.'
172	)
173
174	if matlab_class == 'string':	9✔
175	warn(	9✔
176	'pymatreader cannot import Matlab string variables. '
177	'Please convert these variables to char arrays in Matlab.'
178	)
179	return None	9✔
180
181	if data.dtype == np.dtype('object'):	9✔
182	data_list = [hdf5_object.file[cur_data] for cur_data in data.flatten()]	9✔
183
184	if len(data_list) == 1 and matlab_class == 'cell':	9✔
185	if isinstance(data_list[0], h5py.Group):	9✔
186	return _handle_hdf5_group(data_list[0])	9✔
187
188	matlab_class = data_list[0].attrs.get('MATLAB_class', matlab_class).decode()	9✔
189
190	return _assign_types(data_list[0][()], matlab_class)	9✔
191
192	return _assign_types(_handle_hdf5_list(data_list), matlab_class)	9✔
193
194	return _assign_types(data, matlab_class)	9✔
195
196
197	def _convert_string_hdf5(values: np.ndarray) -> str \| np.ndarray:	9✔
198	if values.size > 1:	9✔
199	return ''.join(chr(c) for c in values.flatten())	9✔
200	else:
201	try:	9✔
202	return chr(int(values))	9✔
203	except TypeError:	9✔
204	return np.array([])	9✔
205
206
207	def _assign_types(	9✔
208	values: np.ndarray \| np.float64 \| dict \| list \| str \| int \| float \| complex \| None, matlab_class: str
209	) -> np.ndarray \| int \| float \| complex \| str \| list \| dict \| None:
210	"""Private function, which assigns correct types to h5py extracted values from _browse_dataset()."""
211	if matlab_class == 'char' and isinstance(values, np.ndarray):	9✔
212	values = np.squeeze(values).T	9✔
213	return _handle_hdf5_strings(values)	9✔
214	elif isinstance(values, np.ndarray):	9✔
215	return _handle_ndarray(values)	9✔
216	elif isinstance(values, np.float64):	9✔
217	return float(values)	×
218	else:
219	return values	9✔
220
221
222	def _handle_ndarray(values: np.ndarray) -> np.ndarray \| int \| float \| complex:	9✔
223	"""Handle conversion of ndarrays."""
224	values = np.squeeze(values).T	9✔
225	if values.dtype.names == ('real', 'imag'):	9✔
226	values = np.array(values.view(complex))	9✔
227
228	if values.size == 1:	9✔
229	return values.item()	9✔
230	else:
231	return values	9✔
232
233
234	def _handle_hdf5_strings(values: np.ndarray) -> str \| np.ndarray \| list[str \| np.ndarray]:	9✔
235	if values.ndim in (0, 1):	9✔
236	return _convert_string_hdf5(values)	9✔
237	elif values.ndim == 2: # noqa PLR2004	9✔
238	return [_convert_string_hdf5(cur_val) for cur_val in values]	9✔
239	else:
NEW 240	raise RuntimeError('String arrays with more than 2 dimensionsare not supported at the moment.')	×
241
242
243	def _parse_scipy_mat_dict(data: dict) -> dict:	9✔
244	"""
245	Parse a scipy.io.matlab.mio5_params.mat_struct dictionary.
246
247	Parameters
248	----------
249	data: dict
250	data to be parsed
251
252	Returns
253	-------
254	dict
255	parsed data
256	"""
257	for key in data: # noqa PLC0206	9✔
258	data[key] = _check_for_scipy_mat_struct(data[key])	9✔
259
260	return data	9✔
261
262
263	def _check_for_scipy_mat_struct(data: dict \| np.ndarray \| MatlabOpaque) -> dict \| np.ndarray \| list \| None:	9✔
264	"""
265	Check all entries of data for occurrences of scipy.io.matlab.mio5_params.mat_struct and convert them.
266
267	Parameters
268	----------
269	data: any
270	data to be checked
271
272	Returns
273	-------
274	object
275	checked and converted data
276	"""
277	if isinstance(data, dict):	9✔
278	return _parse_scipy_mat_dict(data)	9✔
279
280	if isinstance(data, MatlabOpaque):	9✔
281	try:	9✔
282	if data[0][2] == b'string':	9✔
283	warn(	9✔
284	'pymatreader cannot import Matlab string variables. '
285	'Please convert these variables to char arrays '
286	'in Matlab.'
287	)
288	return None	9✔
289	except IndexError:	×
290	pass	×
291	warn(	9✔
292	'Complex objects (like classes) are not supported. '
293	'They are imported on a best effort base '
294	'but your mileage will vary.'
295	)
296
297	if isinstance(data, np.ndarray):	9✔
298	return _handle_scipy_ndarray(data)	9✔
299
300	return data	9✔
301
302
303	def _handle_scipy_ndarray(data: np.ndarray \| MatlabFunction) -> np.ndarray \| list:	9✔
304	if data.dtype == np.dtype('object') and not isinstance(data, MatlabFunction):	9✔
305	as_list = []	9✔
306	for element in data:	9✔
307	as_list.append(_check_for_scipy_mat_struct(element))	9✔
308	data = as_list	9✔
309	elif isinstance(data.dtype.names, tuple):	9✔
310	data = _todict_from_np_struct(data)	9✔
311	data = _check_for_scipy_mat_struct(data)	9✔
312
313	if isinstance(data, np.ndarray):	9✔
314	data = np.array(data)	9✔
315
316	return data	9✔
317
318
319	def _todict_from_np_struct(data: np.ndarray) -> dict[str, np.ndarray \| int \| float \| str \| list]:	9✔
320	data_dict: dict[str, np.ndarray \| int \| float \| str \| list] = {}	9✔
321
322	for cur_field_name in data.dtype.names:	9✔
323	try:	9✔
324	n_items = len(data[cur_field_name])	9✔
325	cur_list = []	9✔
326
327	for idx in np.arange(n_items):	9✔
328	cur_value = data[cur_field_name].item(idx)	9✔
329	cur_value = _check_for_scipy_mat_struct(cur_value)	9✔
330	cur_list.append(cur_value)	9✔
331
332	data_dict[cur_field_name] = cur_list	9✔
333	except TypeError:	9✔
334	cur_value = data[cur_field_name].item(0)	9✔
335	cur_value = _check_for_scipy_mat_struct(cur_value)	9✔
336	data_dict[cur_field_name] = cur_value	9✔
337
338	return data_dict	9✔

obob / pymatreader / 1478

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous