5139219947

pending completion

Build # 5139219947

Build Type

push

github

Committed by

web-flow

Commit Message

Bug fix for __read_csv() method to now read RISMA network data of "header & files" format again (#67)

* PEP8 and typing

reformatted code according to PEP8 style guide and added  missing type

* changed __read_csv

function instead of lambda function inside __read_csv

* pep8

* updated __read_csv

__read_csv had problems with reading faulty sensor files. This was now fixed by introducing **kwargs as additonal arguments

* small correction and pep8

* fix: now all files of format 'header values' can be parsed correctly by pandas

kept the **kwargs in the __read_csv method and added exception 'pd.erros.ParserError as text_exception'

* preparation for sphinx

started preparation for a sphinx documentation

* deleted sphinx files

* test

* removed comments

* added short file describing the original problem and the fix, added test data

* minor revision

* minor revision

* minor revision

* minor revision

* minor revision

* minor revision

* new branch for USDA-ARS debugging

* pep8 style check now done by yapf

* smal adaptions

* did the required changes for pull request

* yapf pep8

* exchanged tuple[] with typing.Tuple[]

* added line to CHANGELOG.rst

* added line to CHANGELOG.rst to correct position this time

* Removed the now ignored directory ".vscode"

* Deleted "RISMA_test_data" directory

Run Details

554 of 670 branches covered (82.69%)

Branch coverage included in aggregate %.

38 of 40 new or added lines in 6 files covered. (95.0%)

50 existing lines in 6 files now uncovered.

1338 of 1472 relevant lines covered (90.9%)

3.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.63

/src/ismn/base.py

# -*- coding: utf-8 -*-
# The MIT License (MIT)
#
# Copyright (c) 2021 TU Wien
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import numpy as np
import zipfile
from collections import OrderedDict
import glob
import fnmatch
import warnings
from pathlib import Path, PurePosixPath
from typing import Union, List


def zip(func):

    def wrapper(cls, *args, **kwargs):
        if not cls.zip:
            raise IOError("Zip archive expected, use @dir functions instead.")
        return func(cls, *args, **kwargs)

    return wrapper


def dir(func):

    def wrapper(cls, *args, **kwargs):
        if cls.zip:
            raise IOError(
                "Unzipped archive expected, use @zip functions instead.")
        return func(cls, *args, **kwargs)

    return wrapper


class IsmnRoot:
    """
    Connection to the zip resp. extracted zip archive downloaded from the
    ismn website. This class only handles file access / requests made by the
    readers, lists files in path and can extract files to temp folders for
    safe reading.

    Attributes
    ----------
    path : Path
        Data directory
    """

    def __init__(self, path):
        """
        Parameters
        ----------
        path : str or Path
            Path to the downloaded zip file or the extracted zip
            directory.
        """
        self.path = Path(path)

        if not self.path.exists():
            raise IOError(f"Archive does not exist: {self.path}")

        self.__cont = None
        self.__isopen = False

        self.open()

    @property
    def isopen(self) -> bool:
        # if data is a zipfile, this indicates if the zip is opened
        return self.__isopen

    @isopen.setter
    def isopen(self, isopen):
        self.__isopen = isopen

    @property
    def root_dir(self) -> Path:
        # the parent directory where the data is stored
        if self.zip:
            return self.path.parent
        else:
            return self.path

    def __repr__(self):
        """Simplified representation of object as string"""
        __type = type(self)
        __zip = "Zip" if self.zip else "Unzipped"
        return f"{__type.__module__}.{__type.__qualname__} {__zip} at {self.path}"

    def __contains__(self, filepath) -> bool:
        """Check if files exists in archive"""
        if self.zip:
            filepath = PurePosixPath(filepath)
            return str(filepath) in self.zip.namelist()
        else:
            path = self.path / filepath
            return path.exists()

    def clean_subpath(self, subpath) -> Union[Path, PurePosixPath]:
        """Check if subpath is a valid path and adapt to archive format and os"""
        subpath = Path(subpath)
        if subpath.parts[0] in ["/", "\\"]:
            warnings.warn("Remove leading (back)slash in passed subpath.")
            subpath = Path(*subpath.parts[1:])

        if self.zip:
            subpath = PurePosixPath(subpath)
        else:
            if not (self.path / Path(subpath)).exists():
                raise ValueError(f"Subpath {subpath} does not exist"
                                 f" in archive {self.path}")

        return subpath

    @property
    def cont(self):
        """Get cont of object, or scan to create cont."""
        if self.__cont is None:
            self.__cont = self.scan()
        return self.__cont

    @zip
    def __scan_zip(self, station_subdirs: bool = True) -> OrderedDict:
        """
        Go through the archive and group station folders in archive
        by network folders. if scan_files is selected, group files by
        station folders.
        """
        cont = {}

        file_list = self.zip.namelist()

        for f in file_list:
            relpath = os.path.split(f)[0]
            if relpath == "":
                continue
            net, stat = os.path.split(relpath)
            if net == "":
                continue
            if station_subdirs:
                stat = relpath
            if net not in cont.keys():
                cont[net] = np.array([])
            if not np.isin([stat], cont[net])[0]:
                cont[net] = np.append(cont[net], stat)

        self.__cont = OrderedDict(sorted(cont.items()))

        return self.cont

    @dir
    def __scan_dir(self, station_subdirs: bool = True) -> OrderedDict:
        """
        Go through the archive and group station folders in archive
        by network folders
        """
        cont = {}

        for f in os.scandir(self.path):
            if f.is_dir():
                if f.path == self.path:
                    continue
                net = os.path.relpath(f.path, self.path)
                if net == "":
                    continue
                for stat in os.scandir(f.path):
                    if stat.is_dir():
                        if net not in cont.keys():
                            cont[net] = np.array([])
                        if station_subdirs:
                            cont[net] = np.append(cont[net],
                                                  Path(net, stat.name))
                        else:
                            cont[net] = np.append(cont[net], stat.name)

        self.__cont = OrderedDict(sorted(cont.items()))

        return self.cont

    @dir
    def __find_files_dir(self,
                         subpath: str = None,
                         fn_templ: str = "*.csv") -> list:
        """
        Find files in the archive or a subdirectory of the archive
        that match to the passed filename template.
        """
        if subpath is None:
            subpath = "**"

        subpath = self.clean_subpath(subpath)

        filenames = glob.glob(str(self.path / subpath / fn_templ))
        filenames = [Path(os.path.relpath(f, self.path)) for f in filenames]
        return filenames

    @zip
    def __find_files_zip(self,
                         subpath: str = None,
                         fn_templ: str = "*.csv") -> list:
        """
        Find files in zip archive that match the passed template and subdir.
        """
        if subpath is None:
            subpath = "**"

        subpath = self.clean_subpath(subpath)

        all_files = np.array(self.zip.namelist())

        filterlist = list(
            filter(
                lambda f: fnmatch.fnmatch(f, f"{subpath}/{fn_templ}"),
                all_files,
            )).copy()

        return filterlist

    def scan(self, station_subdirs=True) -> OrderedDict:
        """
        Go through archive (zip or dir) and group station folders

        Parameters
        ----------
        station_subdirs : bool, optional (default: True)
            Include the station dir as a subdir of network dir.
            If False is selected, station dir is included directly.

        Returns
        -------
        cont : OrderedDict
            Archive content, station dirs grouped by network dirs
        """

        if self.zip:
            return self.__scan_zip(station_subdirs)
        else:
            return self.__scan_dir(station_subdirs)

    def find_files(self, subpath=None, fn_templ="*.csv"):
        """
        List files in archive or a subdirectory of the archive that match
        the passed filename pattern.

        Parameters
        ----------
        subpath: str, optional (default: None)
            Use linux slashes '/' (and no leading '/') to define a subpath
            in the zip file. If None is selected, the whole archive is searched.
        fn_templ: str, optional (default: '*.csv')
            Filename template for files that are searched in the passed dir.

        Returns
        -------
        files : list[str]
            Found files that match the passed template.
        """

        if self.zip:
            return self.__find_files_zip(subpath, fn_templ)
        else:
            return self.__find_files_dir(subpath, fn_templ)

    @zip
    def extract_file(self, file_in_archive, out_path):
        """
        Extract a file from the zip archive

        Parameters
        ----------
        file_in_archive : Path or str
            Use linux slashes '/' (no leading '/') to define a subpath
            Relative path in the archive (network/station/filename)
        out_path : Path or str
            Directory where the extracted file is stored.

        Returns
        -------
        extracted_file : Path
            Path to the the extracted file.
        """
        out_path = Path(out_path)
        file_in_archive = PurePosixPath(file_in_archive)

        file_in_archive = self.clean_subpath(file_in_archive)

        ls = np.array(self.zip.namelist())

        ext = None
        if str(file_in_archive) in ls:  # single file was passed
            ext = self.zip.extract(member=str(file_in_archive), path=out_path)

        return Path(ext)

    @zip
    def extract_dir(self, subdir_in_archive, out_path):
        """
        Extract all files in subdir from the zip archive

        Parameters
        ----------
        subdir_in_archive : str
            Relative path in the archive (network/station or network)
        out_path : str
            Directory where the extracted files are stored.

        Returns
        -------
        extraced_files : str
            Path to the the extracted files.
        """
        out_path = Path(out_path)

        subdir_in_archive = PurePosixPath(subdir_in_archive)
        subdir_in_archive = self.clean_subpath(subdir_in_archive)

        ls = np.array(self.zip.namelist())

        filterlist = list(
            filter(lambda x: x.startswith(str(subdir_in_archive)), ls)).copy()

        self.zip.extractall(members=filterlist, path=out_path)

        return [out_path / f for f in filterlist]

    def open(self):
        # open connection to data archive
        if zipfile.is_zipfile(self.path):
            self.zip = zipfile.ZipFile(self.path, mode="r")
            self.name = self.path.with_suffix("").name
        else:
            self.zip = None
            self.name = self.path.name

        self.isopen = True

    def close(self):
        # close connection to data archive
        if self.zip:  # if not zip, connection is always open
            self.zip.close()
            self.zip = None
            self.isopen = False

    def __enter__(self):
        return self

    def __exit__(self, value_type, value, traceback):
        self.close()

1	# -- coding: utf-8 --
2	# The MIT License (MIT)
3	#
4	# Copyright (c) 2021 TU Wien
5	#
6	# Permission is hereby granted, free of charge, to any person obtaining a copy
7	# of this software and associated documentation files (the "Software"), to deal
8	# in the Software without restriction, including without limitation the rights
9	# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	# copies of the Software, and to permit persons to whom the Software is
11	# furnished to do so, subject to the following conditions:
12	#
13	# The above copyright notice and this permission notice shall be included in all
14	# copies or substantial portions of the Software.
15	#
16	# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19	# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22	# SOFTWARE.
23
24	import os	4✔
25	import numpy as np	4✔
26	import zipfile	4✔
27	from collections import OrderedDict	4✔
28	import glob	4✔
29	import fnmatch	4✔
30	import warnings	4✔
31	from pathlib import Path, PurePosixPath	4✔
32	from typing import Union, List	4✔
33
34
35	def zip(func):	4✔
36
37	def wrapper(cls, args, *kwargs):	4✔
38	if not cls.zip:	4!
39	raise IOError("Zip archive expected, use @dir functions instead.")	×
40	return func(cls, args, *kwargs)	4✔
41
42	return wrapper	4✔
43
44
45	def dir(func):	4✔
46
47	def wrapper(cls, args, *kwargs):	4✔
48	if cls.zip:	4!
49	raise IOError(	×
50	"Unzipped archive expected, use @zip functions instead.")
51	return func(cls, args, *kwargs)	4✔
52
53	return wrapper	4✔
54
55
56	class IsmnRoot:	4✔
57	"""
58	Connection to the zip resp. extracted zip archive downloaded from the
59	ismn website. This class only handles file access / requests made by the
60	readers, lists files in path and can extract files to temp folders for
61	safe reading.
62
63	Attributes
64	----------
65	path : Path
66	Data directory
67	"""
68
69	def __init__(self, path):	4✔
70	"""
71	Parameters
72	----------
73	path : str or Path
74	Path to the downloaded zip file or the extracted zip
75	directory.
76	"""
77	self.path = Path(path)	4✔
78
79	if not self.path.exists():	4!
80	raise IOError(f"Archive does not exist: {self.path}")	×
81
82	self.__cont = None	4✔
83	self.__isopen = False	4✔
84
85	self.open()	4✔
86
87	@property	4✔
88	def isopen(self) -> bool:	4✔
89	# if data is a zipfile, this indicates if the zip is opened
90	return self.__isopen	4✔
91
92	@isopen.setter	4✔
93	def isopen(self, isopen):	4✔
94	self.__isopen = isopen	4✔
95
96	@property	4✔
97	def root_dir(self) -> Path:	4✔
98	# the parent directory where the data is stored
99	if self.zip:	4✔
100	return self.path.parent	4✔
101	else:
102	return self.path	4✔
103
104	def __repr__(self):
105	"""Simplified representation of object as string"""
106	__type = type(self)
107	__zip = "Zip" if self.zip else "Unzipped"
108	return f"{__type.__module__}.{__type.__qualname__} {__zip} at {self.path}"
109
110	def __contains__(self, filepath) -> bool:	4✔
111	"""Check if files exists in archive"""
112	if self.zip:	4✔
113	filepath = PurePosixPath(filepath)	4✔
114	return str(filepath) in self.zip.namelist()	4✔
115	else:
116	path = self.path / filepath	4✔
117	return path.exists()	4✔
118
119	def clean_subpath(self, subpath) -> Union[Path, PurePosixPath]:	4✔
120	"""Check if subpath is a valid path and adapt to archive format and os"""
121	subpath = Path(subpath)	4✔
122	if subpath.parts[0] in ["/", "\\"]:	4!
123	warnings.warn("Remove leading (back)slash in passed subpath.")	×
124	subpath = Path(*subpath.parts[1:])	×
125
126	if self.zip:	4✔
127	subpath = PurePosixPath(subpath)	4✔
128	else:
129	if not (self.path / Path(subpath)).exists():	4!
NEW 130	raise ValueError(f"Subpath {subpath} does not exist"	×
131	f" in archive {self.path}")
132
133	return subpath	4✔
134
135	@property	4✔
136	def cont(self):	4✔
137	"""Get cont of object, or scan to create cont."""
138	if self.__cont is None:	4✔
139	self.__cont = self.scan()	4✔
140	return self.__cont	4✔
141
142	@zip	4✔
143	def __scan_zip(self, station_subdirs: bool = True) -> OrderedDict:	4✔
144	"""
145	Go through the archive and group station folders in archive
146	by network folders. if scan_files is selected, group files by
147	station folders.
148	"""
149	cont = {}	4✔
150
151	file_list = self.zip.namelist()	4✔
152
153	for f in file_list:	4✔
154	relpath = os.path.split(f)[0]	4✔
155	if relpath == "":	4✔
156	continue	4✔
157	net, stat = os.path.split(relpath)	4✔
158	if net == "":	4✔
159	continue	4✔
160	if station_subdirs:	4!
161	stat = relpath	4✔
162	if net not in cont.keys():	4✔
163	cont[net] = np.array([])	4✔
164	if not np.isin([stat], cont[net])[0]:	4✔
165	cont[net] = np.append(cont[net], stat)	4✔
166
167	self.__cont = OrderedDict(sorted(cont.items()))	4✔
168
169	return self.cont	4✔
170
171	@dir	4✔
172	def __scan_dir(self, station_subdirs: bool = True) -> OrderedDict:	4✔
173	"""
174	Go through the archive and group station folders in archive
175	by network folders
176	"""
177	cont = {}	4✔
178
179	for f in os.scandir(self.path):	4✔
180	if f.is_dir():	4✔
181	if f.path == self.path:	4!
UNCOV 182	continue	×
183	net = os.path.relpath(f.path, self.path)	4✔
184	if net == "":	4!
UNCOV 185	continue	×
186	for stat in os.scandir(f.path):	4✔
187	if stat.is_dir():	4!
188	if net not in cont.keys():	4✔
189	cont[net] = np.array([])	4✔
190	if station_subdirs:	4!
191	cont[net] = np.append(cont[net],	4✔
192	Path(net, stat.name))
193	else:
UNCOV 194	cont[net] = np.append(cont[net], stat.name)	×
195
196	self.__cont = OrderedDict(sorted(cont.items()))	4✔
197
198	return self.cont	4✔
199
200	@dir	4✔
201	def __find_files_dir(self,	4✔
202	subpath: str = None,
203	fn_templ: str = "*.csv") -> list:
204	"""
205	Find files in the archive or a subdirectory of the archive
206	that match to the passed filename template.
207	"""
208	if subpath is None:	4!
UNCOV 209	subpath = "**"	×
210
211	subpath = self.clean_subpath(subpath)	4✔
212
213	filenames = glob.glob(str(self.path / subpath / fn_templ))	4✔
214	filenames = [Path(os.path.relpath(f, self.path)) for f in filenames]	4✔
215	return filenames	4✔
216
217	@zip	4✔
218	def __find_files_zip(self,	4✔
219	subpath: str = None,
220	fn_templ: str = "*.csv") -> list:
221	"""
222	Find files in zip archive that match the passed template and subdir.
223	"""
224	if subpath is None:	4!
UNCOV 225	subpath = "**"	×
226
227	subpath = self.clean_subpath(subpath)	4✔
228
229	all_files = np.array(self.zip.namelist())	4✔
230
231	filterlist = list(	4✔
232	filter(
233	lambda f: fnmatch.fnmatch(f, f"{subpath}/{fn_templ}"),
234	all_files,
235	)).copy()
236
237	return filterlist	4✔
238
239	def scan(self, station_subdirs=True) -> OrderedDict:	4✔
240	"""
241	Go through archive (zip or dir) and group station folders
242
243	Parameters
244	----------
245	station_subdirs : bool, optional (default: True)
246	Include the station dir as a subdir of network dir.
247	If False is selected, station dir is included directly.
248
249	Returns
250	-------
251	cont : OrderedDict
252	Archive content, station dirs grouped by network dirs
253	"""
254
255	if self.zip:	4✔
256	return self.__scan_zip(station_subdirs)	4✔
257	else:
258	return self.__scan_dir(station_subdirs)	4✔
259
260	def find_files(self, subpath=None, fn_templ="*.csv"):	4✔
261	"""
262	List files in archive or a subdirectory of the archive that match
263	the passed filename pattern.
264
265	Parameters
266	----------
267	subpath: str, optional (default: None)
268	Use linux slashes '/' (and no leading '/') to define a subpath
269	in the zip file. If None is selected, the whole archive is searched.
270	fn_templ: str, optional (default: '*.csv')
271	Filename template for files that are searched in the passed dir.
272
273	Returns
274	-------
275	files : list[str]
276	Found files that match the passed template.
277	"""
278
279	if self.zip:	4✔
280	return self.__find_files_zip(subpath, fn_templ)	4✔
281	else:
282	return self.__find_files_dir(subpath, fn_templ)	4✔
283
284	@zip	4✔
285	def extract_file(self, file_in_archive, out_path):	4✔
286	"""
287	Extract a file from the zip archive
288
289	Parameters
290	----------
291	file_in_archive : Path or str
292	Use linux slashes '/' (no leading '/') to define a subpath
293	Relative path in the archive (network/station/filename)
294	out_path : Path or str
295	Directory where the extracted file is stored.
296
297	Returns
298	-------
299	extracted_file : Path
300	Path to the the extracted file.
301	"""
302	out_path = Path(out_path)	4✔
303	file_in_archive = PurePosixPath(file_in_archive)	4✔
304
305	file_in_archive = self.clean_subpath(file_in_archive)	4✔
306
307	ls = np.array(self.zip.namelist())	4✔
308
309	ext = None	4✔
310	if str(file_in_archive) in ls: # single file was passed	4!
311	ext = self.zip.extract(member=str(file_in_archive), path=out_path)	4✔
312
313	return Path(ext)	4✔
314
315	@zip	4✔
316	def extract_dir(self, subdir_in_archive, out_path):	4✔
317	"""
318	Extract all files in subdir from the zip archive
319
320	Parameters
321	----------
322	subdir_in_archive : str
323	Relative path in the archive (network/station or network)
324	out_path : str
325	Directory where the extracted files are stored.
326
327	Returns
328	-------
329	extraced_files : str
330	Path to the the extracted files.
331	"""
332	out_path = Path(out_path)	×
333
UNCOV 334	subdir_in_archive = PurePosixPath(subdir_in_archive)	×
335	subdir_in_archive = self.clean_subpath(subdir_in_archive)	×
336
337	ls = np.array(self.zip.namelist())	×
338
UNCOV 339	filterlist = list(	×
340	filter(lambda x: x.startswith(str(subdir_in_archive)), ls)).copy()
341
342	self.zip.extractall(members=filterlist, path=out_path)	×
343
UNCOV 344	return [out_path / f for f in filterlist]	×
345
346	def open(self):	4✔
347	# open connection to data archive
348	if zipfile.is_zipfile(self.path):	4✔
349	self.zip = zipfile.ZipFile(self.path, mode="r")	4✔
350	self.name = self.path.with_suffix("").name	4✔
351	else:
352	self.zip = None	4✔
353	self.name = self.path.name	4✔
354
355	self.isopen = True	4✔
356
357	def close(self):	4✔
358	# close connection to data archive
359	if self.zip: # if not zip, connection is always open	4✔
360	self.zip.close()	4✔
361	self.zip = None	4✔
362	self.isopen = False	4✔
363
364	def __enter__(self):	4✔
UNCOV 365	return self	×
366
367	def __exit__(self, value_type, value, traceback):	4✔
UNCOV 368	self.close()	×

TUW-GEO / ismn / 5139219947

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous