4591698001

Build Type

push

github

Committed by GitHub

Commit Message

Merge pull request #61 from wpreimes/zlib

Run Details

342 of 426 branches covered (80.28%)

Branch coverage included in aggregate %.

5 of 5 new or added lines in 2 files covered. (100.0%)

5 existing lines in 2 files now uncovered.

848 of 959 relevant lines covered (88.43%)

2.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.34

/src/pynetcf/base.py

# Copyright (c) 2023, TU Wien, Department of Geodesy and Geoinformation.
# All rights reserved.

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the Vienna University of Technology,
#     Department of Geodesy and Geoinformation nor the
#     names of its contributors may be used to endorse or promote products
#     derived from this software without specific prior written permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL VIENNA UNIVERSITY OF TECHNOLOGY,
# DEPARTMENT OF GEODESY AND GEOINFORMATION BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
Base classes for reading and writing time series and images in NetCDF files
using Climate Forecast Metadata Conventions (http://cfconventions.org/).
"""

import os
import time
import numpy as np
import netCDF4
import datetime


class DatasetError(Exception):
    pass


class Dataset:
    """
    NetCDF file wrapper class that makes some things easier

    Parameters
    ----------
    filename : string
        filename of netCDF file. If already exiting then it will be opened
        as read only unless the append keyword is set. if the overwrite
        keyword is set then the file will be overwritten
    name : string, optional
        will be written as a global attribute if the file is a new file
    file_format : string, optional
        file format
    mode : string, optional
        access mode. default "r"
        "r" means read-only; no data can be modified.
        "w" means write; a new file is created, an existing file with the
            same name is deleted.
        "a" and "r+" mean append (in analogy with serial files); an existing
            file is opened for reading and writing.
        Appending s to modes w, r+ or a will enable unbuffered shared access
        to NETCDF3_CLASSIC or NETCDF3_64BIT formatted files. Unbuffered
        access may be useful even if you don"t need shared access, since it
        may be faster for programs that don"t access data sequentially.
        This option is ignored for NETCDF4 and NETCDF4_CLASSIC
        formatted files.
    zlib : boolean, optional
        Default True
        if set netCDF compression will be used
    complevel : int, optional
        Default 4
        compression level used from 1(low compression) to 9(high compression)
    autoscale : bool, optional
        If disabled data will not be automatically scaled when reading and
        writing
    automask : bool, optional
        If disabled data will not be masked during reading.
        This means Fill Values will be used instead of NaN.
    """

    def __init__(self,
                 filename,
                 name=None,
                 file_format="NETCDF4",
                 mode="r",
                 zlib=True,
                 complevel=4,
                 autoscale=True,
                 automask=True):

        self.dataset_name = name
        self.filename = filename
        self.file = None
        self.file_format = file_format
        self.buf_len = 0
        self.global_attr = {}
        self.global_attr["id"] = os.path.split(self.filename)[1]
        s = "%Y-%m-%d %H:%M:%S"
        self.global_attr["date_created"] = datetime.datetime.now().strftime(s)
        if self.dataset_name is not None:
            self.global_attr["dataset_name"] = self.dataset_name
        self.zlib = zlib
        self.complevel = complevel
        self.mode = mode
        self.autoscale = autoscale
        self.automask = automask

        if self.mode == "a" and not os.path.exists(self.filename):
            self.mode = "w"
        if self.mode == "w":
            self._create_file_dir()

        try:
            self.dataset = netCDF4.Dataset(self.filename,
                                           self.mode,
                                           format=self.file_format)
        except RuntimeError:
            raise IOError(f"File {self.filename} does not exist")

        self.dataset.set_auto_scale(self.autoscale)
        self.dataset.set_auto_mask(self.automask)

    def _create_file_dir(self):
        """
        Create directory for file to sit in.
        Avoid race condition if multiple instances are
        writing files into the same directory.
        """
        path = os.path.dirname(self.filename)
        if not os.path.exists(path):
            try:
                os.makedirs(path)
            except OSError:
                time.sleep(1)
                self._create_file_dir()

    def _set_global_attr(self):
        """
        Write global attributes to NetCDF file.
        """
        self.dataset.setncatts(self.global_attr)
        self.global_attr = {}

    def create_dim(self, name, n):
        """
        Create dimension for NetCDF file.
        if it does not yet exist

        Parameters
        ----------
        name : str
            Name of the NetCDF dimension.
        n : int
            Size of the dimension.
        """
        if name not in self.dataset.dimensions.keys():
            self.dataset.createDimension(name, size=n)

    def write_var(self,
                  name,
                  data=None,
                  dim=None,
                  attr={},
                  dtype=None,
                  zlib=None,
                  complevel=None,
                  chunksizes=None,
                  **kwargs):
        """
        Create or overwrite values in a NetCDF variable. The data will be
        written to disk once flush or close is called

        Parameters
        ----------
        name : str
            Name of the NetCDF variable.
        data : np.ndarray, optional
            Array containing the data.
            if not given then the variable will be left empty
        dim : tuple, optional
            A tuple containing the dimension names.
        attr : dict, optional
            A dictionary containing the variable attributes.
        dtype: data type, string or numpy.dtype, optional
            if not given data.dtype will be used
        zlib: boolean, optional
            explicit compression for this variable
            if not given then global attribute is used
        complevel: int, optional
            explicit compression level for this variable
            if not given then global attribute is used
        chunksizes : tuple, optional
            chunksizes can be used to manually specify the
            HDF5 chunksizes for each dimension of the variable.
        """

        fill_value = None
        if "_FillValue" in attr:
            fill_value = attr.pop("_FillValue")

        if dtype is None:
            dtype = data.dtype

        if zlib is None:
            zlib = self.zlib
        if not np.issubdtype(dtype, np.number):
            # Only numeric data can be compressed
            zlib = False
        if complevel is None:
            complevel = self.complevel

        if name in self.dataset.variables.keys():
            var = self.dataset.variables[name]
        else:
            var = self.dataset.createVariable(name,
                                              dtype,
                                              dim,
                                              fill_value=fill_value,
                                              zlib=zlib,
                                              complevel=complevel,
                                              chunksizes=chunksizes,
                                              **kwargs)

        for attr_name in attr:
            attr_value = attr[attr_name]
            var.setncattr(attr_name, attr_value)

        var.set_auto_scale(self.autoscale)
        if data is not None:
            var[:] = data

    def append_var(self, name, data, **kwargs):
        """
        append data along unlimited dimension(s) of variable

        Parameters
        ----------
        name : string
            Name of variable to append to.
        data : numpy.array
            Numpy array of correct dimension.

        Raises
        ------
        IOError
            if appending to variable without unlimited dimension
        """
        if name in self.dataset.variables.keys():
            var = self.dataset.variables[name]
            dim_unlimited = []
            key = []
            for index, dim in enumerate(var.dimensions):
                unlimited = self.dataset.dimensions[dim].isunlimited()
                dim_unlimited.append(unlimited)
                if not unlimited:
                    # if the dimension is not unlimited set the slice to :
                    key.append(slice(None, None, None))
                else:
                    # if unlimited set slice of this dimension to
                    # append meaning
                    # [var.shape[index]:]
                    key.append(slice(var.shape[index], None, None))

            dim_unlimited = np.array(dim_unlimited)
            nr_unlimited = np.where(dim_unlimited)[0].size
            key = tuple(key)
            # if there are unlimited dimensions we can do an append
            if nr_unlimited > 0:
                var[key] = data
            else:
                raise IOError(
                    "Cannot append to variable that has no unlimited dimension")
        else:
            self.write_var(name, data, **kwargs)

    def read_var(self, name):
        """
        reads variable from netCDF file

        Parameters
        ----------
        name : string
            name of the variable
        """

        if self.mode in ["r", "r+"]:
            if name in self.dataset.variables.keys():
                return self.dataset.variables[name][:]

    def add_global_attr(self, name, value):
        """
        Add global attribute.

        Parameters
        ----------
        name : str
            Name.
        value : str or number
            Value.
        """
        self.global_attr[name] = value

    def flush(self):
        """
        Flush data to disk.
        """
        if self.dataset is not None:
            if self.mode in ["w", "r+"]:
                self._set_global_attr()
                self.dataset.sync()

    def close(self):
        """
        Flush and close file.
        """
        if self.dataset is not None:
            self.flush()
            self.dataset.close()
            self.dataset = None

    def __enter__(self):
        """
        ContextManager enter.
        """
        return self

    def __exit__(self, value_type, value, traceback):
        """
        ContextManager exit.
        """
        self.close()

    def __del__(self):
        """
        Destructor.
        """
        if hasattr(self, "dataset"):
            self.close()

1	# Copyright (c) 2023, TU Wien, Department of Geodesy and Geoinformation.
2	# All rights reserved.
3
4	# Redistribution and use in source and binary forms, with or without
5	# modification, are permitted provided that the following conditions are met:
6	# * Redistributions of source code must retain the above copyright
7	# notice, this list of conditions and the following disclaimer.
8	# * Redistributions in binary form must reproduce the above copyright
9	# notice, this list of conditions and the following disclaimer in the
10	# documentation and/or other materials provided with the distribution.
11	# * Neither the name of the Vienna University of Technology,
12	# Department of Geodesy and Geoinformation nor the
13	# names of its contributors may be used to endorse or promote products
14	# derived from this software without specific prior written permission.
15
16	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19	# ARE DISCLAIMED. IN NO EVENT SHALL VIENNA UNIVERSITY OF TECHNOLOGY,
20	# DEPARTMENT OF GEODESY AND GEOINFORMATION BE LIABLE FOR ANY
21	# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22	# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24	# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26	# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	"""	3✔
28	Base classes for reading and writing time series and images in NetCDF files
29	using Climate Forecast Metadata Conventions (http://cfconventions.org/).
30	"""
31
32	import os	3✔
33	import time	3✔
34	import numpy as np	3✔
35	import netCDF4	3✔
36	import datetime	3✔
37
38
39	class DatasetError(Exception):	3✔
40	pass	3✔
41
42
43	class Dataset:	3✔
44	"""
45	NetCDF file wrapper class that makes some things easier
46
47	Parameters
48	----------
49	filename : string
50	filename of netCDF file. If already exiting then it will be opened
51	as read only unless the append keyword is set. if the overwrite
52	keyword is set then the file will be overwritten
53	name : string, optional
54	will be written as a global attribute if the file is a new file
55	file_format : string, optional
56	file format
57	mode : string, optional
58	access mode. default "r"
59	"r" means read-only; no data can be modified.
60	"w" means write; a new file is created, an existing file with the
61	same name is deleted.
62	"a" and "r+" mean append (in analogy with serial files); an existing
63	file is opened for reading and writing.
64	Appending s to modes w, r+ or a will enable unbuffered shared access
65	to NETCDF3_CLASSIC or NETCDF3_64BIT formatted files. Unbuffered
66	access may be useful even if you don"t need shared access, since it
67	may be faster for programs that don"t access data sequentially.
68	This option is ignored for NETCDF4 and NETCDF4_CLASSIC
69	formatted files.
70	zlib : boolean, optional
71	Default True
72	if set netCDF compression will be used
73	complevel : int, optional
74	Default 4
75	compression level used from 1(low compression) to 9(high compression)
76	autoscale : bool, optional
77	If disabled data will not be automatically scaled when reading and
78	writing
79	automask : bool, optional
80	If disabled data will not be masked during reading.
81	This means Fill Values will be used instead of NaN.
82	"""
83
84	def __init__(self,	3✔
85	filename,
86	name=None,
87	file_format="NETCDF4",
88	mode="r",
89	zlib=True,
90	complevel=4,
91	autoscale=True,
92	automask=True):
93
94	self.dataset_name = name	3✔
95	self.filename = filename	3✔
96	self.file = None	3✔
97	self.file_format = file_format	3✔
98	self.buf_len = 0	3✔
99	self.global_attr = {}	3✔
100	self.global_attr["id"] = os.path.split(self.filename)[1]	3✔
101	s = "%Y-%m-%d %H:%M:%S"	3✔
102	self.global_attr["date_created"] = datetime.datetime.now().strftime(s)	3✔
103	if self.dataset_name is not None:	3✔
104	self.global_attr["dataset_name"] = self.dataset_name	3✔
105	self.zlib = zlib	3✔
106	self.complevel = complevel	3✔
107	self.mode = mode	3✔
108	self.autoscale = autoscale	3✔
109	self.automask = automask	3✔
110
111	if self.mode == "a" and not os.path.exists(self.filename):	3!
112	self.mode = "w"	×
113	if self.mode == "w":	3✔
114	self._create_file_dir()	3✔
115
116	try:	3✔
117	self.dataset = netCDF4.Dataset(self.filename,	3✔
118	self.mode,
119	format=self.file_format)
120	except RuntimeError:	3✔
121	raise IOError(f"File {self.filename} does not exist")	×
122
123	self.dataset.set_auto_scale(self.autoscale)	3✔
124	self.dataset.set_auto_mask(self.automask)	3✔
125
126	def _create_file_dir(self):	3✔
127	"""
128	Create directory for file to sit in.
129	Avoid race condition if multiple instances are
130	writing files into the same directory.
131	"""
132	path = os.path.dirname(self.filename)	3✔
133	if not os.path.exists(path):	3!
134	try:	×
135	os.makedirs(path)	×
136	except OSError:	×
137	time.sleep(1)	×
138	self._create_file_dir()	×
139
140	def _set_global_attr(self):	3✔
141	"""
142	Write global attributes to NetCDF file.
143	"""
144	self.dataset.setncatts(self.global_attr)	3✔
145	self.global_attr = {}	3✔
146
147	def create_dim(self, name, n):	3✔
148	"""
149	Create dimension for NetCDF file.
150	if it does not yet exist
151
152	Parameters
153	----------
154	name : str
155	Name of the NetCDF dimension.
156	n : int
157	Size of the dimension.
158	"""
159	if name not in self.dataset.dimensions.keys():	3!
160	self.dataset.createDimension(name, size=n)	3✔
161
162	def write_var(self,	3✔
163	name,
164	data=None,
165	dim=None,
166	attr={},
167	dtype=None,
168	zlib=None,
169	complevel=None,
170	chunksizes=None,
171	**kwargs):
172	"""
173	Create or overwrite values in a NetCDF variable. The data will be
174	written to disk once flush or close is called
175
176	Parameters
177	----------
178	name : str
179	Name of the NetCDF variable.
180	data : np.ndarray, optional
181	Array containing the data.
182	if not given then the variable will be left empty
183	dim : tuple, optional
184	A tuple containing the dimension names.
185	attr : dict, optional
186	A dictionary containing the variable attributes.
187	dtype: data type, string or numpy.dtype, optional
188	if not given data.dtype will be used
189	zlib: boolean, optional
190	explicit compression for this variable
191	if not given then global attribute is used
192	complevel: int, optional
193	explicit compression level for this variable
194	if not given then global attribute is used
195	chunksizes : tuple, optional
196	chunksizes can be used to manually specify the
197	HDF5 chunksizes for each dimension of the variable.
198	"""
199
200	fill_value = None	3✔
201	if "_FillValue" in attr:	3✔
202	fill_value = attr.pop("_FillValue")	3✔
203
204	if dtype is None:	3✔
205	dtype = data.dtype	3✔
206
207	if zlib is None:	3!
208	zlib = self.zlib	3✔
209	if not np.issubdtype(dtype, np.number):	3✔
210	# Only numeric data can be compressed
211	zlib = False	3✔
212	if complevel is None:	3!
213	complevel = self.complevel	3✔
214
215	if name in self.dataset.variables.keys():	3✔
216	var = self.dataset.variables[name]	3✔
217	else:
218	var = self.dataset.createVariable(name,	3✔
219	dtype,
220	dim,
221	fill_value=fill_value,
222	zlib=zlib,
223	complevel=complevel,
224	chunksizes=chunksizes,
225	**kwargs)
226
227	for attr_name in attr:	3✔
228	attr_value = attr[attr_name]	3✔
229	var.setncattr(attr_name, attr_value)	3✔
230
231	var.set_auto_scale(self.autoscale)	3✔
232	if data is not None:	3✔
233	var[:] = data	3✔
234
235	def append_var(self, name, data, **kwargs):	3✔
236	"""
237	append data along unlimited dimension(s) of variable
238
239	Parameters
240	----------
241	name : string
242	Name of variable to append to.
243	data : numpy.array
244	Numpy array of correct dimension.
245
246	Raises
247	------
248	IOError
249	if appending to variable without unlimited dimension
250	"""
251	if name in self.dataset.variables.keys():	3!
252	var = self.dataset.variables[name]	3✔
253	dim_unlimited = []	3✔
254	key = []	3✔
255	for index, dim in enumerate(var.dimensions):	3✔
256	unlimited = self.dataset.dimensions[dim].isunlimited()	3✔
257	dim_unlimited.append(unlimited)	3✔
258	if not unlimited:	3✔
259	# if the dimension is not unlimited set the slice to :
260	key.append(slice(None, None, None))	3✔
261	else:
262	# if unlimited set slice of this dimension to
263	# append meaning
264	# [var.shape[index]:]
265	key.append(slice(var.shape[index], None, None))	3✔
266
267	dim_unlimited = np.array(dim_unlimited)	3✔
268	nr_unlimited = np.where(dim_unlimited)[0].size	3✔
269	key = tuple(key)	3✔
270	# if there are unlimited dimensions we can do an append
271	if nr_unlimited > 0:	3!
272	var[key] = data	3✔
273	else:
UNCOV 274	raise IOError(	×
275	"Cannot append to variable that has no unlimited dimension")
276	else:
277	self.write_var(name, data, **kwargs)	×
278
279	def read_var(self, name):	3✔
280	"""
281	reads variable from netCDF file
282
283	Parameters
284	----------
285	name : string
286	name of the variable
287	"""
288
289	if self.mode in ["r", "r+"]:	3!
290	if name in self.dataset.variables.keys():	3!
291	return self.dataset.variables[name][:]	3✔
292
293	def add_global_attr(self, name, value):	3✔
294	"""
295	Add global attribute.
296
297	Parameters
298	----------
299	name : str
300	Name.
301	value : str or number
302	Value.
303	"""
UNCOV 304	self.global_attr[name] = value	×
305
306	def flush(self):	3✔
307	"""
308	Flush data to disk.
309	"""
310	if self.dataset is not None:	3!
311	if self.mode in ["w", "r+"]:	3✔
312	self._set_global_attr()	3✔
313	self.dataset.sync()	3✔
314
315	def close(self):	3✔
316	"""
317	Flush and close file.
318	"""
319	if self.dataset is not None:	3✔
320	self.flush()	3✔
321	self.dataset.close()	3✔
322	self.dataset = None	3✔
323
324	def __enter__(self):	3✔
325	"""
326	ContextManager enter.
327	"""
328	return self	3✔
329
330	def __exit__(self, value_type, value, traceback):	3✔
331	"""
332	ContextManager exit.
333	"""
334	self.close()	3✔
335
336	def __del__(self):	3✔
337	"""
338	Destructor.
339	"""
340	if hasattr(self, "dataset"):	3✔
341	self.close()	3✔

TUW-GEO / pynetcf / 4591698001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous