#223

pending completion

Build # #223

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

<a href="https://github.com/USDA-ARS-NWRC/smrf/commit/<a class=hub.com/USDA-ARS-NWRC/smrf/commit/773d462dd52a0bb19dec02faf346025106dc1853">773d462dd<a href="https://github.com/USDA-ARS-NWRC/smrf/commit/773d462dd52a0bb19dec02faf346025106dc1853">">Merge </a><a class="double-link" href="https://github.com/USDA-ARS-NWRC/smrf/commit/<a class="double-link" href="https://github.com/USDA-ARS-NWRC/smrf/commit/419c9f717c6be14bf9a2947e6224116324466773">419c9f717</a>">419c9f717</a><a href="https://github.com/USDA-ARS-NWRC/smrf/commit/773d462dd52a0bb19dec02faf346025106dc1853"> into 4a1e52430">4a1e52430</a>

Run Details

3 of 3 new or added lines in 2 files covered. (100.0%)

2752 of 3889 relevant lines covered (70.76%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.29

/smrf/data/loadData.py

import logging

import pandas as pd

from smrf.data import mysql_data


class wxdata():
    """
    Class for loading and storing the data, either from
    - CSV file
    - MySQL database
    - Add other sources here

    Inputs to data() are:
    - dataConfig, either the [csv] or [mysql] section
    - start_date, datetime object
    - end_date, datetime object
    - dataType, either 'csv' or 'mysql'

    The data will be loaded into a Pandas dataframe

    """

    variables = ['air_temp',
                 'vapor_pressure',
                 'precip',
                 'wind_speed',
                 'wind_direction',
                 'cloud_factor']

    db_config_vars = ['user',
                      'password',
                      'host',
                      'database',
                      'port',
                      'metadata',
                      'data_table',
                      'station_table',
                      'stations',
                      'client']

    def __init__(self, dataConfig, start_date, end_date, time_zone='UTC',
                                                         dataType=None):

        if dataType is None:
            raise Exception('''loadData.data() must have a specified dataType
                            of "csv" or "mysql"''')

        self.dataConfig = dataConfig
        self.dataType = dataType
        self.start_date = start_date
        self.end_date = end_date
        self.time_zone = time_zone

        self._logger = logging.getLogger(__name__)
        self.stations = self.dataConfig['stations']

        # load the data
        if dataType == 'csv':
            self.load_from_csv()

        elif dataType == 'mysql':
            self.load_from_mysql()
        else:
            raise Exception('Could not resolve dataType')

    def load_from_csv(self):
        """
        Load the data from a csv file
        Fields that are operated on
        - metadata -> dictionary, one for each station,
        must have at least the following:
        primary_id, X, Y, elevation
        - csv data files -> dictionary, one for each time step,
        must have at least the following columns:
        date_time, column names matching metadata.primary_id
        """

        self._logger.info('Reading data coming from CSV files')

        sta = self.stations

        if sta != None:
            msta = ", ".join(sta)
            self._logger.debug('Using only stations {0}'.format(msta))

        # load the data
        v = list(self.variables)
        v.append('metadata')
        for i in v:
            if i in self.dataConfig:

                self._logger.debug('Reading %s...' % self.dataConfig[i])
                if i == 'metadata':
                    dp_final = pd.read_csv(self.dataConfig[i],
                                     index_col='primary_id')
                    #Ensure all stations are all caps.
                    dp_final.index = [s.upper() for s in dp_final.index]

                elif self.dataConfig[i]:
                    dp_full = pd.read_csv(self.dataConfig[i],
                                     index_col='date_time',
                                     parse_dates=[0])
                    dp_full = dp_full.tz_localize(self.time_zone)
                    dp_full.columns = [s.upper() for s in dp_full.columns]

                    if sta is not None:

                        data_sta = dp_full.columns.str.upper()

                        # Grab IDs from user list thats also in Data
                        self.stations = [s for s in data_sta if s in sta]
                        dp = dp_full[dp_full.columns[(data_sta).isin(sta)]]

                    else:
                        dp = dp_full

                    # Only get the desired dates
                    dp_final = dp[self.start_date:self.end_date]

                    if dp_final.empty:
                        raise Exception("No CSV data found for {0}"
                                        "".format(i))

                setattr(self, i, dp_final)

    def load_from_mysql(self):
        """
        Load the data from a mysql database
        """

        self._logger.info('Reading data from MySQL database')

        # open the database connection
        data = mysql_data.database(self.dataConfig['user'],
                                   self.dataConfig['password'],
                                   self.dataConfig['host'],
                                   self.dataConfig['database'],
                                   self.dataConfig['port'])

        # ---------------------------------------------------
        # determine if it's stations or client
        sta = self.stations

        c = None
        stable = None
        if 'client' in self.dataConfig.keys():
            c = self.dataConfig['client']
            stable = self.dataConfig['station_table']

        # Determine what table for the metadata
        mtable = self.dataConfig['metadata']

        # Raise an error if neither stations or client provided
        if (sta is None) & (c is None):
            raise Exception('Error in configuration file for [mysql],'
                            ' must specify either "stations" or "client"')

        self._logger.debug('Loading metadata from table %s' % mtable)

        # ---------------------------------------------------
        # load the metadata
        self.metadata = data.metadata(mtable, station_ids=sta,
                                      client=c, station_table=stable)

        self._logger.debug('%i stations loaded' % self.metadata.shape[0])

        # ---------------------------------------------------
        # get a list of the stations
        station_ids = self.metadata.index.tolist()

        # get the correct column names if specified, along with variable names
        db_var_names = [val for key, val in self.dataConfig.items()
                        if key not in self.db_config_vars]
        variables = [x for x in self.dataConfig.keys()
                     if x not in self.db_config_vars]

        # get the data
        # dp is a dictionary of dataframes
        dp = data.get_data(
            self.dataConfig['data_table'],
            station_ids,
            self.start_date,
            self.end_date, db_var_names
        )

        # go through and extract the data
        for v in variables:
            # MySQL Data is TZ aware. So convert just in case non utc is passed.
            dfv = dp[self.dataConfig[v]].tz_localize(self.time_zone)
            setattr(self, v, dfv)

1	import logging	1✔
2
3	import pandas as pd	1✔
4
5	from smrf.data import mysql_data	1✔
6
7
8	class wxdata():	1✔
9	"""
10	Class for loading and storing the data, either from
11	- CSV file
12	- MySQL database
13	- Add other sources here
14
15	Inputs to data() are:
16	- dataConfig, either the [csv] or [mysql] section
17	- start_date, datetime object
18	- end_date, datetime object
19	- dataType, either 'csv' or 'mysql'
20
21	The data will be loaded into a Pandas dataframe
22
23	"""
24
25	variables = ['air_temp',	1✔
26	'vapor_pressure',
27	'precip',
28	'wind_speed',
29	'wind_direction',
30	'cloud_factor']
31
32	db_config_vars = ['user',	1✔
33	'password',
34	'host',
35	'database',
36	'port',
37	'metadata',
38	'data_table',
39	'station_table',
40	'stations',
41	'client']
42
43	def __init__(self, dataConfig, start_date, end_date, time_zone='UTC',	1✔
44	dataType=None):
45
46	if dataType is None:	1✔
47	raise Exception('''loadData.data() must have a specified dataType	×
48	of "csv" or "mysql"''')
49
50	self.dataConfig = dataConfig	1✔
51	self.dataType = dataType	1✔
52	self.start_date = start_date	1✔
53	self.end_date = end_date	1✔
54	self.time_zone = time_zone	1✔
55
56	self._logger = logging.getLogger(__name__)	1✔
57	self.stations = self.dataConfig['stations']	1✔
58
59	# load the data
60	if dataType == 'csv':	1✔
61	self.load_from_csv()	1✔
62
63	elif dataType == 'mysql':	×
64	self.load_from_mysql()	×
65	else:
66	raise Exception('Could not resolve dataType')	×
67
68	def load_from_csv(self):	1✔
69	"""
70	Load the data from a csv file
71	Fields that are operated on
72	- metadata -> dictionary, one for each station,
73	must have at least the following:
74	primary_id, X, Y, elevation
75	- csv data files -> dictionary, one for each time step,
76	must have at least the following columns:
77	date_time, column names matching metadata.primary_id
78	"""
79
80	self._logger.info('Reading data coming from CSV files')	1✔
81
82	sta = self.stations	1✔
83
84	if sta != None:	1✔
85	msta = ", ".join(sta)	1✔
86	self._logger.debug('Using only stations {0}'.format(msta))	1✔
87
88	# load the data
89	v = list(self.variables)	1✔
90	v.append('metadata')	1✔
91	for i in v:	1✔
92	if i in self.dataConfig:	1✔
93
94	self._logger.debug('Reading %s...' % self.dataConfig[i])	1✔
95	if i == 'metadata':	1✔
96	dp_final = pd.read_csv(self.dataConfig[i],	1✔
97	index_col='primary_id')
98	#Ensure all stations are all caps.
99	dp_final.index = [s.upper() for s in dp_final.index]	1✔
100
101	elif self.dataConfig[i]:	1✔
102	dp_full = pd.read_csv(self.dataConfig[i],	1✔
103	index_col='date_time',
104	parse_dates=[0])
105	dp_full = dp_full.tz_localize(self.time_zone)	1✔
106	dp_full.columns = [s.upper() for s in dp_full.columns]	1✔
107
108	if sta is not None:	1✔
109
110	data_sta = dp_full.columns.str.upper()	1✔
111
112	# Grab IDs from user list thats also in Data
113	self.stations = [s for s in data_sta if s in sta]	1✔
114	dp = dp_full[dp_full.columns[(data_sta).isin(sta)]]	1✔
115
116	else:
117	dp = dp_full	1✔
118
119	# Only get the desired dates
120	dp_final = dp[self.start_date:self.end_date]	1✔
121
122	if dp_final.empty:	1✔
123	raise Exception("No CSV data found for {0}"	1✔
124	"".format(i))
125
126	setattr(self, i, dp_final)	1✔
127
128	def load_from_mysql(self):	1✔
129	"""
130	Load the data from a mysql database
131	"""
132
133	self._logger.info('Reading data from MySQL database')	×
134
135	# open the database connection
136	data = mysql_data.database(self.dataConfig['user'],	×
137	self.dataConfig['password'],
138	self.dataConfig['host'],
139	self.dataConfig['database'],
140	self.dataConfig['port'])
141
142	# ---------------------------------------------------
143	# determine if it's stations or client
144	sta = self.stations	×
145
146	c = None	×
147	stable = None	×
148	if 'client' in self.dataConfig.keys():	×
149	c = self.dataConfig['client']	×
150	stable = self.dataConfig['station_table']	×
151
152	# Determine what table for the metadata
153	mtable = self.dataConfig['metadata']	×
154
155	# Raise an error if neither stations or client provided
156	if (sta is None) & (c is None):	×
157	raise Exception('Error in configuration file for [mysql],'	×
158	' must specify either "stations" or "client"')
159
160	self._logger.debug('Loading metadata from table %s' % mtable)	×
161
162	# ---------------------------------------------------
163	# load the metadata
164	self.metadata = data.metadata(mtable, station_ids=sta,	×
165	client=c, station_table=stable)
166
167	self._logger.debug('%i stations loaded' % self.metadata.shape[0])	×
168
169	# ---------------------------------------------------
170	# get a list of the stations
171	station_ids = self.metadata.index.tolist()	×
172
173	# get the correct column names if specified, along with variable names
174	db_var_names = [val for key, val in self.dataConfig.items()	×
175	if key not in self.db_config_vars]
176	variables = [x for x in self.dataConfig.keys()	×
177	if x not in self.db_config_vars]
178
179	# get the data
180	# dp is a dictionary of dataframes
181	dp = data.get_data(	×
182	self.dataConfig['data_table'],
183	station_ids,
184	self.start_date,
185	self.end_date, db_var_names
186	)
187
188	# go through and extract the data
189	for v in variables:	×
190	# MySQL Data is TZ aware. So convert just in case non utc is passed.
191	dfv = dp[self.dataConfig[v]].tz_localize(self.time_zone)	×
192	setattr(self, v, dfv)	×

USDA-ARS-NWRC / smrf / #223

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous