1238

pending completion

Build # 1238

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #390 from EducationalTestingService/unpin-pandas-and-numpy

Unpin pandas and numpy

Run Details

17 of 17 new or added lines in 4 files covered. (100.0%)

2 existing lines in 1 file now uncovered.

3550 of 3943 relevant lines covered (90.03%)

2.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.2

/rsmtool/reader.py

"""
Classes for reading data files (or dictionaries)
and converting them to DataContainer objects.

:author: Jeremy Biggs (jbiggs@ets.org)
:author: Anastassia Loukina (aloukina@ets.org)
:author: Nitin Madnani (nmadnani@ets.org)

:organization: ETS
"""

import warnings

from functools import partial
from os.path import (abspath,
                     exists,
                     join,
                     splitext)

import pandas as pd

from rsmtool.container import DataContainer


def read_jsonlines(filename, converters=None):
    """
    Read jsonlines from a file.
    Normalize nested jsons with up to one level of nesting

    Parameters
    ----------
    filename: str
        Name of file to read
    converters : dict or None, optional
        A dictionary specifying how the types of the columns
        in the file should be converted. Specified in the same
        format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.

    Returns
    -------
    df : pandas DataFrame
         Data frame containing the data in the given file.
    """

    try:
        df = pd.read_json(filename,
                          orient='records',
                          lines=True,
                          dtype=converters)
    except ValueError:
        raise ValueError("The jsonlines file is not formatted correctly. "
                         "Please check that each line ends with a comma, "
                         "there is no comma at the end of the last line, "
                         "and that all quotes match.")

    # make sure we didn't get a plain json
    if type(df.columns) == pd.RangeIndex:
        raise ValueError("It looks like {} is a simple json file. "
                         "Please check documentation (for the expected "
                         "file format".format(filename))

    dfs = []
    for column in df:
        try:
            df_column = pd.json_normalize(df[column])
        except AttributeError:
            df_column = df[column].copy()

        dfs.append(df_column)

    df = pd.concat(dfs, axis=1)

    return df


def try_to_load_file(filename,
                     converters=None,
                     raise_error=False,
                     raise_warning=False,
                     **kwargs):
    """
    A helper function for reading a single
    file, if it exists. Optionally raise an
    error or warning if the file cannot be found.
    Otherwise, simply return None.

    Parameters
    ----------
    filename : str
        Name of file to read.
    converters : dict or None, optional
        A dictionary specifying how the types of the columns
        in the file should be converted. Specified in the same
        format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
    raise_error : bool, optional
        Raise an error if the file cannot be located.
        Defaults to False.
    raise_warning : bool, optional
        Raise a warning if the file cannot be located
        Defaults to False.

    Returns
    -------
    df : pd.DataFrame or None
        DataFrame containing the data in the given file,
        or None if the file does not exist

    Raises
    ------
    FileNotFoundError
        If `raise_error` is True and file cannot be located.
    """
    if exists(filename):
        return DataReader.read_from_file(filename, converters, **kwargs)

    message = 'The file `{}` could not be located.'.format(filename)
    if raise_error:
        raise FileNotFoundError(message)

    if raise_warning:
        warnings.warn(message)


class DataReader:
    """
    A DataReader class to generate
    DataContainer objects
    """

    def __init__(self,
                 filepaths,
                 framenames,
                 file_converters=None):
        """
        Initialize DataReader object.

        Parameters
        ----------
        filepaths : list of str
            A list of paths to files that will be read into pd.DataFrames.
        filenames : list of str
            A list of names for the pd.DataFrames.
        file_converters : dict of dicts, optional
            A dictionary of file converter dicts.
            Defaults to None

        Raises
        ------
        AssertionError
            If length of filepaths is not equal to length of framenames.
        ValueError
            If any elements in file_converters are not dict.
        NameError
            If file converter name does not exist in the dataset.
        ValueError
            If filepath for a given file is None
        """

        # Default datasets list
        self.datasets = []

        # Make sure filepaths length matches frame names length
        assert len(filepaths) == len(framenames)

        # Make sure that there are no Nones in the filepaths
        if None in filepaths:
            frames_with_no_path = [framenames[i] for i in range(len(framenames))
                                   if filepaths[i] is None]

            raise ValueError("No path specified for "
                             "{}".format(' ,'.join(frames_with_no_path)))

        # Assign names and paths lists
        self.dataset_names = framenames
        self.dataset_paths = filepaths

        # If file_converters exists, then
        # check to make sure it is the correct length
        # and add all elements to file_converters list
        if file_converters is not None:

            # assert len(filepaths) == len(file_converters)

            if not isinstance(file_converters, dict):
                raise ValueError('The `file_converters` argument must be type ``dict``, '
                                 'not ``{}``.'.format(type(file_converters)))

            for file_converter_name in file_converters:

                # Make sure file_converter name is in `dataset_names`
                if file_converter_name not in self.dataset_names:
                    raise NameError('The file converter name ``{}`` '
                                    'does not exist in the '
                                    'dataset names that you '
                                    'passed.'.format(file_converter_name))

                # Make sure file converter is a `dict`
                file_converter = file_converters[file_converter_name]
                if not isinstance(file_converter, dict):
                    raise ValueError('Value for {} must be``dict`` ',
                                     'not {}'.format(file_converter_name,
                                                     type(file_converter)))

        # Default file_converters dict
        self.file_converters = {} if file_converters is None else file_converters

    @staticmethod
    def read_from_file(filename, converters=None, **kwargs):
        """
        Read a CSV/TSV/XLS/XLSX/JSONLINES/SAS7BDAT file and return a data frame.

        Parameters
        ----------
        filename : str
            Name of file to read.
        converters : None, optional
            A dictionary specifying how the types of the columns
            in the file should be converted. Specified in the same
            format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.

        Returns
        -------
        df : pandas DataFrame
            Data frame containing the data in the given file.

        Raises
        ------
        ValueError
            If the file has an extension that we do not support
        pd.parser.CParserError
            If the file is badly formatted or corrupt.

        Note
        ----
        Keyword arguments are passed to the given `pandas`
        IO reader function.
        """

        file_extension = splitext(filename)[1].lower()

        if file_extension in ['.csv', '.tsv']:
            sep = '\t' if file_extension == '.tsv' else ','
            do_read = partial(pd.read_csv, sep=sep, converters=converters)
        elif file_extension in ['.xls', '.xlsx']:
            do_read = partial(pd.read_excel, converters=converters)
        elif file_extension in ['.sas7bdat']:
            if 'encoding' not in kwargs:
                encoding = 'latin-1'
            else:
                encoding = kwargs.pop('encoding')
            do_read = partial(pd.read_sas, encoding=encoding)
        elif file_extension in ['.jsonlines']:
            do_read = partial(read_jsonlines, converters=converters)
        else:
            raise ValueError("RSMTool only supports files in .csv, "
                             ".tsv, .xls/.xlsx, or .sas7bdat format. "
                             "The file should have the extension "
                             "which matches its format. The file you "
                             "passed is: {}.".format(filename))

        # ignore warnings about mixed data types for large files
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=pd.io.common.DtypeWarning)
            try:
                df = do_read(filename, **kwargs)
            except pd.parser.CParserError:
                raise pd.parser.CParserError('Cannot read {}. Please check that it is '
                                             'not corrupt or in an incompatible format. '
                                             '(Try running dos2unix?)'.format(filename))
        return df

    @staticmethod
    def locate_files(filepaths, configdir):
        """
        Try to locate an experiment file, or a list of experiment files.
        If the given path doesn't exist, then maybe the path is relative
        to the path of the config file. If neither exists, then return None.

        Parameters
        ----------
        filepath_or_paths : str or list
            Name of the experiment file we want to locate.
        configdir : str
            Path to the reference configuration directory
            (usually the directory of the config file)

        Returns
        --------
        retval :  str or list
            Absolute path to the experiment file or None
            if the file could not be located. If the `filepaths` argument
            was a string, this method will return a string. Otherwise, it will
            return a list.

        Raises
        ------
        ValueError
            If filepaths  is not a string or list.
        """

        # the feature config file can be in the 'feature' directory
        # at the same level as the main config file

        if not (isinstance(filepaths, str) or
                isinstance(filepaths, list)):

            raise ValueError('The `filepaths` argument must be a '
                             'string or list, not {}.'.format(type(filepaths)))

        if isinstance(filepaths, str):
            filepaths = [filepaths]
            return_string = True
        else:
            return_string = False

        located_paths = []
        for filepath in filepaths:

            retval = None
            alternate_path = abspath(join(configdir, filepath))

            # if the given path exists as is, convert
            # that to an absolute path and return
            if exists(filepath):
                retval = abspath(filepath)

            # otherwise check if it exists relative
            # to the reference directory
            elif exists(alternate_path):
                retval = alternate_path

            located_paths.append(retval)

        if return_string:
            return located_paths[0]

        return located_paths

    def read(self,
             kwargs_dict=None):
        """
        Read all files passed to the constructor.


        Parameters
        ----------
        kwargs_dict : dict of dicts, optional
            Any additional keyword arguments to pass to a particular DataFrame.
            These arguments will be passed to the `pandas` IO reader function.
            Defaults to None.

        Returns
        -------
        datacontainer : DataContainer
            A DataContainer object.
        """

        for idx, set_path in enumerate(self.dataset_paths):

            name = self.dataset_names[idx]
            converter = self.file_converters.get(name, None)

            if not exists(set_path):
                raise FileNotFoundError('The file {} does not exist'.format(set_path))

            if kwargs_dict is not None:
                kwargs = kwargs_dict.get(name, {})
            else:
                kwargs = {}

            dataframe = self.read_from_file(set_path, converter, **kwargs)

            # Add to list of datasets
            self.datasets.append({'name': name.strip(),
                                  'path': set_path,
                                  'frame': dataframe})

        return DataContainer(self.datasets)

1	"""
2	Classes for reading data files (or dictionaries)
3	and converting them to DataContainer objects.
4
5	:author: Jeremy Biggs (jbiggs@ets.org)
6	:author: Anastassia Loukina (aloukina@ets.org)
7	:author: Nitin Madnani (nmadnani@ets.org)
8
9	:organization: ETS
10	"""
11
12	import warnings	5✔
13
14	from functools import partial	5✔
15	from os.path import (abspath,	5✔
16	exists,
17	join,
18	splitext)
19
20	import pandas as pd	5✔
21
22	from rsmtool.container import DataContainer	5✔
23
24
25	def read_jsonlines(filename, converters=None):	5✔
26	"""
27	Read jsonlines from a file.
28	Normalize nested jsons with up to one level of nesting
29
30	Parameters
31	----------
32	filename: str
33	Name of file to read
34	converters : dict or None, optional
35	A dictionary specifying how the types of the columns
36	in the file should be converted. Specified in the same
37	format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
38
39	Returns
40	-------
41	df : pandas DataFrame
42	Data frame containing the data in the given file.
43	"""
44
45	try:	3✔
46	df = pd.read_json(filename,	3✔
47	orient='records',
48	lines=True,
49	dtype=converters)
UNCOV 50	except ValueError:	×
UNCOV 51	raise ValueError("The jsonlines file is not formatted correctly. "	×
52	"Please check that each line ends with a comma, "
53	"there is no comma at the end of the last line, "
54	"and that all quotes match.")
55
56	# make sure we didn't get a plain json
57	if type(df.columns) == pd.RangeIndex:	3✔
58	raise ValueError("It looks like {} is a simple json file. "	1✔
59	"Please check documentation (for the expected "
60	"file format".format(filename))
61
62	dfs = []	3✔
63	for column in df:	3✔
64	try:	3✔
65	df_column = pd.json_normalize(df[column])	3✔
66	except AttributeError:	3✔
67	df_column = df[column].copy()	3✔
68
69	dfs.append(df_column)	3✔
70
71	df = pd.concat(dfs, axis=1)	3✔
72
73	return df	3✔
74
75
76	def try_to_load_file(filename,	5✔
77	converters=None,
78	raise_error=False,
79	raise_warning=False,
80	**kwargs):
81	"""
82	A helper function for reading a single
83	file, if it exists. Optionally raise an
84	error or warning if the file cannot be found.
85	Otherwise, simply return None.
86
87	Parameters
88	----------
89	filename : str
90	Name of file to read.
91	converters : dict or None, optional
92	A dictionary specifying how the types of the columns
93	in the file should be converted. Specified in the same
94	format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
95	raise_error : bool, optional
96	Raise an error if the file cannot be located.
97	Defaults to False.
98	raise_warning : bool, optional
99	Raise a warning if the file cannot be located
100	Defaults to False.
101
102	Returns
103	-------
104	df : pd.DataFrame or None
105	DataFrame containing the data in the given file,
106	or None if the file does not exist
107
108	Raises
109	------
110	FileNotFoundError
111	If `raise_error` is True and file cannot be located.
112	"""
113	if exists(filename):	1✔
114	return DataReader.read_from_file(filename, converters, **kwargs)	×
115
116	message = 'The file `{}` could not be located.'.format(filename)	1✔
117	if raise_error:	1✔
118	raise FileNotFoundError(message)	1✔
119
120	if raise_warning:	1✔
121	warnings.warn(message)	1✔
122
123
124	class DataReader:	5✔
125	"""
126	A DataReader class to generate
127	DataContainer objects
128	"""
129
130	def __init__(self,	5✔
131	filepaths,
132	framenames,
133	file_converters=None):
134	"""
135	Initialize DataReader object.
136
137	Parameters
138	----------
139	filepaths : list of str
140	A list of paths to files that will be read into pd.DataFrames.
141	filenames : list of str
142	A list of names for the pd.DataFrames.
143	file_converters : dict of dicts, optional
144	A dictionary of file converter dicts.
145	Defaults to None
146
147	Raises
148	------
149	AssertionError
150	If length of filepaths is not equal to length of framenames.
151	ValueError
152	If any elements in file_converters are not dict.
153	NameError
154	If file converter name does not exist in the dataset.
155	ValueError
156	If filepath for a given file is None
157	"""
158
159	# Default datasets list
160	self.datasets = []	5✔
161
162	# Make sure filepaths length matches frame names length
163	assert len(filepaths) == len(framenames)	5✔
164
165	# Make sure that there are no Nones in the filepaths
166	if None in filepaths:	5✔
167	frames_with_no_path = [framenames[i] for i in range(len(framenames))	1✔
168	if filepaths[i] is None]
169
170	raise ValueError("No path specified for "	1✔
171	"{}".format(' ,'.join(frames_with_no_path)))
172
173	# Assign names and paths lists
174	self.dataset_names = framenames	5✔
175	self.dataset_paths = filepaths	5✔
176
177	# If file_converters exists, then
178	# check to make sure it is the correct length
179	# and add all elements to file_converters list
180	if file_converters is not None:	5✔
181
182	# assert len(filepaths) == len(file_converters)
183
184	if not isinstance(file_converters, dict):	5✔
185	raise ValueError('The `file_converters` argument must be type ``dict``, '	×
186	'not ``{}``.'.format(type(file_converters)))
187
188	for file_converter_name in file_converters:	5✔
189
190	# Make sure file_converter name is in `dataset_names`
191	if file_converter_name not in self.dataset_names:	5✔
192	raise NameError('The file converter name ``{}`` '	×
193	'does not exist in the '
194	'dataset names that you '
195	'passed.'.format(file_converter_name))
196
197	# Make sure file converter is a `dict`
198	file_converter = file_converters[file_converter_name]	5✔
199	if not isinstance(file_converter, dict):	5✔
200	raise ValueError('Value for {} must be``dict`` ',	×
201	'not {}'.format(file_converter_name,
202	type(file_converter)))
203
204	# Default file_converters dict
205	self.file_converters = {} if file_converters is None else file_converters	5✔
206
207	@staticmethod	5✔
208	def read_from_file(filename, converters=None, **kwargs):	5✔
209	"""
210	Read a CSV/TSV/XLS/XLSX/JSONLINES/SAS7BDAT file and return a data frame.
211
212	Parameters
213	----------
214	filename : str
215	Name of file to read.
216	converters : None, optional
217	A dictionary specifying how the types of the columns
218	in the file should be converted. Specified in the same
219	format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
220
221	Returns
222	-------
223	df : pandas DataFrame
224	Data frame containing the data in the given file.
225
226	Raises
227	------
228	ValueError
229	If the file has an extension that we do not support
230	pd.parser.CParserError
231	If the file is badly formatted or corrupt.
232
233	Note
234	----
235	Keyword arguments are passed to the given `pandas`
236	IO reader function.
237	"""
238
239	file_extension = splitext(filename)[1].lower()	5✔
240
241	if file_extension in ['.csv', '.tsv']:	5✔
242	sep = '\t' if file_extension == '.tsv' else ','	5✔
243	do_read = partial(pd.read_csv, sep=sep, converters=converters)	5✔
244	elif file_extension in ['.xls', '.xlsx']:	4✔
245	do_read = partial(pd.read_excel, converters=converters)	4✔
246	elif file_extension in ['.sas7bdat']:	4✔
247	if 'encoding' not in kwargs:	1✔
248	encoding = 'latin-1'	1✔
249	else:
250	encoding = kwargs.pop('encoding')	×
251	do_read = partial(pd.read_sas, encoding=encoding)	1✔
252	elif file_extension in ['.jsonlines']:	4✔
253	do_read = partial(read_jsonlines, converters=converters)	3✔
254	else:
255	raise ValueError("RSMTool only supports files in .csv, "	2✔
256	".tsv, .xls/.xlsx, or .sas7bdat format. "
257	"The file should have the extension "
258	"which matches its format. The file you "
259	"passed is: {}.".format(filename))
260
261	# ignore warnings about mixed data types for large files
262	with warnings.catch_warnings():	5✔
263	warnings.filterwarnings('ignore', category=pd.io.common.DtypeWarning)	5✔
264	try:	5✔
265	df = do_read(filename, **kwargs)	5✔
266	except pd.parser.CParserError:	×
267	raise pd.parser.CParserError('Cannot read {}. Please check that it is '	×
268	'not corrupt or in an incompatible format. '
269	'(Try running dos2unix?)'.format(filename))
270	return df	5✔
271
272	@staticmethod	5✔
273	def locate_files(filepaths, configdir):
274	"""
275	Try to locate an experiment file, or a list of experiment files.
276	If the given path doesn't exist, then maybe the path is relative
277	to the path of the config file. If neither exists, then return None.
278
279	Parameters
280	----------
281	filepath_or_paths : str or list
282	Name of the experiment file we want to locate.
283	configdir : str
284	Path to the reference configuration directory
285	(usually the directory of the config file)
286
287	Returns
288	--------
289	retval : str or list
290	Absolute path to the experiment file or None
291	if the file could not be located. If the `filepaths` argument
292	was a string, this method will return a string. Otherwise, it will
293	return a list.
294
295	Raises
296	------
297	ValueError
298	If filepaths is not a string or list.
299	"""
300
301	# the feature config file can be in the 'feature' directory
302	# at the same level as the main config file
303
304	if not (isinstance(filepaths, str) or	5✔
305	isinstance(filepaths, list)):
306
307	raise ValueError('The `filepaths` argument must be a '	1✔
308	'string or list, not {}.'.format(type(filepaths)))
309
310	if isinstance(filepaths, str):	5✔
311	filepaths = [filepaths]	5✔
312	return_string = True	5✔
313	else:
314	return_string = False	5✔
315
316	located_paths = []	5✔
317	for filepath in filepaths:	5✔
318
319	retval = None	5✔
320	alternate_path = abspath(join(configdir, filepath))	5✔
321
322	# if the given path exists as is, convert
323	# that to an absolute path and return
324	if exists(filepath):	5✔
325	retval = abspath(filepath)	3✔
326
327	# otherwise check if it exists relative
328	# to the reference directory
329	elif exists(alternate_path):	5✔
330	retval = alternate_path	5✔
331
332	located_paths.append(retval)	5✔
333
334	if return_string:	5✔
335	return located_paths[0]	5✔
336
337	return located_paths	5✔
338
339	def read(self,	5✔
340	kwargs_dict=None):
341	"""
342	Read all files passed to the constructor.
343
344
345	Parameters
346	----------
347	kwargs_dict : dict of dicts, optional
348	Any additional keyword arguments to pass to a particular DataFrame.
349	These arguments will be passed to the `pandas` IO reader function.
350	Defaults to None.
351
352	Returns
353	-------
354	datacontainer : DataContainer
355	A DataContainer object.
356	"""
357
358	for idx, set_path in enumerate(self.dataset_paths):	5✔
359
360	name = self.dataset_names[idx]	5✔
361	converter = self.file_converters.get(name, None)	5✔
362
363	if not exists(set_path):	5✔
364	raise FileNotFoundError('The file {} does not exist'.format(set_path))	×
365
366	if kwargs_dict is not None:	5✔
367	kwargs = kwargs_dict.get(name, {})	1✔
368	else:
369	kwargs = {}	5✔
370
371	dataframe = self.read_from_file(set_path, converter, **kwargs)	5✔
372
373	# Add to list of datasets
374	self.datasets.append({'name': name.strip(),	5✔
375	'path': set_path,
376	'frame': dataframe})
377
378	return DataContainer(self.datasets)	5✔

EducationalTestingService / rsmtool / 1238

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous