6507626273

Committed 13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%

Build # 6507626273

Build Type

Pull #433

github

Committed by

odscjames

Commit Message

New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424

Pull Request Pull Request #433: New "Geo" optional dependencies

Run Details

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.23

/flattentool/tests/test_input_SpreadsheetInput.py

# -*- coding: utf-8 -*-
"""
Tests of SpreadsheetInput class from input.py, and its children.
Tests of unflatten method are in test_input_SpreadsheetInput_unflatten.py
"""
from __future__ import unicode_literals

import datetime
import sys
from collections import OrderedDict
from decimal import Decimal

import pytest
import pytz

from flattentool.input import (
    CSVInput,
    ODSInput,
    SpreadsheetInput,
    XLSXInput,
    convert_type,
)


class ListInput(SpreadsheetInput):
    def __init__(self, sheets, **kwargs):
        self.sheets = sheets
        super(ListInput, self).__init__(**kwargs)

    def get_sheet_lines(self, sheet_name):
        return self.sheets[sheet_name]

    def read_sheets(self):
        self.sub_sheet_names = list(self.sheets.keys())


def test_spreadsheetinput_base_fails():
    spreadsheet_input = SpreadsheetInput()
    with pytest.raises(NotImplementedError):
        spreadsheet_input.read_sheets()
    with pytest.raises(NotImplementedError):
        spreadsheet_input.get_sheet_lines("test")


class TestSuccessfulInput(object):
    def test_csv_input(self, tmpdir):
        main = tmpdir.join("main.csv")
        main.write("colA,colB\ncell1,cell2\ncell3,cell4")
        subsheet = tmpdir.join("subsheet.csv")
        subsheet.write("colC,colD\ncell5,cell6\ncell7,cell8")

        csvinput = CSVInput(input_name=tmpdir.strpath)

        csvinput.read_sheets()

        assert csvinput.sub_sheet_names == ["main", "subsheet"]
        assert list(csvinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(csvinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_xlsx_input(self):
        xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/basic.xlsx")

        xlsxinput.read_sheets()

        assert xlsxinput.sub_sheet_names == ["main", "subsheet"]
        assert list(xlsxinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(xlsxinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_ods_input(self):
        odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/basic.ods")

        odsinput.read_sheets()

        assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]
        assert list(odsinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(odsinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_xlsx_vertical(self):
        xlsxinput = XLSXInput(
            input_name="flattentool/tests/fixtures/xlsx/basic_transpose.xlsx",
            vertical_orientation=True,
        )

        xlsxinput.read_sheets()

        assert xlsxinput.sub_sheet_names == ["main", "subsheet"]
        assert list(xlsxinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(xlsxinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_ods_vertical(self):
        odsinput = ODSInput(
            input_name="flattentool/tests/fixtures/ods/basic_transpose.ods",
            vertical_orientation=True,
        )

        odsinput.read_sheets()

        assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]
        assert list(odsinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(odsinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_xlsx_include_ignore(self):
        xlsxinput = XLSXInput(
            input_name="flattentool/tests/fixtures/xlsx/basic_meta.xlsx",
            include_sheets=["Meta"],
            vertical_orientation=True,
        )
        xlsxinput.read_sheets()
        assert xlsxinput.sub_sheet_names == ["Meta"]
        assert list(xlsxinput.get_sheet_lines("Meta")) == [
            {"a": "a1", "b": "b1", "c": "c1"}
        ]

        xlsxinput = XLSXInput(
            input_name="flattentool/tests/fixtures/xlsx/basic_meta.xlsx",
            exclude_sheets=["Meta"],
        )
        xlsxinput.read_sheets()

        assert xlsxinput.sub_sheet_names == ["main", "subsheet"]
        assert list(xlsxinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(xlsxinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_ods_include_ignore(self):
        odsinput = ODSInput(
            input_name="flattentool/tests/fixtures/ods/basic_meta.ods",
            include_sheets=["Meta"],
            vertical_orientation=True,
        )
        odsinput.read_sheets()
        assert list(odsinput.sub_sheet_names) == ["Meta"]
        assert list(odsinput.get_sheet_lines("Meta")) == [
            {"a": "a1", "b": "b1", "c": "c1"}
        ]

        odsinput = ODSInput(
            input_name="flattentool/tests/fixtures/ods/basic_meta.ods",
            exclude_sheets=["Meta"],
        )
        odsinput.read_sheets()

        assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]
        assert list(odsinput.get_sheet_lines("main")) == [
            {"colA": "cell1", "colB": "cell2"},
            {"colA": "cell3", "colB": "cell4"},
        ]
        assert list(odsinput.get_sheet_lines("subsheet")) == [
            {"colC": "cell5", "colD": "cell6"},
            {"colC": "cell7", "colD": "cell8"},
        ]

    def test_xlsx_input_types(self):
        xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/types.xlsx")

        xlsxinput.read_sheets()

        assert list(xlsxinput.get_sheet_lines("main")) == [
            {
                "colInt": 1,
                "colFloat": 1000.2,
                "colFloatComma": 1000.2,
                "colDate": datetime.datetime(2020, 3, 5),
                "colDateTime": datetime.datetime(
                    2020, 2, 7, 16, 41, 0, 1 if sys.version_info < (3, 6) else 0
                ),
                None: None,
            }
        ]
        assert type(list(xlsxinput.get_sheet_lines("main"))[0]["colInt"]) == int
        assert type(list(xlsxinput.get_sheet_lines("main"))[0]["colFloat"]) == float
        assert (
            type(list(xlsxinput.get_sheet_lines("main"))[0]["colFloatComma"]) == float
        )
        assert xlsxinput.sub_sheet_names == ["main"]

    def test_ods_input_types(self):
        odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/types.ods")

        odsinput.read_sheets()

        assert list(odsinput.get_sheet_lines("main")) == [
            {
                "colInt": 1,
                "colFloat": 1000.2,
                "colFloatComma": 1000.2,
                "colDate": datetime.datetime(2020, 3, 5),
                "colDateTime": datetime.datetime(2020, 2, 7, 16, 41),
            }
        ]
        assert type(list(odsinput.get_sheet_lines("main"))[0]["colInt"]) == int
        assert type(list(odsinput.get_sheet_lines("main"))[0]["colFloatComma"]) == float
        assert list(odsinput.sub_sheet_names) == ["main"]

    def test_xlsx_input_integer2(self):
        xlsxinput = XLSXInput(
            input_name="flattentool/tests/fixtures/xlsx/integer2.xlsx"
        )

        xlsxinput.read_sheets()

        assert list(xlsxinput.get_sheet_lines("Sheet1")) == [
            {"activity-status/@code": 2}
        ]
        # This is a float, but is converted to an int in the unflatten step, see
        # test_input_SpreadsheetInput_unflatten.py
        # 'Basic with float'
        assert (
            type(list(xlsxinput.get_sheet_lines("Sheet1"))[0]["activity-status/@code"])
            == float
        )
        assert xlsxinput.sub_sheet_names == ["Sheet1"]

    def test_ods_input_integer2(self):
        odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/integer2.ods")

        odsinput.read_sheets()

        assert list(odsinput.get_sheet_lines("Sheet1")) == [
            {"activity-status/@code": 2}
        ]
        assert (
            type(list(odsinput.get_sheet_lines("Sheet1"))[0]["activity-status/@code"])
            == int
        )
        assert list(odsinput.sub_sheet_names) == ["Sheet1"]

    def test_xlsx_input_formula(self):
        """When a formula is present, we should use the value, rather than the
        formula itself."""

        xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/formula.xlsx")

        xlsxinput.read_sheets()

        assert xlsxinput.sub_sheet_names == ["main", "subsheet"]
        assert list(xlsxinput.get_sheet_lines("main")) == [
            {"colA": 1, "colB": 2},
            {"colA": 2, "colB": 4},
        ]
        assert list(xlsxinput.get_sheet_lines("subsheet")) == [
            {"colC": 3, "colD": 9},
            {"colC": 4, "colD": 12},
        ]

    def test_bad_xlsx(self):
        """XLSX file that is not a XLSX"""

        xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/file.xlsx")

        try:
            xlsxinput.read_sheets()
        except Exception as e:
            assert (
                str(e)
                == "The supplied file has extension .xlsx but isn't an XLSX file."
            )
            return

        assert False, "No Exception Raised"

    def test_ods_input_formula(self):
        """When a formula is present, we should use the value, rather than the
        formula itself."""

        odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/formula.ods")

        odsinput.read_sheets()

        assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]
        assert list(odsinput.get_sheet_lines("main")) == [
            OrderedDict([("colA", 1), ("colB", 2)]),
            OrderedDict([("colA", 2), ("colB", 4)]),
        ]
        assert list(odsinput.get_sheet_lines("subsheet")) == [
            OrderedDict([("colC", 3), ("colD", 9)]),
            OrderedDict([("colC", 4), ("colD", 12)]),
        ]

    def test_xlsx_empty_column_header(self):
        xlsxinput = XLSXInput(
            input_name="flattentool/tests/fixtures/xlsx/empty_column_header.xlsx"
        )

        xlsxinput.read_sheets()

        assert list(xlsxinput.sub_sheet_names) == ["main"]
        assert list(xlsxinput.get_sheet_lines("main")) == [
            {"colA": "cell1", None: None},
            {"colA": "cell3", None: None},
        ]

    def test_ods_empty_column_header(self):
        odsinput = ODSInput(
            input_name="flattentool/tests/fixtures/ods/empty_column_header.ods"
        )

        odsinput.read_sheets()

        assert list(odsinput.sub_sheet_names) == ["main"]
        assert list(odsinput.get_sheet_lines("main")) == [
            {"colA": "cell1"},
            {"colA": "cell3"},
        ]


class TestInputFailure(object):
    def test_csv_no_directory(self):
        csvinput = CSVInput(input_name="nonesensedirectory")
        with pytest.raises(FileNotFoundError):
            csvinput.read_sheets()

    def test_xlsx_no_file(self, tmpdir):
        xlsxinput = XLSXInput(input_name=tmpdir.join("test.xlsx").strpath)
        with pytest.raises(FileNotFoundError):
            xlsxinput.read_sheets()

    def test_ods_no_file(self, tmpdir):
        odsinput = ODSInput(input_name=tmpdir.join("test.ods").strpath)
        if sys.version > "3":
            with pytest.raises(FileNotFoundError):
                odsinput.read_sheets()
        else:
            with pytest.raises(IOError):
                odsinput.read_sheets()


class TestUnicodeInput(object):
    def test_csv_input_utf8(self, tmpdir):
        main = tmpdir.join("main.csv")
        main.write_text("colA\néαГ😼𝒞人", encoding="utf8")
        csvinput = CSVInput(input_name=tmpdir.strpath)  # defaults to utf8
        csvinput.read_sheets()
        assert list(csvinput.get_sheet_lines("main")) == [{"colA": "éαГ😼𝒞人"}]
        assert csvinput.sub_sheet_names == ["main"]

    def test_csv_input_latin1(self, tmpdir):
        main = tmpdir.join("main.csv")
        main.write_text("colA\né", encoding="latin-1")
        csvinput = CSVInput(input_name=tmpdir.strpath)
        csvinput.encoding = "latin-1"
        csvinput.read_sheets()
        assert list(csvinput.get_sheet_lines("main")) == [{"colA": "é"}]
        assert csvinput.sub_sheet_names == ["main"]

    @pytest.mark.xfail(
        sys.version_info < (3, 0),
        reason="Python 2 CSV readers does not support UTF-16 (or any encodings with null bytes",
    )
    def test_csv_input_utf16(self, tmpdir):
        main = tmpdir.join("main.csv")
        main.write_text("colA\néαГ😼𝒞人", encoding="utf16")
        csvinput = CSVInput(input_name=tmpdir.strpath)
        csvinput.encoding = "utf16"
        csvinput.read_sheets()
        assert list(csvinput.get_sheet_lines("main")) == [{"colA": "éαГ😼𝒞人"}]
        assert csvinput.sub_sheet_names == ["main"]

    def test_xlsx_input_utf8(self):
        """This is an xlsx file saved by OpenOffice. It seems to use UTF8 internally."""
        xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/unicode.xlsx")

        xlsxinput.read_sheets()
        assert list(xlsxinput.get_sheet_lines("main"))[0]["id"] == "éαГ😼𝒞人"

    def test_ods_input_utf8(self):
        """This is an ods file saved by OpenOffice. It seems to use UTF8 internally."""
        odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/unicode.ods")

        odsinput.read_sheets()
        assert list(odsinput.get_sheet_lines("main"))[0]["id"] == "éαГ😼𝒞人"


def test_convert_type(recwarn):
    si = SpreadsheetInput()  # noqa
    assert convert_type("", "somestring") == "somestring"
    # If not type is specified, ints are kept as ints...
    assert convert_type("", 3) == 3

    # ... but all other objects are converted to strings
    class NotAString(object):
        def __str__(self):
            return "string representation"

    assert NotAString() != "string representation"
    assert convert_type("", NotAString()) == "string representation"
    assert convert_type("string", NotAString()) == "string representation"

    assert convert_type("string", 3) == "3"
    assert convert_type("number", "3") == Decimal("3")
    assert convert_type("number", "1.2") == Decimal("1.2")
    assert convert_type("integer", "3") == 3
    assert convert_type("integer", 3) == 3

    assert convert_type("boolean", "TRUE") is True
    assert convert_type("boolean", "True") is True
    assert convert_type("boolean", 1) is True
    assert convert_type("boolean", "1") is True
    assert convert_type("boolean", "FALSE") is False
    assert convert_type("boolean", "False") is False
    assert convert_type("boolean", 0) is False
    assert convert_type("boolean", "0") is False
    convert_type("boolean", 2)
    assert 'Unrecognised value for boolean: "2"' in str(
        recwarn.pop(UserWarning).message
    )
    convert_type("boolean", "test")
    assert 'Unrecognised value for boolean: "test"' in str(
        recwarn.pop(UserWarning).message
    )

    convert_type("integer", "test")
    assert 'Non-integer value "test"' in str(recwarn.pop(UserWarning).message)

    convert_type("number", "test")
    assert 'Non-numeric value "test"' in str(recwarn.pop(UserWarning).message)

    assert convert_type("string", "") is None
    assert convert_type("number", "") is None
    assert convert_type("integer", "") is None
    assert convert_type("array", "") is None
    assert convert_type("boolean", "") is None
    assert convert_type("string", None) is None
    assert convert_type("number", None) is None
    assert convert_type("integer", None) is None
    assert convert_type("array", None) is None
    assert convert_type("boolean", None) is None

    for type_string in ["array", "string_array", "array_array", "number_array"]:
        assert convert_type(type_string, "one") == ["one"]
        assert convert_type(type_string, "one;two") == ["one", "two"]
        assert convert_type(type_string, "one,two;three,four") == [
            ["one", "two"],
            ["three", "four"],
        ]
    assert 'Non-numeric value "one"' in str(recwarn.pop(UserWarning).message)
    assert 'Non-numeric value "one;two"' in str(recwarn.pop(UserWarning).message)
    assert 'Non-numeric value "one,two;three,four"' in str(
        recwarn.pop(UserWarning).message
    )
    assert convert_type("number_array", "1") == [1]
    assert convert_type("number_array", "1;2") == [1, 2]
    assert convert_type("number_array", "1,2;3,4") == [[1, 2], [3, 4]]

    with pytest.raises(ValueError) as e:
        convert_type("notatype", "test")
    assert 'Unrecognised type: "notatype"' in str(e)

    assert (
        convert_type("string", datetime.datetime(2015, 1, 1))
        == "2015-01-01T00:00:00+00:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 1, 1)) == "2015-01-01T00:00:00+00:00"
    )
    assert (
        convert_type("string", datetime.datetime(2015, 1, 1, 13, 37, 59))
        == "2015-01-01T13:37:59+00:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 1, 1, 13, 37, 59))
        == "2015-01-01T13:37:59+00:00"
    )

    timezone = pytz.timezone("Europe/London")
    assert (
        convert_type("string", datetime.datetime(2015, 1, 1), timezone)
        == "2015-01-01T00:00:00+00:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 1, 1), timezone)
        == "2015-01-01T00:00:00+00:00"
    )
    assert (
        convert_type("string", datetime.datetime(2015, 1, 1, 13, 37, 59), timezone)
        == "2015-01-01T13:37:59+00:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 1, 1, 13, 37, 59), timezone)
        == "2015-01-01T13:37:59+00:00"
    )
    assert (
        convert_type("string", datetime.datetime(2015, 6, 1), timezone)
        == "2015-06-01T00:00:00+01:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 6, 1), timezone)
        == "2015-06-01T00:00:00+01:00"
    )
    assert (
        convert_type("string", datetime.datetime(2015, 6, 1, 13, 37, 59), timezone)
        == "2015-06-01T13:37:59+01:00"
    )
    assert (
        convert_type("", datetime.datetime(2015, 6, 1, 13, 37, 59), timezone)
        == "2015-06-01T13:37:59+01:00"
    )
    assert len(recwarn) == 0


@pytest.mark.geo
def test_convert_type_geojson(recwarn):
    assert convert_type(
        "geojson", "POINT (53.486434 -2.239353)", convert_flags={"wkt": True}
    ) == {
        "type": "Point",
        "coordinates": [53.486434, -2.239353],
    }
    assert convert_type(
        "geojson",
        "LINESTRING (-0.173 5.626, -0.178 5.807, -0.112 5.971, -0.211 5.963, -0.321 6.17, -0.488 6.29, -0.56 6.421, -0.752 6.533, -0.867 6.607, -1.101 6.585, -1.304 6.623, -1.461 6.727, -1.628 6.713)",
        convert_flags={"wkt": True},
    ) == {
        "type": "LineString",
        "coordinates": [
            [-0.173, 5.626],
            [-0.178, 5.807],
            [-0.112, 5.971],
            [-0.211, 5.963],
            [-0.321, 6.170],
            [-0.488, 6.290],
            [-0.560, 6.421],
            [-0.752, 6.533],
            [-0.867, 6.607],
            [-1.101, 6.585],
            [-1.304, 6.623],
            [-1.461, 6.727],
            [-1.628, 6.713],
        ],
    }
    assert len(recwarn) == 0

1	# -- coding: utf-8 --
2	"""	10✔
3	Tests of SpreadsheetInput class from input.py, and its children.
4	Tests of unflatten method are in test_input_SpreadsheetInput_unflatten.py
5	"""
6	from __future__ import unicode_literals	10✔
7
8	import datetime	10✔
9	import sys	10✔
10	from collections import OrderedDict	10✔
11	from decimal import Decimal	10✔
12
13	import pytest	10✔
14	import pytz	10✔
15
16	from flattentool.input import (	10✔
17	CSVInput,
18	ODSInput,
19	SpreadsheetInput,
20	XLSXInput,
21	convert_type,
22	)
23
24
25	class ListInput(SpreadsheetInput):	10✔
26	def __init__(self, sheets, **kwargs):	10✔
27	self.sheets = sheets	10✔
28	super(ListInput, self).__init__(**kwargs)	10✔
29
30	def get_sheet_lines(self, sheet_name):	10✔
31	return self.sheets[sheet_name]	10✔
32
33	def read_sheets(self):	10✔
34	self.sub_sheet_names = list(self.sheets.keys())	10✔
35
36
37	def test_spreadsheetinput_base_fails():	10✔
38	spreadsheet_input = SpreadsheetInput()	×
39	with pytest.raises(NotImplementedError):	×
40	spreadsheet_input.read_sheets()	×
41	with pytest.raises(NotImplementedError):	×
42	spreadsheet_input.get_sheet_lines("test")	×
43
44
45	class TestSuccessfulInput(object):	10✔
46	def test_csv_input(self, tmpdir):	10✔
47	main = tmpdir.join("main.csv")	×
48	main.write("colA,colB\ncell1,cell2\ncell3,cell4")	×
49	subsheet = tmpdir.join("subsheet.csv")	×
50	subsheet.write("colC,colD\ncell5,cell6\ncell7,cell8")	×
51
52	csvinput = CSVInput(input_name=tmpdir.strpath)	×
53
54	csvinput.read_sheets()	×
55
56	assert csvinput.sub_sheet_names == ["main", "subsheet"]	×
57	assert list(csvinput.get_sheet_lines("main")) == [	×
58	{"colA": "cell1", "colB": "cell2"},
59	{"colA": "cell3", "colB": "cell4"},
60	]
61	assert list(csvinput.get_sheet_lines("subsheet")) == [	×
62	{"colC": "cell5", "colD": "cell6"},
63	{"colC": "cell7", "colD": "cell8"},
64	]
65
66	def test_xlsx_input(self):	10✔
67	xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/basic.xlsx")	×
68
69	xlsxinput.read_sheets()	×
70
71	assert xlsxinput.sub_sheet_names == ["main", "subsheet"]	×
72	assert list(xlsxinput.get_sheet_lines("main")) == [	×
73	{"colA": "cell1", "colB": "cell2"},
74	{"colA": "cell3", "colB": "cell4"},
75	]
76	assert list(xlsxinput.get_sheet_lines("subsheet")) == [	×
77	{"colC": "cell5", "colD": "cell6"},
78	{"colC": "cell7", "colD": "cell8"},
79	]
80
81	def test_ods_input(self):	10✔
82	odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/basic.ods")	×
83
84	odsinput.read_sheets()	×
85
86	assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]	×
87	assert list(odsinput.get_sheet_lines("main")) == [	×
88	{"colA": "cell1", "colB": "cell2"},
89	{"colA": "cell3", "colB": "cell4"},
90	]
91	assert list(odsinput.get_sheet_lines("subsheet")) == [	×
92	{"colC": "cell5", "colD": "cell6"},
93	{"colC": "cell7", "colD": "cell8"},
94	]
95
96	def test_xlsx_vertical(self):	10✔
97	xlsxinput = XLSXInput(	×
98	input_name="flattentool/tests/fixtures/xlsx/basic_transpose.xlsx",
99	vertical_orientation=True,
100	)
101
102	xlsxinput.read_sheets()	×
103
104	assert xlsxinput.sub_sheet_names == ["main", "subsheet"]	×
105	assert list(xlsxinput.get_sheet_lines("main")) == [	×
106	{"colA": "cell1", "colB": "cell2"},
107	{"colA": "cell3", "colB": "cell4"},
108	]
109	assert list(xlsxinput.get_sheet_lines("subsheet")) == [	×
110	{"colC": "cell5", "colD": "cell6"},
111	{"colC": "cell7", "colD": "cell8"},
112	]
113
114	def test_ods_vertical(self):	10✔
115	odsinput = ODSInput(	×
116	input_name="flattentool/tests/fixtures/ods/basic_transpose.ods",
117	vertical_orientation=True,
118	)
119
120	odsinput.read_sheets()	×
121
122	assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]	×
123	assert list(odsinput.get_sheet_lines("main")) == [	×
124	{"colA": "cell1", "colB": "cell2"},
125	{"colA": "cell3", "colB": "cell4"},
126	]
127	assert list(odsinput.get_sheet_lines("subsheet")) == [	×
128	{"colC": "cell5", "colD": "cell6"},
129	{"colC": "cell7", "colD": "cell8"},
130	]
131
132	def test_xlsx_include_ignore(self):	10✔
133	xlsxinput = XLSXInput(	×
134	input_name="flattentool/tests/fixtures/xlsx/basic_meta.xlsx",
135	include_sheets=["Meta"],
136	vertical_orientation=True,
137	)
138	xlsxinput.read_sheets()	×
139	assert xlsxinput.sub_sheet_names == ["Meta"]	×
140	assert list(xlsxinput.get_sheet_lines("Meta")) == [	×
141	{"a": "a1", "b": "b1", "c": "c1"}
142	]
143
144	xlsxinput = XLSXInput(	×
145	input_name="flattentool/tests/fixtures/xlsx/basic_meta.xlsx",
146	exclude_sheets=["Meta"],
147	)
148	xlsxinput.read_sheets()	×
149
150	assert xlsxinput.sub_sheet_names == ["main", "subsheet"]	×
151	assert list(xlsxinput.get_sheet_lines("main")) == [	×
152	{"colA": "cell1", "colB": "cell2"},
153	{"colA": "cell3", "colB": "cell4"},
154	]
155	assert list(xlsxinput.get_sheet_lines("subsheet")) == [	×
156	{"colC": "cell5", "colD": "cell6"},
157	{"colC": "cell7", "colD": "cell8"},
158	]
159
160	def test_ods_include_ignore(self):	10✔
161	odsinput = ODSInput(	×
162	input_name="flattentool/tests/fixtures/ods/basic_meta.ods",
163	include_sheets=["Meta"],
164	vertical_orientation=True,
165	)
166	odsinput.read_sheets()	×
167	assert list(odsinput.sub_sheet_names) == ["Meta"]	×
168	assert list(odsinput.get_sheet_lines("Meta")) == [	×
169	{"a": "a1", "b": "b1", "c": "c1"}
170	]
171
172	odsinput = ODSInput(	×
173	input_name="flattentool/tests/fixtures/ods/basic_meta.ods",
174	exclude_sheets=["Meta"],
175	)
176	odsinput.read_sheets()	×
177
178	assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]	×
179	assert list(odsinput.get_sheet_lines("main")) == [	×
180	{"colA": "cell1", "colB": "cell2"},
181	{"colA": "cell3", "colB": "cell4"},
182	]
183	assert list(odsinput.get_sheet_lines("subsheet")) == [	×
184	{"colC": "cell5", "colD": "cell6"},
185	{"colC": "cell7", "colD": "cell8"},
186	]
187
188	def test_xlsx_input_types(self):	10✔
189	xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/types.xlsx")	×
190
191	xlsxinput.read_sheets()	×
192
193	assert list(xlsxinput.get_sheet_lines("main")) == [	×
194	{
195	"colInt": 1,
196	"colFloat": 1000.2,
197	"colFloatComma": 1000.2,
198	"colDate": datetime.datetime(2020, 3, 5),
199	"colDateTime": datetime.datetime(
200	2020, 2, 7, 16, 41, 0, 1 if sys.version_info < (3, 6) else 0
201	),
202	None: None,
203	}
204	]
205	assert type(list(xlsxinput.get_sheet_lines("main"))[0]["colInt"]) == int	×
206	assert type(list(xlsxinput.get_sheet_lines("main"))[0]["colFloat"]) == float	×
207	assert (	×
208	type(list(xlsxinput.get_sheet_lines("main"))[0]["colFloatComma"]) == float
209	)
210	assert xlsxinput.sub_sheet_names == ["main"]	×
211
212	def test_ods_input_types(self):	10✔
213	odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/types.ods")	×
214
215	odsinput.read_sheets()	×
216
217	assert list(odsinput.get_sheet_lines("main")) == [	×
218	{
219	"colInt": 1,
220	"colFloat": 1000.2,
221	"colFloatComma": 1000.2,
222	"colDate": datetime.datetime(2020, 3, 5),
223	"colDateTime": datetime.datetime(2020, 2, 7, 16, 41),
224	}
225	]
226	assert type(list(odsinput.get_sheet_lines("main"))[0]["colInt"]) == int	×
227	assert type(list(odsinput.get_sheet_lines("main"))[0]["colFloatComma"]) == float	×
228	assert list(odsinput.sub_sheet_names) == ["main"]	×
229
230	def test_xlsx_input_integer2(self):	10✔
231	xlsxinput = XLSXInput(	×
232	input_name="flattentool/tests/fixtures/xlsx/integer2.xlsx"
233	)
234
235	xlsxinput.read_sheets()	×
236
237	assert list(xlsxinput.get_sheet_lines("Sheet1")) == [	×
238	{"activity-status/@code": 2}
239	]
240	# This is a float, but is converted to an int in the unflatten step, see
241	# test_input_SpreadsheetInput_unflatten.py
242	# 'Basic with float'
243	assert (	×
244	type(list(xlsxinput.get_sheet_lines("Sheet1"))[0]["activity-status/@code"])
245	== float
246	)
247	assert xlsxinput.sub_sheet_names == ["Sheet1"]	×
248
249	def test_ods_input_integer2(self):	10✔
250	odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/integer2.ods")	×
251
252	odsinput.read_sheets()	×
253
254	assert list(odsinput.get_sheet_lines("Sheet1")) == [	×
255	{"activity-status/@code": 2}
256	]
257	assert (	×
258	type(list(odsinput.get_sheet_lines("Sheet1"))[0]["activity-status/@code"])
259	== int
260	)
261	assert list(odsinput.sub_sheet_names) == ["Sheet1"]	×
262
263	def test_xlsx_input_formula(self):	10✔
264	"""When a formula is present, we should use the value, rather than the
265	formula itself."""
266
267	xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/formula.xlsx")	×
268
269	xlsxinput.read_sheets()	×
270
271	assert xlsxinput.sub_sheet_names == ["main", "subsheet"]	×
272	assert list(xlsxinput.get_sheet_lines("main")) == [	×
273	{"colA": 1, "colB": 2},
274	{"colA": 2, "colB": 4},
275	]
276	assert list(xlsxinput.get_sheet_lines("subsheet")) == [	×
277	{"colC": 3, "colD": 9},
278	{"colC": 4, "colD": 12},
279	]
280
281	def test_bad_xlsx(self):	10✔
282	"""XLSX file that is not a XLSX"""
283
284	xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/file.xlsx")	×
285
286	try:	×
287	xlsxinput.read_sheets()	×
288	except Exception as e:	×
289	assert (	×
290	str(e)
291	== "The supplied file has extension .xlsx but isn't an XLSX file."
292	)
293	return	×
294
295	assert False, "No Exception Raised"	×
296
297	def test_ods_input_formula(self):	10✔
298	"""When a formula is present, we should use the value, rather than the
299	formula itself."""
300
301	odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/formula.ods")	×
302
303	odsinput.read_sheets()	×
304
305	assert list(odsinput.sub_sheet_names) == ["main", "subsheet"]	×
306	assert list(odsinput.get_sheet_lines("main")) == [	×
307	OrderedDict([("colA", 1), ("colB", 2)]),
308	OrderedDict([("colA", 2), ("colB", 4)]),
309	]
310	assert list(odsinput.get_sheet_lines("subsheet")) == [	×
311	OrderedDict([("colC", 3), ("colD", 9)]),
312	OrderedDict([("colC", 4), ("colD", 12)]),
313	]
314
315	def test_xlsx_empty_column_header(self):	10✔
316	xlsxinput = XLSXInput(	×
317	input_name="flattentool/tests/fixtures/xlsx/empty_column_header.xlsx"
318	)
319
320	xlsxinput.read_sheets()	×
321
322	assert list(xlsxinput.sub_sheet_names) == ["main"]	×
323	assert list(xlsxinput.get_sheet_lines("main")) == [	×
324	{"colA": "cell1", None: None},
325	{"colA": "cell3", None: None},
326	]
327
328	def test_ods_empty_column_header(self):	10✔
329	odsinput = ODSInput(	×
330	input_name="flattentool/tests/fixtures/ods/empty_column_header.ods"
331	)
332
333	odsinput.read_sheets()	×
334
335	assert list(odsinput.sub_sheet_names) == ["main"]	×
336	assert list(odsinput.get_sheet_lines("main")) == [	×
337	{"colA": "cell1"},
338	{"colA": "cell3"},
339	]
340
341
342	class TestInputFailure(object):	10✔
343	def test_csv_no_directory(self):	10✔
344	csvinput = CSVInput(input_name="nonesensedirectory")	×
345	with pytest.raises(FileNotFoundError):	×
346	csvinput.read_sheets()	×
347
348	def test_xlsx_no_file(self, tmpdir):	10✔
349	xlsxinput = XLSXInput(input_name=tmpdir.join("test.xlsx").strpath)	×
350	with pytest.raises(FileNotFoundError):	×
351	xlsxinput.read_sheets()	×
352
353	def test_ods_no_file(self, tmpdir):	10✔
354	odsinput = ODSInput(input_name=tmpdir.join("test.ods").strpath)	×
355	if sys.version > "3":	×
356	with pytest.raises(FileNotFoundError):	×
357	odsinput.read_sheets()	×
358	else:
359	with pytest.raises(IOError):	×
360	odsinput.read_sheets()	×
361
362
363	class TestUnicodeInput(object):	10✔
364	def test_csv_input_utf8(self, tmpdir):	10✔
365	main = tmpdir.join("main.csv")	×
366	main.write_text("colA\néαГ😼𝒞人", encoding="utf8")	×
367	csvinput = CSVInput(input_name=tmpdir.strpath) # defaults to utf8	×
368	csvinput.read_sheets()	×
369	assert list(csvinput.get_sheet_lines("main")) == [{"colA": "éαГ😼𝒞人"}]	×
370	assert csvinput.sub_sheet_names == ["main"]	×
371
372	def test_csv_input_latin1(self, tmpdir):	10✔
373	main = tmpdir.join("main.csv")	×
374	main.write_text("colA\né", encoding="latin-1")	×
375	csvinput = CSVInput(input_name=tmpdir.strpath)	×
376	csvinput.encoding = "latin-1"	×
377	csvinput.read_sheets()	×
378	assert list(csvinput.get_sheet_lines("main")) == [{"colA": "é"}]	×
379	assert csvinput.sub_sheet_names == ["main"]	×
380
381	@pytest.mark.xfail(	10✔
382	sys.version_info < (3, 0),
383	reason="Python 2 CSV readers does not support UTF-16 (or any encodings with null bytes",
384	)
385	def test_csv_input_utf16(self, tmpdir):	8✔
386	main = tmpdir.join("main.csv")	×
387	main.write_text("colA\néαГ😼𝒞人", encoding="utf16")	×
388	csvinput = CSVInput(input_name=tmpdir.strpath)	×
389	csvinput.encoding = "utf16"	×
390	csvinput.read_sheets()	×
391	assert list(csvinput.get_sheet_lines("main")) == [{"colA": "éαГ😼𝒞人"}]	×
392	assert csvinput.sub_sheet_names == ["main"]	×
393
394	def test_xlsx_input_utf8(self):	10✔
395	"""This is an xlsx file saved by OpenOffice. It seems to use UTF8 internally."""
396	xlsxinput = XLSXInput(input_name="flattentool/tests/fixtures/xlsx/unicode.xlsx")	×
397
398	xlsxinput.read_sheets()	×
399	assert list(xlsxinput.get_sheet_lines("main"))[0]["id"] == "éαГ😼𝒞人"	×
400
401	def test_ods_input_utf8(self):	10✔
402	"""This is an ods file saved by OpenOffice. It seems to use UTF8 internally."""
403	odsinput = ODSInput(input_name="flattentool/tests/fixtures/ods/unicode.ods")	×
404
405	odsinput.read_sheets()	×
406	assert list(odsinput.get_sheet_lines("main"))[0]["id"] == "éαГ😼𝒞人"	×
407
408
409	def test_convert_type(recwarn):	10✔
410	si = SpreadsheetInput() # noqa	×
411	assert convert_type("", "somestring") == "somestring"	×
412	# If not type is specified, ints are kept as ints...
413	assert convert_type("", 3) == 3	×
414
415	# ... but all other objects are converted to strings
416	class NotAString(object):	×
417	def __str__(self):	×
418	return "string representation"	×
419
420	assert NotAString() != "string representation"	×
421	assert convert_type("", NotAString()) == "string representation"	×
422	assert convert_type("string", NotAString()) == "string representation"	×
423
424	assert convert_type("string", 3) == "3"	×
425	assert convert_type("number", "3") == Decimal("3")	×
426	assert convert_type("number", "1.2") == Decimal("1.2")	×
427	assert convert_type("integer", "3") == 3	×
428	assert convert_type("integer", 3) == 3	×
429
430	assert convert_type("boolean", "TRUE") is True	×
431	assert convert_type("boolean", "True") is True	×
432	assert convert_type("boolean", 1) is True	×
433	assert convert_type("boolean", "1") is True	×
434	assert convert_type("boolean", "FALSE") is False	×
435	assert convert_type("boolean", "False") is False	×
436	assert convert_type("boolean", 0) is False	×
437	assert convert_type("boolean", "0") is False	×
438	convert_type("boolean", 2)	×
439	assert 'Unrecognised value for boolean: "2"' in str(	×
440	recwarn.pop(UserWarning).message
441	)
442	convert_type("boolean", "test")	×
443	assert 'Unrecognised value for boolean: "test"' in str(	×
444	recwarn.pop(UserWarning).message
445	)
446
447	convert_type("integer", "test")	×
448	assert 'Non-integer value "test"' in str(recwarn.pop(UserWarning).message)	×
449
450	convert_type("number", "test")	×
451	assert 'Non-numeric value "test"' in str(recwarn.pop(UserWarning).message)	×
452
453	assert convert_type("string", "") is None	×
454	assert convert_type("number", "") is None	×
455	assert convert_type("integer", "") is None	×
456	assert convert_type("array", "") is None	×
457	assert convert_type("boolean", "") is None	×
458	assert convert_type("string", None) is None	×
459	assert convert_type("number", None) is None	×
460	assert convert_type("integer", None) is None	×
461	assert convert_type("array", None) is None	×
462	assert convert_type("boolean", None) is None	×
463
464	for type_string in ["array", "string_array", "array_array", "number_array"]:	×
465	assert convert_type(type_string, "one") == ["one"]	×
466	assert convert_type(type_string, "one;two") == ["one", "two"]	×
467	assert convert_type(type_string, "one,two;three,four") == [	×
468	["one", "two"],
469	["three", "four"],
470	]
471	assert 'Non-numeric value "one"' in str(recwarn.pop(UserWarning).message)	×
472	assert 'Non-numeric value "one;two"' in str(recwarn.pop(UserWarning).message)	×
473	assert 'Non-numeric value "one,two;three,four"' in str(	×
474	recwarn.pop(UserWarning).message
475	)
476	assert convert_type("number_array", "1") == [1]	×
477	assert convert_type("number_array", "1;2") == [1, 2]	×
478	assert convert_type("number_array", "1,2;3,4") == [[1, 2], [3, 4]]	×
479
480	with pytest.raises(ValueError) as e:	×
481	convert_type("notatype", "test")	×
482	assert 'Unrecognised type: "notatype"' in str(e)	×
483
484	assert (	×
485	convert_type("string", datetime.datetime(2015, 1, 1))
486	== "2015-01-01T00:00:00+00:00"
487	)
488	assert (	×
489	convert_type("", datetime.datetime(2015, 1, 1)) == "2015-01-01T00:00:00+00:00"
490	)
491	assert (	×
492	convert_type("string", datetime.datetime(2015, 1, 1, 13, 37, 59))
493	== "2015-01-01T13:37:59+00:00"
494	)
495	assert (	×
496	convert_type("", datetime.datetime(2015, 1, 1, 13, 37, 59))
497	== "2015-01-01T13:37:59+00:00"
498	)
499
500	timezone = pytz.timezone("Europe/London")	×
501	assert (	×
502	convert_type("string", datetime.datetime(2015, 1, 1), timezone)
503	== "2015-01-01T00:00:00+00:00"
504	)
505	assert (	×
506	convert_type("", datetime.datetime(2015, 1, 1), timezone)
507	== "2015-01-01T00:00:00+00:00"
508	)
509	assert (	×
510	convert_type("string", datetime.datetime(2015, 1, 1, 13, 37, 59), timezone)
511	== "2015-01-01T13:37:59+00:00"
512	)
513	assert (	×
514	convert_type("", datetime.datetime(2015, 1, 1, 13, 37, 59), timezone)
515	== "2015-01-01T13:37:59+00:00"
516	)
517	assert (	×
518	convert_type("string", datetime.datetime(2015, 6, 1), timezone)
519	== "2015-06-01T00:00:00+01:00"
520	)
521	assert (	×
522	convert_type("", datetime.datetime(2015, 6, 1), timezone)
523	== "2015-06-01T00:00:00+01:00"
524	)
525	assert (	×
526	convert_type("string", datetime.datetime(2015, 6, 1, 13, 37, 59), timezone)
527	== "2015-06-01T13:37:59+01:00"
528	)
529	assert (	×
530	convert_type("", datetime.datetime(2015, 6, 1, 13, 37, 59), timezone)
531	== "2015-06-01T13:37:59+01:00"
532	)
533	assert len(recwarn) == 0	×
534
535
536	@pytest.mark.geo	10✔
537	def test_convert_type_geojson(recwarn):	8✔
538	assert convert_type(	10✔
539	"geojson", "POINT (53.486434 -2.239353)", convert_flags={"wkt": True}
540	) == {
541	"type": "Point",
542	"coordinates": [53.486434, -2.239353],
543	}
544	assert convert_type(	10✔
545	"geojson",
546	"LINESTRING (-0.173 5.626, -0.178 5.807, -0.112 5.971, -0.211 5.963, -0.321 6.17, -0.488 6.29, -0.56 6.421, -0.752 6.533, -0.867 6.607, -1.101 6.585, -1.304 6.623, -1.461 6.727, -1.628 6.713)",
547	convert_flags={"wkt": True},
548	) == {
549	"type": "LineString",
550	"coordinates": [
551	[-0.173, 5.626],
552	[-0.178, 5.807],
553	[-0.112, 5.971],
554	[-0.211, 5.963],
555	[-0.321, 6.170],
556	[-0.488, 6.290],
557	[-0.560, 6.421],
558	[-0.752, 6.533],
559	[-0.867, 6.607],
560	[-1.101, 6.585],
561	[-1.304, 6.623],
562	[-1.461, 6.727],
563	[-1.628, 6.713],
564	],
565	}
566	assert len(recwarn) == 0	10✔

OpenDataServices / flatten-tool / 6507626273

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous