6507626273

Committed 13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%

Build # 6507626273

Build Type

Pull #433

github

Committed by

odscjames

Commit Message

New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424

Pull Request Pull Request #433: New "Geo" optional dependencies

Run Details

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.92

/flattentool/tests/test_schema_parser.py

from collections import OrderedDict

import pytest

from flattentool.schema import SchemaParser, get_property_type_set, is_ref_local
from flattentool.sheet import Sheet

type_string = {"type": "string"}


def test_sub_sheet_list_like():
    # SubSheet object should be appendable and iterable...
    # .append() is used in json_input.py at https://github.com/OpenDataServices/flatten-tool/blob/master/flattentool/json_input.py#L33
    sub_sheet = Sheet()
    assert list(sub_sheet) == []
    sub_sheet.append("a")
    sub_sheet.append("b")
    assert list(sub_sheet) == ["a", "b"]
    # ... but also has an add_field method, which also appends
    sub_sheet.add_field("c")
    assert list(sub_sheet) == ["a", "b", "c"]
    # but with the option to add an id_field, which appears at the start of the list
    sub_sheet.add_field("d", id_field=True)
    assert list(sub_sheet) == ["d", "a", "b", "c"]


def test_get_property_type_set():
    assert get_property_type_set({"type": "a"}) == set(["a"])
    assert get_property_type_set({"type": ["a"]}) == set(["a"])
    assert get_property_type_set({"type": ["a", "b"]}) == set(["a", "b"])


def test_filename_and_dict_error(tmpdir):
    """A value error should be raised if both schema_filename and
    root_schema_dict are supplied to SchemaParser"""
    tmpfile = tmpdir.join("test_schema.json")
    tmpfile.write("{}")
    with pytest.raises(ValueError):
        SchemaParser(schema_filename=tmpfile.strpath, root_schema_dict={})
    # Supplying neither should also raise a ValueError
    with pytest.raises(ValueError):
        SchemaParser()


def test_references_followed(tmpdir):
    """JSON references should be followed when a JSON file is read."""
    tmpfile = tmpdir.join("test_schema.json")
    tmpfile.write('{"a":{"$ref":"#/b"}, "b":"c"}')
    parser = SchemaParser(schema_filename=tmpfile.strpath)
    assert parser.root_schema_dict["a"] == "c"


def test_order_preserved(tmpdir):
    """Order should be preserved when a JSON file is read."""
    tmpfile = tmpdir.join("test_schema.json")
    tmpfile.write('{"a":{}, "c":{}, "b":{}, "d":{}}')
    parser = SchemaParser(schema_filename=tmpfile.strpath)
    assert list(parser.root_schema_dict.keys()) == ["a", "c", "b", "d"]


def test_main_sheet_basic():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": type_string,
                # type is allowed to be empty, and we should assume string
                "Btest": {},
            }
        }
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["Atest", "Btest"])


def test_main_sheet_nested():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {"type": "object", "properties": {"Ctest": type_string}}
            }
        }
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["Atest/Ctest"])


def test_sub_sheet():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "items": {"type": "object", "properties": {"Btest": type_string}},
                },
            }
        }
    )
    parser.parse()
    assert set(parser.main_sheet) == set([])
    assert set(parser.sub_sheets) == set(["Atest"])
    assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Btest"]


def object_in_array_example_properties(parent_name, child_name):
    return {
        "id": type_string,
        parent_name: {
            "type": "array",
            "items": {"type": "object", "properties": {child_name: type_string}},
        },
    }


class TestSubSheetParentID(object):
    def test_parent_is_object(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "Atest": {
                        "type": "object",
                        "properties": object_in_array_example_properties(
                            "Btest", "Ctest"
                        ),
                    }
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["Atest/id"])
        assert set(parser.sub_sheets) == set(["Ate_Btest"])
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "Atest/id",
            "Atest/Btest/0/Ctest",
        ]

    def test_parent_is_array(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "Atest": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": object_in_array_example_properties(
                                "Btest", "Ctest"
                            ),
                        },
                    }
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set()
        assert set(parser.sub_sheets) == set(["Atest", "Ate_Btest"])
        assert list(parser.sub_sheets["Atest"]) == ["Atest/0/id"]
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "Atest/0/id",
            "Atest/0/Btest/0/Ctest",
        ]

    def test_two_parents(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": OrderedDict(
                    [
                        (
                            "Atest",
                            {
                                "type": "array",
                                "items": {
                                    "type": "object",
                                    "properties": object_in_array_example_properties(
                                        "Btest", "Ctest"
                                    ),
                                },
                            },
                        ),
                        (
                            "Dtest",
                            {
                                "type": "array",
                                "items": {
                                    "type": "object",
                                    "properties": object_in_array_example_properties(
                                        "Btest", "Etest"
                                    ),
                                },
                            },
                        ),
                    ]
                )
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set()
        assert set(parser.sub_sheets) == set(
            ["Atest", "Dtest", "Ate_Btest", "Dte_Btest"]
        )
        assert list(parser.sub_sheets["Atest"]) == ["Atest/0/id"]
        assert list(parser.sub_sheets["Dtest"]) == ["Dtest/0/id"]
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "Atest/0/id",
            "Atest/0/Btest/0/Ctest",
        ]
        assert list(parser.sub_sheets["Dte_Btest"]) == [
            "Dtest/0/id",
            "Dtest/0/Btest/0/Etest",
        ]

    def test_parent_is_object_nested(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "Atest": {
                        "type": "object",
                        "properties": {
                            "Btest": {
                                "type": "object",
                                "properties": object_in_array_example_properties(
                                    "Btest", "Ctest"
                                ),
                            }
                        },
                    }
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["Atest/Btest/id"])
        assert set(parser.sub_sheets) == set(["Ate_Bte_Btest"])
        assert list(parser.sub_sheets["Ate_Bte_Btest"]) == [
            "Atest/Btest/id",
            "Atest/Btest/Btest/0/Ctest",
        ]


class TestSubSheetMainID(object):
    def test_parent_is_object(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "id": type_string,
                    "Atest": {
                        "type": "object",
                        "properties": object_in_array_example_properties(
                            "Btest", "Ctest"
                        ),
                    },
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["id", "Atest/id"])
        assert set(parser.sub_sheets) == set(["Ate_Btest"])
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "id",
            "Atest/id",
            "Atest/Btest/0/Ctest",
        ]

    def test_parent_is_array(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "id": type_string,
                    "Atest": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": object_in_array_example_properties(
                                "Btest", "Ctest"
                            ),
                        },
                    },
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["id"])
        assert set(parser.sub_sheets) == set(["Atest", "Ate_Btest"])
        assert list(parser.sub_sheets["Atest"]) == ["id", "Atest/0/id"]
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "id",
            "Atest/0/id",
            "Atest/0/Btest/0/Ctest",
        ]

    def test_two_parents(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": OrderedDict(
                    [
                        ("id", type_string),
                        (
                            "Atest",
                            {
                                "type": "array",
                                "items": {
                                    "type": "object",
                                    "properties": object_in_array_example_properties(
                                        "Btest", "Ctest"
                                    ),
                                },
                            },
                        ),
                        (
                            "Dtest",
                            {
                                "type": "array",
                                "items": {
                                    "type": "object",
                                    "properties": object_in_array_example_properties(
                                        "Btest", "Etest"
                                    ),
                                },
                            },
                        ),
                    ]
                )
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["id"])
        assert set(parser.sub_sheets) == set(
            ["Atest", "Dtest", "Ate_Btest", "Dte_Btest"]
        )
        assert list(parser.sub_sheets["Atest"]) == ["id", "Atest/0/id"]
        assert list(parser.sub_sheets["Dtest"]) == ["id", "Dtest/0/id"]
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "id",
            "Atest/0/id",
            "Atest/0/Btest/0/Ctest",
        ]
        assert list(parser.sub_sheets["Dte_Btest"]) == [
            "id",
            "Dtest/0/id",
            "Dtest/0/Btest/0/Etest",
        ]

    def test_custom_main_sheet_name(self):
        parser = SchemaParser(
            root_schema_dict={
                "properties": {
                    "id": type_string,
                    "Atest": {
                        "type": "object",
                        "properties": object_in_array_example_properties(
                            "Btest", "Ctest"
                        ),
                    },
                }
            }
        )
        parser.parse()
        assert set(parser.main_sheet) == set(["id", "Atest/id"])
        assert set(parser.sub_sheets) == set(["Ate_Btest"])
        assert list(parser.sub_sheets["Ate_Btest"]) == [
            "id",
            "Atest/id",
            "Atest/Btest/0/Ctest",
        ]


@pytest.mark.parametrize("type_", ["string", "number"])
def test_simple_array(type_):
    parser = SchemaParser(
        root_schema_dict={
            "properties": {"Atest": {"type": "array", "items": {"type": type_}}}
        }
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["Atest"])


@pytest.mark.parametrize("type_", ["string", "number"])
def test_nested_simple_array(type_):
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "items": {"type": "array", "items": {"type": type_}},
                }
            }
        }
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["Atest"])


def test_references_sheet_names(tmpdir):
    """
    The referenced name used to be used for the sheet name,
    but is NOT any more.

    """
    tmpfile = tmpdir.join("test_schema.json")
    tmpfile.write(
        """{
        "properties": { "Atest": {
            "type": "array",
            "items": {"$ref": "#/Btest"}
        } },
        "Btest": { "type": "object", "properties": {"Ctest":{"type": "string"}} }
    }"""
    )
    parser = SchemaParser(schema_filename=tmpfile.strpath)
    parser.parse()
    assert set(parser.sub_sheets) == set(["Atest"])  # used to be Btest
    assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Ctest"]


def test_rollup():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "rollUp": ["Btest"],
                    "items": {
                        "type": "object",
                        "properties": {"Btest": type_string, "Ctest": type_string},
                    },
                },
            }
        },
        rollup=True,
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["Atest/0/Btest"])
    assert set(parser.sub_sheets) == set(["Atest"])
    assert set(parser.sub_sheets["Atest"]) == set(["Atest/0/Btest", "Atest/0/Ctest"])


def test_bad_rollup(recwarn):
    """
    When rollUp is specified, but the field is missing in the schema, we expect
    a warning.

    """
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "rollUp": ["Btest"],
                    "items": {"type": "object", "properties": {"Ctest": type_string}},
                },
            }
        },
        rollup=True,
    )
    parser.parse()

    w = recwarn.pop(UserWarning)
    assert "Btest in rollUp but not in schema" in str(w.message)

    assert set(parser.main_sheet) == set()
    assert set(parser.sub_sheets) == set(["Atest"])
    assert set(parser.sub_sheets["Atest"]) == set(["Atest/0/Ctest"])


def test_sub_sheet_custom_id():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "items": {"type": "object", "properties": {"Btest": type_string}},
                },
            }
        },
        root_id="custom",
    )
    parser.parse()
    assert set(parser.main_sheet) == set([])
    assert set(parser.sub_sheets) == set(["Atest"])
    assert list(parser.sub_sheets["Atest"]) == ["custom", "Atest/0/Btest"]


def test_sub_sheet_empty_string_root_id():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "items": {"type": "object", "properties": {"Btest": type_string}},
                },
            }
        },
        root_id="",
    )
    parser.parse()
    assert set(parser.main_sheet) == set([])
    assert set(parser.sub_sheets) == set(["Atest"])
    assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Btest"]


@pytest.mark.parametrize("use_titles", [True, False])
def test_use_titles(recwarn, use_titles):
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "title": "ATitle",
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {"Btest": {"type": "string", "title": "BTitle"}},
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    assert len(recwarn) == 0
    if use_titles:
        assert set(parser.main_sheet) == set(["CTitle"])
        assert set(parser.sub_sheets) == set(["ATitle"])
        assert list(parser.sub_sheets["ATitle"]) == ["ATitle:BTitle"]

    # Array title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {"Btest": {"type": "string", "title": "BTitle"}},
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    if use_titles:
        assert set(parser.main_sheet) == set(["CTitle"])
        assert set(parser.sub_sheets) == set(["Atest"])
        assert list(parser.sub_sheets["Atest"]) == []
        assert len(recwarn) == 1
        w = recwarn.pop(UserWarning)
        assert "Field Atest does not have a title" in str(w.message)
    else:
        assert len(recwarn) == 0

    # Object containing array title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Xtest": {
                    "type": "object",
                    "properties": {
                        "Atest": {
                            "type": "array",
                            "title": "ATitle",
                            "items": {
                                "type": "object",
                                "properties": {
                                    "Btest": {"type": "string", "title": "BTitle"}
                                },
                            },
                        }
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    if use_titles:
        assert set(parser.main_sheet) == set(["CTitle"])
        assert set(parser.sub_sheets) == set(["Xte_Atest"])
        assert list(parser.sub_sheets["Xte_Atest"]) == []
        assert len(recwarn) == 1
        w = recwarn.pop(UserWarning)
        assert "Field Xtest/Atest/0/Btest is missing a title" in str(w.message)
    else:
        assert len(recwarn) == 0


@pytest.mark.parametrize("use_titles", [True, False])
def test_use_titles3(recwarn, use_titles):
    # Array containing a nested object title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "title": "ATitle",
                    "items": {
                        "type": "object",
                        "properties": {
                            "Btest": {
                                "type": "object",
                                "properties": {
                                    "Ctest": {"type": "string", "title": "CTitle"}
                                },
                            }
                        },
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    if use_titles:
        assert set(parser.main_sheet) == set(["CTitle"])
        assert set(parser.sub_sheets) == set(["ATitle"])
        assert list(parser.sub_sheets["ATitle"]) == []
        assert len(recwarn) == 1
        w = recwarn.pop(UserWarning)
        assert "Field Atest/0/Btest/Ctest is missing a title" in str(w.message)
    else:
        assert len(recwarn) == 0


@pytest.mark.parametrize("use_titles", [True, False])
def test_use_titles2(recwarn, use_titles):
    # Object containing object title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Xtest": {
                    "type": "object",
                    "properties": {
                        "Atest": {
                            "type": "object",
                            "title": "ATitle",
                            "properties": {
                                "Btest": {"type": "string", "title": "BTitle"}
                            },
                        }
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    if use_titles:
        assert set(parser.main_sheet) == set(["CTitle"])
        assert set(parser.sub_sheets) == set([])
        assert len(recwarn) == 1
        w = recwarn.pop(UserWarning)
        assert "Field Xtest/Atest/Btest does not have a title, skipping" in str(
            w.message
        )
    else:
        assert len(recwarn) == 0

    # Main sheet title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "title": "ATitle",
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {"Btest": {"type": "string", "title": "BTitle"}},
                    },
                },
                "Ctest": {"type": "string"},
            }
        },
        use_titles=use_titles,
    )
    parser.parse()
    if use_titles:
        assert set(parser.main_sheet) == set([])
        assert set(parser.sub_sheets) == set(["ATitle"])
        assert list(parser.sub_sheets["ATitle"]) == ["ATitle:BTitle"]
        assert len(recwarn) == 1
        w = recwarn.pop(UserWarning)
        assert "Field Ctest does not have a title" in str(w.message)
    else:
        assert len(recwarn) == 0


def test_use_titles5(recwarn):
    # Child sheet title missing
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "title": "ATitle",
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {"Btest": {"type": "string"}},
                    },
                },
                "Ctest": {"type": "string", "title": "CTitle"},
            }
        },
        use_titles=True,
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["CTitle"])
    assert set(parser.sub_sheets) == set(["ATitle"])
    assert list(parser.sub_sheets["ATitle"]) == []
    w = recwarn.pop(UserWarning)
    assert "Field Atest/0/Btest is missing a title" in str(w.message)


def test_titles_rollup():
    parser = SchemaParser(
        root_schema_dict={
            "properties": {
                "Atest": {
                    "type": "array",
                    "title": "ATitle",
                    "rollUp": ["Btest"],
                    "items": {
                        "type": "object",
                        "properties": {
                            "Btest": {
                                "type": "string",
                                "title": "BTitle",
                            },
                            "Ctest": {
                                "type": "string",
                                "title": "CTitle",
                            },
                        },
                    },
                },
            }
        },
        rollup=True,
        use_titles=True,
    )
    parser.parse()
    assert set(parser.main_sheet) == set(["ATitle:BTitle"])
    assert set(parser.sub_sheets) == set(["ATitle"])
    assert set(parser.sub_sheets["ATitle"]) == set(["ATitle:BTitle", "ATitle:CTitle"])


def test_schema_from_uri(httpserver):
    httpserver.serve_content('{"a":{"$ref":"#/b"}, "b":"c"}', 404)
    parser = SchemaParser(schema_filename=httpserver.url)
    assert parser.root_schema_dict["a"] == "c"


test_json_loader_local_refs_disabled_is_ref_local_data_returns_true = [
    (
        "file:///home/odsc/work/flatten-tool/examples/create-template/refs/definitions.json#/definition/address"
    ),
    ("definitions.json#/definition/address"),
]


@pytest.mark.parametrize(
    "data", test_json_loader_local_refs_disabled_is_ref_local_data_returns_true
)
def test_json_loader_local_refs_disabled_is_ref_local_true(data):
    assert True == is_ref_local(data)


test_json_loader_local_refs_disabled_is_ref_local_data_returns_false = [
    (
        "https://raw.githubusercontent.com/openownership/data-standard/master/schema/beneficial-ownership-statements.json"
    ),
    (
        "http://raw.githubusercontent.com/openownership/data-standard/master/schema/beneficial-ownership-statements.json"
    ),
]


@pytest.mark.parametrize(  # noqa
    "data", test_json_loader_local_refs_disabled_is_ref_local_data_returns_false
)
def test_json_loader_local_refs_disabled_is_ref_local_true(data):  # noqa
    assert False == is_ref_local(data)

1	from collections import OrderedDict	10✔
2
3	import pytest	10✔
4
5	from flattentool.schema import SchemaParser, get_property_type_set, is_ref_local	10✔
6	from flattentool.sheet import Sheet	10✔
7
8	type_string = {"type": "string"}	10✔
9
10
11	def test_sub_sheet_list_like():	10✔
12	# SubSheet object should be appendable and iterable...
13	# .append() is used in json_input.py at https://github.com/OpenDataServices/flatten-tool/blob/master/flattentool/json_input.py#L33
14	sub_sheet = Sheet()	×
15	assert list(sub_sheet) == []	×
16	sub_sheet.append("a")	×
17	sub_sheet.append("b")	×
18	assert list(sub_sheet) == ["a", "b"]	×
19	# ... but also has an add_field method, which also appends
20	sub_sheet.add_field("c")	×
21	assert list(sub_sheet) == ["a", "b", "c"]	×
22	# but with the option to add an id_field, which appears at the start of the list
23	sub_sheet.add_field("d", id_field=True)	×
24	assert list(sub_sheet) == ["d", "a", "b", "c"]	×
25
26
27	def test_get_property_type_set():	10✔
28	assert get_property_type_set({"type": "a"}) == set(["a"])	×
29	assert get_property_type_set({"type": ["a"]}) == set(["a"])	×
30	assert get_property_type_set({"type": ["a", "b"]}) == set(["a", "b"])	×
31
32
33	def test_filename_and_dict_error(tmpdir):	10✔
34	"""A value error should be raised if both schema_filename and
35	root_schema_dict are supplied to SchemaParser"""
36	tmpfile = tmpdir.join("test_schema.json")	×
37	tmpfile.write("{}")	×
38	with pytest.raises(ValueError):	×
39	SchemaParser(schema_filename=tmpfile.strpath, root_schema_dict={})	×
40	# Supplying neither should also raise a ValueError
41	with pytest.raises(ValueError):	×
42	SchemaParser()	×
43
44
45	def test_references_followed(tmpdir):	10✔
46	"""JSON references should be followed when a JSON file is read."""
47	tmpfile = tmpdir.join("test_schema.json")	×
48	tmpfile.write('{"a":{"$ref":"#/b"}, "b":"c"}')	×
49	parser = SchemaParser(schema_filename=tmpfile.strpath)	×
50	assert parser.root_schema_dict["a"] == "c"	×
51
52
53	def test_order_preserved(tmpdir):	10✔
54	"""Order should be preserved when a JSON file is read."""
55	tmpfile = tmpdir.join("test_schema.json")	×
56	tmpfile.write('{"a":{}, "c":{}, "b":{}, "d":{}}')	×
57	parser = SchemaParser(schema_filename=tmpfile.strpath)	×
58	assert list(parser.root_schema_dict.keys()) == ["a", "c", "b", "d"]	×
59
60
61	def test_main_sheet_basic():	10✔
62	parser = SchemaParser(	×
63	root_schema_dict={
64	"properties": {
65	"Atest": type_string,
66	# type is allowed to be empty, and we should assume string
67	"Btest": {},
68	}
69	}
70	)
71	parser.parse()	×
72	assert set(parser.main_sheet) == set(["Atest", "Btest"])	×
73
74
75	def test_main_sheet_nested():	10✔
76	parser = SchemaParser(	×
77	root_schema_dict={
78	"properties": {
79	"Atest": {"type": "object", "properties": {"Ctest": type_string}}
80	}
81	}
82	)
83	parser.parse()	×
84	assert set(parser.main_sheet) == set(["Atest/Ctest"])	×
85
86
87	def test_sub_sheet():	10✔
88	parser = SchemaParser(	×
89	root_schema_dict={
90	"properties": {
91	"Atest": {
92	"type": "array",
93	"items": {"type": "object", "properties": {"Btest": type_string}},
94	},
95	}
96	}
97	)
98	parser.parse()	×
99	assert set(parser.main_sheet) == set([])	×
100	assert set(parser.sub_sheets) == set(["Atest"])	×
101	assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Btest"]	×
102
103
104	def object_in_array_example_properties(parent_name, child_name):	10✔
105	return {	×
106	"id": type_string,
107	parent_name: {
108	"type": "array",
109	"items": {"type": "object", "properties": {child_name: type_string}},
110	},
111	}
112
113
114	class TestSubSheetParentID(object):	10✔
115	def test_parent_is_object(self):	10✔
116	parser = SchemaParser(	×
117	root_schema_dict={
118	"properties": {
119	"Atest": {
120	"type": "object",
121	"properties": object_in_array_example_properties(
122	"Btest", "Ctest"
123	),
124	}
125	}
126	}
127	)
128	parser.parse()	×
129	assert set(parser.main_sheet) == set(["Atest/id"])	×
130	assert set(parser.sub_sheets) == set(["Ate_Btest"])	×
131	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
132	"Atest/id",
133	"Atest/Btest/0/Ctest",
134	]
135
136	def test_parent_is_array(self):	10✔
137	parser = SchemaParser(	×
138	root_schema_dict={
139	"properties": {
140	"Atest": {
141	"type": "array",
142	"items": {
143	"type": "object",
144	"properties": object_in_array_example_properties(
145	"Btest", "Ctest"
146	),
147	},
148	}
149	}
150	}
151	)
152	parser.parse()	×
153	assert set(parser.main_sheet) == set()	×
154	assert set(parser.sub_sheets) == set(["Atest", "Ate_Btest"])	×
155	assert list(parser.sub_sheets["Atest"]) == ["Atest/0/id"]	×
156	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
157	"Atest/0/id",
158	"Atest/0/Btest/0/Ctest",
159	]
160
161	def test_two_parents(self):	10✔
162	parser = SchemaParser(	×
163	root_schema_dict={
164	"properties": OrderedDict(
165	[
166	(
167	"Atest",
168	{
169	"type": "array",
170	"items": {
171	"type": "object",
172	"properties": object_in_array_example_properties(
173	"Btest", "Ctest"
174	),
175	},
176	},
177	),
178	(
179	"Dtest",
180	{
181	"type": "array",
182	"items": {
183	"type": "object",
184	"properties": object_in_array_example_properties(
185	"Btest", "Etest"
186	),
187	},
188	},
189	),
190	]
191	)
192	}
193	)
194	parser.parse()	×
195	assert set(parser.main_sheet) == set()	×
196	assert set(parser.sub_sheets) == set(	×
197	["Atest", "Dtest", "Ate_Btest", "Dte_Btest"]
198	)
199	assert list(parser.sub_sheets["Atest"]) == ["Atest/0/id"]	×
200	assert list(parser.sub_sheets["Dtest"]) == ["Dtest/0/id"]	×
201	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
202	"Atest/0/id",
203	"Atest/0/Btest/0/Ctest",
204	]
205	assert list(parser.sub_sheets["Dte_Btest"]) == [	×
206	"Dtest/0/id",
207	"Dtest/0/Btest/0/Etest",
208	]
209
210	def test_parent_is_object_nested(self):	10✔
211	parser = SchemaParser(	×
212	root_schema_dict={
213	"properties": {
214	"Atest": {
215	"type": "object",
216	"properties": {
217	"Btest": {
218	"type": "object",
219	"properties": object_in_array_example_properties(
220	"Btest", "Ctest"
221	),
222	}
223	},
224	}
225	}
226	}
227	)
228	parser.parse()	×
229	assert set(parser.main_sheet) == set(["Atest/Btest/id"])	×
230	assert set(parser.sub_sheets) == set(["Ate_Bte_Btest"])	×
231	assert list(parser.sub_sheets["Ate_Bte_Btest"]) == [	×
232	"Atest/Btest/id",
233	"Atest/Btest/Btest/0/Ctest",
234	]
235
236
237	class TestSubSheetMainID(object):	10✔
238	def test_parent_is_object(self):	10✔
239	parser = SchemaParser(	×
240	root_schema_dict={
241	"properties": {
242	"id": type_string,
243	"Atest": {
244	"type": "object",
245	"properties": object_in_array_example_properties(
246	"Btest", "Ctest"
247	),
248	},
249	}
250	}
251	)
252	parser.parse()	×
253	assert set(parser.main_sheet) == set(["id", "Atest/id"])	×
254	assert set(parser.sub_sheets) == set(["Ate_Btest"])	×
255	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
256	"id",
257	"Atest/id",
258	"Atest/Btest/0/Ctest",
259	]
260
261	def test_parent_is_array(self):	10✔
262	parser = SchemaParser(	×
263	root_schema_dict={
264	"properties": {
265	"id": type_string,
266	"Atest": {
267	"type": "array",
268	"items": {
269	"type": "object",
270	"properties": object_in_array_example_properties(
271	"Btest", "Ctest"
272	),
273	},
274	},
275	}
276	}
277	)
278	parser.parse()	×
279	assert set(parser.main_sheet) == set(["id"])	×
280	assert set(parser.sub_sheets) == set(["Atest", "Ate_Btest"])	×
281	assert list(parser.sub_sheets["Atest"]) == ["id", "Atest/0/id"]	×
282	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
283	"id",
284	"Atest/0/id",
285	"Atest/0/Btest/0/Ctest",
286	]
287
288	def test_two_parents(self):	10✔
289	parser = SchemaParser(	×
290	root_schema_dict={
291	"properties": OrderedDict(
292	[
293	("id", type_string),
294	(
295	"Atest",
296	{
297	"type": "array",
298	"items": {
299	"type": "object",
300	"properties": object_in_array_example_properties(
301	"Btest", "Ctest"
302	),
303	},
304	},
305	),
306	(
307	"Dtest",
308	{
309	"type": "array",
310	"items": {
311	"type": "object",
312	"properties": object_in_array_example_properties(
313	"Btest", "Etest"
314	),
315	},
316	},
317	),
318	]
319	)
320	}
321	)
322	parser.parse()	×
323	assert set(parser.main_sheet) == set(["id"])	×
324	assert set(parser.sub_sheets) == set(	×
325	["Atest", "Dtest", "Ate_Btest", "Dte_Btest"]
326	)
327	assert list(parser.sub_sheets["Atest"]) == ["id", "Atest/0/id"]	×
328	assert list(parser.sub_sheets["Dtest"]) == ["id", "Dtest/0/id"]	×
329	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
330	"id",
331	"Atest/0/id",
332	"Atest/0/Btest/0/Ctest",
333	]
334	assert list(parser.sub_sheets["Dte_Btest"]) == [	×
335	"id",
336	"Dtest/0/id",
337	"Dtest/0/Btest/0/Etest",
338	]
339
340	def test_custom_main_sheet_name(self):	10✔
341	parser = SchemaParser(	×
342	root_schema_dict={
343	"properties": {
344	"id": type_string,
345	"Atest": {
346	"type": "object",
347	"properties": object_in_array_example_properties(
348	"Btest", "Ctest"
349	),
350	},
351	}
352	}
353	)
354	parser.parse()	×
355	assert set(parser.main_sheet) == set(["id", "Atest/id"])	×
356	assert set(parser.sub_sheets) == set(["Ate_Btest"])	×
357	assert list(parser.sub_sheets["Ate_Btest"]) == [	×
358	"id",
359	"Atest/id",
360	"Atest/Btest/0/Ctest",
361	]
362
363
364	@pytest.mark.parametrize("type_", ["string", "number"])	10✔
365	def test_simple_array(type_):	8✔
366	parser = SchemaParser(	×
367	root_schema_dict={
368	"properties": {"Atest": {"type": "array", "items": {"type": type_}}}
369	}
370	)
371	parser.parse()	×
372	assert set(parser.main_sheet) == set(["Atest"])	×
373
374
375	@pytest.mark.parametrize("type_", ["string", "number"])	10✔
376	def test_nested_simple_array(type_):	8✔
377	parser = SchemaParser(	×
378	root_schema_dict={
379	"properties": {
380	"Atest": {
381	"type": "array",
382	"items": {"type": "array", "items": {"type": type_}},
383	}
384	}
385	}
386	)
387	parser.parse()	×
388	assert set(parser.main_sheet) == set(["Atest"])	×
389
390
391	def test_references_sheet_names(tmpdir):	10✔
392	"""
393	The referenced name used to be used for the sheet name,
394	but is NOT any more.
395
396	"""
397	tmpfile = tmpdir.join("test_schema.json")	×
398	tmpfile.write(	×
399	"""{
400	"properties": { "Atest": {
401	"type": "array",
402	"items": {"$ref": "#/Btest"}
403	} },
404	"Btest": { "type": "object", "properties": {"Ctest":{"type": "string"}} }
405	}"""
406	)
407	parser = SchemaParser(schema_filename=tmpfile.strpath)	×
408	parser.parse()	×
409	assert set(parser.sub_sheets) == set(["Atest"]) # used to be Btest	×
410	assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Ctest"]	×
411
412
413	def test_rollup():	10✔
414	parser = SchemaParser(	×
415	root_schema_dict={
416	"properties": {
417	"Atest": {
418	"type": "array",
419	"rollUp": ["Btest"],
420	"items": {
421	"type": "object",
422	"properties": {"Btest": type_string, "Ctest": type_string},
423	},
424	},
425	}
426	},
427	rollup=True,
428	)
429	parser.parse()	×
430	assert set(parser.main_sheet) == set(["Atest/0/Btest"])	×
431	assert set(parser.sub_sheets) == set(["Atest"])	×
432	assert set(parser.sub_sheets["Atest"]) == set(["Atest/0/Btest", "Atest/0/Ctest"])	×
433
434
435	def test_bad_rollup(recwarn):	10✔
436	"""
437	When rollUp is specified, but the field is missing in the schema, we expect
438	a warning.
439
440	"""
441	parser = SchemaParser(	×
442	root_schema_dict={
443	"properties": {
444	"Atest": {
445	"type": "array",
446	"rollUp": ["Btest"],
447	"items": {"type": "object", "properties": {"Ctest": type_string}},
448	},
449	}
450	},
451	rollup=True,
452	)
453	parser.parse()	×
454
455	w = recwarn.pop(UserWarning)	×
456	assert "Btest in rollUp but not in schema" in str(w.message)	×
457
458	assert set(parser.main_sheet) == set()	×
459	assert set(parser.sub_sheets) == set(["Atest"])	×
460	assert set(parser.sub_sheets["Atest"]) == set(["Atest/0/Ctest"])	×
461
462
463	def test_sub_sheet_custom_id():	10✔
464	parser = SchemaParser(	×
465	root_schema_dict={
466	"properties": {
467	"Atest": {
468	"type": "array",
469	"items": {"type": "object", "properties": {"Btest": type_string}},
470	},
471	}
472	},
473	root_id="custom",
474	)
475	parser.parse()	×
476	assert set(parser.main_sheet) == set([])	×
477	assert set(parser.sub_sheets) == set(["Atest"])	×
478	assert list(parser.sub_sheets["Atest"]) == ["custom", "Atest/0/Btest"]	×
479
480
481	def test_sub_sheet_empty_string_root_id():	10✔
482	parser = SchemaParser(	×
483	root_schema_dict={
484	"properties": {
485	"Atest": {
486	"type": "array",
487	"items": {"type": "object", "properties": {"Btest": type_string}},
488	},
489	}
490	},
491	root_id="",
492	)
493	parser.parse()	×
494	assert set(parser.main_sheet) == set([])	×
495	assert set(parser.sub_sheets) == set(["Atest"])	×
496	assert list(parser.sub_sheets["Atest"]) == ["Atest/0/Btest"]	×
497
498
499	@pytest.mark.parametrize("use_titles", [True, False])	10✔
500	def test_use_titles(recwarn, use_titles):	8✔
501	parser = SchemaParser(	×
502	root_schema_dict={
503	"properties": {
504	"Atest": {
505	"title": "ATitle",
506	"type": "array",
507	"items": {
508	"type": "object",
509	"properties": {"Btest": {"type": "string", "title": "BTitle"}},
510	},
511	},
512	"Ctest": {"type": "string", "title": "CTitle"},
513	}
514	},
515	use_titles=use_titles,
516	)
517	parser.parse()	×
518	assert len(recwarn) == 0	×
519	if use_titles:	×
520	assert set(parser.main_sheet) == set(["CTitle"])	×
521	assert set(parser.sub_sheets) == set(["ATitle"])	×
522	assert list(parser.sub_sheets["ATitle"]) == ["ATitle:BTitle"]	×
523
524	# Array title missing
525	parser = SchemaParser(	×
526	root_schema_dict={
527	"properties": {
528	"Atest": {
529	"type": "array",
530	"items": {
531	"type": "object",
532	"properties": {"Btest": {"type": "string", "title": "BTitle"}},
533	},
534	},
535	"Ctest": {"type": "string", "title": "CTitle"},
536	}
537	},
538	use_titles=use_titles,
539	)
540	parser.parse()	×
541	if use_titles:	×
542	assert set(parser.main_sheet) == set(["CTitle"])	×
543	assert set(parser.sub_sheets) == set(["Atest"])	×
544	assert list(parser.sub_sheets["Atest"]) == []	×
545	assert len(recwarn) == 1	×
546	w = recwarn.pop(UserWarning)	×
547	assert "Field Atest does not have a title" in str(w.message)	×
548	else:
549	assert len(recwarn) == 0	×
550
551	# Object containing array title missing
552	parser = SchemaParser(	×
553	root_schema_dict={
554	"properties": {
555	"Xtest": {
556	"type": "object",
557	"properties": {
558	"Atest": {
559	"type": "array",
560	"title": "ATitle",
561	"items": {
562	"type": "object",
563	"properties": {
564	"Btest": {"type": "string", "title": "BTitle"}
565	},
566	},
567	}
568	},
569	},
570	"Ctest": {"type": "string", "title": "CTitle"},
571	}
572	},
573	use_titles=use_titles,
574	)
575	parser.parse()	×
576	if use_titles:	×
577	assert set(parser.main_sheet) == set(["CTitle"])	×
578	assert set(parser.sub_sheets) == set(["Xte_Atest"])	×
579	assert list(parser.sub_sheets["Xte_Atest"]) == []	×
580	assert len(recwarn) == 1	×
581	w = recwarn.pop(UserWarning)	×
582	assert "Field Xtest/Atest/0/Btest is missing a title" in str(w.message)	×
583	else:
584	assert len(recwarn) == 0	×
585
586
587	@pytest.mark.parametrize("use_titles", [True, False])	10✔
588	def test_use_titles3(recwarn, use_titles):	8✔
589	# Array containing a nested object title missing
590	parser = SchemaParser(	×
591	root_schema_dict={
592	"properties": {
593	"Atest": {
594	"type": "array",
595	"title": "ATitle",
596	"items": {
597	"type": "object",
598	"properties": {
599	"Btest": {
600	"type": "object",
601	"properties": {
602	"Ctest": {"type": "string", "title": "CTitle"}
603	},
604	}
605	},
606	},
607	},
608	"Ctest": {"type": "string", "title": "CTitle"},
609	}
610	},
611	use_titles=use_titles,
612	)
613	parser.parse()	×
614	if use_titles:	×
615	assert set(parser.main_sheet) == set(["CTitle"])	×
616	assert set(parser.sub_sheets) == set(["ATitle"])	×
617	assert list(parser.sub_sheets["ATitle"]) == []	×
618	assert len(recwarn) == 1	×
619	w = recwarn.pop(UserWarning)	×
620	assert "Field Atest/0/Btest/Ctest is missing a title" in str(w.message)	×
621	else:
622	assert len(recwarn) == 0	×
623
624
625	@pytest.mark.parametrize("use_titles", [True, False])	10✔
626	def test_use_titles2(recwarn, use_titles):	8✔
627	# Object containing object title missing
628	parser = SchemaParser(	×
629	root_schema_dict={
630	"properties": {
631	"Xtest": {
632	"type": "object",
633	"properties": {
634	"Atest": {
635	"type": "object",
636	"title": "ATitle",
637	"properties": {
638	"Btest": {"type": "string", "title": "BTitle"}
639	},
640	}
641	},
642	},
643	"Ctest": {"type": "string", "title": "CTitle"},
644	}
645	},
646	use_titles=use_titles,
647	)
648	parser.parse()	×
649	if use_titles:	×
650	assert set(parser.main_sheet) == set(["CTitle"])	×
651	assert set(parser.sub_sheets) == set([])	×
652	assert len(recwarn) == 1	×
653	w = recwarn.pop(UserWarning)	×
654	assert "Field Xtest/Atest/Btest does not have a title, skipping" in str(	×
655	w.message
656	)
657	else:
658	assert len(recwarn) == 0	×
659
660	# Main sheet title missing
661	parser = SchemaParser(	×
662	root_schema_dict={
663	"properties": {
664	"Atest": {
665	"title": "ATitle",
666	"type": "array",
667	"items": {
668	"type": "object",
669	"properties": {"Btest": {"type": "string", "title": "BTitle"}},
670	},
671	},
672	"Ctest": {"type": "string"},
673	}
674	},
675	use_titles=use_titles,
676	)
677	parser.parse()	×
678	if use_titles:	×
679	assert set(parser.main_sheet) == set([])	×
680	assert set(parser.sub_sheets) == set(["ATitle"])	×
681	assert list(parser.sub_sheets["ATitle"]) == ["ATitle:BTitle"]	×
682	assert len(recwarn) == 1	×
683	w = recwarn.pop(UserWarning)	×
684	assert "Field Ctest does not have a title" in str(w.message)	×
685	else:
686	assert len(recwarn) == 0	×
687
688
689	def test_use_titles5(recwarn):	10✔
690	# Child sheet title missing
691	parser = SchemaParser(	×
692	root_schema_dict={
693	"properties": {
694	"Atest": {
695	"title": "ATitle",
696	"type": "array",
697	"items": {
698	"type": "object",
699	"properties": {"Btest": {"type": "string"}},
700	},
701	},
702	"Ctest": {"type": "string", "title": "CTitle"},
703	}
704	},
705	use_titles=True,
706	)
707	parser.parse()	×
708	assert set(parser.main_sheet) == set(["CTitle"])	×
709	assert set(parser.sub_sheets) == set(["ATitle"])	×
710	assert list(parser.sub_sheets["ATitle"]) == []	×
711	w = recwarn.pop(UserWarning)	×
712	assert "Field Atest/0/Btest is missing a title" in str(w.message)	×
713
714
715	def test_titles_rollup():	10✔
716	parser = SchemaParser(	×
717	root_schema_dict={
718	"properties": {
719	"Atest": {
720	"type": "array",
721	"title": "ATitle",
722	"rollUp": ["Btest"],
723	"items": {
724	"type": "object",
725	"properties": {
726	"Btest": {
727	"type": "string",
728	"title": "BTitle",
729	},
730	"Ctest": {
731	"type": "string",
732	"title": "CTitle",
733	},
734	},
735	},
736	},
737	}
738	},
739	rollup=True,
740	use_titles=True,
741	)
742	parser.parse()	×
743	assert set(parser.main_sheet) == set(["ATitle:BTitle"])	×
744	assert set(parser.sub_sheets) == set(["ATitle"])	×
745	assert set(parser.sub_sheets["ATitle"]) == set(["ATitle:BTitle", "ATitle:CTitle"])	×
746
747
748	def test_schema_from_uri(httpserver):	10✔
749	httpserver.serve_content('{"a":{"$ref":"#/b"}, "b":"c"}', 404)	×
750	parser = SchemaParser(schema_filename=httpserver.url)	×
751	assert parser.root_schema_dict["a"] == "c"	×
752
753
754	test_json_loader_local_refs_disabled_is_ref_local_data_returns_true = [	10✔
755	(
756	"file:///home/odsc/work/flatten-tool/examples/create-template/refs/definitions.json#/definition/address"
757	),
758	("definitions.json#/definition/address"),
759	]
760
761
762	@pytest.mark.parametrize(	10✔
763	"data", test_json_loader_local_refs_disabled_is_ref_local_data_returns_true
764	)
765	def test_json_loader_local_refs_disabled_is_ref_local_true(data):	8✔
766	assert True == is_ref_local(data)	×
767
768
769	test_json_loader_local_refs_disabled_is_ref_local_data_returns_false = [	10✔
770	(
771	"https://raw.githubusercontent.com/openownership/data-standard/master/schema/beneficial-ownership-statements.json"
772	),
773	(
774	"http://raw.githubusercontent.com/openownership/data-standard/master/schema/beneficial-ownership-statements.json"
775	),
776	]
777
778
779	@pytest.mark.parametrize( # noqa	10✔
780	"data", test_json_loader_local_refs_disabled_is_ref_local_data_returns_false
781	)
782	def test_json_loader_local_refs_disabled_is_ref_local_true(data): # noqa	8✔
783	assert False == is_ref_local(data)	×

OpenDataServices / flatten-tool / 6507626273

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous