6507626273

Committed 13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%

Build # 6507626273

Build Type

Pull #433

github

Committed by

odscjames

Commit Message

New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424

Pull Request Pull Request #433: New "Geo" optional dependencies

Run Details

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.86

/flattentool/tests/test_input_SpreadsheetInput_unflatten_mulitplesheets.py

# -*- coding: utf-8 -*-
"""
Tests of unflatten method of the SpreadsheetInput class from input.py

Tests that only apply for multiple sheets.
"""
from __future__ import unicode_literals

from collections import OrderedDict

import pytest

from flattentool.schema import SchemaParser

from .test_input_SpreadsheetInput import ListInput
from .test_input_SpreadsheetInput_unflatten import (
    ROOT_ID_PARAMS,
    create_schema,
    inject_root_id,
)

testdata_multiplesheets = [
    (
        "Basic sub sheet",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                },
                {
                    "ROOT_ID": 1,
                    "id": 3,
                },
            ],
            "testArr": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testArr/0/testC": "3",
                },
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testArr/0/testC": "4",
                },
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testArr": [
                    {"testC": "3"},
                    {"testC": "4"},
                ],
            },
            {"ROOT_ID": 1, "id": 3},
        ],
        [],
        True,
    ),
    (
        "Nested sub sheet (with id)",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testB/id": 3,
                    "testB/testC": 4,
                }
            ],
            "tes_subField": [
                # It used to be necessary to supply testA/id in this
                # situation, but now it's optional
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testB/id": 3,
                    "testB/subField/0/testD": 5,
                }
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testB": {"id": 3, "testC": 4, "subField": [{"testD": 5}]},
            }
        ],
        [],
        True,
    ),
    (
        "Nested sub sheet (without id)",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testB/id": 3,
                    "testB/testC": 4,
                }
            ],
            "sub": [
                # It used to be necessary to supply testA/id in this
                # situation, but now it's optional
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testB/subField/0/testD": 5,
                }
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testB": {"id": 3, "testC": 4, "subField": [{"testD": 5}]},
            }
        ],
        [],
        False,
    ),
    (
        "Basic two sub sheets",
        OrderedDict(
            [
                (
                    "custom_main",
                    [
                        OrderedDict(
                            [
                                ("ROOT_ID", 1),
                                ("id", 2),
                            ]
                        ),
                        OrderedDict(
                            [
                                ("ROOT_ID", 1),
                                ("id", 6),
                            ]
                        ),
                    ],
                ),
                (
                    "sub1Field",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                            "sub1Field/0/id": 3,
                            "sub1Field/0/testA": 4,
                        }
                    ],
                ),
                (
                    "sub_sub2Field",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                            "sub1Field/0/id": 3,
                            "sub1Field/0/sub2Field/0/testB": 5,
                        }
                    ],
                ),
            ]
        ),
        [
            OrderedDict(
                [
                    ("ROOT_ID", 1),
                    ("id", 2),
                    ("sub1Field", [{"id": 3, "testA": 4, "sub2Field": [{"testB": 5}]}]),
                ]
            ),
            {"ROOT_ID": 1, "id": 6},
        ],
        [],
        True,
    ),
    (
        "Nested id",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                }
            ],
            "subField": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "subField/0/id": 3,
                    "subField/0/testA/id": 4,
                }
            ],
        },
        [{"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": {"id": 4}}]}],
        [],
        True,
    ),
    (
        "Missing columns",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                }
            ],
            "sub": [
                {
                    "ROOT_ID": 1,
                    "id": "",
                    "subField/0/id": 3,
                    "subField/0/testA": 4,
                },
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "subField/0/id": 3,
                    "subField/0/testA": 5,
                },
            ],
        },
        [
            {"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": 5}]},
            {"ROOT_ID": 1, "subField": [{"id": 3, "testA": 4}]},
        ],
        [],
        False,
    ),
    (
        "Unmatched id",
        OrderedDict(
            [
                (
                    "custom_main",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                        }
                    ],
                ),
                (
                    "sub",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 100,
                            "subField/0/id": 3,
                            "subField/0/testA": 4,
                        },
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                            "subField/0/id": 3,
                            "subField/0/testA": 5,
                        },
                    ],
                ),
            ]
        ),
        [
            {"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": 5}]},
            {"ROOT_ID": 1, "id": 100, "subField": [{"id": 3, "testA": 4}]},
        ],
        [],
        False,
    ),
    (
        "Test same rollup",
        {
            "main": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testC": 3,
                    "testArr/0/id": "4",
                    "testArr/0/testB": "5",
                },
                {
                    "ROOT_ID": 6,
                    "id": 7,
                    "testC": 8,
                    "testArr/0/testB": "9",
                },
            ],
            "testArr": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testArr/0/id": "4",
                    "testArr/0/testB": "5",
                },
                {
                    "ROOT_ID": 6,
                    "id": 7,
                    "testArr/0/testB": "9",
                },
            ],
        },
        [
            {"ROOT_ID": 1, "id": 2, "testC": 3, "testArr": [{"id": "4", "testB": "5"}]},
            {
                "ROOT_ID": 6,
                "id": 7,
                "testC": 8,
                "testArr": [
                    {"testB": "9"},
                    {"testB": "9"}
                    # We have duplicates here because there's no ID to merge these
                    # on. This is different to the old behaviour. Issue filed at
                    # https://github.com/OpenDataServices/flatten-tool/issues/99
                ],
            },
        ],
        [],
        False,
    ),
    (
        "Test conflicting rollup",
        OrderedDict(
            [
                (
                    "main",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                            "testArr/0/id": "3",
                            "testArr/0/testB": "4",
                        }
                    ],
                ),
                (
                    "testArr",
                    [
                        {
                            "ROOT_ID": 1,
                            "id": 2,
                            "testArr/0/id": "3",
                            "testArr/0/testB": "5",
                        }
                    ],
                ),
            ]
        ),
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testArr": [
                    {
                        "id": "3",
                        "testB": "4"
                        # (Since sheets are parsed in the order they appear, and the first value is used).
                    }
                ],
            }
        ],
        [
            'Conflict when merging field "testB" for ROOT_ID "1", id "2" in sheet testA: "4" != "5"'
        ],
        False,
    ),
    (
        "Unflatten empty",
        {
            "custom_main": [],
            "subsheet": [
                {
                    "ROOT_ID": "",
                    "id": "",
                    "testA": "",
                    "testB": "",
                    "testC": "",
                    "testD": "",
                }
            ],
        },
        [],
        [],
        False,
    ),
]


testdata_multiplesheets_pointer = [
    (
        "with schema",
        {
            "custom_main": [{"ROOT_ID": 1, "id": "2", "testA": 3}],
            "sub": [
                {
                    "ROOT_ID": 1,
                    "id": 2,
                    "testArr/testB": 4,  # test that we can infer this an array from schema
                }
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,  # check that we join correctly when this gets converted to an
                # integer because of the schema type
                "testA": 3,
                "testArr": [{"testB": "4"}],
            }
        ],
        [],
    )
]


testdata_multiplesheets_titles = [
    (
        "Basic sub sheet",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                },
                {
                    "ROOT_ID": 1,
                    "Identifier": 3,
                },
            ],
            "Arr title": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "Arr title:C title": "3",
                },
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "Arr title:C title": "4",
                },
            ],
            "arr_not_in_schema": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "arr_not_in_schema/0/testD": "5",
                },
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "arr_not_in_schema/0/testD": "6",
                },
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testArr": [
                    {"testC": "3"},
                    {"testC": "4"},
                ],
                "arr_not_in_schema": [{"testD": "5"}, {"testD": "6"}],
            },
            {"ROOT_ID": 1, "id": 3},
        ],
        [],
        True,
    ),
    (
        "Nested sub sheet (with id)",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "B title:Identifier": 3,
                    "B title:C title": 4,
                }
            ],
            "B t_Sub title": [
                # It used to be necessary to supply testA/id in this
                # situation, but now it's optional
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "B title:Identifier": 3,
                    "B title:Sub title:E title": 5,
                }
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testB": {"id": 3, "testC": 4, "subField": [{"testE": 5}]},
            }
        ],
        [],
        True,
    ),
    (
        "Nested sub sheet (without id)",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "B title:Identifier": 3,
                    "B title:C title": 4,
                }
            ],
            "sub": [
                # It used to be necessary to supply testA/id in this
                # situation, but now it's optional
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "B title:Sub title:E title": 5,
                }
            ],
        },
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testB": {"id": 3, "testC": 4, "subField": [{"testE": 5}]},
            }
        ],
        [],
        False,
    ),
    (
        "Basic two sub sheets",
        OrderedDict(
            [
                (
                    "custom_main",
                    [
                        OrderedDict(
                            [
                                ("ROOT_ID", 1),
                                ("Identifier", 2),
                            ]
                        ),
                        OrderedDict(
                            [
                                ("ROOT_ID", 1),
                                ("Identifier", 6),
                            ]
                        ),
                    ],
                ),
                (
                    "Arr title",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                            "Arr title:Identifier": "3",
                            "Arr title:B title": "4",
                        }
                    ],
                ),
                (
                    "Arr_Nest title",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                            "Arr title:Identifier": "3",
                            "Arr title:Nest title:D title": "5",
                        }
                    ],
                ),
            ]
        ),
        [
            OrderedDict(
                [
                    ("ROOT_ID", 1),
                    ("id", 2),
                    (
                        "testArr",
                        [{"id": "3", "testB": "4", "testNest": [{"testD": "5"}]}],
                    ),
                ]
            ),
            {"ROOT_ID": 1, "id": 6},
        ],
        [],
        True,
    ),
    (
        "Nested id",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                }
            ],
            "Arr title": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "Arr title:Identifier": "3",
                    "Arr title:NestObj title:Identifier": "4",
                }
            ],
        },
        [{"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testNestObj": {"id": "4"}}]}],
        [],
        True,
    ),
    (
        "Missing columns",
        {
            "custom_main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                }
            ],
            "sub": [
                {
                    "ROOT_ID": 1,
                    "Identifier": "",
                    "Arr title:Identifier": 3,
                    "Arr title:B title": 4,
                },
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "Arr title:Identifier": 3,
                    "Arr title:B title": 5,
                },
            ],
        },
        [
            {"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testB": "5"}]},
            {"ROOT_ID": 1, "testArr": [{"id": "3", "testB": "4"}]},
        ],
        [],
        False,
    ),
    (
        "Unmatched id",
        OrderedDict(
            [
                (
                    "custom_main",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                        }
                    ],
                ),
                (
                    "sub",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 100,
                            "Arr title:Identifier": 3,
                            "Arr title:B title": 4,
                        },
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                            "Arr title:Identifier": 3,
                            "Arr title:B title": 5,
                        },
                    ],
                ),
            ]
        ),
        [
            {"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testB": "5"}]},
            {"ROOT_ID": 1, "id": 100, "testArr": [{"id": "3", "testB": "4"}]},
        ],
        [],
        False,
    ),
    (
        "Test same rollup",
        {
            "main": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "A title": 3,
                    "Arr title:Identifier": 4,
                    "Arr title:B title": 5,
                },
                {
                    "ROOT_ID": 6,
                    "Identifier": 7,
                    "A title": 8,
                    "Arr title:B title": 9,
                },
            ],
            "testArr": [
                {
                    "ROOT_ID": 1,
                    "Identifier": 2,
                    "Arr title:Identifier": 4,
                    "Arr title:B title": 5,
                },
                {
                    "ROOT_ID": 6,
                    "Identifier": 7,
                    "Arr title:B title": 9,
                },
            ],
        },
        [
            {"ROOT_ID": 1, "id": 2, "testA": 3, "testArr": [{"id": "4", "testB": "5"}]},
            {
                "ROOT_ID": 6,
                "id": 7,
                "testA": 8,
                "testArr": [
                    {"testB": "9"},
                    {"testB": "9"}
                    # We have duplicates here because there's no ID to merge these
                    # on. This is different to the old behaviour. Issue filed at
                    # https://github.com/OpenDataServices/flatten-tool/issues/99
                ],
            },
        ],
        [],
        False,
    ),
    (
        "Test conflicting rollup",
        OrderedDict(
            [
                (
                    "main",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                            "Arr title:Identifier": "3",
                            "Arr title:B title": "4",
                        }
                    ],
                ),
                (
                    "testArr",
                    [
                        {
                            "ROOT_ID": 1,
                            "Identifier": 2,
                            "Arr title:Identifier": "3",
                            "Arr title:B title": "5",
                        }
                    ],
                ),
            ]
        ),
        [
            {
                "ROOT_ID": 1,
                "id": 2,
                "testArr": [
                    {
                        "id": "3",
                        "testB": "4"
                        # (Since sheets are parsed in the order they appear, and the first value is used).
                    }
                ],
            }
        ],
        [
            'Conflict when merging field "testB" for ROOT_ID "1", id "2" in sheet testA: "4" != "5"'
        ],
        False,
    ),
    (
        "Unflatten empty",
        {
            "custom_main": [],
            "subsheet": [
                {
                    "ROOT_ID": "",
                    "Identifier": "",
                    "A title": "",
                    "U title": "",
                }
            ],
        },
        [],
        [],
        False,
    ),
]


@pytest.mark.parametrize("convert_titles", [True, False])
@pytest.mark.parametrize("use_schema", [True, False])
@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)
@pytest.mark.parametrize(
    "comment,input_dict,expected_output_list,warning_messages,reversible",
    testdata_multiplesheets,
)
def test_unflatten(
    convert_titles,
    use_schema,
    root_id,
    root_id_kwargs,
    input_dict,
    expected_output_list,
    recwarn,
    comment,
    warning_messages,
    reversible,
):
    extra_kwargs = {"convert_titles": convert_titles}
    extra_kwargs.update(root_id_kwargs)
    spreadsheet_input = ListInput(
        sheets=OrderedDict(
            [
                (sheet_name, [inject_root_id(root_id, line) for line in lines])
                for sheet_name, lines in input_dict.items()
            ]
        ),
        **extra_kwargs
    )
    spreadsheet_input.read_sheets()

    parser = SchemaParser(
        root_schema_dict=create_schema(root_id) if use_schema else {"properties": {}},
        root_id=root_id,
        rollup=True,
    )
    parser.parse()
    spreadsheet_input.parser = parser

    expected_output_list = [
        inject_root_id(root_id, expected_output_dict)
        for expected_output_dict in expected_output_list
    ]
    assert list(spreadsheet_input.unflatten()) == expected_output_list


@pytest.mark.parametrize("convert_titles", [True, False])
@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)
@pytest.mark.parametrize(
    "comment,input_dict,expected_output_list,warning_messages",
    testdata_multiplesheets_pointer,
)
def test_unflatten_pointer(
    convert_titles,
    root_id,
    root_id_kwargs,
    input_dict,
    expected_output_list,
    recwarn,
    comment,
    warning_messages,
):
    return test_unflatten(
        convert_titles=convert_titles,
        use_schema=True,
        root_id=root_id,
        root_id_kwargs=root_id_kwargs,
        input_dict=input_dict,
        expected_output_list=expected_output_list,
        recwarn=recwarn,
        comment=comment,
        warning_messages=warning_messages,
        reversible=False,
    )


@pytest.mark.parametrize(
    "comment,input_dict,expected_output_list,warning_messages,reversible",
    testdata_multiplesheets_titles,
)
@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)
def test_unflatten_titles(
    root_id,
    root_id_kwargs,
    input_dict,
    expected_output_list,
    recwarn,
    comment,
    warning_messages,
    reversible,
):
    """
    Essentially the same as test unflatten, except that convert_titles and
    use_schema are always true, as both of these are needed to convert titles
    properly. (and runs with different test data).
    """
    if root_id != "":
        # Skip all tests with a root ID for now, as this is broken
        # https://github.com/OpenDataServices/flatten-tool/issues/84
        pytest.skip()
    return test_unflatten(
        convert_titles=True,
        use_schema=True,
        root_id=root_id,
        root_id_kwargs=root_id_kwargs,
        input_dict=input_dict,
        expected_output_list=expected_output_list,
        recwarn=recwarn,
        comment=comment,
        warning_messages=warning_messages,
        reversible=reversible,
    )

1	# -- coding: utf-8 --
2	"""	10✔
3	Tests of unflatten method of the SpreadsheetInput class from input.py
4
5	Tests that only apply for multiple sheets.
6	"""
7	from __future__ import unicode_literals	10✔
8
9	from collections import OrderedDict	10✔
10
11	import pytest	10✔
12
13	from flattentool.schema import SchemaParser	10✔
14
15	from .test_input_SpreadsheetInput import ListInput	10✔
16	from .test_input_SpreadsheetInput_unflatten import (	10✔
17	ROOT_ID_PARAMS,
18	create_schema,
19	inject_root_id,
20	)
21
22	testdata_multiplesheets = [	10✔
23	(
24	"Basic sub sheet",
25	{
26	"custom_main": [
27	{
28	"ROOT_ID": 1,
29	"id": 2,
30	},
31	{
32	"ROOT_ID": 1,
33	"id": 3,
34	},
35	],
36	"testArr": [
37	{
38	"ROOT_ID": 1,
39	"id": 2,
40	"testArr/0/testC": "3",
41	},
42	{
43	"ROOT_ID": 1,
44	"id": 2,
45	"testArr/0/testC": "4",
46	},
47	],
48	},
49	[
50	{
51	"ROOT_ID": 1,
52	"id": 2,
53	"testArr": [
54	{"testC": "3"},
55	{"testC": "4"},
56	],
57	},
58	{"ROOT_ID": 1, "id": 3},
59	],
60	[],
61	True,
62	),
63	(
64	"Nested sub sheet (with id)",
65	{
66	"custom_main": [
67	{
68	"ROOT_ID": 1,
69	"id": 2,
70	"testB/id": 3,
71	"testB/testC": 4,
72	}
73	],
74	"tes_subField": [
75	# It used to be necessary to supply testA/id in this
76	# situation, but now it's optional
77	{
78	"ROOT_ID": 1,
79	"id": 2,
80	"testB/id": 3,
81	"testB/subField/0/testD": 5,
82	}
83	],
84	},
85	[
86	{
87	"ROOT_ID": 1,
88	"id": 2,
89	"testB": {"id": 3, "testC": 4, "subField": [{"testD": 5}]},
90	}
91	],
92	[],
93	True,
94	),
95	(
96	"Nested sub sheet (without id)",
97	{
98	"custom_main": [
99	{
100	"ROOT_ID": 1,
101	"id": 2,
102	"testB/id": 3,
103	"testB/testC": 4,
104	}
105	],
106	"sub": [
107	# It used to be necessary to supply testA/id in this
108	# situation, but now it's optional
109	{
110	"ROOT_ID": 1,
111	"id": 2,
112	"testB/subField/0/testD": 5,
113	}
114	],
115	},
116	[
117	{
118	"ROOT_ID": 1,
119	"id": 2,
120	"testB": {"id": 3, "testC": 4, "subField": [{"testD": 5}]},
121	}
122	],
123	[],
124	False,
125	),
126	(
127	"Basic two sub sheets",
128	OrderedDict(
129	[
130	(
131	"custom_main",
132	[
133	OrderedDict(
134	[
135	("ROOT_ID", 1),
136	("id", 2),
137	]
138	),
139	OrderedDict(
140	[
141	("ROOT_ID", 1),
142	("id", 6),
143	]
144	),
145	],
146	),
147	(
148	"sub1Field",
149	[
150	{
151	"ROOT_ID": 1,
152	"id": 2,
153	"sub1Field/0/id": 3,
154	"sub1Field/0/testA": 4,
155	}
156	],
157	),
158	(
159	"sub_sub2Field",
160	[
161	{
162	"ROOT_ID": 1,
163	"id": 2,
164	"sub1Field/0/id": 3,
165	"sub1Field/0/sub2Field/0/testB": 5,
166	}
167	],
168	),
169	]
170	),
171	[
172	OrderedDict(
173	[
174	("ROOT_ID", 1),
175	("id", 2),
176	("sub1Field", [{"id": 3, "testA": 4, "sub2Field": [{"testB": 5}]}]),
177	]
178	),
179	{"ROOT_ID": 1, "id": 6},
180	],
181	[],
182	True,
183	),
184	(
185	"Nested id",
186	{
187	"custom_main": [
188	{
189	"ROOT_ID": 1,
190	"id": 2,
191	}
192	],
193	"subField": [
194	{
195	"ROOT_ID": 1,
196	"id": 2,
197	"subField/0/id": 3,
198	"subField/0/testA/id": 4,
199	}
200	],
201	},
202	[{"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": {"id": 4}}]}],
203	[],
204	True,
205	),
206	(
207	"Missing columns",
208	{
209	"custom_main": [
210	{
211	"ROOT_ID": 1,
212	"id": 2,
213	}
214	],
215	"sub": [
216	{
217	"ROOT_ID": 1,
218	"id": "",
219	"subField/0/id": 3,
220	"subField/0/testA": 4,
221	},
222	{
223	"ROOT_ID": 1,
224	"id": 2,
225	"subField/0/id": 3,
226	"subField/0/testA": 5,
227	},
228	],
229	},
230	[
231	{"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": 5}]},
232	{"ROOT_ID": 1, "subField": [{"id": 3, "testA": 4}]},
233	],
234	[],
235	False,
236	),
237	(
238	"Unmatched id",
239	OrderedDict(
240	[
241	(
242	"custom_main",
243	[
244	{
245	"ROOT_ID": 1,
246	"id": 2,
247	}
248	],
249	),
250	(
251	"sub",
252	[
253	{
254	"ROOT_ID": 1,
255	"id": 100,
256	"subField/0/id": 3,
257	"subField/0/testA": 4,
258	},
259	{
260	"ROOT_ID": 1,
261	"id": 2,
262	"subField/0/id": 3,
263	"subField/0/testA": 5,
264	},
265	],
266	),
267	]
268	),
269	[
270	{"ROOT_ID": 1, "id": 2, "subField": [{"id": 3, "testA": 5}]},
271	{"ROOT_ID": 1, "id": 100, "subField": [{"id": 3, "testA": 4}]},
272	],
273	[],
274	False,
275	),
276	(
277	"Test same rollup",
278	{
279	"main": [
280	{
281	"ROOT_ID": 1,
282	"id": 2,
283	"testC": 3,
284	"testArr/0/id": "4",
285	"testArr/0/testB": "5",
286	},
287	{
288	"ROOT_ID": 6,
289	"id": 7,
290	"testC": 8,
291	"testArr/0/testB": "9",
292	},
293	],
294	"testArr": [
295	{
296	"ROOT_ID": 1,
297	"id": 2,
298	"testArr/0/id": "4",
299	"testArr/0/testB": "5",
300	},
301	{
302	"ROOT_ID": 6,
303	"id": 7,
304	"testArr/0/testB": "9",
305	},
306	],
307	},
308	[
309	{"ROOT_ID": 1, "id": 2, "testC": 3, "testArr": [{"id": "4", "testB": "5"}]},
310	{
311	"ROOT_ID": 6,
312	"id": 7,
313	"testC": 8,
314	"testArr": [
315	{"testB": "9"},
316	{"testB": "9"}
317	# We have duplicates here because there's no ID to merge these
318	# on. This is different to the old behaviour. Issue filed at
319	# https://github.com/OpenDataServices/flatten-tool/issues/99
320	],
321	},
322	],
323	[],
324	False,
325	),
326	(
327	"Test conflicting rollup",
328	OrderedDict(
329	[
330	(
331	"main",
332	[
333	{
334	"ROOT_ID": 1,
335	"id": 2,
336	"testArr/0/id": "3",
337	"testArr/0/testB": "4",
338	}
339	],
340	),
341	(
342	"testArr",
343	[
344	{
345	"ROOT_ID": 1,
346	"id": 2,
347	"testArr/0/id": "3",
348	"testArr/0/testB": "5",
349	}
350	],
351	),
352	]
353	),
354	[
355	{
356	"ROOT_ID": 1,
357	"id": 2,
358	"testArr": [
359	{
360	"id": "3",
361	"testB": "4"
362	# (Since sheets are parsed in the order they appear, and the first value is used).
363	}
364	],
365	}
366	],
367	[
368	'Conflict when merging field "testB" for ROOT_ID "1", id "2" in sheet testA: "4" != "5"'
369	],
370	False,
371	),
372	(
373	"Unflatten empty",
374	{
375	"custom_main": [],
376	"subsheet": [
377	{
378	"ROOT_ID": "",
379	"id": "",
380	"testA": "",
381	"testB": "",
382	"testC": "",
383	"testD": "",
384	}
385	],
386	},
387	[],
388	[],
389	False,
390	),
391	]
392
393
394	testdata_multiplesheets_pointer = [	10✔
395	(
396	"with schema",
397	{
398	"custom_main": [{"ROOT_ID": 1, "id": "2", "testA": 3}],
399	"sub": [
400	{
401	"ROOT_ID": 1,
402	"id": 2,
403	"testArr/testB": 4, # test that we can infer this an array from schema
404	}
405	],
406	},
407	[
408	{
409	"ROOT_ID": 1,
410	"id": 2, # check that we join correctly when this gets converted to an
411	# integer because of the schema type
412	"testA": 3,
413	"testArr": [{"testB": "4"}],
414	}
415	],
416	[],
417	)
418	]
419
420
421	testdata_multiplesheets_titles = [	10✔
422	(
423	"Basic sub sheet",
424	{
425	"custom_main": [
426	{
427	"ROOT_ID": 1,
428	"Identifier": 2,
429	},
430	{
431	"ROOT_ID": 1,
432	"Identifier": 3,
433	},
434	],
435	"Arr title": [
436	{
437	"ROOT_ID": 1,
438	"Identifier": 2,
439	"Arr title:C title": "3",
440	},
441	{
442	"ROOT_ID": 1,
443	"Identifier": 2,
444	"Arr title:C title": "4",
445	},
446	],
447	"arr_not_in_schema": [
448	{
449	"ROOT_ID": 1,
450	"Identifier": 2,
451	"arr_not_in_schema/0/testD": "5",
452	},
453	{
454	"ROOT_ID": 1,
455	"Identifier": 2,
456	"arr_not_in_schema/0/testD": "6",
457	},
458	],
459	},
460	[
461	{
462	"ROOT_ID": 1,
463	"id": 2,
464	"testArr": [
465	{"testC": "3"},
466	{"testC": "4"},
467	],
468	"arr_not_in_schema": [{"testD": "5"}, {"testD": "6"}],
469	},
470	{"ROOT_ID": 1, "id": 3},
471	],
472	[],
473	True,
474	),
475	(
476	"Nested sub sheet (with id)",
477	{
478	"custom_main": [
479	{
480	"ROOT_ID": 1,
481	"Identifier": 2,
482	"B title:Identifier": 3,
483	"B title:C title": 4,
484	}
485	],
486	"B t_Sub title": [
487	# It used to be necessary to supply testA/id in this
488	# situation, but now it's optional
489	{
490	"ROOT_ID": 1,
491	"Identifier": 2,
492	"B title:Identifier": 3,
493	"B title:Sub title:E title": 5,
494	}
495	],
496	},
497	[
498	{
499	"ROOT_ID": 1,
500	"id": 2,
501	"testB": {"id": 3, "testC": 4, "subField": [{"testE": 5}]},
502	}
503	],
504	[],
505	True,
506	),
507	(
508	"Nested sub sheet (without id)",
509	{
510	"custom_main": [
511	{
512	"ROOT_ID": 1,
513	"Identifier": 2,
514	"B title:Identifier": 3,
515	"B title:C title": 4,
516	}
517	],
518	"sub": [
519	# It used to be necessary to supply testA/id in this
520	# situation, but now it's optional
521	{
522	"ROOT_ID": 1,
523	"Identifier": 2,
524	"B title:Sub title:E title": 5,
525	}
526	],
527	},
528	[
529	{
530	"ROOT_ID": 1,
531	"id": 2,
532	"testB": {"id": 3, "testC": 4, "subField": [{"testE": 5}]},
533	}
534	],
535	[],
536	False,
537	),
538	(
539	"Basic two sub sheets",
540	OrderedDict(
541	[
542	(
543	"custom_main",
544	[
545	OrderedDict(
546	[
547	("ROOT_ID", 1),
548	("Identifier", 2),
549	]
550	),
551	OrderedDict(
552	[
553	("ROOT_ID", 1),
554	("Identifier", 6),
555	]
556	),
557	],
558	),
559	(
560	"Arr title",
561	[
562	{
563	"ROOT_ID": 1,
564	"Identifier": 2,
565	"Arr title:Identifier": "3",
566	"Arr title:B title": "4",
567	}
568	],
569	),
570	(
571	"Arr_Nest title",
572	[
573	{
574	"ROOT_ID": 1,
575	"Identifier": 2,
576	"Arr title:Identifier": "3",
577	"Arr title:Nest title:D title": "5",
578	}
579	],
580	),
581	]
582	),
583	[
584	OrderedDict(
585	[
586	("ROOT_ID", 1),
587	("id", 2),
588	(
589	"testArr",
590	[{"id": "3", "testB": "4", "testNest": [{"testD": "5"}]}],
591	),
592	]
593	),
594	{"ROOT_ID": 1, "id": 6},
595	],
596	[],
597	True,
598	),
599	(
600	"Nested id",
601	{
602	"custom_main": [
603	{
604	"ROOT_ID": 1,
605	"Identifier": 2,
606	}
607	],
608	"Arr title": [
609	{
610	"ROOT_ID": 1,
611	"Identifier": 2,
612	"Arr title:Identifier": "3",
613	"Arr title:NestObj title:Identifier": "4",
614	}
615	],
616	},
617	[{"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testNestObj": {"id": "4"}}]}],
618	[],
619	True,
620	),
621	(
622	"Missing columns",
623	{
624	"custom_main": [
625	{
626	"ROOT_ID": 1,
627	"Identifier": 2,
628	}
629	],
630	"sub": [
631	{
632	"ROOT_ID": 1,
633	"Identifier": "",
634	"Arr title:Identifier": 3,
635	"Arr title:B title": 4,
636	},
637	{
638	"ROOT_ID": 1,
639	"Identifier": 2,
640	"Arr title:Identifier": 3,
641	"Arr title:B title": 5,
642	},
643	],
644	},
645	[
646	{"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testB": "5"}]},
647	{"ROOT_ID": 1, "testArr": [{"id": "3", "testB": "4"}]},
648	],
649	[],
650	False,
651	),
652	(
653	"Unmatched id",
654	OrderedDict(
655	[
656	(
657	"custom_main",
658	[
659	{
660	"ROOT_ID": 1,
661	"Identifier": 2,
662	}
663	],
664	),
665	(
666	"sub",
667	[
668	{
669	"ROOT_ID": 1,
670	"Identifier": 100,
671	"Arr title:Identifier": 3,
672	"Arr title:B title": 4,
673	},
674	{
675	"ROOT_ID": 1,
676	"Identifier": 2,
677	"Arr title:Identifier": 3,
678	"Arr title:B title": 5,
679	},
680	],
681	),
682	]
683	),
684	[
685	{"ROOT_ID": 1, "id": 2, "testArr": [{"id": "3", "testB": "5"}]},
686	{"ROOT_ID": 1, "id": 100, "testArr": [{"id": "3", "testB": "4"}]},
687	],
688	[],
689	False,
690	),
691	(
692	"Test same rollup",
693	{
694	"main": [
695	{
696	"ROOT_ID": 1,
697	"Identifier": 2,
698	"A title": 3,
699	"Arr title:Identifier": 4,
700	"Arr title:B title": 5,
701	},
702	{
703	"ROOT_ID": 6,
704	"Identifier": 7,
705	"A title": 8,
706	"Arr title:B title": 9,
707	},
708	],
709	"testArr": [
710	{
711	"ROOT_ID": 1,
712	"Identifier": 2,
713	"Arr title:Identifier": 4,
714	"Arr title:B title": 5,
715	},
716	{
717	"ROOT_ID": 6,
718	"Identifier": 7,
719	"Arr title:B title": 9,
720	},
721	],
722	},
723	[
724	{"ROOT_ID": 1, "id": 2, "testA": 3, "testArr": [{"id": "4", "testB": "5"}]},
725	{
726	"ROOT_ID": 6,
727	"id": 7,
728	"testA": 8,
729	"testArr": [
730	{"testB": "9"},
731	{"testB": "9"}
732	# We have duplicates here because there's no ID to merge these
733	# on. This is different to the old behaviour. Issue filed at
734	# https://github.com/OpenDataServices/flatten-tool/issues/99
735	],
736	},
737	],
738	[],
739	False,
740	),
741	(
742	"Test conflicting rollup",
743	OrderedDict(
744	[
745	(
746	"main",
747	[
748	{
749	"ROOT_ID": 1,
750	"Identifier": 2,
751	"Arr title:Identifier": "3",
752	"Arr title:B title": "4",
753	}
754	],
755	),
756	(
757	"testArr",
758	[
759	{
760	"ROOT_ID": 1,
761	"Identifier": 2,
762	"Arr title:Identifier": "3",
763	"Arr title:B title": "5",
764	}
765	],
766	),
767	]
768	),
769	[
770	{
771	"ROOT_ID": 1,
772	"id": 2,
773	"testArr": [
774	{
775	"id": "3",
776	"testB": "4"
777	# (Since sheets are parsed in the order they appear, and the first value is used).
778	}
779	],
780	}
781	],
782	[
783	'Conflict when merging field "testB" for ROOT_ID "1", id "2" in sheet testA: "4" != "5"'
784	],
785	False,
786	),
787	(
788	"Unflatten empty",
789	{
790	"custom_main": [],
791	"subsheet": [
792	{
793	"ROOT_ID": "",
794	"Identifier": "",
795	"A title": "",
796	"U title": "",
797	}
798	],
799	},
800	[],
801	[],
802	False,
803	),
804	]
805
806
807	@pytest.mark.parametrize("convert_titles", [True, False])	10✔
808	@pytest.mark.parametrize("use_schema", [True, False])	10✔
809	@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)	10✔
810	@pytest.mark.parametrize(	10✔
811	"comment,input_dict,expected_output_list,warning_messages,reversible",
812	testdata_multiplesheets,
813	)
814	def test_unflatten(	8✔
815	convert_titles,
816	use_schema,
817	root_id,
818	root_id_kwargs,
819	input_dict,
820	expected_output_list,
821	recwarn,
822	comment,
823	warning_messages,
824	reversible,
825	):
826	extra_kwargs = {"convert_titles": convert_titles}	×
827	extra_kwargs.update(root_id_kwargs)	×
828	spreadsheet_input = ListInput(	×
829	sheets=OrderedDict(
830	[
831	(sheet_name, [inject_root_id(root_id, line) for line in lines])
832	for sheet_name, lines in input_dict.items()
833	]
834	),
835	**extra_kwargs
836	)
837	spreadsheet_input.read_sheets()	×
838
839	parser = SchemaParser(	×
840	root_schema_dict=create_schema(root_id) if use_schema else {"properties": {}},
841	root_id=root_id,
842	rollup=True,
843	)
844	parser.parse()	×
845	spreadsheet_input.parser = parser	×
846
847	expected_output_list = [	×
848	inject_root_id(root_id, expected_output_dict)
849	for expected_output_dict in expected_output_list
850	]
851	assert list(spreadsheet_input.unflatten()) == expected_output_list	×
852
853
854	@pytest.mark.parametrize("convert_titles", [True, False])	10✔
855	@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)	10✔
856	@pytest.mark.parametrize(	10✔
857	"comment,input_dict,expected_output_list,warning_messages",
858	testdata_multiplesheets_pointer,
859	)
860	def test_unflatten_pointer(	8✔
861	convert_titles,
862	root_id,
863	root_id_kwargs,
864	input_dict,
865	expected_output_list,
866	recwarn,
867	comment,
868	warning_messages,
869	):
870	return test_unflatten(	×
871	convert_titles=convert_titles,
872	use_schema=True,
873	root_id=root_id,
874	root_id_kwargs=root_id_kwargs,
875	input_dict=input_dict,
876	expected_output_list=expected_output_list,
877	recwarn=recwarn,
878	comment=comment,
879	warning_messages=warning_messages,
880	reversible=False,
881	)
882
883
884	@pytest.mark.parametrize(	10✔
885	"comment,input_dict,expected_output_list,warning_messages,reversible",
886	testdata_multiplesheets_titles,
887	)
888	@pytest.mark.parametrize("root_id,root_id_kwargs", ROOT_ID_PARAMS)	10✔
889	def test_unflatten_titles(	8✔
890	root_id,
891	root_id_kwargs,
892	input_dict,
893	expected_output_list,
894	recwarn,
895	comment,
896	warning_messages,
897	reversible,
898	):
899	"""
900	Essentially the same as test unflatten, except that convert_titles and
901	use_schema are always true, as both of these are needed to convert titles
902	properly. (and runs with different test data).
903	"""
904	if root_id != "":	×
905	# Skip all tests with a root ID for now, as this is broken
906	# https://github.com/OpenDataServices/flatten-tool/issues/84
907	pytest.skip()	×
908	return test_unflatten(	×
909	convert_titles=True,
910	use_schema=True,
911	root_id=root_id,
912	root_id_kwargs=root_id_kwargs,
913	input_dict=input_dict,
914	expected_output_list=expected_output_list,
915	recwarn=recwarn,
916	comment=comment,
917	warning_messages=warning_messages,
918	reversible=reversible,
919	)

OpenDataServices / flatten-tool / 6507626273

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous