6507626273

Committed 13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%

Build # 6507626273

Build Type

Pull #433

github

Committed by

odscjames

Commit Message

New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424

Pull Request Pull Request #433: New "Geo" optional dependencies

Run Details

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

11.76

/flattentool/tests/test_xml_input.py

from flattentool.json_input import (
    JSONParser,
    dicts_to_list_of_dicts,
    list_dict_consistency,
    lists_of_dicts_paths,
)


def test_xml_empty():
    parser = JSONParser(
        json_filename="flattentool/tests/fixtures/empty.xml",
        root_list_path="iati-activity",
        schema_parser=None,
        root_id="",
        xml=True,
        id_name="iati-identifier",
    )
    assert list(parser.main_sheet) == []
    assert list(parser.main_sheet.lines) == []
    assert parser.sub_sheets == {}


def test_xml_basic_example():
    parser = JSONParser(
        json_filename="examples/iati/expected.xml",
        root_list_path="iati-activity",
        schema_parser=None,
        root_id="",
        xml=True,
        id_name="iati-identifier",
    )
    assert list(parser.main_sheet) == [
        "iati-identifier",
        "@last-updated-datetime",
        "reporting-org/@ref",
        "reporting-org/@type",
        "reporting-org/narrative",
        "title/narrative",
        "description/narrative",
        "participating-org/@ref",
        "participating-org/@role",
        "activity-status/@code",
        "activity-date/@iso-date",
        "activity-date/@type",
    ]
    assert list(parser.main_sheet.lines) == [
        {
            "@last-updated-datetime": "2011-10-01T00:00:00+00:00",
            "activity-date/@type": "1",
            "reporting-org/narrative": "Organisation name",
            "participating-org/@ref": "AA-AAA-123456789",
            "title/narrative": "A title",
            "participating-org/@role": "1",
            "reporting-org/@ref": "AA-AAA-123456789",
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "reporting-org/@type": "40",
            "description/narrative": "A description",
            "activity-date/@iso-date": "2011-10-01",
            "activity-status/@code": "2",
        },
        {
            "@last-updated-datetime": "2016-01-01T00:00:00+00:00",
            "activity-date/@type": "2",
            "reporting-org/narrative": "Organisation name",
            "participating-org/@ref": "AA-AAA-123456789",
            "title/narrative": "Another title",
            "participating-org/@role": "1",
            "reporting-org/@ref": "AA-AAA-123456789",
            "iati-identifier": "AA-AAA-123456789-ABC124",
            "reporting-org/@type": "40",
            "description/narrative": "Another description",
            "activity-date/@iso-date": "2016-01-01",
            "activity-status/@code": "3",
        },
    ]
    assert set(parser.sub_sheets.keys()) == set(["transaction", "recipient-country"])
    assert list(parser.sub_sheets["transaction"]) == [
        "iati-identifier",
        "transaction/0/transaction-type/@code",
        "transaction/0/transaction-date/@iso-date",
        "transaction/0/value/@value-date",
        "transaction/0/value",
    ]
    assert list(parser.sub_sheets["transaction"].lines) == [
        {
            "transaction/0/value/@value-date": "2012-01-01",
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "transaction/0/transaction-date/@iso-date": "2012-01-01",
            "transaction/0/value": "10",
            "transaction/0/transaction-type/@code": "2",
        },
        {
            "transaction/0/value/@value-date": "2012-03-03",
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "transaction/0/transaction-date/@iso-date": "2012-03-03",
            "transaction/0/value": "20",
            "transaction/0/transaction-type/@code": "3",
        },
        {
            "transaction/0/value/@value-date": "2013-04-04",
            "iati-identifier": "AA-AAA-123456789-ABC124",
            "transaction/0/transaction-date/@iso-date": "2013-04-04",
            "transaction/0/value": "30",
            "transaction/0/transaction-type/@code": "2",
        },
        {
            "transaction/0/value/@value-date": "2013-05-05",
            "iati-identifier": "AA-AAA-123456789-ABC124",
            "transaction/0/transaction-date/@iso-date": "2013-05-05",
            "transaction/0/value": "40",
            "transaction/0/transaction-type/@code": "3",
        },
    ]
    assert list(parser.sub_sheets["recipient-country"]) == [
        "iati-identifier",
        "recipient-country/0/@code",
        "recipient-country/0/@percentage",
    ]
    assert list(parser.sub_sheets["recipient-country"].lines) == [
        {
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "recipient-country/0/@code": "AF",
            "recipient-country/0/@percentage": "40",
        },
        {
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "recipient-country/0/@code": "XK",
            "recipient-country/0/@percentage": "60",
        },
        {
            "iati-identifier": "AA-AAA-123456789-ABC124",
            "recipient-country/0/@code": "AG",
            "recipient-country/0/@percentage": "30",
        },
        {
            "iati-identifier": "AA-AAA-123456789-ABC124",
            "recipient-country/0/@code": "XK",
            "recipient-country/0/@percentage": "70",
        },
    ]


def test_varyin_transaction_count():
    parser = JSONParser(
        json_filename="flattentool/tests/fixtures/varying_transaction_count.xml",
        root_list_path="iati-activity",
        schema_parser=None,
        root_id="",
        xml=True,
        id_name="iati-identifier",
    )
    assert list(parser.main_sheet) == ["iati-identifier"]
    assert list(parser.main_sheet.lines) == [
        {"iati-identifier": "AA-AAA-123456789-ABC123"},
        {"iati-identifier": "AA-AAA-123456789-ABC124"},
        {"iati-identifier": "AA-AAA-123456789-ABC125"},
    ]
    assert set(parser.sub_sheets.keys()) == set(["transaction"])
    assert list(parser.sub_sheets["transaction"]) == [
        "iati-identifier",
        "transaction/0/transaction-date/@iso-date",
        "transaction/0/value/@value-date",
        "transaction/0/value",
    ]
    assert list(parser.sub_sheets["transaction"].lines) == [
        {
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "transaction/0/value/@value-date": "2012-01-01",
            "transaction/0/transaction-date/@iso-date": "2012-01-01",
            "transaction/0/value": "10",
        },
        {
            "iati-identifier": "AA-AAA-123456789-ABC123",
            "transaction/0/value/@value-date": "2012-02-02",
            "transaction/0/transaction-date/@iso-date": "2012-02-02",
            "transaction/0/value": "20",
        },
        {
            "iati-identifier": "AA-AAA-123456789-ABC125",
            "transaction/0/value/@value-date": "2012-03-03",
            "transaction/0/transaction-date/@iso-date": "2012-03-03",
            "transaction/0/value": "30",
        },
    ]


def test_lists_of_dicts_paths():
    assert list(lists_of_dicts_paths({})) == []
    assert list(lists_of_dicts_paths({"a": [{}]})) == [("a",)]
    assert list(lists_of_dicts_paths({"a": [{"d": "str1"}]})) == [("a",)]
    assert list(lists_of_dicts_paths({"a": [{"b": [{"d": "str1"}]}]})) == [
        ("a",),
        ("a", "b"),
    ]
    assert list(lists_of_dicts_paths({"a": [{"b": {"d": "str1"}}]})) == [("a",)]
    assert list(lists_of_dicts_paths({"a": [{"b": {"d": "str1"}}, {"b": [{}]}]})) == [
        ("a",),
        ("a", "b"),
    ]
    assert list(lists_of_dicts_paths({"a": {"b": {"c": [{"d": "str1"}]}}})) == [
        ("a", "b", "c")
    ]


def test_dicts_to_list_of_dicts():
    xml_dict = {"a": {"b": {"c": {"d": "aStr"}}}}
    dicts_to_list_of_dicts({("x", "y", "z"), ("a", "b", "c")}, xml_dict)
    assert xml_dict == {"a": {"b": {"c": [{"d": "aStr"}]}}}


def test_list_dict_consistency():
    xml_dict = {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}

    xml_dict = {"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}

    xml_dict = {"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}

    # Wrapped in a dict

    xml_dict = {"c": {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}}

    xml_dict = {"c": {"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}}

    xml_dict = {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}}

    # Wrapped in a list of dicts

    xml_dict = {"c": [{"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": [{"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}]}

    xml_dict = {"c": [{"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}]}

    xml_dict = {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}]}
    list_dict_consistency(xml_dict)
    assert xml_dict == {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}]}


def test_xml_whitespace():
    try:
        parser = JSONParser(
            json_filename="flattentool/tests/fixtures/narrative_whitespace.xml",
            root_list_path="iati-activity",
            schema_parser=None,
            root_id="",
            xml=True,
            id_name="iati-identifier",
        )
        assert parser
    except TypeError as e:
        raise e

1	from flattentool.json_input import (	10✔
2	JSONParser,
3	dicts_to_list_of_dicts,
4	list_dict_consistency,
5	lists_of_dicts_paths,
6	)
7
8
9	def test_xml_empty():	10✔
10	parser = JSONParser(	×
11	json_filename="flattentool/tests/fixtures/empty.xml",
12	root_list_path="iati-activity",
13	schema_parser=None,
14	root_id="",
15	xml=True,
16	id_name="iati-identifier",
17	)
18	assert list(parser.main_sheet) == []	×
19	assert list(parser.main_sheet.lines) == []	×
20	assert parser.sub_sheets == {}	×
21
22
23	def test_xml_basic_example():	10✔
24	parser = JSONParser(	×
25	json_filename="examples/iati/expected.xml",
26	root_list_path="iati-activity",
27	schema_parser=None,
28	root_id="",
29	xml=True,
30	id_name="iati-identifier",
31	)
32	assert list(parser.main_sheet) == [	×
33	"iati-identifier",
34	"@last-updated-datetime",
35	"reporting-org/@ref",
36	"reporting-org/@type",
37	"reporting-org/narrative",
38	"title/narrative",
39	"description/narrative",
40	"participating-org/@ref",
41	"participating-org/@role",
42	"activity-status/@code",
43	"activity-date/@iso-date",
44	"activity-date/@type",
45	]
46	assert list(parser.main_sheet.lines) == [	×
47	{
48	"@last-updated-datetime": "2011-10-01T00:00:00+00:00",
49	"activity-date/@type": "1",
50	"reporting-org/narrative": "Organisation name",
51	"participating-org/@ref": "AA-AAA-123456789",
52	"title/narrative": "A title",
53	"participating-org/@role": "1",
54	"reporting-org/@ref": "AA-AAA-123456789",
55	"iati-identifier": "AA-AAA-123456789-ABC123",
56	"reporting-org/@type": "40",
57	"description/narrative": "A description",
58	"activity-date/@iso-date": "2011-10-01",
59	"activity-status/@code": "2",
60	},
61	{
62	"@last-updated-datetime": "2016-01-01T00:00:00+00:00",
63	"activity-date/@type": "2",
64	"reporting-org/narrative": "Organisation name",
65	"participating-org/@ref": "AA-AAA-123456789",
66	"title/narrative": "Another title",
67	"participating-org/@role": "1",
68	"reporting-org/@ref": "AA-AAA-123456789",
69	"iati-identifier": "AA-AAA-123456789-ABC124",
70	"reporting-org/@type": "40",
71	"description/narrative": "Another description",
72	"activity-date/@iso-date": "2016-01-01",
73	"activity-status/@code": "3",
74	},
75	]
76	assert set(parser.sub_sheets.keys()) == set(["transaction", "recipient-country"])	×
77	assert list(parser.sub_sheets["transaction"]) == [	×
78	"iati-identifier",
79	"transaction/0/transaction-type/@code",
80	"transaction/0/transaction-date/@iso-date",
81	"transaction/0/value/@value-date",
82	"transaction/0/value",
83	]
84	assert list(parser.sub_sheets["transaction"].lines) == [	×
85	{
86	"transaction/0/value/@value-date": "2012-01-01",
87	"iati-identifier": "AA-AAA-123456789-ABC123",
88	"transaction/0/transaction-date/@iso-date": "2012-01-01",
89	"transaction/0/value": "10",
90	"transaction/0/transaction-type/@code": "2",
91	},
92	{
93	"transaction/0/value/@value-date": "2012-03-03",
94	"iati-identifier": "AA-AAA-123456789-ABC123",
95	"transaction/0/transaction-date/@iso-date": "2012-03-03",
96	"transaction/0/value": "20",
97	"transaction/0/transaction-type/@code": "3",
98	},
99	{
100	"transaction/0/value/@value-date": "2013-04-04",
101	"iati-identifier": "AA-AAA-123456789-ABC124",
102	"transaction/0/transaction-date/@iso-date": "2013-04-04",
103	"transaction/0/value": "30",
104	"transaction/0/transaction-type/@code": "2",
105	},
106	{
107	"transaction/0/value/@value-date": "2013-05-05",
108	"iati-identifier": "AA-AAA-123456789-ABC124",
109	"transaction/0/transaction-date/@iso-date": "2013-05-05",
110	"transaction/0/value": "40",
111	"transaction/0/transaction-type/@code": "3",
112	},
113	]
114	assert list(parser.sub_sheets["recipient-country"]) == [	×
115	"iati-identifier",
116	"recipient-country/0/@code",
117	"recipient-country/0/@percentage",
118	]
119	assert list(parser.sub_sheets["recipient-country"].lines) == [	×
120	{
121	"iati-identifier": "AA-AAA-123456789-ABC123",
122	"recipient-country/0/@code": "AF",
123	"recipient-country/0/@percentage": "40",
124	},
125	{
126	"iati-identifier": "AA-AAA-123456789-ABC123",
127	"recipient-country/0/@code": "XK",
128	"recipient-country/0/@percentage": "60",
129	},
130	{
131	"iati-identifier": "AA-AAA-123456789-ABC124",
132	"recipient-country/0/@code": "AG",
133	"recipient-country/0/@percentage": "30",
134	},
135	{
136	"iati-identifier": "AA-AAA-123456789-ABC124",
137	"recipient-country/0/@code": "XK",
138	"recipient-country/0/@percentage": "70",
139	},
140	]
141
142
143	def test_varyin_transaction_count():	10✔
144	parser = JSONParser(	×
145	json_filename="flattentool/tests/fixtures/varying_transaction_count.xml",
146	root_list_path="iati-activity",
147	schema_parser=None,
148	root_id="",
149	xml=True,
150	id_name="iati-identifier",
151	)
152	assert list(parser.main_sheet) == ["iati-identifier"]	×
153	assert list(parser.main_sheet.lines) == [	×
154	{"iati-identifier": "AA-AAA-123456789-ABC123"},
155	{"iati-identifier": "AA-AAA-123456789-ABC124"},
156	{"iati-identifier": "AA-AAA-123456789-ABC125"},
157	]
158	assert set(parser.sub_sheets.keys()) == set(["transaction"])	×
159	assert list(parser.sub_sheets["transaction"]) == [	×
160	"iati-identifier",
161	"transaction/0/transaction-date/@iso-date",
162	"transaction/0/value/@value-date",
163	"transaction/0/value",
164	]
165	assert list(parser.sub_sheets["transaction"].lines) == [	×
166	{
167	"iati-identifier": "AA-AAA-123456789-ABC123",
168	"transaction/0/value/@value-date": "2012-01-01",
169	"transaction/0/transaction-date/@iso-date": "2012-01-01",
170	"transaction/0/value": "10",
171	},
172	{
173	"iati-identifier": "AA-AAA-123456789-ABC123",
174	"transaction/0/value/@value-date": "2012-02-02",
175	"transaction/0/transaction-date/@iso-date": "2012-02-02",
176	"transaction/0/value": "20",
177	},
178	{
179	"iati-identifier": "AA-AAA-123456789-ABC125",
180	"transaction/0/value/@value-date": "2012-03-03",
181	"transaction/0/transaction-date/@iso-date": "2012-03-03",
182	"transaction/0/value": "30",
183	},
184	]
185
186
187	def test_lists_of_dicts_paths():	10✔
188	assert list(lists_of_dicts_paths({})) == []	×
189	assert list(lists_of_dicts_paths({"a": [{}]})) == [("a",)]	×
190	assert list(lists_of_dicts_paths({"a": [{"d": "str1"}]})) == [("a",)]	×
191	assert list(lists_of_dicts_paths({"a": [{"b": [{"d": "str1"}]}]})) == [	×
192	("a",),
193	("a", "b"),
194	]
195	assert list(lists_of_dicts_paths({"a": [{"b": {"d": "str1"}}]})) == [("a",)]	×
196	assert list(lists_of_dicts_paths({"a": [{"b": {"d": "str1"}}, {"b": [{}]}]})) == [	×
197	("a",),
198	("a", "b"),
199	]
200	assert list(lists_of_dicts_paths({"a": {"b": {"c": [{"d": "str1"}]}}})) == [	×
201	("a", "b", "c")
202	]
203
204
205	def test_dicts_to_list_of_dicts():	10✔
206	xml_dict = {"a": {"b": {"c": {"d": "aStr"}}}}	×
207	dicts_to_list_of_dicts({("x", "y", "z"), ("a", "b", "c")}, xml_dict)	×
208	assert xml_dict == {"a": {"b": {"c": [{"d": "aStr"}]}}}	×
209
210
211	def test_list_dict_consistency():	10✔
212	xml_dict = {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}	×
213	list_dict_consistency(xml_dict)	×
214	assert xml_dict == {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}	×
215
216	xml_dict = {"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}	×
217	list_dict_consistency(xml_dict)	×
218	assert xml_dict == {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}	×
219
220	xml_dict = {"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}	×
221	list_dict_consistency(xml_dict)	×
222	assert xml_dict == {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}	×
223
224	# Wrapped in a dict
225
226	xml_dict = {"c": {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}}	×
227	list_dict_consistency(xml_dict)	×
228	assert xml_dict == {"c": {"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}}	×
229
230	xml_dict = {"c": {"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}}	×
231	list_dict_consistency(xml_dict)	×
232	assert xml_dict == {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}}	×
233
234	xml_dict = {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}}	×
235	list_dict_consistency(xml_dict)	×
236	assert xml_dict == {"c": {"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}}	×
237
238	# Wrapped in a list of dicts
239
240	xml_dict = {"c": [{"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}]}	×
241	list_dict_consistency(xml_dict)	×
242	assert xml_dict == {"c": [{"a": [{"b": {"d": "str1"}}, {"b": {"d": "str2"}}]}]}	×
243
244	xml_dict = {"c": [{"a": [{"b": {"d": "str1"}}, {"b": [{"d": "str2"}]}]}]}	×
245	list_dict_consistency(xml_dict)	×
246	assert xml_dict == {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}]}	×
247
248	xml_dict = {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": {"d": "str2"}}]}]}	×
249	list_dict_consistency(xml_dict)	×
250	assert xml_dict == {"c": [{"a": [{"b": [{"d": "str1"}]}, {"b": [{"d": "str2"}]}]}]}	×
251
252
253	def test_xml_whitespace():	10✔
254	try:	×
255	parser = JSONParser(	×
256	json_filename="flattentool/tests/fixtures/narrative_whitespace.xml",
257	root_list_path="iati-activity",
258	schema_parser=None,
259	root_id="",
260	xml=True,
261	id_name="iati-identifier",
262	)
263	assert parser	×
264	except TypeError as e:	×
265	raise e	×

OpenDataServices / flatten-tool / 6507626273

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous