• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

OpenDataServices / flatten-tool / 6507626273

13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%
6507626273

Pull #433

github

odscjames
New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424
Pull Request #433: New "Geo" optional dependencies

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.99
/flattentool/cli.py
1
from __future__ import print_function
10✔
2

3
import argparse
10✔
4
import sys
10✔
5
import warnings
10✔
6

7
from flattentool import create_template, flatten, unflatten
10✔
8
from flattentool.input import FORMATS as INPUT_FORMATS
10✔
9
from flattentool.json_input import BadlyFormedJSONError
10✔
10
from flattentool.output import FORMATS as OUTPUT_FORMATS
10✔
11

12
"""
4✔
13
This file does most of the work of the flatten-tool commandline command.
14

15
It takes any commandline arguments, and passes them to a function in
16
``__init__.py``.
17

18
It is callable via the ``flatten-tool`` executable in the directory below, or
19
using ``python -m flattentool.cli``.
20

21
"""
22

23

24
def create_parser():
10✔
25
    """
26
    Create an argparse ArgumentParser for our commandline arguments
27

28
    Defaults are not set here, but rather given in the appropriate function.
29

30
    (This is split out as it's own function primarily so it can be tested.)
31

32
    """
33

34
    parser = argparse.ArgumentParser()
10✔
35
    subparsers = parser.add_subparsers(dest="subparser_name")
10✔
36

37
    output_formats = sorted(OUTPUT_FORMATS) + ["all"]
10✔
38
    input_formats = sorted(INPUT_FORMATS)
10✔
39

40
    parser.add_argument(
10✔
41
        "-v",
42
        "--verbose",
43
        action="store_true",
44
        help="Print detailed output when warnings or errors occur.",
45
    )
46

47
    parser_create_template = subparsers.add_parser(
10✔
48
        "create-template", help="Create a template from the given schema"
49
    )
50
    parser_create_template.add_argument(
10✔
51
        "-s",
52
        "--schema",
53
        help="Path to the schema file you want to use to create the template",
54
        required=True,
55
    )
56
    parser_create_template.add_argument(
10✔
57
        "-f",
58
        "--output-format",
59
        help="Type of template you want to create. Defaults to all available options",
60
        choices=output_formats,
61
    )
62
    parser_create_template.add_argument(
10✔
63
        "-m",
64
        "--main-sheet-name",
65
        help="The name of the main sheet, as seen in the first tab of the spreadsheet for example. Defaults to main",
66
    )
67
    parser_create_template.add_argument(
10✔
68
        "-o",
69
        "--output-name",
70
        help="Name of the outputted file. Will have an extension appended if format is all.",
71
    )
72
    parser_create_template.add_argument(
10✔
73
        "--rollup",
74
        action="store_true",
75
        help='"Roll up" columns from subsheets into the main sheet if they are specified in a rollUp attribute in the schema.',
76
    )
77
    parser_create_template.add_argument(
10✔
78
        "-r", "--root-id", help="Root ID of the data format, e.g. ocid for OCDS"
79
    )
80
    parser_create_template.add_argument(
10✔
81
        "--use-titles", action="store_true", help="Convert titles."
82
    )
83
    parser_create_template.add_argument(
10✔
84
        "--disable-local-refs",
85
        action="store_true",
86
        help="Disable local refs when parsing JSON Schema.",
87
    )
88
    parser_create_template.add_argument(
10✔
89
        "--no-deprecated-fields",
90
        action="store_true",
91
        help="Exclude Fields marked as deprecated in the JSON Schema.",
92
    )
93
    parser_create_template.add_argument(
10✔
94
        "--truncation-length",
95
        type=int,
96
        default=3,
97
        help="The length of components of sub-sheet names (default 3).",
98
    )
99
    parser_create_template.add_argument(
10✔
100
        "--line-terminator",
101
        help="The line terminator to use when writing CSV files: CRLF or LF",
102
    )
103
    parser_create_template.add_argument(
10✔
104
        "--convert-wkt",
105
        action="store_true",
106
        help="Enable conversion of WKT to geojson",
107
    )
108

109
    parser_flatten = subparsers.add_parser("flatten", help="Flatten a JSON file")
10✔
110
    parser_flatten.add_argument("input_name", help="Name of the input JSON file.")
10✔
111
    parser_flatten.add_argument("-s", "--schema", help="Path to a relevant schema.")
10✔
112
    parser_flatten.add_argument(
10✔
113
        "-f",
114
        "--output-format",
115
        help="Type of template you want to create. Defaults to all available options",
116
        choices=output_formats,
117
    )
118
    parser_flatten.add_argument(
10✔
119
        "--xml", action="store_true", help="Use XML as the input format"
120
    )
121
    parser_flatten.add_argument(
10✔
122
        "--id-name", help="String to use for the identifier key, defaults to 'id'"
123
    )
124
    parser_flatten.add_argument(
10✔
125
        "-m",
126
        "--main-sheet-name",
127
        help="The name of the main sheet, as seen in the first tab of the spreadsheet for example. Defaults to main",
128
    )
129
    parser_flatten.add_argument(
10✔
130
        "-o",
131
        "--output-name",
132
        help="Name of the outputted file. Will have an extension appended if format is all.",
133
    )
134
    parser_flatten.add_argument(
10✔
135
        "--root-list-path", help="Path of the root list, defaults to main"
136
    )
137
    parser_flatten.add_argument(
10✔
138
        "--rollup",
139
        nargs="?",
140
        const=True,
141
        action="append",
142
        help='"Roll up" columns from subsheets into the main sheet. Pass one or more JSON paths directly, or a file with one JSON path per line, or no value and use a schema containing (a) rollUp attribute(s). Schema takes precedence if both direct input and schema with rollUps are present.',
143
    )
144
    parser_flatten.add_argument(
10✔
145
        "-r", "--root-id", help="Root ID of the data format, e.g. ocid for OCDS"
146
    )
147
    parser_flatten.add_argument(
10✔
148
        "--use-titles",
149
        action="store_true",
150
        help="Convert titles. Requires a schema to be specified.",
151
    )
152
    parser_flatten.add_argument(
10✔
153
        "--truncation-length",
154
        type=int,
155
        default=3,
156
        help="The length of components of sub-sheet names (default 3).",
157
    )
158
    parser_flatten.add_argument(
10✔
159
        "--root-is-list",
160
        action="store_true",
161
        help="The root element is a list. --root-list-path and meta data will be ignored.",
162
    )
163
    parser_flatten.add_argument(
10✔
164
        "--sheet-prefix",
165
        help="A string to prefix to the start of every sheet (or file) name.",
166
    )
167
    parser_flatten.add_argument(
10✔
168
        "--filter-field",
169
        help="Data Filter - only data with this will be processed. Use with --filter-value",
170
    )
171
    parser_flatten.add_argument(
10✔
172
        "--filter-value",
173
        help="Data Filter - only data with this will be processed. Use with --filter-field",
174
    )
175
    parser_flatten.add_argument(
10✔
176
        "--preserve-fields",
177
        help="Only these fields will be processed. Pass a file with JSON paths to be preserved one per line.",
178
    )
179
    parser_flatten.add_argument(
10✔
180
        "--disable-local-refs",
181
        action="store_true",
182
        help="Disable local refs when parsing JSON Schema.",
183
    )
184
    parser_flatten.add_argument(
10✔
185
        "--remove-empty-schema-columns",
186
        action="store_true",
187
        help="When using flatten with a schema, remove columns and sheets from the output that contain no data.",
188
    )
189
    parser_flatten.add_argument(
10✔
190
        "--line-terminator",
191
        help="The line terminator to use when writing CSV files: CRLF or LF",
192
    )
193
    parser_flatten.add_argument(
10✔
194
        "--convert-wkt",
195
        action="store_true",
196
        help="Enable conversion of geojson to WKT",
197
    )
198
    parser_unflatten = subparsers.add_parser(
10✔
199
        "unflatten", help="Unflatten a spreadsheet"
200
    )
201
    parser_unflatten.add_argument(
10✔
202
        "input_name", help="Name of the input file or directory."
203
    )
204
    parser_unflatten.add_argument(
10✔
205
        "-f",
206
        "--input-format",
207
        help="File format of input file or directory.",
208
        choices=input_formats,
209
        required=True,
210
    )
211
    parser_unflatten.add_argument(
10✔
212
        "--xml", action="store_true", help="Use XML as the output format"
213
    )
214
    parser_unflatten.add_argument(
10✔
215
        "--id-name", help="String to use for the identifier key, defaults to 'id'"
216
    )
217
    parser_unflatten.add_argument(
10✔
218
        "-b",
219
        "--base-json",
220
        help="A base json file to populate with the unflattened data.",
221
    )
222
    parser_unflatten.add_argument(
10✔
223
        "-m",
224
        "--root-list-path",
225
        help="The path in the JSON that will contain the unflattened list. Defaults to main.",
226
    )
227
    parser_unflatten.add_argument(
10✔
228
        "-e",
229
        "--encoding",
230
        help="Encoding of the input file(s) (only relevant for CSV). This can be any encoding recognised by Python. Defaults to utf8.",
231
    )
232
    parser_unflatten.add_argument(
10✔
233
        "-o",
234
        "--output-name",
235
        help="Name of the outputted file. Will have an extension appended as appropriate.",
236
    )
237
    parser_unflatten.add_argument(
10✔
238
        "-c",
239
        "--cell-source-map",
240
        help="Path to write a cell source map to. Will have an extension appended as appropriate.",
241
    )
242
    parser_unflatten.add_argument(
10✔
243
        "-a",
244
        "--heading-source-map",
245
        help="Path to write a heading source map to. Will have an extension appended as appropriate.",
246
    )
247
    parser_unflatten.add_argument(
10✔
248
        "--timezone-name",
249
        help="Name of the timezone, defaults to UTC. Should be in tzdata format, e.g. Europe/London",
250
    )
251
    parser_unflatten.add_argument(
10✔
252
        "-r", "--root-id", help="Root ID of the data format, e.g. ocid for OCDS"
253
    )
254
    parser_unflatten.add_argument("-s", "--schema", help="Path to a relevant schema.")
10✔
255
    parser_unflatten.add_argument(
10✔
256
        "--convert-titles",
257
        action="store_true",
258
        help="Convert titles. Requires a schema to be specified.",
259
    )
260
    parser_unflatten.add_argument(
10✔
261
        "--vertical-orientation",
262
        action="store_true",
263
        help="Read spreadsheet so that headings are in the first column and data is read vertically. Only for XLSX not CSV",
264
    )
265
    parser_unflatten.add_argument(
10✔
266
        "--metatab-name",
267
        help="If supplied will assume there is a metadata tab with the given name",
268
    )
269
    parser_unflatten.add_argument(
10✔
270
        "--metatab-schema", help="The jsonschema of the metadata tab"
271
    )
272
    parser_unflatten.add_argument(
10✔
273
        "--metatab-only", action="store_true", help="Parse the metatab and nothing else"
274
    )
275
    parser_unflatten.add_argument(
10✔
276
        "--metatab-vertical-orientation",
277
        action="store_true",
278
        help="Read metatab so that headings are in the first column and data is read vertically. Only for XLSX not CSV",
279
    )
280
    parser_unflatten.add_argument(
10✔
281
        "--xml-schema",
282
        dest="xml_schemas",
283
        metavar="XML_SCHEMA",
284
        nargs="*",
285
        help="Path to one or more XML schemas (used for sorting)",
286
    )
287
    parser_unflatten.add_argument(
10✔
288
        "--default-configuration",
289
        help="Comma separated list of default parsing commands for all sheets. Only for XLSX not CSV",
290
    )
291
    parser_unflatten.add_argument(
10✔
292
        "--root-is-list",
293
        action="store_true",
294
        help="The root element is a list. --root-list-path and meta data will be ignored.",
295
    )
296
    parser_unflatten.add_argument(
10✔
297
        "--disable-local-refs",
298
        action="store_true",
299
        help="Disable local refs when parsing JSON Schema.",
300
    )
301
    parser_unflatten.add_argument(
10✔
302
        "--xml-comment",
303
        required=False,
304
        default="XML generated by flatten-tool",
305
        help="String comment of what generates the xml file",
306
    )
307
    parser_unflatten.add_argument(
10✔
308
        "--convert-wkt",
309
        action="store_true",
310
        help="Enable conversion of WKT to geojson",
311
    )
312

313
    return parser
10✔
314

315

316
def kwargs_from_parsed_args(args):
10✔
317
    """
318
    Transforms argparse's parsed args object into a dictionary to be passed  as
319
    kwargs.
320

321
    """
322
    return {k: v for k, v in vars(args).items() if v is not None}
10✔
323

324

325
def non_verbose_error_handler(type, value, traceback):
10✔
326
    if type == BadlyFormedJSONError:
×
327
        sys.stderr.write("JSON error: {}\n".format(value))
×
328
    else:
329
        sys.stderr.write(str(value) + "\n")
×
330

331

332
default_warning_formatter = warnings.formatwarning
10✔
333

334

335
def non_verbose_warning_formatter(message, category, filename, lineno, line=None):
10✔
336
    if issubclass(category, UserWarning):
×
337
        return str(message) + "\n"
×
338
    else:
339
        return default_warning_formatter(message, category, filename, lineno, line)
×
340

341

342
def main():
10✔
343
    """
344
    Use ``create_parser`` to get the commandline arguments, and pass them to
345
    the appropriate function in __init__.py (create_template, flatten or
346
    unflatten).
347

348
    """
349
    parser = create_parser()
10✔
350
    # Store the supplied arguments in args
351
    args = parser.parse_args()
10✔
352

353
    if args.subparser_name is None:
10✔
354
        parser.print_help()
×
355
        return
×
356

357
    if not args.verbose:
10✔
358
        sys.excepthook = non_verbose_error_handler
10✔
359
        warnings.formatwarning = non_verbose_warning_formatter
10✔
360

361
    if args.subparser_name == "create-template":
10✔
362
        # Pass the arguments to the create_template function
363
        # If the schema file does not exist we catch it in this exception
364
        try:
10✔
365
            # Note: Ensures that empty arguments are not passed to the create_template function
366
            create_template(**kwargs_from_parsed_args(args))
10✔
367
        except (OSError, IOError) as e:
×
368
            print(str(e))
×
369
            return
×
370
    elif args.subparser_name == "flatten":
10✔
371
        flatten(**kwargs_from_parsed_args(args))
10✔
372
    elif args.subparser_name == "unflatten":
10✔
373
        unflatten(**kwargs_from_parsed_args(args))
10✔
374

375

376
if __name__ == "__main__":
10✔
377
    main()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc