18357161143

Committed 08 Oct 2025 08:26PM UTC coverage: 84.603% (+0.3%) from 84.325%

Build # 18357161143

Build Type

push

github

Committed by

web-flow

Commit Message

Add progress bars to bulk processing scripts (#341)

`grammarinator-parse`, `grammarinator-decode` and `grammarinator-generate`
now support progress bars to provide visual feedback during bulk processing
and make it easier to track overall progress.

Coverage Stats

23 of 38 new or added lines in 3 files covered. (60.53%)

2187 of 2585 relevant lines covered (84.6%)

0.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.77

/grammarinator/parse.py

# Copyright (c) 2018-2025 Renata Hodovan, Akos Kiss.
#
# Licensed under the BSD 3-Clause License
# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
# This file may not be copied, modified, or distributed except
# according to those terms.

import os

from argparse import ArgumentParser
from multiprocessing import Pool
from os.path import exists, join

import enlighten

from antlerinator import add_antlr_argument, process_antlr_argument
from inators.arg import add_log_level_argument, add_sys_path_argument, add_sys_recursion_limit_argument, add_version_argument, process_log_level_argument, process_sys_path_argument, process_sys_recursion_limit_argument

from .cli import add_disable_cleanup_argument, add_encoding_argument, add_encoding_errors_argument, add_tree_format_argument, add_jobs_argument, import_list, init_logging, iter_files, logger, process_tree_format_argument
from .pkgdata import __version__
from .runtime import RuleSize
from .tool import DefaultPopulation, ParserTool


def process_args(args):
    for grammar in args.grammar:
        if not exists(grammar):
            raise ValueError(f'{grammar} does not exist.')

    if not args.parser_dir:
        args.parser_dir = join(args.out, 'grammars')

    args.transformer = import_list(args.transformer)


def execute():
    parser = ArgumentParser(description='Grammarinator: Parser',
                            epilog="""
                            The tool parses files with ANTLR v4 grammars, builds Grammarinator-
                            compatible tree representations from them and saves them for further
                            reuse.
                            """)
    parser.add_argument('grammar', metavar='FILE', nargs='+',
                        help='ANTLR grammar files describing the expected format of input to parse.')
    parser.add_argument('-i', '--input', metavar='FILE', nargs='+',
                        help='input files or directories to process.')
    parser.add_argument('--glob', metavar='PATTERN', nargs='+',
                        help='wildcard patterns for input files to process (supported wildcards: ?, *, **, [])')
    parser.add_argument('-r', '--rule', metavar='NAME',
                        help='name of the rule to start parsing with (default: first parser rule).')
    parser.add_argument('-t', '--transformer', metavar='NAME', action='append', default=[],
                        help='reference to a transformer (in package.module.function format) to postprocess the parsed tree.')
    parser.add_argument('--hidden', metavar='NAME', action='append', default=[],
                        help='list of hidden tokens to be built into the parsed tree.')
    parser.add_argument('--max-depth', type=int, default=RuleSize.max.depth,
                        help='maximum expected tree depth (deeper tests will be discarded (default: %(default)f)).')
    parser.add_argument('--strict', action='store_true',
                        help='discard tests that contain syntax errors.')
    parser.add_argument('-o', '--out', metavar='DIR', default=os.getcwd(),
                        help='directory to save the trees (default: %(default)s).')
    parser.add_argument('--parser-dir', metavar='DIR',
                        help='directory to save the parser grammars (default: <OUTDIR>/grammars).')
    parser.add_argument('--lib', metavar='DIR',
                        help='alternative location of import grammars.')
    add_tree_format_argument(parser)
    add_encoding_argument(parser, help='input file encoding (default: %(default)s).')
    add_encoding_errors_argument(parser)
    add_disable_cleanup_argument(parser)
    add_jobs_argument(parser)
    add_antlr_argument(parser)
    add_sys_path_argument(parser)
    add_sys_recursion_limit_argument(parser)
    add_log_level_argument(parser, short_alias=())
    add_version_argument(parser, version=__version__)
    args = parser.parse_args()

    init_logging()
    process_log_level_argument(args, logger)
    process_sys_path_argument(args)
    process_sys_recursion_limit_argument(args)
    process_antlr_argument(args)
    process_tree_format_argument(args)
    try:
        process_args(args)
    except ValueError as e:
        parser.error(e)

    files = list(iter_files(args))
    with enlighten.get_manager() as progress_manager:
        progress_bar = progress_manager.counter(total=len(files), desc='Parsing', unit='file')

        with ParserTool(grammars=args.grammar, hidden=args.hidden, transformers=args.transformer, parser_dir=args.parser_dir, antlr=args.antlr, rule=args.rule,
                        population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, strict=args.strict,
                        lib_dir=args.lib, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser_tool:
            if args.jobs > 1:
                with Pool(args.jobs) as pool:
                    for _ in pool.imap_unordered(parser_tool.parse, files):
                        progress_bar.update()
            else:
                for fn in files:
                    parser_tool.parse(fn)
                    progress_bar.update()


if __name__ == '__main__':
    execute()

1	# Copyright (c) 2018-2025 Renata Hodovan, Akos Kiss.
2	#
3	# Licensed under the BSD 3-Clause License
4	# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
5	# This file may not be copied, modified, or distributed except
6	# according to those terms.
7
8	import os	1✔
9
10	from argparse import ArgumentParser	1✔
11	from multiprocessing import Pool	1✔
12	from os.path import exists, join	1✔
13
14	import enlighten	1✔
15
16	from antlerinator import add_antlr_argument, process_antlr_argument	1✔
17	from inators.arg import add_log_level_argument, add_sys_path_argument, add_sys_recursion_limit_argument, add_version_argument, process_log_level_argument, process_sys_path_argument, process_sys_recursion_limit_argument	1✔
18
19	from .cli import add_disable_cleanup_argument, add_encoding_argument, add_encoding_errors_argument, add_tree_format_argument, add_jobs_argument, import_list, init_logging, iter_files, logger, process_tree_format_argument	1✔
20	from .pkgdata import __version__	1✔
21	from .runtime import RuleSize	1✔
22	from .tool import DefaultPopulation, ParserTool	1✔
23
24
25	def process_args(args):	1✔
26	for grammar in args.grammar:	1✔
27	if not exists(grammar):	1✔
28	raise ValueError(f'{grammar} does not exist.')	×
29
30	if not args.parser_dir:	1✔
31	args.parser_dir = join(args.out, 'grammars')	1✔
32
33	args.transformer = import_list(args.transformer)	1✔
34
35
36	def execute():	1✔
37	parser = ArgumentParser(description='Grammarinator: Parser',	1✔
38	epilog="""
39	The tool parses files with ANTLR v4 grammars, builds Grammarinator-
40	compatible tree representations from them and saves them for further
41	reuse.
42	""")
43	parser.add_argument('grammar', metavar='FILE', nargs='+',	1✔
44	help='ANTLR grammar files describing the expected format of input to parse.')
45	parser.add_argument('-i', '--input', metavar='FILE', nargs='+',	1✔
46	help='input files or directories to process.')
47	parser.add_argument('--glob', metavar='PATTERN', nargs='+',	1✔
48	help='wildcard patterns for input files to process (supported wildcards: ?, , *, [])')
49	parser.add_argument('-r', '--rule', metavar='NAME',	1✔
50	help='name of the rule to start parsing with (default: first parser rule).')
51	parser.add_argument('-t', '--transformer', metavar='NAME', action='append', default=[],	1✔
52	help='reference to a transformer (in package.module.function format) to postprocess the parsed tree.')
53	parser.add_argument('--hidden', metavar='NAME', action='append', default=[],	1✔
54	help='list of hidden tokens to be built into the parsed tree.')
55	parser.add_argument('--max-depth', type=int, default=RuleSize.max.depth,	1✔
56	help='maximum expected tree depth (deeper tests will be discarded (default: %(default)f)).')
57	parser.add_argument('--strict', action='store_true',	1✔
58	help='discard tests that contain syntax errors.')
59	parser.add_argument('-o', '--out', metavar='DIR', default=os.getcwd(),	1✔
60	help='directory to save the trees (default: %(default)s).')
61	parser.add_argument('--parser-dir', metavar='DIR',	1✔
62	help='directory to save the parser grammars (default: <OUTDIR>/grammars).')
63	parser.add_argument('--lib', metavar='DIR',	1✔
64	help='alternative location of import grammars.')
65	add_tree_format_argument(parser)	1✔
66	add_encoding_argument(parser, help='input file encoding (default: %(default)s).')	1✔
67	add_encoding_errors_argument(parser)	1✔
68	add_disable_cleanup_argument(parser)	1✔
69	add_jobs_argument(parser)	1✔
70	add_antlr_argument(parser)	1✔
71	add_sys_path_argument(parser)	1✔
72	add_sys_recursion_limit_argument(parser)	1✔
73	add_log_level_argument(parser, short_alias=())	1✔
74	add_version_argument(parser, version=__version__)	1✔
75	args = parser.parse_args()	1✔
76
77	init_logging()	1✔
78	process_log_level_argument(args, logger)	1✔
79	process_sys_path_argument(args)	1✔
80	process_sys_recursion_limit_argument(args)	1✔
81	process_antlr_argument(args)	1✔
82	process_tree_format_argument(args)	1✔
83	try:	1✔
84	process_args(args)	1✔
85	except ValueError as e:	×
86	parser.error(e)	×
87
88	files = list(iter_files(args))	1✔
89	with enlighten.get_manager() as progress_manager:	1✔
90	progress_bar = progress_manager.counter(total=len(files), desc='Parsing', unit='file')	1✔
91
92	with ParserTool(grammars=args.grammar, hidden=args.hidden, transformers=args.transformer, parser_dir=args.parser_dir, antlr=args.antlr, rule=args.rule,	1✔
93	population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, strict=args.strict,
94	lib_dir=args.lib, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser_tool:
95	if args.jobs > 1:	1✔
NEW 96	with Pool(args.jobs) as pool:	×
NEW 97	for _ in pool.imap_unordered(parser_tool.parse, files):	×
NEW 98	progress_bar.update()	×
99	else:
100	for fn in files:	1✔
101	parser_tool.parse(fn)	1✔
102	progress_bar.update()	1✔
103
104
105	if __name__ == '__main__':	1✔
106	execute()	1✔

renatahodovan / grammarinator / 18357161143

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous