570066763

Committed 16 Feb 2021 - 0:19 coverage: 88.377% (-0.08%) from 88.457%

Build Type

github-actions

Committed by GitHub

Commit Message

Merge ae9870f03 into f91ef1bae

Pull Request Pull Request #31: Switch to MyST-Parser to extract from Markdown

Run Details

4 of 4 new or added lines in 1 file covered. (100.0%)

4 existing lines in 1 file now uncovered.

768 of 869 relevant lines covered (88.38%)

10.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.7

/ocdsextensionregistry/cli/commands/generate_pot_files.py

import logging
import subprocess
from contextlib import closing, contextmanager
from glob import glob
from pathlib import Path
from tempfile import TemporaryDirectory

import sphinx
from babel.messages.catalog import Catalog
from babel.messages.extract import extract, pathmatch
from babel.messages.pofile import write_po
from docutils.parsers.rst import directives
from ocds_babel.extract import extract_codelist, extract_extension_metadata, extract_schema
from sphinx.application import Sphinx
from sphinx.util.docutils import docutils_namespace
from sphinx.util.osutil import cd

from ocdsextensionregistry import EXTENSION_VERSIONS_DATA, EXTENSIONS_DATA
from ocdsextensionregistry.cli.commands.base import BaseCommand

# patch_docutils is added in Sphinx 1.6.
if sphinx.version_info >= (1, 6):
    from sphinx.util.docutils import patch_docutils
else:
    @contextmanager
    def patch_docutils(confdir=None):
        yield

logger = logging.getLogger('ocdsextensionregistry')


class Command(BaseCommand):
    name = 'generate-pot-files'
    help = 'generates POT files (message catalogs) for versions of extensions'

    def add_arguments(self):
        self.add_argument('output_directory',
                          help='the directory in which to write the output')
        self.add_argument('versions', nargs='*',
                          help="the versions of extensions to process (e.g. 'bids' or 'lots==master')")
        self.add_argument('-v', '--verbose', action='store_true',
                          help='print verbose output')
        self.add_argument('--extensions-url', default=EXTENSIONS_DATA,
                          help="the URL of the registry's extensions.csv")
        self.add_argument('--extension-versions-url', default=EXTENSION_VERSIONS_DATA,
                          help="the URL of the registry's extension_versions.csv")
        self.add_argument('--versions-dir',
                          help="a directory containing versions of extensions")

    def handle(self):
        output_directory = Path(self.args.output_directory)

        if self.args.versions_dir:
            versions_directory = Path(self.args.versions_dir)

        # We simulate pybabel and sphinx-build commands. Variable names are chosen to match upstream code.

        # For sphinx-build, the code path is:
        #
        # * bin/sphinx-build calls main() in sphinx.cmd.build, which calls build_main(), which calls Sphinx(���).build(���)

        if sphinx.version_info >= (1, 6):
            # https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings
            warning_type = 'image.not_readable'
        else:
            # https://sphinx.readthedocs.io/en/1.5/config.html
            warning_type = 'image.nonlocal_uri'

        kwargs = {
            # sphinx-build -E ���
            'freshenv': True,
            # sphinx-build -D suppress_warnings=image.not_readable ���
            'confoverrides': {
                # 'contents' is the default in Sphinx<2. 'index' is the default in Sphinx>=2.
                'master_doc': 'index',
                'suppress_warnings': [warning_type],
            },
        }
        # Verbose is useful for debugging.
        if not self.args.verbose:
            # sphinx-build -q ���
            kwargs['status'] = None

        # For pybabel, the code path is:
        #
        # * bin/pybabel calls main() in babel.messages.frontend
        # * main() calls CommandLineInterface().run(sys.argv)
        # * CommandLineInterface() calls extract_messages(), which:
        #   1. Reads the input path and method map from command-line options
        #   2. Instantiates a catalog
        #   3. Calls extract_from_dir() in babel.messages.extract to extract messages
        #   4. extract_from_dir() calls check_and_call_extract_file() to find the method in the method map
        #   5. check_and_call_extract_file() calls extract_from_file() to open a file for extraction
        #   6. extract_from_file() calls extract() to extract messages
        #   7. Adds the messages to the catalog
        #   8. Writes a POT file

        options = {
            'headers': 'Title,Description,Extension',
            'ignore': 'currency.csv',
        }

        # 1. Reads the input path and method map from command-line options
        arguments = [
            # pybabel extract -F babel_ocds_codelist.cfg . -o $(POT_DIR)/$(DOMAIN_PREFIX)codelists.pot
            ('codelists.pot', [
                ('codelists/*.csv', extract_codelist, options),
            ]),
            # pybabel extract -F babel_ocds_schema.cfg . -o $(POT_DIR)/$(DOMAIN_PREFIX)schema.pot
            ('schema.pot', [
                ('*-schema.json', extract_schema, None),
                ('extension.json', extract_extension_metadata, None),
            ]),
        ]

        for version in self.versions():
            if self.args.versions_dir:
                download_dir = versions_directory / version.id / version.version
                if not download_dir.is_dir():
                    logger.warning('Not processing {}=={} (not in {})'.format(
                        version.id, version.version, versions_directory))
                    continue
                version.download_url = download_dir.as_uri()
            else:
                if not version.download_url:
                    logger.warning('Not processing {}=={} (no Download URL)'.format(
                        version.id, version.version))
                    continue

            outdir = output_directory / version.id / version.version

            outdir.mkdir(parents=True, exist_ok=True)

            # See the `files` method of `ExtensionVersion` for similar code.
            with closing(version.zipfile()) as zipfile:
                names = zipfile.namelist()
                start = len(names[0])

                for output_file, method_map in arguments:
                    # 2. Instantiates a catalog
                    catalog = Catalog()

                    # 3. Calls extract_from_dir() in babel.messages.extract to extract messages
                    for name in names[1:]:
                        filename = name[start:]

                        # 4. extract_from_dir() calls check_and_call_extract_file()
                        for pattern, method, options in method_map:
                            if not pathmatch(pattern, filename):
                                continue

                            # 5. check_and_call_extract_file() calls extract_from_file()
                            with zipfile.open(name) as fileobj:
                                # 6. extract_from_file() calls extract() to extract messages
                                for lineno, message, comments, context in extract(method, fileobj, options=options):
                                    # 7. Adds the messages to the catalog
                                    catalog.add(message, None, [(filename, lineno)],
                                                auto_comments=comments, context=context)

                            break

                    # 8. Writes a POT file
                    if catalog:
                        with open(outdir / output_file, 'wb') as outfile:
                            write_po(outfile, catalog)

                # This section is equivalent to running:
                #
                # echo -e '.. toctree::\n   :hidden:\n\n   README' > index.rst
                # sphinx-build -v -b gettext -a -E -C -D extensions=myst_parser . outdir
                # msgcat outdir/*.pot
                with TemporaryDirectory() as srcdir:
                    infos = zipfile.infolist()
                    start = len(infos[0].filename)

                    for info in infos[1:]:
                        filename = info.filename[start:]
                        if filename == 'README.md':
                            # This avoids writing an unnecessary directory.
                            info.filename = filename
                            zipfile.extract(info, srcdir)
                            break

                    with cd(srcdir):
                        # Eliminates a warning, without changing the output.
                        with open('index.rst', 'w') as f:
                            f.write('.. toctree::\n   :hidden:\n\n   README')

                        # Sphinx's config.py pop()'s extensions.
                        # https://github.com/sphinx-doc/sphinx/issues/6848
                        kwargs['confoverrides']['extensions'] = ['myst_parser']

                        with patch_docutils(), docutils_namespace():
                            # sphinx-build -b gettext $(DOCS_DIR) $(POT_DIR)
                            app = Sphinx('.', None, 'outdir', '.', 'gettext', **kwargs)
                            # sphinx-build -a ���
                            app.build(True)

                        # https://stackoverflow.com/questions/15408348
                        content = subprocess.run(['msgcat', *glob('outdir/*.pot')],
                                                 check=True, stdout=subprocess.PIPE).stdout

                with open(outdir / 'docs.pot', 'wb') as f:
                    f.write(content)

1	import logging	12×
2	import subprocess	12×
3	from contextlib import closing, contextmanager	12×
4	from glob import glob	12×
5	from pathlib import Path	12×
6	from tempfile import TemporaryDirectory	12×
7
8	import sphinx	12×
9	from babel.messages.catalog import Catalog	12×
10	from babel.messages.extract import extract, pathmatch	12×
11	from babel.messages.pofile import write_po	12×
12	from docutils.parsers.rst import directives	12×
13	from ocds_babel.extract import extract_codelist, extract_extension_metadata, extract_schema	12×
14	from sphinx.application import Sphinx	12×
15	from sphinx.util.docutils import docutils_namespace	12×
16	from sphinx.util.osutil import cd	12×
17
18	from ocdsextensionregistry import EXTENSION_VERSIONS_DATA, EXTENSIONS_DATA	12×
19	from ocdsextensionregistry.cli.commands.base import BaseCommand	12×
20
21	# patch_docutils is added in Sphinx 1.6.
22	if sphinx.version_info >= (1, 6):	12×
23	from sphinx.util.docutils import patch_docutils	12×
24	else:
UNCOV 25	@contextmanager	!
UNCOV 26	def patch_docutils(confdir=None):	!
UNCOV 27	yield	!
28
29	logger = logging.getLogger('ocdsextensionregistry')	12×
30
31
32	class Command(BaseCommand):	12×
33	name = 'generate-pot-files'	12×
34	help = 'generates POT files (message catalogs) for versions of extensions'	12×
35
36	def add_arguments(self):	12×
37	self.add_argument('output_directory',	12×
38	help='the directory in which to write the output')
39	self.add_argument('versions', nargs='*',	12×
40	help="the versions of extensions to process (e.g. 'bids' or 'lots==master')")
41	self.add_argument('-v', '--verbose', action='store_true',	12×
42	help='print verbose output')
43	self.add_argument('--extensions-url', default=EXTENSIONS_DATA,	12×
44	help="the URL of the registry's extensions.csv")
45	self.add_argument('--extension-versions-url', default=EXTENSION_VERSIONS_DATA,	12×
46	help="the URL of the registry's extension_versions.csv")
47	self.add_argument('--versions-dir',	12×
48	help="a directory containing versions of extensions")
49
50	def handle(self):	12×
51	output_directory = Path(self.args.output_directory)	12×
52
53	if self.args.versions_dir:	12×
54	versions_directory = Path(self.args.versions_dir)	12×
55
56	# We simulate pybabel and sphinx-build commands. Variable names are chosen to match upstream code.
57
58	# For sphinx-build, the code path is:
59	#
60	# * bin/sphinx-build calls main() in sphinx.cmd.build, which calls build_main(), which calls Sphinx(��).build(��)
61
62	if sphinx.version_info >= (1, 6):	12×
63	# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings
64	warning_type = 'image.not_readable'	12×
65	else:
66	# https://sphinx.readthedocs.io/en/1.5/config.html
UNCOV 67	warning_type = 'image.nonlocal_uri'	!
68
69	kwargs = {	12×
70	# sphinx-build -E ��
71	'freshenv': True,
72	# sphinx-build -D suppress_warnings=image.not_readable ��
73	'confoverrides': {
74	# 'contents' is the default in Sphinx<2. 'index' is the default in Sphinx>=2.
75	'master_doc': 'index',
76	'suppress_warnings': [warning_type],
77	},
78	}
79	# Verbose is useful for debugging.
80	if not self.args.verbose:	12×
81	# sphinx-build -q ��
82	kwargs['status'] = None	12×
83
84	# For pybabel, the code path is:
85	#
86	# * bin/pybabel calls main() in babel.messages.frontend
87	# * main() calls CommandLineInterface().run(sys.argv)
88	# * CommandLineInterface() calls extract_messages(), which:
89	# 1. Reads the input path and method map from command-line options
90	# 2. Instantiates a catalog
91	# 3. Calls extract_from_dir() in babel.messages.extract to extract messages
92	# 4. extract_from_dir() calls check_and_call_extract_file() to find the method in the method map
93	# 5. check_and_call_extract_file() calls extract_from_file() to open a file for extraction
94	# 6. extract_from_file() calls extract() to extract messages
95	# 7. Adds the messages to the catalog
96	# 8. Writes a POT file
97
98	options = {	12×
99	'headers': 'Title,Description,Extension',
100	'ignore': 'currency.csv',
101	}
102
103	# 1. Reads the input path and method map from command-line options
104	arguments = [	12×
105	# pybabel extract -F babel_ocds_codelist.cfg . -o $(POT_DIR)/$(DOMAIN_PREFIX)codelists.pot
106	('codelists.pot', [
107	('codelists/*.csv', extract_codelist, options),
108	]),
109	# pybabel extract -F babel_ocds_schema.cfg . -o $(POT_DIR)/$(DOMAIN_PREFIX)schema.pot
110	('schema.pot', [
111	('*-schema.json', extract_schema, None),
112	('extension.json', extract_extension_metadata, None),
113	]),
114	]
115
116	for version in self.versions():	12×
117	if self.args.versions_dir:	12×
118	download_dir = versions_directory / version.id / version.version	12×
119	if not download_dir.is_dir():	12×
120	logger.warning('Not processing {}=={} (not in {})'.format(	12×
121	version.id, version.version, versions_directory))
122	continue	12×
123	version.download_url = download_dir.as_uri()	12×
124	else:
125	if not version.download_url:	12×
126	logger.warning('Not processing {}=={} (no Download URL)'.format(	12×
127	version.id, version.version))
128	continue	12×
129
130	outdir = output_directory / version.id / version.version	12×
131
132	outdir.mkdir(parents=True, exist_ok=True)	12×
133
134	# See the `files` method of `ExtensionVersion` for similar code.
135	with closing(version.zipfile()) as zipfile:	12×
136	names = zipfile.namelist()	12×
137	start = len(names[0])	12×
138
139	for output_file, method_map in arguments:	12×
140	# 2. Instantiates a catalog
141	catalog = Catalog()	12×
142
143	# 3. Calls extract_from_dir() in babel.messages.extract to extract messages
144	for name in names[1:]:	12×
145	filename = name[start:]	12×
146
147	# 4. extract_from_dir() calls check_and_call_extract_file()
148	for pattern, method, options in method_map:	12×
149	if not pathmatch(pattern, filename):	12×
150	continue	12×
151
152	# 5. check_and_call_extract_file() calls extract_from_file()
153	with zipfile.open(name) as fileobj:	12×
154	# 6. extract_from_file() calls extract() to extract messages
155	for lineno, message, comments, context in extract(method, fileobj, options=options):	12×
156	# 7. Adds the messages to the catalog
157	catalog.add(message, None, [(filename, lineno)],	12×
158	auto_comments=comments, context=context)
159
160	break	12×
161
162	# 8. Writes a POT file
163	if catalog:	12×
164	with open(outdir / output_file, 'wb') as outfile:	12×
165	write_po(outfile, catalog)	12×
166
167	# This section is equivalent to running:
168	#
169	# echo -e '.. toctree::\n :hidden:\n\n README' > index.rst
170	# sphinx-build -v -b gettext -a -E -C -D extensions=myst_parser . outdir
171	# msgcat outdir/*.pot
172	with TemporaryDirectory() as srcdir:	12×
173	infos = zipfile.infolist()	12×
174	start = len(infos[0].filename)	12×
175
176	for info in infos[1:]:	12×
177	filename = info.filename[start:]	12×
178	if filename == 'README.md':	12×
179	# This avoids writing an unnecessary directory.
180	info.filename = filename	12×
181	zipfile.extract(info, srcdir)	12×
182	break	12×
183
184	with cd(srcdir):	12×
185	# Eliminates a warning, without changing the output.
186	with open('index.rst', 'w') as f:	12×
187	f.write('.. toctree::\n :hidden:\n\n README')	12×
188
189	# Sphinx's config.py pop()'s extensions.
190	# https://github.com/sphinx-doc/sphinx/issues/6848
191	kwargs['confoverrides']['extensions'] = ['myst_parser']	12×
192
193	with patch_docutils(), docutils_namespace():	12×
194	# sphinx-build -b gettext $(DOCS_DIR) $(POT_DIR)
195	app = Sphinx('.', None, 'outdir', '.', 'gettext', **kwargs)	12×
196	# sphinx-build -a ��
197	app.build(True)	12×
198
199	# https://stackoverflow.com/questions/15408348
200	content = subprocess.run(['msgcat', glob('outdir/.pot')],	12×
201	check=True, stdout=subprocess.PIPE).stdout
202
203	with open(outdir / 'docs.pot', 'wb') as f:	12×
204	f.write(content)	12×

open-contracting / extension_registry.py / 570066763

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous