22740642519

Committed 05 Mar 2026 11:00PM UTC coverage: 52.677% (-40.3%) from 92.931%

Build # 22740642519

Build Type

Pull #23157

github

Committed by

web-flow

Commit Message

Merge 2aa18e6d4 into f0030f5e7

Pull Request Pull Request #23157: [pants ng] Partition source files by config.

Run Details

31678 of 60136 relevant lines covered (52.68%)

0.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.27

/src/python/pants/backend/python/typecheck/mypy/subsystem.py

# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
from collections.abc import Iterable
from dataclasses import dataclass
from enum import StrEnum

from pants.backend.python.subsystems.python_tool_base import PythonToolBase
from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.target_types import (
    ConsoleScript,
    InterpreterConstraintsField,
    PythonRequirementsField,
    PythonResolveField,
    PythonSourceField,
)
from pants.backend.python.typecheck.mypy.skip_field import SkipMyPyField
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints
from pants.backend.python.util_rules.pex_requirements import PexRequirements
from pants.backend.python.util_rules.python_sources import (
    PythonSourceFilesRequest,
    prepare_python_sources,
)
from pants.core.goals.resolves import ExportableTool
from pants.core.util_rules.config_files import ConfigFilesRequest, find_config_file
from pants.engine.addresses import UnparsedAddressInputs
from pants.engine.fs import EMPTY_DIGEST, Digest, FileContent
from pants.engine.internals.graph import resolve_unparsed_address_inputs
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import get_digest_contents
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import FieldSet, Target, TransitiveTargetsRequest
from pants.engine.unions import UnionRule
from pants.option.option_types import (
    ArgsListOption,
    BoolOption,
    EnumOption,
    FileOption,
    SkipOption,
    TargetListOption,
)
from pants.util.docutil import doc_url
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet
from pants.util.strutil import softwrap

logger = logging.getLogger(__name__)


class MyPyCacheMode(StrEnum):
    sqlite = "sqlite"
    none = "none"


@dataclass(frozen=True)
class MyPyFieldSet(FieldSet):
    required_fields = (PythonSourceField,)

    sources: PythonSourceField
    resolve: PythonResolveField
    interpreter_constraints: InterpreterConstraintsField

    @classmethod
    def opt_out(cls, tgt: Target) -> bool:
        return tgt.get(SkipMyPyField).value


# --------------------------------------------------------------------------------------
# Subsystem
# --------------------------------------------------------------------------------------


class MyPy(PythonToolBase):
    options_scope = "mypy"
    name = "MyPy"
    help_short = "The MyPy Python type checker (http://mypy-lang.org/)."

    default_main = ConsoleScript("mypy")
    default_requirements = ["mypy>=0.961,<2"]

    # See `mypy/rules.py`. We only use these default constraints in some situations.
    register_interpreter_constraints = True

    default_lockfile_resource = ("pants.backend.python.typecheck.mypy", "mypy.lock")

    skip = SkipOption("check")
    args = ArgsListOption(example="--python-version 3.7 --disallow-any-expr")
    config = FileOption(
        default=None,
        advanced=True,
        help=lambda cls: softwrap(
            f"""
            Path to a config file understood by MyPy
            (https://mypy.readthedocs.io/en/stable/config_file.html).

            Setting this option will disable `[{cls.options_scope}].config_discovery`. Use
            this option if the config is located in a non-standard location.
            """
        ),
    )
    config_discovery = BoolOption(
        default=True,
        advanced=True,
        help=lambda cls: softwrap(
            f"""
            If true, Pants will include any relevant config files during runs
            (`mypy.ini`, `.mypy.ini`, and `setup.cfg`).

            Use `[{cls.options_scope}].config` instead if your config is in a non-standard location.
            """
        ),
    )
    _source_plugins = TargetListOption(
        advanced=True,
        help=softwrap(
            f"""
            An optional list of `python_sources` target addresses to load first-party plugins.

            You must also set `plugins = path.to.module` in your `mypy.ini`, and
            set the `[mypy].config` option in your `pants.toml`.

            To instead load third-party plugins, set the option `[mypy].install_from_resolve`
            to a resolve whose lockfile includes those plugins, and set the `plugins` option
            in `mypy.ini`.  See {doc_url("docs/python/goals/check")}.
            """
        ),
    )
    cache_mode = EnumOption(
        default=MyPyCacheMode.sqlite,
        advanced=True,
        help=softwrap(
            """
            `sqlite`: Default. Uses mypy's SQLite cache.

            `none`: Disables caching entirely (--cache-dir=/dev/null). Much
            slower.  Intended as an "escape valve" if you believe you are
            encountering a Pants or mypy related bug.
            """
        ),
    )

    @property
    def config_request(self) -> ConfigFilesRequest:
        # Refer to https://mypy.readthedocs.io/en/stable/config_file.html.
        return ConfigFilesRequest(
            specified=self.config,
            specified_option_name=f"{self.options_scope}.config",
            discovery=self.config_discovery,
            check_existence=["mypy.ini", ".mypy.ini"],
            check_content={"setup.cfg": b"[mypy", "pyproject.toml": b"[tool.mypy"},
        )

    @property
    def source_plugins(self) -> UnparsedAddressInputs:
        return UnparsedAddressInputs(
            self._source_plugins,
            owning_address=None,
            description_of_origin=f"the option `[{self.options_scope}].source_plugins`",
        )

    def check_and_warn_if_python_version_configured(self, config: FileContent | None) -> bool:
        """Determine if we can dynamically set `--python-version` and warn if not."""
        configured = []
        if config and b"python_version" in config.content:
            configured.append(
                softwrap(
                    f"""
                    `python_version` in {config.path} (which is used because of either config
                    discovery or the `[mypy].config` option)
                    """
                )
            )
        if "--py2" in self.args:
            configured.append("`--py2` in the `--mypy-args` option")
        if any(arg.startswith("--python-version") for arg in self.args):
            configured.append("`--python-version` in the `--mypy-args` option")
        if configured:
            formatted_configured = " and you set ".join(configured)
            logger.warning(
                softwrap(
                    f"""
                    You set {formatted_configured}. Normally, Pants would automatically set this
                    for you based on your code's interpreter constraints
                    ({doc_url("docs/python/overview/interpreter-compatibility")}). Instead, it will
                    use what you set.

                    (Allowing Pants to automatically set the option allows Pants to partition your
                    targets by their constraints, so that, for example, you can run MyPy on
                    Python 2-only code and Python 3-only code at the same time. It also allows Pants
                    to leverage MyPy's cache, making subsequent runs of MyPy very fast.
                    In the future, this feature may no longer work.)
                    """
                )
            )
        return bool(configured)


# --------------------------------------------------------------------------------------
# Config files
# --------------------------------------------------------------------------------------


@dataclass(frozen=True)
class MyPyConfigFile:
    digest: Digest
    _python_version_configured: bool

    def python_version_to_autoset(
        self, interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str]
    ) -> str | None:
        """If the user did not already set `--python-version`, return the major.minor version to
        use."""
        if self._python_version_configured:
            return None
        return interpreter_constraints.minimum_python_version(interpreter_universe)


@rule
async def setup_mypy_config(mypy: MyPy) -> MyPyConfigFile:
    config_files = await find_config_file(mypy.config_request)
    digest_contents = await get_digest_contents(config_files.snapshot.digest)
    python_version_configured = mypy.check_and_warn_if_python_version_configured(
        digest_contents[0] if digest_contents else None
    )
    return MyPyConfigFile(config_files.snapshot.digest, python_version_configured)


# --------------------------------------------------------------------------------------
# First party plugins
# --------------------------------------------------------------------------------------


@dataclass(frozen=True)
class MyPyFirstPartyPlugins:
    requirement_strings: FrozenOrderedSet[str]
    sources_digest: Digest
    source_roots: tuple[str, ...]


@rule(desc="Prepare [mypy].source_plugins", level=LogLevel.DEBUG)
async def mypy_first_party_plugins(
    mypy: MyPy,
) -> MyPyFirstPartyPlugins:
    if not mypy.source_plugins:
        return MyPyFirstPartyPlugins(FrozenOrderedSet(), EMPTY_DIGEST, ())

    plugin_target_addresses = await resolve_unparsed_address_inputs(
        mypy.source_plugins, **implicitly()
    )
    transitive_targets = await transitive_targets_get(
        TransitiveTargetsRequest(plugin_target_addresses), **implicitly()
    )

    requirements = PexRequirements.req_strings_from_requirement_fields(
        (
            plugin_tgt[PythonRequirementsField]
            for plugin_tgt in transitive_targets.closure
            if plugin_tgt.has_field(PythonRequirementsField)
        ),
    )

    sources = await prepare_python_sources(
        PythonSourceFilesRequest(transitive_targets.closure), **implicitly()
    )
    return MyPyFirstPartyPlugins(
        requirement_strings=requirements,
        sources_digest=sources.source_files.snapshot.digest,
        source_roots=sources.source_roots,
    )


# --------------------------------------------------------------------------------------
# Interpreter constraints
# --------------------------------------------------------------------------------------


async def _mypy_interpreter_constraints(
    mypy: MyPy, python_setup: PythonSetup
) -> InterpreterConstraints:
    constraints = mypy.interpreter_constraints
    if mypy.options.is_default("interpreter_constraints"):
        code_constraints = await _find_all_unique_interpreter_constraints(
            python_setup, MyPyFieldSet
        )
        if code_constraints.requires_python38_or_newer(python_setup.interpreter_versions_universe):
            constraints = code_constraints
    return constraints


def rules():
    return (
        *collect_rules(),
        UnionRule(ExportableTool, MyPy),
    )

1	# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	1✔
5
6	import logging	1✔
7	from collections.abc import Iterable	1✔
8	from dataclasses import dataclass	1✔
9	from enum import StrEnum	1✔
10
11	from pants.backend.python.subsystems.python_tool_base import PythonToolBase	1✔
12	from pants.backend.python.subsystems.setup import PythonSetup	1✔
13	from pants.backend.python.target_types import (	1✔
14	ConsoleScript,
15	InterpreterConstraintsField,
16	PythonRequirementsField,
17	PythonResolveField,
18	PythonSourceField,
19	)
20	from pants.backend.python.typecheck.mypy.skip_field import SkipMyPyField	1✔
21	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	1✔
22	from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints	1✔
23	from pants.backend.python.util_rules.pex_requirements import PexRequirements	1✔
24	from pants.backend.python.util_rules.python_sources import (	1✔
25	PythonSourceFilesRequest,
26	prepare_python_sources,
27	)
28	from pants.core.goals.resolves import ExportableTool	1✔
29	from pants.core.util_rules.config_files import ConfigFilesRequest, find_config_file	1✔
30	from pants.engine.addresses import UnparsedAddressInputs	1✔
31	from pants.engine.fs import EMPTY_DIGEST, Digest, FileContent	1✔
32	from pants.engine.internals.graph import resolve_unparsed_address_inputs	1✔
33	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	1✔
34	from pants.engine.intrinsics import get_digest_contents	1✔
35	from pants.engine.rules import collect_rules, implicitly, rule	1✔
36	from pants.engine.target import FieldSet, Target, TransitiveTargetsRequest	1✔
37	from pants.engine.unions import UnionRule	1✔
38	from pants.option.option_types import (	1✔
39	ArgsListOption,
40	BoolOption,
41	EnumOption,
42	FileOption,
43	SkipOption,
44	TargetListOption,
45	)
46	from pants.util.docutil import doc_url	1✔
47	from pants.util.logging import LogLevel	1✔
48	from pants.util.ordered_set import FrozenOrderedSet	1✔
49	from pants.util.strutil import softwrap	1✔
50
51	logger = logging.getLogger(__name__)	1✔
52
53
54	class MyPyCacheMode(StrEnum):	1✔
55	sqlite = "sqlite"	1✔
56	none = "none"	1✔
57
58
59	@dataclass(frozen=True)	1✔
60	class MyPyFieldSet(FieldSet):	1✔
61	required_fields = (PythonSourceField,)	1✔
62
63	sources: PythonSourceField	1✔
64	resolve: PythonResolveField	1✔
65	interpreter_constraints: InterpreterConstraintsField	1✔
66
67	@classmethod	1✔
68	def opt_out(cls, tgt: Target) -> bool:	1✔
69	return tgt.get(SkipMyPyField).value	×
70
71
72	# --------------------------------------------------------------------------------------
73	# Subsystem
74	# --------------------------------------------------------------------------------------
75
76
77	class MyPy(PythonToolBase):	1✔
78	options_scope = "mypy"	1✔
79	name = "MyPy"	1✔
80	help_short = "The MyPy Python type checker (http://mypy-lang.org/)."	1✔
81
82	default_main = ConsoleScript("mypy")	1✔
83	default_requirements = ["mypy>=0.961,<2"]	1✔
84
85	# See `mypy/rules.py`. We only use these default constraints in some situations.
86	register_interpreter_constraints = True	1✔
87
88	default_lockfile_resource = ("pants.backend.python.typecheck.mypy", "mypy.lock")	1✔
89
90	skip = SkipOption("check")	1✔
91	args = ArgsListOption(example="--python-version 3.7 --disallow-any-expr")	1✔
92	config = FileOption(	1✔
93	default=None,
94	advanced=True,
95	help=lambda cls: softwrap(
96	f"""
97	Path to a config file understood by MyPy
98	(https://mypy.readthedocs.io/en/stable/config_file.html).
99
100	Setting this option will disable `[{cls.options_scope}].config_discovery`. Use
101	this option if the config is located in a non-standard location.
102	"""
103	),
104	)
105	config_discovery = BoolOption(	1✔
106	default=True,
107	advanced=True,
108	help=lambda cls: softwrap(
109	f"""
110	If true, Pants will include any relevant config files during runs
111	(`mypy.ini`, `.mypy.ini`, and `setup.cfg`).
112
113	Use `[{cls.options_scope}].config` instead if your config is in a non-standard location.
114	"""
115	),
116	)
117	_source_plugins = TargetListOption(	1✔
118	advanced=True,
119	help=softwrap(
120	f"""
121	An optional list of `python_sources` target addresses to load first-party plugins.
122
123	You must also set `plugins = path.to.module` in your `mypy.ini`, and
124	set the `[mypy].config` option in your `pants.toml`.
125
126	To instead load third-party plugins, set the option `[mypy].install_from_resolve`
127	to a resolve whose lockfile includes those plugins, and set the `plugins` option
128	in `mypy.ini`. See {doc_url("docs/python/goals/check")}.
129	"""
130	),
131	)
132	cache_mode = EnumOption(	1✔
133	default=MyPyCacheMode.sqlite,
134	advanced=True,
135	help=softwrap(
136	"""
137	`sqlite`: Default. Uses mypy's SQLite cache.
138
139	`none`: Disables caching entirely (--cache-dir=/dev/null). Much
140	slower. Intended as an "escape valve" if you believe you are
141	encountering a Pants or mypy related bug.
142	"""
143	),
144	)
145
146	@property	1✔
147	def config_request(self) -> ConfigFilesRequest:	1✔
148	# Refer to https://mypy.readthedocs.io/en/stable/config_file.html.
149	return ConfigFilesRequest(	1✔
150	specified=self.config,
151	specified_option_name=f"{self.options_scope}.config",
152	discovery=self.config_discovery,
153	check_existence=["mypy.ini", ".mypy.ini"],
154	check_content={"setup.cfg": b"[mypy", "pyproject.toml": b"[tool.mypy"},
155	)
156
157	@property	1✔
158	def source_plugins(self) -> UnparsedAddressInputs:	1✔
159	return UnparsedAddressInputs(	1✔
160	self._source_plugins,
161	owning_address=None,
162	description_of_origin=f"the option `[{self.options_scope}].source_plugins`",
163	)
164
165	def check_and_warn_if_python_version_configured(self, config: FileContent \| None) -> bool:	1✔
166	"""Determine if we can dynamically set `--python-version` and warn if not."""
167	configured = []	1✔
168	if config and b"python_version" in config.content:	1✔
169	configured.append(	×
170	softwrap(
171	f"""
172	`python_version` in {config.path} (which is used because of either config
173	discovery or the `[mypy].config` option)
174	"""
175	)
176	)
177	if "--py2" in self.args:	1✔
178	configured.append("`--py2` in the `--mypy-args` option")	×
179	if any(arg.startswith("--python-version") for arg in self.args):	1✔
180	configured.append("`--python-version` in the `--mypy-args` option")	×
181	if configured:	1✔
182	formatted_configured = " and you set ".join(configured)	×
183	logger.warning(	×
184	softwrap(
185	f"""
186	You set {formatted_configured}. Normally, Pants would automatically set this
187	for you based on your code's interpreter constraints
188	({doc_url("docs/python/overview/interpreter-compatibility")}). Instead, it will
189	use what you set.
190
191	(Allowing Pants to automatically set the option allows Pants to partition your
192	targets by their constraints, so that, for example, you can run MyPy on
193	Python 2-only code and Python 3-only code at the same time. It also allows Pants
194	to leverage MyPy's cache, making subsequent runs of MyPy very fast.
195	In the future, this feature may no longer work.)
196	"""
197	)
198	)
199	return bool(configured)	1✔
200
201
202	# --------------------------------------------------------------------------------------
203	# Config files
204	# --------------------------------------------------------------------------------------
205
206
207	@dataclass(frozen=True)	1✔
208	class MyPyConfigFile:	1✔
209	digest: Digest	1✔
210	_python_version_configured: bool	1✔
211
212	def python_version_to_autoset(	1✔
213	self, interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str]
214	) -> str \| None:
215	"""If the user did not already set `--python-version`, return the major.minor version to
216	use."""
217	if self._python_version_configured:	1✔
218	return None	×
219	return interpreter_constraints.minimum_python_version(interpreter_universe)	1✔
220
221
222	@rule	1✔
223	async def setup_mypy_config(mypy: MyPy) -> MyPyConfigFile:	1✔
224	config_files = await find_config_file(mypy.config_request)	1✔
225	digest_contents = await get_digest_contents(config_files.snapshot.digest)	1✔
226	python_version_configured = mypy.check_and_warn_if_python_version_configured(	1✔
227	digest_contents[0] if digest_contents else None
228	)
229	return MyPyConfigFile(config_files.snapshot.digest, python_version_configured)	1✔
230
231
232	# --------------------------------------------------------------------------------------
233	# First party plugins
234	# --------------------------------------------------------------------------------------
235
236
237	@dataclass(frozen=True)	1✔
238	class MyPyFirstPartyPlugins:	1✔
239	requirement_strings: FrozenOrderedSet[str]	1✔
240	sources_digest: Digest	1✔
241	source_roots: tuple[str, ...]	1✔
242
243
244	@rule(desc="Prepare [mypy].source_plugins", level=LogLevel.DEBUG)	1✔
245	async def mypy_first_party_plugins(	1✔
246	mypy: MyPy,
247	) -> MyPyFirstPartyPlugins:
248	if not mypy.source_plugins:	1✔
249	return MyPyFirstPartyPlugins(FrozenOrderedSet(), EMPTY_DIGEST, ())	×
250
251	plugin_target_addresses = await resolve_unparsed_address_inputs(	1✔
252	mypy.source_plugins, **implicitly()
253	)
254	transitive_targets = await transitive_targets_get(	1✔
255	TransitiveTargetsRequest(plugin_target_addresses), **implicitly()
256	)
257
258	requirements = PexRequirements.req_strings_from_requirement_fields(	1✔
259	(
260	plugin_tgt[PythonRequirementsField]
261	for plugin_tgt in transitive_targets.closure
262	if plugin_tgt.has_field(PythonRequirementsField)
263	),
264	)
265
266	sources = await prepare_python_sources(	1✔
267	PythonSourceFilesRequest(transitive_targets.closure), **implicitly()
268	)
269	return MyPyFirstPartyPlugins(	1✔
270	requirement_strings=requirements,
271	sources_digest=sources.source_files.snapshot.digest,
272	source_roots=sources.source_roots,
273	)
274
275
276	# --------------------------------------------------------------------------------------
277	# Interpreter constraints
278	# --------------------------------------------------------------------------------------
279
280
281	async def _mypy_interpreter_constraints(	1✔
282	mypy: MyPy, python_setup: PythonSetup
283	) -> InterpreterConstraints:
284	constraints = mypy.interpreter_constraints	×
285	if mypy.options.is_default("interpreter_constraints"):	×
286	code_constraints = await _find_all_unique_interpreter_constraints(	×
287	python_setup, MyPyFieldSet
288	)
289	if code_constraints.requires_python38_or_newer(python_setup.interpreter_versions_universe):	×
290	constraints = code_constraints	×
291	return constraints	×
292
293
294	def rules():	1✔
295	return (	1✔
296	*collect_rules(),
297	UnionRule(ExportableTool, MyPy),
298	)

pantsbuild / pants / 22740642519

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous