22507851448

Committed 27 Feb 2026 11:28PM UTC coverage: 92.928% (-0.007%) from 92.935%

Build # 22507851448

Build Type

push

github

Committed by

web-flow

Commit Message

silence new HdrHistogram induced deprecation warning on Python 3.14 (#23144)

No more:
```
/usr/lib/python3.14/ctypes/_endian.py:33: DeprecationWarning: Due to '_pack_', the
'ExternalHeader' Structure will use memory layout compatible with MSVC (Windows). If this is intended, set _layout_ to 'ms'. The
 implicit default is deprecated and slated to become an error in Python 3.19.
  super().__setattr__(attrname, value)
/usr/lib/python3.14/ctypes/_endian.py:33: DeprecationWarning: Due to '_pack_', the 'PayloadHeader' Structure will use memory
layout compatible with MSVC (Windows). If this is intended, set _layout_ to 'ms'. The implicit default is deprecated and slated
to become an error in Python 3.19.
  super().__setattr__(attrname, value)
```

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

71 existing lines in 12 files now uncovered.

90912 of 97831 relevant lines covered (92.93%)

4.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.14

/src/python/pants/backend/python/typecheck/mypy/rules.py

# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import dataclasses
from collections.abc import Iterable
from dataclasses import dataclass
from hashlib import sha256
from textwrap import dedent  # noqa: PNT20

import packaging

from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.typecheck.mypy.subsystem import (
    MyPy,
    MyPyConfigFile,
    MyPyFieldSet,
    MyPyFirstPartyPlugins,
)
from pants.backend.python.util_rules import pex_from_targets
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.partition import (
    _partition_by_interpreter_constraints_and_resolve,
)
from pants.backend.python.util_rules.pex import (
    PexRequest,
    VenvPex,
    VenvPexProcess,
    create_pex,
    create_venv_pex,
    determine_venv_pex_resolve_info,
    setup_venv_pex_process,
)
from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest
from pants.backend.python.util_rules.python_sources import (
    PythonSourceFilesRequest,
    prepare_python_sources,
)
from pants.base.build_root import BuildRoot
from pants.core.goals.check import (
    REPORT_DIR,
    CheckRequest,
    CheckResult,
    CheckResults,
    CheckSubsystem,
)
from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files
from pants.core.util_rules.system_binaries import (
    CpBinary,
    LnBinary,
    MkdirBinary,
    MktempBinary,
    MvBinary,
)
from pants.engine.collection import Collection
from pants.engine.fs import CreateDigest, FileContent, MergeDigests, RemovePrefix
from pants.engine.internals.graph import resolve_coarsened_targets as coarsened_targets_get
from pants.engine.intrinsics import create_digest, execute_process, merge_digests, remove_prefix
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.target import CoarsenedTargets, CoarsenedTargetsRequest
from pants.engine.unions import UnionRule
from pants.option.global_options import GlobalOptions
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet, OrderedSet
from pants.util.strutil import pluralize, shell_quote


@dataclass(frozen=True)
class MyPyPartition:
    field_sets: FrozenOrderedSet[MyPyFieldSet]
    root_targets: CoarsenedTargets
    resolve_description: str | None
    interpreter_constraints: InterpreterConstraints

    def description(self) -> str:
        ics = str(sorted(str(c) for c in self.interpreter_constraints))
        return f"{self.resolve_description}, {ics}" if self.resolve_description else ics


class MyPyPartitions(Collection[MyPyPartition]):
    pass


class MyPyRequest(CheckRequest):
    field_set_type = MyPyFieldSet
    tool_name = MyPy.options_scope


async def _generate_argv(
    mypy: MyPy,
    *,
    pex: VenvPex,
    cache_dir: str,
    venv_python: str,
    file_list_path: str,
    python_version: str | None,
) -> tuple[str, ...]:
    args = [pex.pex.argv0, f"--python-executable={venv_python}", *mypy.args]
    if mypy.config:
        args.append(f"--config-file={mypy.config}")
    if python_version:
        args.append(f"--python-version={python_version}")

    mypy_pex_info = await determine_venv_pex_resolve_info(pex)
    mypy_info = mypy_pex_info.find("mypy")
    assert mypy_info is not None
    if mypy_info.version > packaging.version.Version("0.700") and python_version is not None:
        # Skip mtime checks because we don't propagate mtime when materializing the sandbox, so the
        # mtime checks will always fail otherwise.
        args.append("--skip-cache-mtime-check")
        # See "__run_wrapper.sh" below for explanation
        args.append("--sqlite-cache")  # Added in v 0.660
        args.extend(("--cache-dir", cache_dir))
    else:
        # Don't bother caching
        args.append("--cache-dir=/dev/null")
    args.append(f"@{file_list_path}")
    return tuple(args)


def determine_python_files(files: Iterable[str]) -> tuple[str, ...]:
    """We run over all .py and .pyi files, but .pyi files take precedence.

    MyPy will error if we say to run over the same module with both its .py and .pyi files, so we
    must be careful to only use the .pyi stub.
    """
    result: OrderedSet[str] = OrderedSet()
    for f in files:
        if f.endswith(".pyi"):
            py_file = f[:-1]  # That is, strip the `.pyi` suffix to be `.py`.
            result.discard(py_file)
            result.add(f)
        elif f.endswith(".py"):
            pyi_file = f + "i"
            if pyi_file not in result:
                result.add(f)
        else:
            result.add(f)

    return tuple(result)


@rule
async def mypy_typecheck_partition(
    partition: MyPyPartition,
    config_file: MyPyConfigFile,
    first_party_plugins: MyPyFirstPartyPlugins,
    build_root: BuildRoot,
    mypy: MyPy,
    check_subsystem: CheckSubsystem,
    python_setup: PythonSetup,
    mkdir: MkdirBinary,
    mktemp: MktempBinary,
    cp: CpBinary,
    mv: MvBinary,
    ln: LnBinary,
    global_options: GlobalOptions,
) -> CheckResult:
    # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
    # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
    # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
    # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
    # 3.8+ is not installed on the machine.
    tool_interpreter_constraints = (
        partition.interpreter_constraints
        if (
            mypy.options.is_default("interpreter_constraints")
            and partition.interpreter_constraints.requires_python38_or_newer(
                python_setup.interpreter_versions_universe
            )
        )
        else mypy.interpreter_constraints
    )

    roots_sources_get = determine_source_files(
        SourceFilesRequest(fs.sources for fs in partition.field_sets)
    )

    # See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
    requirements_pex_get = create_pex(
        **implicitly(
            RequirementsPexRequest(
                (fs.address for fs in partition.field_sets),
                hardcoded_interpreter_constraints=partition.interpreter_constraints,
            )
        )
    )

    mypy_pex_get = create_venv_pex(
        **implicitly(
            mypy.to_pex_request(
                interpreter_constraints=tool_interpreter_constraints,
                extra_requirements=first_party_plugins.requirement_strings,
            )
        )
    )

    (
        roots_sources,
        mypy_pex,
        requirements_pex,
    ) = await concurrently(
        roots_sources_get,
        mypy_pex_get,
        requirements_pex_get,
    )

    python_files = determine_python_files(roots_sources.snapshot.files)
    file_list_path = "__files.txt"
    file_list_digest_request = create_digest(
        CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())])
    )

    # This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to
    # use this venv by setting `--python-executable`. Note that this Python interpreter is
    # different than what we run MyPy with.
    #
    # We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than
    # `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and
    # `run`.
    requirements_venv_pex_request = create_venv_pex(
        **implicitly(
            PexRequest(
                output_filename="requirements_venv.pex",
                internal_only=True,
                pex_path=[requirements_pex],
                interpreter_constraints=partition.interpreter_constraints,
            )
        )
    )
    closure_sources_get = prepare_python_sources(
        PythonSourceFilesRequest(partition.root_targets.closure()), **implicitly()
    )

    closure_sources, requirements_venv_pex, file_list_digest = await concurrently(
        closure_sources_get, requirements_venv_pex_request, file_list_digest_request
    )

    py_version = config_file.python_version_to_autoset(
        partition.interpreter_constraints, python_setup.interpreter_versions_universe
    )
    named_cache_dir = ".cache/mypy_cache"
    mypy_cache_dir = f"{named_cache_dir}/{sha256(build_root.path.encode()).hexdigest()}"
    if partition.resolve_description:
        mypy_cache_dir += f"/{partition.resolve_description}"
    run_cache_dir = ".tmp_cache/mypy_cache"
    argv = await _generate_argv(
        mypy,
        pex=mypy_pex,
        venv_python=requirements_venv_pex.python.argv0,
        cache_dir=run_cache_dir,
        file_list_path=file_list_path,
        python_version=py_version,
    )

    script_runner_digest = await create_digest(
        CreateDigest(
            [
                FileContent(
                    "__mypy_runner.sh",
                    dedent(
                        f"""\
                            # We want to leverage the MyPy cache for fast incremental runs of MyPy.
                            # Pants exposes "append_only_caches" we can leverage, but with the caveat
                            # that it requires either only appending files, or multiprocess-safe access.
                            #
                            # MyPy guarantees neither, but there's workarounds!
                            #
                            # By default, MyPy uses 2 cache files per source file, which introduces a
                            # whole slew of race conditions. We can minimize the race conditions by
                            # using MyPy's SQLite cache. MyPy still has race conditions when using the
                            # db, as it issues at least 2 single-row queries per source file at different
                            # points in time (therefore SQLite's own safety guarantees don't apply).
                            #
                            # Our workaround depends on whether we can hardlink between the sandbox
                            # and cache or not.
                            #
                            # If we can hardlink (this means the two sides of the link are on the
                            # same filesystem), then after mypy runs, we hardlink from the sandbox
                            # to a temp file in the named cache, then atomically rename it into place.
                            #
                            # If we can't hardlink, we resort to copying the result to a temp file
                            # in the named cache, and finally doing an atomic mv from the tempfile
                            # to the real one.
                            #
                            # In either case, the result is an atomic replacement of the "old" named
                            # cache db, such that old references (via opened file descriptors) are
                            # still valid, but new references use the new contents.
                            #
                            # There is a chance of multiple processes thrashing on the cache, leaving
                            # it in a state that doesn't reflect reality at the current point in time,
                            # and forcing other processes to do potentially done work. This strategy
                            # still provides a net benefit because the cache is generally _mostly_
                            # valid (it includes entries for the standard library, and 3rdparty deps,
                            # among 1stparty sources), and even in the worst case
                            # (every single file has changed) the overhead of missing the cache each
                            # query should be small when compared to the work being done of typechecking.
                            #
                            # Lastly, we expect that since this is run through Pants which attempts
                            # to partition MyPy runs by python version (which the DB is independent
                            # for different versions) and uses a one-process-at-a-time daemon by default,
                            # multiple MyPy processes operating on a single db cache should be rare.

                            NAMED_CACHE_DIR="{mypy_cache_dir}/{py_version}"
                            NAMED_CACHE_DB="$NAMED_CACHE_DIR/cache.db"
                            SANDBOX_CACHE_DIR="{run_cache_dir}/{py_version}"
                            SANDBOX_CACHE_DB="$SANDBOX_CACHE_DIR/cache.db"

                            {mkdir.path} -p "$NAMED_CACHE_DIR" > /dev/null 2>&1
                            {mkdir.path} -p "$SANDBOX_CACHE_DIR" > /dev/null 2>&1
                            {cp.path} "$NAMED_CACHE_DB" "$SANDBOX_CACHE_DB" > /dev/null 2>&1

                            {" ".join(shell_quote(arg) for arg in argv)}
                            EXIT_CODE=$?

                            # Only update the cache on successful runs (exit code 0 or 1).
                            # Exit code 2 indicates a crash or internal error, which may have
                            # left the cache in an inconsistent state.
                            # See https://github.com/python/mypy/issues/6003 for exit codes
                            if [ $EXIT_CODE -le 1 ]; then
                                if LN_TMP=$({mktemp.path} -u "$NAMED_CACHE_DB.tmp.XXXXXX") &&
                                   {ln.path} "$SANDBOX_CACHE_DB" "$LN_TMP" > /dev/null 2>&1; then
                                    {mv.path} "$LN_TMP" "$NAMED_CACHE_DB" > /dev/null 2>&1
                                else
                                    CP_TMP=$({mktemp.path} "$NAMED_CACHE_DB.tmp.XXXXXX") &&
                                        {cp.path} "$SANDBOX_CACHE_DB" "$CP_TMP" > /dev/null 2>&1 &&
                                        {mv.path} "$CP_TMP" "$NAMED_CACHE_DB" > /dev/null 2>&1
                                fi
                            fi

                            exit $EXIT_CODE
                        """
                    ).encode(),
                    is_executable=True,
                )
            ]
        )
    )

    merged_input_files = await merge_digests(
        MergeDigests(
            [
                file_list_digest,
                first_party_plugins.sources_digest,
                closure_sources.source_files.snapshot.digest,
                requirements_venv_pex.digest,
                config_file.digest,
                script_runner_digest,
            ]
        )
    )

    env = {
        "PEX_EXTRA_SYS_PATH": ":".join(first_party_plugins.source_roots),
        "MYPYPATH": ":".join(closure_sources.source_roots),
        # Always emit colors to improve cache hit rates, the results are post-processed to match the
        # global setting
        "MYPY_FORCE_COLOR": "1",
        # Mypy needs to know the terminal so it can use appropriate escape sequences. ansi is a
        # reasonable lowest common denominator for the sort of escapes mypy uses (NB. TERM=xterm
        # uses some additional codes that colors.strip_color doesn't remove).
        "TERM": "ansi",
        # Force a fixed terminal width. This is effectively infinite, disabling mypy's
        # builtin truncation and line wrapping. Terminals do an acceptable job of soft-wrapping
        # diagnostic text and source code is typically already hard-wrapped to a limited width.
        # (Unique random number to make it easier to search for the source of this setting.)
        "MYPY_FORCE_TERMINAL_WIDTH": "642092230765939",
    }

    process = await setup_venv_pex_process(
        VenvPexProcess(
            mypy_pex,
            input_digest=merged_input_files,
            extra_env=env,
            output_directories=(REPORT_DIR,),
            description=f"Run MyPy on {pluralize(len(python_files), 'file')}.",
            level=LogLevel.DEBUG,
            cache_scope=check_subsystem.default_process_cache_scope,
            append_only_caches={"mypy_cache": named_cache_dir},
        ),
        **implicitly(),
    )
    process = dataclasses.replace(process, argv=("./__mypy_runner.sh",))
    result = await execute_process(process, **implicitly())
    report = await remove_prefix(RemovePrefix(result.output_digest, REPORT_DIR))
    return CheckResult.from_fallible_process_result(
        result,
        partition_description=partition.description(),
        report=report,
        output_simplifier=global_options.output_simplifier(),
    )


@rule(desc="Determine if necessary to partition MyPy input", level=LogLevel.DEBUG)
async def mypy_determine_partitions(
    request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup
) -> MyPyPartitions:
    resolve_and_interpreter_constraints_to_field_sets = (
        _partition_by_interpreter_constraints_and_resolve(request.field_sets, python_setup)
    )
    coarsened_targets = await coarsened_targets_get(
        CoarsenedTargetsRequest(field_set.address for field_set in request.field_sets),
        **implicitly(),
    )
    coarsened_targets_by_address = coarsened_targets.by_address()

    return MyPyPartitions(
        MyPyPartition(
            FrozenOrderedSet(field_sets),
            CoarsenedTargets(
                OrderedSet(
                    coarsened_targets_by_address[field_set.address] for field_set in field_sets
                )
            ),
            resolve if len(python_setup.resolves) > 1 else None,
            interpreter_constraints or mypy.interpreter_constraints,
        )
        for (resolve, interpreter_constraints), field_sets in sorted(
            resolve_and_interpreter_constraints_to_field_sets.items()
        )
    )


@rule(desc="Typecheck using MyPy", level=LogLevel.DEBUG)
async def mypy_typecheck(request: MyPyRequest, mypy: MyPy) -> CheckResults:
    if mypy.skip:
        return CheckResults([], checker_name=request.tool_name)

    partitions = await mypy_determine_partitions(request, **implicitly())
    partitioned_results = await concurrently(
        mypy_typecheck_partition(partition, **implicitly()) for partition in partitions
    )
    return CheckResults(partitioned_results, checker_name=request.tool_name)


def rules():
    return [
        *collect_rules(),
        UnionRule(CheckRequest, MyPyRequest),
        *pex_from_targets.rules(),
    ]

1	# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	3✔
5
6	import dataclasses	3✔
7	from collections.abc import Iterable	3✔
8	from dataclasses import dataclass	3✔
9	from hashlib import sha256	3✔
10	from textwrap import dedent # noqa: PNT20	3✔
11
12	import packaging	3✔
13
14	from pants.backend.python.subsystems.setup import PythonSetup	3✔
15	from pants.backend.python.typecheck.mypy.subsystem import (	3✔
16	MyPy,
17	MyPyConfigFile,
18	MyPyFieldSet,
19	MyPyFirstPartyPlugins,
20	)
21	from pants.backend.python.util_rules import pex_from_targets	3✔
22	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	3✔
23	from pants.backend.python.util_rules.partition import (	3✔
24	_partition_by_interpreter_constraints_and_resolve,
25	)
26	from pants.backend.python.util_rules.pex import (	3✔
27	PexRequest,
28	VenvPex,
29	VenvPexProcess,
30	create_pex,
31	create_venv_pex,
32	determine_venv_pex_resolve_info,
33	setup_venv_pex_process,
34	)
35	from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest	3✔
36	from pants.backend.python.util_rules.python_sources import (	3✔
37	PythonSourceFilesRequest,
38	prepare_python_sources,
39	)
40	from pants.base.build_root import BuildRoot	3✔
41	from pants.core.goals.check import (	3✔
42	REPORT_DIR,
43	CheckRequest,
44	CheckResult,
45	CheckResults,
46	CheckSubsystem,
47	)
48	from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files	3✔
49	from pants.core.util_rules.system_binaries import (	3✔
50	CpBinary,
51	LnBinary,
52	MkdirBinary,
53	MktempBinary,
54	MvBinary,
55	)
56	from pants.engine.collection import Collection	3✔
57	from pants.engine.fs import CreateDigest, FileContent, MergeDigests, RemovePrefix	3✔
58	from pants.engine.internals.graph import resolve_coarsened_targets as coarsened_targets_get	3✔
59	from pants.engine.intrinsics import create_digest, execute_process, merge_digests, remove_prefix	3✔
60	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	3✔
61	from pants.engine.target import CoarsenedTargets, CoarsenedTargetsRequest	3✔
62	from pants.engine.unions import UnionRule	3✔
63	from pants.option.global_options import GlobalOptions	3✔
64	from pants.util.logging import LogLevel	3✔
65	from pants.util.ordered_set import FrozenOrderedSet, OrderedSet	3✔
66	from pants.util.strutil import pluralize, shell_quote	3✔
67
68
69	@dataclass(frozen=True)	3✔
70	class MyPyPartition:	3✔
71	field_sets: FrozenOrderedSet[MyPyFieldSet]	3✔
72	root_targets: CoarsenedTargets	3✔
73	resolve_description: str \| None	3✔
74	interpreter_constraints: InterpreterConstraints	3✔
75
76	def description(self) -> str:	3✔
77	ics = str(sorted(str(c) for c in self.interpreter_constraints))	3✔
78	return f"{self.resolve_description}, {ics}" if self.resolve_description else ics	3✔
79
80
81	class MyPyPartitions(Collection[MyPyPartition]):	3✔
82	pass	3✔
83
84
85	class MyPyRequest(CheckRequest):	3✔
86	field_set_type = MyPyFieldSet	3✔
87	tool_name = MyPy.options_scope	3✔
88
89
90	async def _generate_argv(	3✔
91	mypy: MyPy,
92	*,
93	pex: VenvPex,
94	cache_dir: str,
95	venv_python: str,
96	file_list_path: str,
97	python_version: str \| None,
98	) -> tuple[str, ...]:
99	args = [pex.pex.argv0, f"--python-executable={venv_python}", *mypy.args]	3✔
100	if mypy.config:	3✔
101	args.append(f"--config-file={mypy.config}")	1✔
102	if python_version:	3✔
103	args.append(f"--python-version={python_version}")	3✔
104
105	mypy_pex_info = await determine_venv_pex_resolve_info(pex)	3✔
106	mypy_info = mypy_pex_info.find("mypy")	3✔
107	assert mypy_info is not None	3✔
108	if mypy_info.version > packaging.version.Version("0.700") and python_version is not None:	3✔
109	# Skip mtime checks because we don't propagate mtime when materializing the sandbox, so the
110	# mtime checks will always fail otherwise.
111	args.append("--skip-cache-mtime-check")	3✔
112	# See "__run_wrapper.sh" below for explanation
113	args.append("--sqlite-cache") # Added in v 0.660	3✔
114	args.extend(("--cache-dir", cache_dir))	3✔
115	else:
116	# Don't bother caching
UNCOV 117	args.append("--cache-dir=/dev/null")	×
118	args.append(f"@{file_list_path}")	3✔
119	return tuple(args)	3✔
120
121
122	def determine_python_files(files: Iterable[str]) -> tuple[str, ...]:	3✔
123	"""We run over all .py and .pyi files, but .pyi files take precedence.
124
125	MyPy will error if we say to run over the same module with both its .py and .pyi files, so we
126	must be careful to only use the .pyi stub.
127	"""
128	result: OrderedSet[str] = OrderedSet()	3✔
129	for f in files:	3✔
130	if f.endswith(".pyi"):	3✔
131	py_file = f[:-1] # That is, strip the `.pyi` suffix to be `.py`.	1✔
132	result.discard(py_file)	1✔
133	result.add(f)	1✔
134	elif f.endswith(".py"):	3✔
135	pyi_file = f + "i"	3✔
136	if pyi_file not in result:	3✔
137	result.add(f)	3✔
138	else:
139	result.add(f)	1✔
140
141	return tuple(result)	3✔
142
143
144	@rule	3✔
145	async def mypy_typecheck_partition(	3✔
146	partition: MyPyPartition,
147	config_file: MyPyConfigFile,
148	first_party_plugins: MyPyFirstPartyPlugins,
149	build_root: BuildRoot,
150	mypy: MyPy,
151	check_subsystem: CheckSubsystem,
152	python_setup: PythonSetup,
153	mkdir: MkdirBinary,
154	mktemp: MktempBinary,
155	cp: CpBinary,
156	mv: MvBinary,
157	ln: LnBinary,
158	global_options: GlobalOptions,
159	) -> CheckResult:
160	# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
161	# and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
162	# Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
163	# loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
164	# 3.8+ is not installed on the machine.
165	tool_interpreter_constraints = (	3✔
166	partition.interpreter_constraints
167	if (
168	mypy.options.is_default("interpreter_constraints")
169	and partition.interpreter_constraints.requires_python38_or_newer(
170	python_setup.interpreter_versions_universe
171	)
172	)
173	else mypy.interpreter_constraints
174	)
175
176	roots_sources_get = determine_source_files(	3✔
177	SourceFilesRequest(fs.sources for fs in partition.field_sets)
178	)
179
180	# See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
181	requirements_pex_get = create_pex(	3✔
182	**implicitly(
183	RequirementsPexRequest(
184	(fs.address for fs in partition.field_sets),
185	hardcoded_interpreter_constraints=partition.interpreter_constraints,
186	)
187	)
188	)
189
190	mypy_pex_get = create_venv_pex(	3✔
191	**implicitly(
192	mypy.to_pex_request(
193	interpreter_constraints=tool_interpreter_constraints,
194	extra_requirements=first_party_plugins.requirement_strings,
195	)
196	)
197	)
198
199	(	3✔
200	roots_sources,
201	mypy_pex,
202	requirements_pex,
203	) = await concurrently(
204	roots_sources_get,
205	mypy_pex_get,
206	requirements_pex_get,
207	)
208
209	python_files = determine_python_files(roots_sources.snapshot.files)	3✔
210	file_list_path = "__files.txt"	3✔
211	file_list_digest_request = create_digest(	3✔
212	CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())])
213	)
214
215	# This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to
216	# use this venv by setting `--python-executable`. Note that this Python interpreter is
217	# different than what we run MyPy with.
218	#
219	# We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than
220	# `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and
221	# `run`.
222	requirements_venv_pex_request = create_venv_pex(	3✔
223	**implicitly(
224	PexRequest(
225	output_filename="requirements_venv.pex",
226	internal_only=True,
227	pex_path=[requirements_pex],
228	interpreter_constraints=partition.interpreter_constraints,
229	)
230	)
231	)
232	closure_sources_get = prepare_python_sources(	3✔
233	PythonSourceFilesRequest(partition.root_targets.closure()), **implicitly()
234	)
235
236	closure_sources, requirements_venv_pex, file_list_digest = await concurrently(	3✔
237	closure_sources_get, requirements_venv_pex_request, file_list_digest_request
238	)
239
240	py_version = config_file.python_version_to_autoset(	3✔
241	partition.interpreter_constraints, python_setup.interpreter_versions_universe
242	)
243	named_cache_dir = ".cache/mypy_cache"	3✔
244	mypy_cache_dir = f"{named_cache_dir}/{sha256(build_root.path.encode()).hexdigest()}"	3✔
245	if partition.resolve_description:	3✔
246	mypy_cache_dir += f"/{partition.resolve_description}"	1✔
247	run_cache_dir = ".tmp_cache/mypy_cache"	3✔
248	argv = await _generate_argv(	3✔
249	mypy,
250	pex=mypy_pex,
251	venv_python=requirements_venv_pex.python.argv0,
252	cache_dir=run_cache_dir,
253	file_list_path=file_list_path,
254	python_version=py_version,
255	)
256
257	script_runner_digest = await create_digest(	3✔
258	CreateDigest(
259	[
260	FileContent(
261	"__mypy_runner.sh",
262	dedent(
263	f"""\
264	# We want to leverage the MyPy cache for fast incremental runs of MyPy.
265	# Pants exposes "append_only_caches" we can leverage, but with the caveat
266	# that it requires either only appending files, or multiprocess-safe access.
267	#
268	# MyPy guarantees neither, but there's workarounds!
269	#
270	# By default, MyPy uses 2 cache files per source file, which introduces a
271	# whole slew of race conditions. We can minimize the race conditions by
272	# using MyPy's SQLite cache. MyPy still has race conditions when using the
273	# db, as it issues at least 2 single-row queries per source file at different
274	# points in time (therefore SQLite's own safety guarantees don't apply).
275	#
276	# Our workaround depends on whether we can hardlink between the sandbox
277	# and cache or not.
278	#
279	# If we can hardlink (this means the two sides of the link are on the
280	# same filesystem), then after mypy runs, we hardlink from the sandbox
281	# to a temp file in the named cache, then atomically rename it into place.
282	#
283	# If we can't hardlink, we resort to copying the result to a temp file
284	# in the named cache, and finally doing an atomic mv from the tempfile
285	# to the real one.
286	#
287	# In either case, the result is an atomic replacement of the "old" named
288	# cache db, such that old references (via opened file descriptors) are
289	# still valid, but new references use the new contents.
290	#
291	# There is a chance of multiple processes thrashing on the cache, leaving
292	# it in a state that doesn't reflect reality at the current point in time,
293	# and forcing other processes to do potentially done work. This strategy
294	# still provides a net benefit because the cache is generally _mostly_
295	# valid (it includes entries for the standard library, and 3rdparty deps,
296	# among 1stparty sources), and even in the worst case
297	# (every single file has changed) the overhead of missing the cache each
298	# query should be small when compared to the work being done of typechecking.
299	#
300	# Lastly, we expect that since this is run through Pants which attempts
301	# to partition MyPy runs by python version (which the DB is independent
302	# for different versions) and uses a one-process-at-a-time daemon by default,
303	# multiple MyPy processes operating on a single db cache should be rare.
304
305	NAMED_CACHE_DIR="{mypy_cache_dir}/{py_version}"
306	NAMED_CACHE_DB="$NAMED_CACHE_DIR/cache.db"
307	SANDBOX_CACHE_DIR="{run_cache_dir}/{py_version}"
308	SANDBOX_CACHE_DB="$SANDBOX_CACHE_DIR/cache.db"
309
310	{mkdir.path} -p "$NAMED_CACHE_DIR" > /dev/null 2>&1
311	{mkdir.path} -p "$SANDBOX_CACHE_DIR" > /dev/null 2>&1
312	{cp.path} "$NAMED_CACHE_DB" "$SANDBOX_CACHE_DB" > /dev/null 2>&1
313
314	{" ".join(shell_quote(arg) for arg in argv)}
315	EXIT_CODE=$?
316
317	# Only update the cache on successful runs (exit code 0 or 1).
318	# Exit code 2 indicates a crash or internal error, which may have
319	# left the cache in an inconsistent state.
320	# See https://github.com/python/mypy/issues/6003 for exit codes
321	if [ $EXIT_CODE -le 1 ]; then
322	if LN_TMP=$({mktemp.path} -u "$NAMED_CACHE_DB.tmp.XXXXXX") &&
323	{ln.path} "$SANDBOX_CACHE_DB" "$LN_TMP" > /dev/null 2>&1; then
324	{mv.path} "$LN_TMP" "$NAMED_CACHE_DB" > /dev/null 2>&1
325	else
326	CP_TMP=$({mktemp.path} "$NAMED_CACHE_DB.tmp.XXXXXX") &&
327	{cp.path} "$SANDBOX_CACHE_DB" "$CP_TMP" > /dev/null 2>&1 &&
328	{mv.path} "$CP_TMP" "$NAMED_CACHE_DB" > /dev/null 2>&1
329	fi
330	fi
331
332	exit $EXIT_CODE
333	"""
334	).encode(),
335	is_executable=True,
336	)
337	]
338	)
339	)
340
341	merged_input_files = await merge_digests(	3✔
342	MergeDigests(
343	[
344	file_list_digest,
345	first_party_plugins.sources_digest,
346	closure_sources.source_files.snapshot.digest,
347	requirements_venv_pex.digest,
348	config_file.digest,
349	script_runner_digest,
350	]
351	)
352	)
353
354	env = {	3✔
355	"PEX_EXTRA_SYS_PATH": ":".join(first_party_plugins.source_roots),
356	"MYPYPATH": ":".join(closure_sources.source_roots),
357	# Always emit colors to improve cache hit rates, the results are post-processed to match the
358	# global setting
359	"MYPY_FORCE_COLOR": "1",
360	# Mypy needs to know the terminal so it can use appropriate escape sequences. ansi is a
361	# reasonable lowest common denominator for the sort of escapes mypy uses (NB. TERM=xterm
362	# uses some additional codes that colors.strip_color doesn't remove).
363	"TERM": "ansi",
364	# Force a fixed terminal width. This is effectively infinite, disabling mypy's
365	# builtin truncation and line wrapping. Terminals do an acceptable job of soft-wrapping
366	# diagnostic text and source code is typically already hard-wrapped to a limited width.
367	# (Unique random number to make it easier to search for the source of this setting.)
368	"MYPY_FORCE_TERMINAL_WIDTH": "642092230765939",
369	}
370
371	process = await setup_venv_pex_process(	3✔
372	VenvPexProcess(
373	mypy_pex,
374	input_digest=merged_input_files,
375	extra_env=env,
376	output_directories=(REPORT_DIR,),
377	description=f"Run MyPy on {pluralize(len(python_files), 'file')}.",
378	level=LogLevel.DEBUG,
379	cache_scope=check_subsystem.default_process_cache_scope,
380	append_only_caches={"mypy_cache": named_cache_dir},
381	),
382	**implicitly(),
383	)
384	process = dataclasses.replace(process, argv=("./__mypy_runner.sh",))	3✔
385	result = await execute_process(process, **implicitly())	3✔
386	report = await remove_prefix(RemovePrefix(result.output_digest, REPORT_DIR))	3✔
387	return CheckResult.from_fallible_process_result(	3✔
388	result,
389	partition_description=partition.description(),
390	report=report,
391	output_simplifier=global_options.output_simplifier(),
392	)
393
394
395	@rule(desc="Determine if necessary to partition MyPy input", level=LogLevel.DEBUG)	3✔
396	async def mypy_determine_partitions(	3✔
397	request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup
398	) -> MyPyPartitions:
399	resolve_and_interpreter_constraints_to_field_sets = (	3✔
400	_partition_by_interpreter_constraints_and_resolve(request.field_sets, python_setup)
401	)
402	coarsened_targets = await coarsened_targets_get(	3✔
403	CoarsenedTargetsRequest(field_set.address for field_set in request.field_sets),
404	**implicitly(),
405	)
406	coarsened_targets_by_address = coarsened_targets.by_address()	3✔
407
408	return MyPyPartitions(	3✔
409	MyPyPartition(
410	FrozenOrderedSet(field_sets),
411	CoarsenedTargets(
412	OrderedSet(
413	coarsened_targets_by_address[field_set.address] for field_set in field_sets
414	)
415	),
416	resolve if len(python_setup.resolves) > 1 else None,
417	interpreter_constraints or mypy.interpreter_constraints,
418	)
419	for (resolve, interpreter_constraints), field_sets in sorted(
420	resolve_and_interpreter_constraints_to_field_sets.items()
421	)
422	)
423
424
425	@rule(desc="Typecheck using MyPy", level=LogLevel.DEBUG)	3✔
426	async def mypy_typecheck(request: MyPyRequest, mypy: MyPy) -> CheckResults:	3✔
427	if mypy.skip:	3✔
428	return CheckResults([], checker_name=request.tool_name)	1✔
429
430	partitions = await mypy_determine_partitions(request, **implicitly())	3✔
431	partitioned_results = await concurrently(	3✔
432	mypy_typecheck_partition(partition, **implicitly()) for partition in partitions
433	)
434	return CheckResults(partitioned_results, checker_name=request.tool_name)	3✔
435
436
437	def rules():	3✔
438	return [	3✔
439	*collect_rules(),
440	UnionRule(CheckRequest, MyPyRequest),
441	*pex_from_targets.rules(),
442	]

pantsbuild / pants / 22507851448

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous