22197409014

Committed 19 Feb 2026 07:46PM UTC coverage: 80.366% (+0.005%) from 80.361%

Build # 22197409014

Build Type

Pull #23109

github

Committed by

web-flow

Commit Message

Merge 5c0479330 into 99b47f0fb

Pull Request Pull Request #23109: cache source and --force for type checkers

Run Details

38 of 51 new or added lines in 8 files covered. (74.51%)

2 existing lines in 2 files now uncovered.

78867 of 98135 relevant lines covered (80.37%)

3.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.48

/src/python/pants/backend/python/typecheck/pyright/rules.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import dataclasses
import json
import logging
import os
import shlex
from collections.abc import Iterable
from dataclasses import dataclass, replace

import toml

from pants.backend.javascript.subsystems import nodejs_tool
from pants.backend.javascript.subsystems.nodejs import NodeJS
from pants.backend.javascript.subsystems.nodejs_tool import prepare_tool_process
from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.target_types import (
    InterpreterConstraintsField,
    PythonResolveField,
    PythonSourceField,
)
from pants.backend.python.typecheck.pyright.skip_field import SkipPyrightField
from pants.backend.python.typecheck.pyright.subsystem import Pyright
from pants.backend.python.util_rules import pex_from_targets
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.partition import (
    _partition_by_interpreter_constraints_and_resolve,
)
from pants.backend.python.util_rules.pex import (
    PexRequest,
    VenvPexProcess,
    VenvPexRequest,
    create_pex,
    create_venv_pex,
)
from pants.backend.python.util_rules.pex_environment import PexEnvironment
from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest
from pants.backend.python.util_rules.python_sources import (
    PythonSourceFilesRequest,
    prepare_python_sources,
)
from pants.core.goals.check import CheckRequest, CheckResult, CheckResults, CheckSubsystem
from pants.core.util_rules import config_files
from pants.core.util_rules.config_files import ConfigFiles, find_config_file
from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files
from pants.core.util_rules.system_binaries import CatBinary, ShBinary
from pants.engine.collection import Collection
from pants.engine.fs import CreateDigest, FileContent
from pants.engine.internals.graph import resolve_coarsened_targets as coarsened_targets_get
from pants.engine.internals.native_engine import Digest, MergeDigests
from pants.engine.internals.selectors import concurrently
from pants.engine.intrinsics import (
    create_digest,
    execute_process,
    get_digest_contents,
    merge_digests,
)
from pants.engine.process import ProcessCacheScope, execute_process_or_raise
from pants.engine.rules import Rule, collect_rules, implicitly, rule
from pants.engine.target import CoarsenedTargets, CoarsenedTargetsRequest, FieldSet, Target
from pants.engine.unions import UnionRule
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet, OrderedSet
from pants.util.strutil import pluralize

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class PyrightFieldSet(FieldSet):
    required_fields = (PythonSourceField,)

    sources: PythonSourceField
    resolve: PythonResolveField
    interpreter_constraints: InterpreterConstraintsField

    @classmethod
    def opt_out(cls, tgt: Target) -> bool:
        return tgt.get(SkipPyrightField).value


class PyrightRequest(CheckRequest):
    field_set_type = PyrightFieldSet
    tool_name = Pyright.options_scope


@dataclass(frozen=True)
class PyrightPartition:
    field_sets: FrozenOrderedSet[PyrightFieldSet]
    root_targets: CoarsenedTargets
    resolve_description: str | None
    interpreter_constraints: InterpreterConstraints

    def description(self) -> str:
        ics = str(sorted(str(c) for c in self.interpreter_constraints))
        return f"{self.resolve_description}, {ics}" if self.resolve_description else ics


class PyrightPartitions(Collection[PyrightPartition]):
    pass


async def _patch_config_file(
    config_files: ConfigFiles, venv_dir: str, source_roots: Iterable[str]
) -> Digest:
    """Patch the Pyright config file to use the incoming venv directory (from
    requirements_venv_pex). If there is no config file, create a dummy pyrightconfig.json with the
    `venv` key populated.

    The incoming venv directory works alongside the `--venvpath` CLI argument.

    Additionally, add source roots to the `extraPaths` key in the config file.
    """

    source_roots_list = list(source_roots)
    if not config_files.snapshot.files:
        # venv workaround as per: https://github.com/microsoft/pyright/issues/4051
        generated_config: dict[str, str | list[str]] = {
            "venv": venv_dir,
            "extraPaths": source_roots_list,
        }
        return await create_digest(
            CreateDigest(
                [
                    FileContent(
                        "pyrightconfig.json",
                        json.dumps(generated_config).encode(),
                    )
                ]
            )
        )

    config_contents = await get_digest_contents(config_files.snapshot.digest)
    new_files: list[FileContent] = []
    for file in config_contents:
        # This only supports a single json config file in the root of the project
        # https://github.com/pantsbuild/pants/issues/17816 tracks supporting multiple config files and workspaces
        if file.path == "pyrightconfig.json":
            json_config = json.loads(file.content)
            json_config["venv"] = venv_dir
            json_extra_paths: list[str] = json_config.get("extraPaths", [])
            json_config["extraPaths"] = list(OrderedSet(json_extra_paths + source_roots_list))
            new_content = json.dumps(json_config).encode()
            new_files.append(replace(file, content=new_content))

        # This only supports a single pyproject.toml file in the root of the project
        # https://github.com/pantsbuild/pants/issues/17816 tracks supporting multiple config files and workspaces
        elif file.path == "pyproject.toml":
            toml_config = toml.loads(file.content.decode())
            pyright_config = toml_config["tool"]["pyright"]
            pyright_config["venv"] = venv_dir
            toml_extra_paths: list[str] = pyright_config.get("extraPaths", [])
            pyright_config["extraPaths"] = list(OrderedSet(toml_extra_paths + source_roots_list))
            new_content = toml.dumps(toml_config).encode()
            new_files.append(replace(file, content=new_content))

    return await create_digest(CreateDigest(new_files))


@rule(
    desc="Pyright typecheck each partition based on its interpreter_constraints",
    level=LogLevel.DEBUG,
)
async def pyright_typecheck_partition(
    partition: PyrightPartition,
    pyright: Pyright,
    check_subsystem: CheckSubsystem,
    pex_environment: PexEnvironment,
    nodejs: NodeJS,
    sh_binary: ShBinary,
    cat_binary: CatBinary,
) -> CheckResult:
    root_sources_get = determine_source_files(
        SourceFilesRequest(fs.sources for fs in partition.field_sets)
    )

    # Grab the closure of the root source files to be typechecked
    transitive_sources_get = prepare_python_sources(
        PythonSourceFilesRequest(partition.root_targets.closure()), **implicitly()
    )

    # See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
    requirements_pex_get = create_pex(
        **implicitly(
            RequirementsPexRequest(
                (fs.address for fs in partition.field_sets),
                hardcoded_interpreter_constraints=partition.interpreter_constraints,
            )
        )
    )

    # Look for any/all of the Pyright configuration files (the config is modified below
    # for the `venv` workaround)
    config_files_get = find_config_file(pyright.config_request())

    root_sources, transitive_sources, requirements_pex, config_files = await concurrently(
        root_sources_get,
        transitive_sources_get,
        requirements_pex_get,
        config_files_get,
    )

    # This is a workaround for https://github.com/pantsbuild/pants/issues/19946.
    # complete_pex_env needs to be created here so that the test `test_passing_cache_clear`
    # test can pass using the appropriate caching directory.
    # See https://github.com/pantsbuild/pants/pull/19430#discussion_r1337851780
    # for more discussion.
    complete_pex_env = pex_environment.in_workspace()
    requirements_pex_request = PexRequest(
        output_filename="requirements_venv.pex",
        internal_only=True,
        pex_path=[requirements_pex],
        interpreter_constraints=partition.interpreter_constraints,
    )
    requirements_venv_pex = await create_venv_pex(
        VenvPexRequest(requirements_pex_request, complete_pex_env), **implicitly()
    )

    # Force the requirements venv to materialize always by running a no-op.
    # This operation must be called with `ProcessCacheScope.SESSION`
    # so that it runs every time.
    _ = await execute_process_or_raise(
        **implicitly(
            VenvPexProcess(
                requirements_venv_pex,
                description="Force venv to materialize",
                argv=["-c", "''"],
                cache_scope=ProcessCacheScope.PER_SESSION,
            )
        )
    )

    # Patch the config file to use the venv directory from the requirements pex,
    # and add source roots to the `extraPaths` key in the config file.
    patched_config_digest = await _patch_config_file(
        config_files, requirements_venv_pex.venv_rel_dir, transitive_sources.source_roots
    )

    # Prepare the process with as much information as we currently have. This will give us the
    # process's cwd, which we need in order to calculate the relative paths to the input files.
    # We will then manually tweak the argv before actually running.
    input_digest = await merge_digests(
        MergeDigests(
            [
                transitive_sources.source_files.snapshot.digest,
                requirements_venv_pex.digest,
                patched_config_digest,
            ]
        )
    )
    process = await prepare_tool_process(
        pyright.request(
            args=(
                f"--venvpath={complete_pex_env.pex_root}",  # Used with `venv` in config
                *pyright.args,  # User-added arguments
                "-",  # Read input file paths from stdin
            ),
            input_digest=input_digest,
            description=f"Run Pyright on {pluralize(len(root_sources.snapshot.files), 'file')}.",
            level=LogLevel.DEBUG,
        ),
        **implicitly(),
    )

    # We must use relative paths, because we don't know the abspath of the sandbox the process
    # will run in, and `{chroot}` interpolation only works on argv, not on the contents of
    # __files.txt (see below). Pyright interprets relpaths as relative to its cwd, so we
    # prepend the appropriate prefix to each file path.
    input_path_prefix = os.path.relpath(".", process.working_directory)
    input_files = [os.path.join(input_path_prefix, file) for file in root_sources.snapshot.files]

    # We prefer to pass the list of input files via stdin, as large numbers of files can cause us
    # to exceed the max command line length.  See https://github.com/pantsbuild/pants/issues/22779.
    # However Pyright, weirdly, splits stdin on spaces as well as newlines. So we can't pass input
    # file paths via stdin if any of them contain spaces.
    file_with_spaces = next((file for file in root_sources.snapshot.files if " " in file), None)
    if file_with_spaces:
        # Fall back to passing paths as args and hope we don't exceed the max command line length.
        process = dataclasses.replace(process, argv=(*process.argv[0:-1], *input_files))
    else:
        # Write the input files out to a text file.
        file_list_path = "__files.txt"
        file_list_content = "\n".join(input_files).encode()
        file_list_digest = await create_digest(
            CreateDigest([FileContent(file_list_path, file_list_content)])
        )
        input_digest = await merge_digests(
            MergeDigests(
                [
                    process.input_digest,
                    file_list_digest,
                ]
            )
        )
        # Run the underlying process inside a shell script that cats the file list to stdin.
        shell_script = f"{cat_binary.path} {os.path.join(input_path_prefix, file_list_path)} | {shlex.join(process.argv)}"
        process = dataclasses.replace(
            process, argv=(sh_binary.path, "-c", shell_script), input_digest=input_digest
        )

    process = dataclasses.replace(process, cache_scope=check_subsystem.default_process_cache_scope)

    result = await execute_process(process, **implicitly())
    if result.exit_code == 249 and file_with_spaces:
        logger.error(
            f"Found input files with spaces in their names, including: {file_with_spaces}. "
            "Due to a bug in Pyright this means that the number of input files Pants can pass to "
            "Pyright is limited, and exceeding that limit causes it to crash with exit code 249. "
            "Please reach out to the Pants team if this happens: "
            "https://www.pantsbuild.org/community/getting-help."
        )
    return CheckResult.from_fallible_process_result(
        result,
        partition_description=partition.description(),
    )


@rule(
    desc="Determine if it is necessary to partition Pyright's input (interpreter_constraints and resolves)",
    level=LogLevel.DEBUG,
)
async def pyright_determine_partitions(
    request: PyrightRequest,
    pyright: Pyright,
    python_setup: PythonSetup,
) -> PyrightPartitions:
    resolve_and_interpreter_constraints_to_field_sets = (
        _partition_by_interpreter_constraints_and_resolve(request.field_sets, python_setup)
    )

    coarsened_targets = await coarsened_targets_get(
        CoarsenedTargetsRequest(field_set.address for field_set in request.field_sets),
        **implicitly(),
    )
    coarsened_targets_by_address = coarsened_targets.by_address()

    return PyrightPartitions(
        PyrightPartition(
            FrozenOrderedSet(field_sets),
            CoarsenedTargets(
                OrderedSet(
                    coarsened_targets_by_address[field_set.address] for field_set in field_sets
                )
            ),
            resolve if len(python_setup.resolves) > 1 else None,
            interpreter_constraints or pyright.interpreter_constraints,
        )
        for (resolve, interpreter_constraints), field_sets in sorted(
            resolve_and_interpreter_constraints_to_field_sets.items()
        )
    )


@rule(desc="Typecheck using Pyright", level=LogLevel.DEBUG)
async def pyright_typecheck(
    request: PyrightRequest,
    pyright: Pyright,
) -> CheckResults:
    if pyright.skip:
        return CheckResults([], checker_name=request.tool_name)

    # Explicitly excluding `pyright` as a function argument to `pyright_determine_partitions` and `pyright_typecheck_partition`
    # as it throws "TypeError: unhashable type: 'Pyright'"
    partitions = await pyright_determine_partitions(request, **implicitly())
    partitioned_results = await concurrently(
        pyright_typecheck_partition(partition, **implicitly()) for partition in partitions
    )
    return CheckResults(
        partitioned_results,
        checker_name=request.tool_name,
    )


def rules() -> Iterable[Rule | UnionRule]:
    return (
        *collect_rules(),
        *config_files.rules(),
        *pex_from_targets.rules(),
        *nodejs_tool.rules(),
        UnionRule(CheckRequest, PyrightRequest),
    )

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	1✔
5
6	import dataclasses	1✔
7	import json	1✔
8	import logging	1✔
9	import os	1✔
10	import shlex	1✔
11	from collections.abc import Iterable	1✔
12	from dataclasses import dataclass, replace	1✔
13
14	import toml	1✔
15
16	from pants.backend.javascript.subsystems import nodejs_tool	1✔
17	from pants.backend.javascript.subsystems.nodejs import NodeJS	1✔
18	from pants.backend.javascript.subsystems.nodejs_tool import prepare_tool_process	1✔
19	from pants.backend.python.subsystems.setup import PythonSetup	1✔
20	from pants.backend.python.target_types import (	1✔
21	InterpreterConstraintsField,
22	PythonResolveField,
23	PythonSourceField,
24	)
25	from pants.backend.python.typecheck.pyright.skip_field import SkipPyrightField	1✔
26	from pants.backend.python.typecheck.pyright.subsystem import Pyright	1✔
27	from pants.backend.python.util_rules import pex_from_targets	1✔
28	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	1✔
29	from pants.backend.python.util_rules.partition import (	1✔
30	_partition_by_interpreter_constraints_and_resolve,
31	)
32	from pants.backend.python.util_rules.pex import (	1✔
33	PexRequest,
34	VenvPexProcess,
35	VenvPexRequest,
36	create_pex,
37	create_venv_pex,
38	)
39	from pants.backend.python.util_rules.pex_environment import PexEnvironment	1✔
40	from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest	1✔
41	from pants.backend.python.util_rules.python_sources import (	1✔
42	PythonSourceFilesRequest,
43	prepare_python_sources,
44	)
45	from pants.core.goals.check import CheckRequest, CheckResult, CheckResults, CheckSubsystem	1✔
46	from pants.core.util_rules import config_files	1✔
47	from pants.core.util_rules.config_files import ConfigFiles, find_config_file	1✔
48	from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files	1✔
49	from pants.core.util_rules.system_binaries import CatBinary, ShBinary	1✔
50	from pants.engine.collection import Collection	1✔
51	from pants.engine.fs import CreateDigest, FileContent	1✔
52	from pants.engine.internals.graph import resolve_coarsened_targets as coarsened_targets_get	1✔
53	from pants.engine.internals.native_engine import Digest, MergeDigests	1✔
54	from pants.engine.internals.selectors import concurrently	1✔
55	from pants.engine.intrinsics import (	1✔
56	create_digest,
57	execute_process,
58	get_digest_contents,
59	merge_digests,
60	)
61	from pants.engine.process import ProcessCacheScope, execute_process_or_raise	1✔
62	from pants.engine.rules import Rule, collect_rules, implicitly, rule	1✔
63	from pants.engine.target import CoarsenedTargets, CoarsenedTargetsRequest, FieldSet, Target	1✔
64	from pants.engine.unions import UnionRule	1✔
65	from pants.util.logging import LogLevel	1✔
66	from pants.util.ordered_set import FrozenOrderedSet, OrderedSet	1✔
67	from pants.util.strutil import pluralize	1✔
68
69	logger = logging.getLogger(__name__)	1✔
70
71
72	@dataclass(frozen=True)	1✔
73	class PyrightFieldSet(FieldSet):	1✔
74	required_fields = (PythonSourceField,)	1✔
75
76	sources: PythonSourceField	1✔
77	resolve: PythonResolveField	1✔
78	interpreter_constraints: InterpreterConstraintsField	1✔
79
80	@classmethod	1✔
81	def opt_out(cls, tgt: Target) -> bool:	1✔
82	return tgt.get(SkipPyrightField).value	×
83
84
85	class PyrightRequest(CheckRequest):	1✔
86	field_set_type = PyrightFieldSet	1✔
87	tool_name = Pyright.options_scope	1✔
88
89
90	@dataclass(frozen=True)	1✔
91	class PyrightPartition:	1✔
92	field_sets: FrozenOrderedSet[PyrightFieldSet]	1✔
93	root_targets: CoarsenedTargets	1✔
94	resolve_description: str \| None	1✔
95	interpreter_constraints: InterpreterConstraints	1✔
96
97	def description(self) -> str:	1✔
98	ics = str(sorted(str(c) for c in self.interpreter_constraints))	1✔
99	return f"{self.resolve_description}, {ics}" if self.resolve_description else ics	1✔
100
101
102	class PyrightPartitions(Collection[PyrightPartition]):	1✔
103	pass	1✔
104
105
106	async def _patch_config_file(	1✔
107	config_files: ConfigFiles, venv_dir: str, source_roots: Iterable[str]
108	) -> Digest:
109	"""Patch the Pyright config file to use the incoming venv directory (from
110	requirements_venv_pex). If there is no config file, create a dummy pyrightconfig.json with the
111	`venv` key populated.
112
113	The incoming venv directory works alongside the `--venvpath` CLI argument.
114
115	Additionally, add source roots to the `extraPaths` key in the config file.
116	"""
117
118	source_roots_list = list(source_roots)	×
119	if not config_files.snapshot.files:	×
120	# venv workaround as per: https://github.com/microsoft/pyright/issues/4051
121	generated_config: dict[str, str \| list[str]] = {	×
122	"venv": venv_dir,
123	"extraPaths": source_roots_list,
124	}
125	return await create_digest(	×
126	CreateDigest(
127	[
128	FileContent(
129	"pyrightconfig.json",
130	json.dumps(generated_config).encode(),
131	)
132	]
133	)
134	)
135
136	config_contents = await get_digest_contents(config_files.snapshot.digest)	×
137	new_files: list[FileContent] = []	×
138	for file in config_contents:	×
139	# This only supports a single json config file in the root of the project
140	# https://github.com/pantsbuild/pants/issues/17816 tracks supporting multiple config files and workspaces
141	if file.path == "pyrightconfig.json":	×
142	json_config = json.loads(file.content)	×
143	json_config["venv"] = venv_dir	×
144	json_extra_paths: list[str] = json_config.get("extraPaths", [])	×
145	json_config["extraPaths"] = list(OrderedSet(json_extra_paths + source_roots_list))	×
146	new_content = json.dumps(json_config).encode()	×
147	new_files.append(replace(file, content=new_content))	×
148
149	# This only supports a single pyproject.toml file in the root of the project
150	# https://github.com/pantsbuild/pants/issues/17816 tracks supporting multiple config files and workspaces
151	elif file.path == "pyproject.toml":	×
152	toml_config = toml.loads(file.content.decode())	×
153	pyright_config = toml_config["tool"]["pyright"]	×
154	pyright_config["venv"] = venv_dir	×
155	toml_extra_paths: list[str] = pyright_config.get("extraPaths", [])	×
156	pyright_config["extraPaths"] = list(OrderedSet(toml_extra_paths + source_roots_list))	×
157	new_content = toml.dumps(toml_config).encode()	×
158	new_files.append(replace(file, content=new_content))	×
159
160	return await create_digest(CreateDigest(new_files))	×
161
162
163	@rule(	1✔
164	desc="Pyright typecheck each partition based on its interpreter_constraints",
165	level=LogLevel.DEBUG,
166	)
167	async def pyright_typecheck_partition(	1✔
168	partition: PyrightPartition,
169	pyright: Pyright,
170	check_subsystem: CheckSubsystem,
171	pex_environment: PexEnvironment,
172	nodejs: NodeJS,
173	sh_binary: ShBinary,
174	cat_binary: CatBinary,
175	) -> CheckResult:
176	root_sources_get = determine_source_files(	×
177	SourceFilesRequest(fs.sources for fs in partition.field_sets)
178	)
179
180	# Grab the closure of the root source files to be typechecked
181	transitive_sources_get = prepare_python_sources(	×
182	PythonSourceFilesRequest(partition.root_targets.closure()), **implicitly()
183	)
184
185	# See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
186	requirements_pex_get = create_pex(	×
187	**implicitly(
188	RequirementsPexRequest(
189	(fs.address for fs in partition.field_sets),
190	hardcoded_interpreter_constraints=partition.interpreter_constraints,
191	)
192	)
193	)
194
195	# Look for any/all of the Pyright configuration files (the config is modified below
196	# for the `venv` workaround)
197	config_files_get = find_config_file(pyright.config_request())	×
198
199	root_sources, transitive_sources, requirements_pex, config_files = await concurrently(	×
200	root_sources_get,
201	transitive_sources_get,
202	requirements_pex_get,
203	config_files_get,
204	)
205
206	# This is a workaround for https://github.com/pantsbuild/pants/issues/19946.
207	# complete_pex_env needs to be created here so that the test `test_passing_cache_clear`
208	# test can pass using the appropriate caching directory.
209	# See https://github.com/pantsbuild/pants/pull/19430#discussion_r1337851780
210	# for more discussion.
211	complete_pex_env = pex_environment.in_workspace()	×
212	requirements_pex_request = PexRequest(	×
213	output_filename="requirements_venv.pex",
214	internal_only=True,
215	pex_path=[requirements_pex],
216	interpreter_constraints=partition.interpreter_constraints,
217	)
218	requirements_venv_pex = await create_venv_pex(	×
219	VenvPexRequest(requirements_pex_request, complete_pex_env), **implicitly()
220	)
221
222	# Force the requirements venv to materialize always by running a no-op.
223	# This operation must be called with `ProcessCacheScope.SESSION`
224	# so that it runs every time.
225	_ = await execute_process_or_raise(	×
226	**implicitly(
227	VenvPexProcess(
228	requirements_venv_pex,
229	description="Force venv to materialize",
230	argv=["-c", "''"],
231	cache_scope=ProcessCacheScope.PER_SESSION,
232	)
233	)
234	)
235
236	# Patch the config file to use the venv directory from the requirements pex,
237	# and add source roots to the `extraPaths` key in the config file.
238	patched_config_digest = await _patch_config_file(	×
239	config_files, requirements_venv_pex.venv_rel_dir, transitive_sources.source_roots
240	)
241
242	# Prepare the process with as much information as we currently have. This will give us the
243	# process's cwd, which we need in order to calculate the relative paths to the input files.
244	# We will then manually tweak the argv before actually running.
245	input_digest = await merge_digests(	×
246	MergeDigests(
247	[
248	transitive_sources.source_files.snapshot.digest,
249	requirements_venv_pex.digest,
250	patched_config_digest,
251	]
252	)
253	)
254	process = await prepare_tool_process(	×
255	pyright.request(
256	args=(
257	f"--venvpath={complete_pex_env.pex_root}", # Used with `venv` in config
258	*pyright.args, # User-added arguments
259	"-", # Read input file paths from stdin
260	),
261	input_digest=input_digest,
262	description=f"Run Pyright on {pluralize(len(root_sources.snapshot.files), 'file')}.",
263	level=LogLevel.DEBUG,
264	),
265	**implicitly(),
266	)
267
268	# We must use relative paths, because we don't know the abspath of the sandbox the process
269	# will run in, and `{chroot}` interpolation only works on argv, not on the contents of
270	# __files.txt (see below). Pyright interprets relpaths as relative to its cwd, so we
271	# prepend the appropriate prefix to each file path.
272	input_path_prefix = os.path.relpath(".", process.working_directory)	×
273	input_files = [os.path.join(input_path_prefix, file) for file in root_sources.snapshot.files]	×
274
275	# We prefer to pass the list of input files via stdin, as large numbers of files can cause us
276	# to exceed the max command line length. See https://github.com/pantsbuild/pants/issues/22779.
277	# However Pyright, weirdly, splits stdin on spaces as well as newlines. So we can't pass input
278	# file paths via stdin if any of them contain spaces.
279	file_with_spaces = next((file for file in root_sources.snapshot.files if " " in file), None)	×
280	if file_with_spaces:	×
281	# Fall back to passing paths as args and hope we don't exceed the max command line length.
282	process = dataclasses.replace(process, argv=(process.argv[0:-1], input_files))	×
283	else:
284	# Write the input files out to a text file.
285	file_list_path = "__files.txt"	×
286	file_list_content = "\n".join(input_files).encode()	×
287	file_list_digest = await create_digest(	×
288	CreateDigest([FileContent(file_list_path, file_list_content)])
289	)
290	input_digest = await merge_digests(	×
291	MergeDigests(
292	[
293	process.input_digest,
294	file_list_digest,
295	]
296	)
297	)
298	# Run the underlying process inside a shell script that cats the file list to stdin.
299	shell_script = f"{cat_binary.path} {os.path.join(input_path_prefix, file_list_path)} \| {shlex.join(process.argv)}"	×
300	process = dataclasses.replace(	×
301	process, argv=(sh_binary.path, "-c", shell_script), input_digest=input_digest
302	)
303
NEW 304	process = dataclasses.replace(process, cache_scope=check_subsystem.default_process_cache_scope)	×
305
306	result = await execute_process(process, **implicitly())	×
307	if result.exit_code == 249 and file_with_spaces:	×
308	logger.error(	×
309	f"Found input files with spaces in their names, including: {file_with_spaces}. "
310	"Due to a bug in Pyright this means that the number of input files Pants can pass to "
311	"Pyright is limited, and exceeding that limit causes it to crash with exit code 249. "
312	"Please reach out to the Pants team if this happens: "
313	"https://www.pantsbuild.org/community/getting-help."
314	)
315	return CheckResult.from_fallible_process_result(	×
316	result,
317	partition_description=partition.description(),
318	)
319
320
321	@rule(	1✔
322	desc="Determine if it is necessary to partition Pyright's input (interpreter_constraints and resolves)",
323	level=LogLevel.DEBUG,
324	)
325	async def pyright_determine_partitions(	1✔
326	request: PyrightRequest,
327	pyright: Pyright,
328	python_setup: PythonSetup,
329	) -> PyrightPartitions:
330	resolve_and_interpreter_constraints_to_field_sets = (	×
331	_partition_by_interpreter_constraints_and_resolve(request.field_sets, python_setup)
332	)
333
334	coarsened_targets = await coarsened_targets_get(	×
335	CoarsenedTargetsRequest(field_set.address for field_set in request.field_sets),
336	**implicitly(),
337	)
338	coarsened_targets_by_address = coarsened_targets.by_address()	×
339
340	return PyrightPartitions(	×
341	PyrightPartition(
342	FrozenOrderedSet(field_sets),
343	CoarsenedTargets(
344	OrderedSet(
345	coarsened_targets_by_address[field_set.address] for field_set in field_sets
346	)
347	),
348	resolve if len(python_setup.resolves) > 1 else None,
349	interpreter_constraints or pyright.interpreter_constraints,
350	)
351	for (resolve, interpreter_constraints), field_sets in sorted(
352	resolve_and_interpreter_constraints_to_field_sets.items()
353	)
354	)
355
356
357	@rule(desc="Typecheck using Pyright", level=LogLevel.DEBUG)	1✔
358	async def pyright_typecheck(	1✔
359	request: PyrightRequest,
360	pyright: Pyright,
361	) -> CheckResults:
362	if pyright.skip:	×
363	return CheckResults([], checker_name=request.tool_name)	×
364
365	# Explicitly excluding `pyright` as a function argument to `pyright_determine_partitions` and `pyright_typecheck_partition`
366	# as it throws "TypeError: unhashable type: 'Pyright'"
367	partitions = await pyright_determine_partitions(request, **implicitly())	×
368	partitioned_results = await concurrently(	×
369	pyright_typecheck_partition(partition, **implicitly()) for partition in partitions
370	)
371	return CheckResults(	×
372	partitioned_results,
373	checker_name=request.tool_name,
374	)
375
376
377	def rules() -> Iterable[Rule \| UnionRule]:	1✔
378	return (	1✔
379	*collect_rules(),
380	*config_files.rules(),
381	*pex_from_targets.rules(),
382	*nodejs_tool.rules(),
383	UnionRule(CheckRequest, PyrightRequest),
384	)

pantsbuild / pants / 22197409014

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous