24604025132

Committed 18 Apr 2026 11:49AM UTC coverage: 92.478% (-0.4%) from 92.924%

Build # 24604025132

Build Type

Pull #23268

github

Committed by

web-flow

Commit Message

Merge c60f47029 into a92bc34b6

Pull Request Pull Request #23268: perf: Remove python coroutine/trampoline overhead in awaits for ~22% faster `dependencies` goal

Coverage Stats

31 of 37 new or added lines in 4 files covered. (83.78%)

443 existing lines in 21 files now uncovered.

91210 of 98629 relevant lines covered (92.48%)

4.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.47

/src/python/pants/core/goals/export_test.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os
import subprocess
from pathlib import Path

from _pytest.monkeypatch import MonkeyPatch

from pants.base.build_root import BuildRoot
from pants.core.environments.target_types import EnvironmentField
from pants.core.goals.export import (
    Export,
    ExportRequest,
    ExportResult,
    ExportResults,
    ExportSubsystem,
    PostProcessingCommand,
    export_goal,
    warn_exported_bin_conflicts,
)
from pants.core.util_rules.distdir import DistDir
from pants.engine.addresses import Address
from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest
from pants.engine.fs import CreateDigest, Digest, FileContent, Workspace
from pants.engine.process import InteractiveProcess, InteractiveProcessResult
from pants.engine.rules import QueryRule
from pants.engine.target import Target, Targets
from pants.engine.unions import UnionMembership, UnionRule
from pants.testutil.option_util import create_subsystem
from pants.testutil.rule_runner import RuleRunner, mock_console, run_rule_with_mocks


class MockTarget(Target):
    alias = "target"
    core_fields = (EnvironmentField,)


def make_target(path: str, target_name: str, environment_name: str | None = None) -> Target:
    return MockTarget(
        {EnvironmentField.alias: environment_name}, Address(path, target_name=target_name)
    )


class MockExportRequest(ExportRequest):
    pass


def mock_export(
    digest: Digest,
    post_processing_cmds: tuple[PostProcessingCommand, ...],
    resolve: str,
) -> ExportResult:
    return ExportResult(
        description=f"mock export for {resolve}",
        reldir="mock",
        digest=digest,
        post_processing_cmds=post_processing_cmds,
        resolve=resolve,
    )


def _mock_run(rule_runner: RuleRunner, ip: InteractiveProcess) -> InteractiveProcessResult:
    """This is still necessary for writing files, which uses a `cp` process."""
    subprocess.check_call(
        ip.process.argv,
        stderr=subprocess.STDOUT,
        env={
            "PATH": os.environ.get("PATH", ""),
            "DIGEST_ROOT": os.path.join(rule_runner.build_root, "dist", "export", "mock"),
        },
    )
    return InteractiveProcessResult(0)


def list_files_with_paths(directory):
    file_paths = []
    for root, dirs, files in os.walk(directory):
        for file in files:
            full_path = os.path.join(root, file)
            file_paths.append(full_path)
    return file_paths


def run_export_rule(
    rule_runner: RuleRunner,
    monkeypatch: MonkeyPatch,
    resolves: list[str] | None = None,
    binaries: list[str] | None = None,
) -> tuple[int, str]:
    resolves = resolves or []
    binaries = binaries or []
    has_post_processing_commands = bool(resolves)
    union_membership = UnionMembership.from_rules([UnionRule(ExportRequest, MockExportRequest)])
    with open(os.path.join(rule_runner.build_root, "somefile"), "wb") as fp:
        fp.write(b"SOMEFILE")

    def noop():
        pass

    monkeypatch.setattr("pants.engine.intrinsics.task_side_effected", noop)
    with mock_console(rule_runner.options_bootstrapper) as (console, stdio_reader):

        def do_mock_export(__implicitly: tuple) -> ExportResults:
            req, typ = next(iter(__implicitly[0].items()))
            assert typ == ExportRequest

            if resolves:
                digest = rule_runner.request(
                    Digest, [CreateDigest([FileContent("foo/bar", b"BAR")])]
                )
                return ExportResults(
                    (
                        mock_export(
                            digest,
                            (
                                PostProcessingCommand(
                                    ["cp", "{digest_root}/foo/bar", "{digest_root}/foo/bar1"]
                                ),
                                PostProcessingCommand(
                                    ["cp", "{digest_root}/foo/bar", "{digest_root}/foo/bar2"]
                                ),
                            ),
                            resolves[0],
                        ),
                    )
                )
            if binaries:
                digest = rule_runner.request(
                    Digest,
                    [
                        CreateDigest(
                            [FileContent(f"bins/{binary}/{binary}", b"BAR") for binary in binaries]
                        )
                    ],
                )
                return ExportResults(mock_export(digest, (), binary) for binary in binaries)
            raise Exception("No resolves or binaries specified")

        result: Export = run_rule_with_mocks(
            export_goal,
            rule_args=[
                console,
                Targets([]),
                Workspace(rule_runner.scheduler, _enforce_effects=False),
                union_membership,
                BuildRoot(),
                DistDir(relpath=Path("dist")),
                create_subsystem(ExportSubsystem, resolve=resolves, bin=binaries),
            ],
            # TODO: Create a rule_runner.call() method that invokes by-name, and use that to
            #  replace these rule_runner.request() by-type calls.
            mock_calls={
                "pants.engine.intrinsics.add_prefix": lambda *xs: rule_runner.request(Digest, xs),
                "pants.core.goals.export.export": do_mock_export,
                **(
                    {
                        "pants.engine.intrinsics._interactive_process": lambda ip: _mock_run(
                            rule_runner, ip
                        )
                    }
                    if has_post_processing_commands
                    else {}
                ),
                "pants.engine.intrinsics.merge_digests": lambda *iv: rule_runner.request(
                    Digest, iv
                ),
                "pants.core.util_rules.env_vars.environment_vars_subset": lambda *iv: rule_runner.request(
                    EnvironmentVars, iv
                ),
                "pants.engine.intrinsics.create_digest": lambda *iv: rule_runner.request(
                    Digest, iv
                ),
            },
            union_membership=union_membership,
        )
        return result.exit_code, stdio_reader.get_stdout()


def test_run_export_rule_resolve(monkeypatch) -> None:
    rule_runner = RuleRunner(
        rules=[
            UnionRule(ExportRequest, MockExportRequest),
            QueryRule(Digest, [CreateDigest]),
            QueryRule(EnvironmentVars, [EnvironmentVarsRequest]),
            QueryRule(InteractiveProcessResult, [InteractiveProcess]),
        ],
        target_types=[MockTarget],
    )
    exit_code, stdout = run_export_rule(rule_runner, monkeypatch, resolves=["resolve"])
    assert exit_code == 0
    assert "Wrote mock export for resolve to dist/export/mock" in stdout
    for filename in ["bar", "bar1", "bar2"]:
        expected_dist_path = os.path.join(
            rule_runner.build_root, "dist", "export", "mock", "foo", filename
        )
        assert os.path.isfile(expected_dist_path)
        with open(expected_dist_path, "rb") as fp:
            assert fp.read() == b"BAR"


def test_run_export_rule_binary(monkeypatch) -> None:
    rule_runner = RuleRunner(
        rules=[
            UnionRule(ExportRequest, MockExportRequest),
            QueryRule(Digest, [CreateDigest]),
            QueryRule(EnvironmentVars, [EnvironmentVarsRequest]),
            QueryRule(InteractiveProcessResult, [InteractiveProcess]),
        ],
        target_types=[MockTarget],
    )
    exit_code, stdout = run_export_rule(rule_runner, monkeypatch, binaries=["mybin"])
    assert exit_code == 0
    assert "Wrote mock export for mybin to dist/export/mock" in stdout
    for filename in ["mybin"]:
        expected_dist_path = os.path.join(
            rule_runner.build_root, "dist", "export", "mock", "bins", filename, filename
        )

        assert os.path.isfile(expected_dist_path)
        with open(expected_dist_path, "rb") as fp:
            assert fp.read() == b"BAR"


def test_warn_exported_bin_conflict() -> None:
    found_warnings = warn_exported_bin_conflicts(
        {
            "bin0": ["r0"],
            "bin1": ["r1"],
            "bin2": ["r0", "r1", "r2"],
        }
    )

    assert len(found_warnings) == 1, "did not detect the right number of conflicts"

    found_warning = found_warnings[0]
    assert "r0 was exported" in found_warning, "did not export from the correct resolve"
    assert "r1, r2" in found_warning, "did not report the other resolves correctly"

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	1✔
5
6	import os	1✔
7	import subprocess	1✔
8	from pathlib import Path	1✔
9
10	from _pytest.monkeypatch import MonkeyPatch	1✔
11
12	from pants.base.build_root import BuildRoot	1✔
13	from pants.core.environments.target_types import EnvironmentField	1✔
14	from pants.core.goals.export import (	1✔
15	Export,
16	ExportRequest,
17	ExportResult,
18	ExportResults,
19	ExportSubsystem,
20	PostProcessingCommand,
21	export_goal,
22	warn_exported_bin_conflicts,
23	)
24	from pants.core.util_rules.distdir import DistDir	1✔
25	from pants.engine.addresses import Address	1✔
26	from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest	1✔
27	from pants.engine.fs import CreateDigest, Digest, FileContent, Workspace	1✔
28	from pants.engine.process import InteractiveProcess, InteractiveProcessResult	1✔
29	from pants.engine.rules import QueryRule	1✔
30	from pants.engine.target import Target, Targets	1✔
31	from pants.engine.unions import UnionMembership, UnionRule	1✔
32	from pants.testutil.option_util import create_subsystem	1✔
33	from pants.testutil.rule_runner import RuleRunner, mock_console, run_rule_with_mocks	1✔
34
35
36	class MockTarget(Target):	1✔
37	alias = "target"	1✔
38	core_fields = (EnvironmentField,)	1✔
39
40
41	def make_target(path: str, target_name: str, environment_name: str \| None = None) -> Target:	1✔
42	return MockTarget(	×
43	{EnvironmentField.alias: environment_name}, Address(path, target_name=target_name)
44	)
45
46
47	class MockExportRequest(ExportRequest):	1✔
48	pass	1✔
49
50
51	def mock_export(	1✔
52	digest: Digest,
53	post_processing_cmds: tuple[PostProcessingCommand, ...],
54	resolve: str,
55	) -> ExportResult:
UNCOV 56	return ExportResult(	×
57	description=f"mock export for {resolve}",
58	reldir="mock",
59	digest=digest,
60	post_processing_cmds=post_processing_cmds,
61	resolve=resolve,
62	)
63
64
65	def _mock_run(rule_runner: RuleRunner, ip: InteractiveProcess) -> InteractiveProcessResult:	1✔
66	"""This is still necessary for writing files, which uses a `cp` process."""
UNCOV 67	subprocess.check_call(	×
68	ip.process.argv,
69	stderr=subprocess.STDOUT,
70	env={
71	"PATH": os.environ.get("PATH", ""),
72	"DIGEST_ROOT": os.path.join(rule_runner.build_root, "dist", "export", "mock"),
73	},
74	)
UNCOV 75	return InteractiveProcessResult(0)	×
76
77
78	def list_files_with_paths(directory):	1✔
79	file_paths = []	×
80	for root, dirs, files in os.walk(directory):	×
81	for file in files:	×
82	full_path = os.path.join(root, file)	×
83	file_paths.append(full_path)	×
84	return file_paths	×
85
86
87	def run_export_rule(	1✔
88	rule_runner: RuleRunner,
89	monkeypatch: MonkeyPatch,
90	resolves: list[str] \| None = None,
91	binaries: list[str] \| None = None,
92	) -> tuple[int, str]:
93	resolves = resolves or []	1✔
94	binaries = binaries or []	1✔
95	has_post_processing_commands = bool(resolves)	1✔
96	union_membership = UnionMembership.from_rules([UnionRule(ExportRequest, MockExportRequest)])	1✔
97	with open(os.path.join(rule_runner.build_root, "somefile"), "wb") as fp:	1✔
98	fp.write(b"SOMEFILE")	1✔
99
100	def noop():	1✔
UNCOV 101	pass	×
102
103	monkeypatch.setattr("pants.engine.intrinsics.task_side_effected", noop)	1✔
104	with mock_console(rule_runner.options_bootstrapper) as (console, stdio_reader):	1✔
105
106	def do_mock_export(__implicitly: tuple) -> ExportResults:	1✔
UNCOV 107	req, typ = next(iter(__implicitly[0].items()))	×
UNCOV 108	assert typ == ExportRequest	×
109
UNCOV 110	if resolves:	×
UNCOV 111	digest = rule_runner.request(	×
112	Digest, [CreateDigest([FileContent("foo/bar", b"BAR")])]
113	)
UNCOV 114	return ExportResults(	×
115	(
116	mock_export(
117	digest,
118	(
119	PostProcessingCommand(
120	["cp", "{digest_root}/foo/bar", "{digest_root}/foo/bar1"]
121	),
122	PostProcessingCommand(
123	["cp", "{digest_root}/foo/bar", "{digest_root}/foo/bar2"]
124	),
125	),
126	resolves[0],
127	),
128	)
129	)
UNCOV 130	if binaries:	×
UNCOV 131	digest = rule_runner.request(	×
132	Digest,
133	[
134	CreateDigest(
135	[FileContent(f"bins/{binary}/{binary}", b"BAR") for binary in binaries]
136	)
137	],
138	)
UNCOV 139	return ExportResults(mock_export(digest, (), binary) for binary in binaries)	×
140	raise Exception("No resolves or binaries specified")	×
141
142	result: Export = run_rule_with_mocks(	1✔
143	export_goal,
144	rule_args=[
145	console,
146	Targets([]),
147	Workspace(rule_runner.scheduler, _enforce_effects=False),
148	union_membership,
149	BuildRoot(),
150	DistDir(relpath=Path("dist")),
151	create_subsystem(ExportSubsystem, resolve=resolves, bin=binaries),
152	],
153	# TODO: Create a rule_runner.call() method that invokes by-name, and use that to
154	# replace these rule_runner.request() by-type calls.
155	mock_calls={
156	"pants.engine.intrinsics.add_prefix": lambda *xs: rule_runner.request(Digest, xs),
157	"pants.core.goals.export.export": do_mock_export,
158	**(
159	{
160	"pants.engine.intrinsics._interactive_process": lambda ip: _mock_run(
161	rule_runner, ip
162	)
163	}
164	if has_post_processing_commands
165	else {}
166	),
167	"pants.engine.intrinsics.merge_digests": lambda *iv: rule_runner.request(
168	Digest, iv
169	),
170	"pants.core.util_rules.env_vars.environment_vars_subset": lambda *iv: rule_runner.request(
171	EnvironmentVars, iv
172	),
173	"pants.engine.intrinsics.create_digest": lambda *iv: rule_runner.request(
174	Digest, iv
175	),
176	},
177	union_membership=union_membership,
178	)
179	return result.exit_code, stdio_reader.get_stdout()	1✔
180
181
182	def test_run_export_rule_resolve(monkeypatch) -> None:	1✔
183	rule_runner = RuleRunner(	1✔
184	rules=[
185	UnionRule(ExportRequest, MockExportRequest),
186	QueryRule(Digest, [CreateDigest]),
187	QueryRule(EnvironmentVars, [EnvironmentVarsRequest]),
188	QueryRule(InteractiveProcessResult, [InteractiveProcess]),
189	],
190	target_types=[MockTarget],
191	)
192	exit_code, stdout = run_export_rule(rule_runner, monkeypatch, resolves=["resolve"])	1✔
UNCOV 193	assert exit_code == 0	×
UNCOV 194	assert "Wrote mock export for resolve to dist/export/mock" in stdout	×
UNCOV 195	for filename in ["bar", "bar1", "bar2"]:	×
UNCOV 196	expected_dist_path = os.path.join(	×
197	rule_runner.build_root, "dist", "export", "mock", "foo", filename
198	)
UNCOV 199	assert os.path.isfile(expected_dist_path)	×
UNCOV 200	with open(expected_dist_path, "rb") as fp:	×
UNCOV 201	assert fp.read() == b"BAR"	×
202
203
204	def test_run_export_rule_binary(monkeypatch) -> None:	1✔
205	rule_runner = RuleRunner(	1✔
206	rules=[
207	UnionRule(ExportRequest, MockExportRequest),
208	QueryRule(Digest, [CreateDigest]),
209	QueryRule(EnvironmentVars, [EnvironmentVarsRequest]),
210	QueryRule(InteractiveProcessResult, [InteractiveProcess]),
211	],
212	target_types=[MockTarget],
213	)
214	exit_code, stdout = run_export_rule(rule_runner, monkeypatch, binaries=["mybin"])	1✔
UNCOV 215	assert exit_code == 0	×
UNCOV 216	assert "Wrote mock export for mybin to dist/export/mock" in stdout	×
UNCOV 217	for filename in ["mybin"]:	×
UNCOV 218	expected_dist_path = os.path.join(	×
219	rule_runner.build_root, "dist", "export", "mock", "bins", filename, filename
220	)
221
UNCOV 222	assert os.path.isfile(expected_dist_path)	×
UNCOV 223	with open(expected_dist_path, "rb") as fp:	×
UNCOV 224	assert fp.read() == b"BAR"	×
225
226
227	def test_warn_exported_bin_conflict() -> None:	1✔
228	found_warnings = warn_exported_bin_conflicts(	1✔
229	{
230	"bin0": ["r0"],
231	"bin1": ["r1"],
232	"bin2": ["r0", "r1", "r2"],
233	}
234	)
235
236	assert len(found_warnings) == 1, "did not detect the right number of conflicts"	1✔
237
238	found_warning = found_warnings[0]	1✔
239	assert "r0 was exported" in found_warning, "did not export from the correct resolve"	1✔
240	assert "r1, r2" in found_warning, "did not report the other resolves correctly"	1✔

pantsbuild / pants / 24604025132

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous