25443604553

Committed 06 May 2026 03:05PM UTC coverage: 92.879% (-0.04%) from 92.915%

Build # 25443604553

Build Type

push

github

Committed by

web-flow

Commit Message

[pants_ng] Scaffolding for a pants_ng mode. (#23319)

In this mode the command line is parsed as an
NG invocation, and dispatched appropriately.

Of course at the moment there are no
implementations to dispatch to. That will follow.

This does expose a new option, `pants_ng` to users. 
There is a big warning not to set it, but we're not trying
to hide that we're working on a new thing, so I am
comfortable with this.

Coverage Stats

25 of 76 new or added lines in 9 files covered. (32.89%)

1294 existing lines in 76 files now uncovered.

92234 of 99306 relevant lines covered (92.88%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.83

/src/python/pants/backend/python/macros/python_requirements.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os
from collections.abc import Callable, Iterator
from itertools import chain
from typing import Any

import toml

from pants.backend.python.macros.common_fields import (
    ModuleMappingField,
    RequirementsOverrideField,
    TypeStubsModuleMappingField,
)
from pants.backend.python.macros.common_requirements_rule import _generate_requirements
from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.target_types import PythonRequirementResolveField, PythonRequirementTarget
from pants.engine.rules import collect_rules, rule
from pants.engine.target import (
    COMMON_TARGET_FIELDS,
    GeneratedTargets,
    GenerateTargetsRequest,
    SingleSourceField,
    TargetGenerator,
)
from pants.engine.unions import UnionMembership, UnionRule
from pants.util.logging import LogLevel
from pants.util.pip_requirement import PipRequirement
from pants.util.requirements import parse_requirements_file
from pants.util.strutil import help_text, softwrap


def parse_pyproject_toml(pyproject_toml: str, *, rel_path: str) -> Iterator[PipRequirement]:
    parsed: dict[str, Any] = toml.loads(pyproject_toml)
    deps_vals: list[str] = parsed.get("project", {}).get("dependencies", [])
    optional_dependencies: dict[str, list[str]] = parsed.get("project", {}).get(
        "optional-dependencies", {}
    )
    if not deps_vals and not optional_dependencies:
        raise KeyError(
            softwrap(
                "No section `project.dependencies` or `project.optional-dependencies` "
                f"found in {rel_path}"
            )
        )
    for dep in deps_vals:
        dep, _, _ = dep.partition("--")
        dep = dep.strip().rstrip("\\")
        if not dep or dep.startswith(("#", "-")):
            continue
        yield PipRequirement.parse(dep, description_of_origin=rel_path)
    for dep in chain.from_iterable(optional_dependencies.values()):
        dep, _, _ = dep.partition("--")
        dep = dep.strip().rstrip("\\")
        if not dep or dep.startswith(("#", "-")):
            continue
        req = PipRequirement.parse(dep, description_of_origin=rel_path)
        yield req


class PythonRequirementsSourceField(SingleSourceField):
    default = "requirements.txt"
    required = False


class PythonRequirementsTargetGenerator(TargetGenerator):
    alias = "python_requirements"
    help = help_text(
        """
        Generate a `python_requirement` for each entry in a requirements.txt-style or PEP 621
        compliant `pyproject.toml` file. The choice of parser for the `source` field is determined
        by the file name. If the `source` field ends with `pyproject.toml`, then the file is
        assumed to be a PEP 621 compliant file. Any other file name uses the requirements.txt-style
        parser.

        Further details about pip-style requirements files are available from the PyPA documentation:
        https://pip.pypa.io/en/latest/reference/requirements-file-format/. However, pip options like
        `--hash` are (for now) ignored.

        Pants will not follow `-r reqs.txt` lines. Instead, add a dedicated `python_requirements`
        target generator for that additional requirements file.

        Further details about PEP 621 and `pyproject.toml` files are available from the PEP itself:
        https://peps.python.org/pep-0621/. If the `project.optional-dependencies` table is
        included, Pants will save the key/name of the optional dependency group as a tag on the
        generated `python_requirement`.
        """
    )
    generated_target_cls = PythonRequirementTarget
    # Note that this does not have a `dependencies` field.
    core_fields = (
        *COMMON_TARGET_FIELDS,
        ModuleMappingField,
        TypeStubsModuleMappingField,
        PythonRequirementsSourceField,
        RequirementsOverrideField,
    )
    copied_fields = COMMON_TARGET_FIELDS
    moved_fields = (PythonRequirementResolveField,)


class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest):
    generate_from = PythonRequirementsTargetGenerator


@rule(
    desc=(
        "Generate `python_requirement` targets from requirements.txt or PEP 621 compliant "
        "pyproject.toml"
    ),
    level=LogLevel.DEBUG,
)
async def generate_from_python_requirement(
    request: GenerateFromPythonRequirementsRequest,
    union_membership: UnionMembership,
    python_setup: PythonSetup,
) -> GeneratedTargets:
    generator = request.generator
    requirements_rel_path = generator[PythonRequirementsSourceField].value
    callback: Callable[[bytes, str], Iterator[PipRequirement]]
    if os.path.basename(requirements_rel_path) == "pyproject.toml":
        callback = parse_pyproject_callback
    else:
        callback = parse_requirements_callback
    result = await _generate_requirements(
        request,
        union_membership,
        python_setup,
        parse_requirements_callback=callback,
    )
    return GeneratedTargets(request.generator, result)


def parse_requirements_callback(file_contents: bytes, file_path: str) -> Iterator[PipRequirement]:
    return parse_requirements_file(file_contents.decode(), rel_path=file_path)


def parse_pyproject_callback(file_contents: bytes, file_path: str) -> Iterator[PipRequirement]:
    return parse_pyproject_toml(file_contents.decode(), rel_path=file_path)


def rules():
    return (
        *collect_rules(),
        UnionRule(GenerateTargetsRequest, GenerateFromPythonRequirementsRequest),
    )

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	6✔
5
6	import os	6✔
7	from collections.abc import Callable, Iterator	6✔
8	from itertools import chain	6✔
9	from typing import Any	6✔
10
11	import toml	6✔
12
13	from pants.backend.python.macros.common_fields import (	6✔
14	ModuleMappingField,
15	RequirementsOverrideField,
16	TypeStubsModuleMappingField,
17	)
18	from pants.backend.python.macros.common_requirements_rule import _generate_requirements	6✔
19	from pants.backend.python.subsystems.setup import PythonSetup	6✔
20	from pants.backend.python.target_types import PythonRequirementResolveField, PythonRequirementTarget	6✔
21	from pants.engine.rules import collect_rules, rule	6✔
22	from pants.engine.target import (	6✔
23	COMMON_TARGET_FIELDS,
24	GeneratedTargets,
25	GenerateTargetsRequest,
26	SingleSourceField,
27	TargetGenerator,
28	)
29	from pants.engine.unions import UnionMembership, UnionRule	6✔
30	from pants.util.logging import LogLevel	6✔
31	from pants.util.pip_requirement import PipRequirement	6✔
32	from pants.util.requirements import parse_requirements_file	6✔
33	from pants.util.strutil import help_text, softwrap	6✔
34
35
36	def parse_pyproject_toml(pyproject_toml: str, *, rel_path: str) -> Iterator[PipRequirement]:	6✔
37	parsed: dict[str, Any] = toml.loads(pyproject_toml)	2✔
38	deps_vals: list[str] = parsed.get("project", {}).get("dependencies", [])	2✔
39	optional_dependencies: dict[str, list[str]] = parsed.get("project", {}).get(	2✔
40	"optional-dependencies", {}
41	)
42	if not deps_vals and not optional_dependencies:	2✔
43	raise KeyError(	1✔
44	softwrap(
45	"No section `project.dependencies` or `project.optional-dependencies` "
46	f"found in {rel_path}"
47	)
48	)
49	for dep in deps_vals:	2✔
50	dep, _, _ = dep.partition("--")	2✔
51	dep = dep.strip().rstrip("\\")	2✔
52	if not dep or dep.startswith(("#", "-")):	2✔
53	continue	×
54	yield PipRequirement.parse(dep, description_of_origin=rel_path)	2✔
55	for dep in chain.from_iterable(optional_dependencies.values()):	2✔
UNCOV 56	dep, _, _ = dep.partition("--")	1✔
UNCOV 57	dep = dep.strip().rstrip("\\")	1✔
UNCOV 58	if not dep or dep.startswith(("#", "-")):	1✔
59	continue	×
UNCOV 60	req = PipRequirement.parse(dep, description_of_origin=rel_path)	1✔
UNCOV 61	yield req	1✔
62
63
64	class PythonRequirementsSourceField(SingleSourceField):	6✔
65	default = "requirements.txt"	6✔
66	required = False	6✔
67
68
69	class PythonRequirementsTargetGenerator(TargetGenerator):	6✔
70	alias = "python_requirements"	6✔
71	help = help_text(	6✔
72	"""
73	Generate a `python_requirement` for each entry in a requirements.txt-style or PEP 621
74	compliant `pyproject.toml` file. The choice of parser for the `source` field is determined
75	by the file name. If the `source` field ends with `pyproject.toml`, then the file is
76	assumed to be a PEP 621 compliant file. Any other file name uses the requirements.txt-style
77	parser.
78
79	Further details about pip-style requirements files are available from the PyPA documentation:
80	https://pip.pypa.io/en/latest/reference/requirements-file-format/. However, pip options like
81	`--hash` are (for now) ignored.
82
83	Pants will not follow `-r reqs.txt` lines. Instead, add a dedicated `python_requirements`
84	target generator for that additional requirements file.
85
86	Further details about PEP 621 and `pyproject.toml` files are available from the PEP itself:
87	https://peps.python.org/pep-0621/. If the `project.optional-dependencies` table is
88	included, Pants will save the key/name of the optional dependency group as a tag on the
89	generated `python_requirement`.
90	"""
91	)
92	generated_target_cls = PythonRequirementTarget	6✔
93	# Note that this does not have a `dependencies` field.
94	core_fields = (	6✔
95	*COMMON_TARGET_FIELDS,
96	ModuleMappingField,
97	TypeStubsModuleMappingField,
98	PythonRequirementsSourceField,
99	RequirementsOverrideField,
100	)
101	copied_fields = COMMON_TARGET_FIELDS	6✔
102	moved_fields = (PythonRequirementResolveField,)	6✔
103
104
105	class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest):	6✔
106	generate_from = PythonRequirementsTargetGenerator	6✔
107
108
109	@rule(	6✔
110	desc=(
111	"Generate `python_requirement` targets from requirements.txt or PEP 621 compliant "
112	"pyproject.toml"
113	),
114	level=LogLevel.DEBUG,
115	)
116	async def generate_from_python_requirement(	6✔
117	request: GenerateFromPythonRequirementsRequest,
118	union_membership: UnionMembership,
119	python_setup: PythonSetup,
120	) -> GeneratedTargets:
121	generator = request.generator	3✔
122	requirements_rel_path = generator[PythonRequirementsSourceField].value	3✔
123	callback: Callable[[bytes, str], Iterator[PipRequirement]]
124	if os.path.basename(requirements_rel_path) == "pyproject.toml":	3✔
UNCOV 125	callback = parse_pyproject_callback	1✔
126	else:
127	callback = parse_requirements_callback	3✔
128	result = await _generate_requirements(	3✔
129	request,
130	union_membership,
131	python_setup,
132	parse_requirements_callback=callback,
133	)
134	return GeneratedTargets(request.generator, result)	3✔
135
136
137	def parse_requirements_callback(file_contents: bytes, file_path: str) -> Iterator[PipRequirement]:	6✔
138	return parse_requirements_file(file_contents.decode(), rel_path=file_path)	3✔
139
140
141	def parse_pyproject_callback(file_contents: bytes, file_path: str) -> Iterator[PipRequirement]:	6✔
UNCOV 142	return parse_pyproject_toml(file_contents.decode(), rel_path=file_path)	1✔
143
144
145	def rules():	6✔
146	return (	6✔
147	*collect_rules(),
148	UnionRule(GenerateTargetsRequest, GenerateFromPythonRequirementsRequest),
149	)

pantsbuild / pants / 25443604553

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous