22285099215

Committed 22 Feb 2026 08:52PM UTC coverage: 75.854% (-17.1%) from 92.936%

Build # 22285099215

Build Type

Pull #23121

github

Committed by

web-flow

Commit Message

Merge c7299df9c into ba8359840

Pull Request Pull Request #23121: fix issue with optional fields in dependency validator

Run Details

28 of 29 new or added lines in 2 files covered. (96.55%)

11174 existing lines in 400 files now uncovered.

53694 of 70786 relevant lines covered (75.85%)

1.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.15

/src/python/pants/backend/openapi/dependency_inference.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import contextlib
import json
import os.path
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any

import yaml

from pants.backend.openapi.target_types import (
    OPENAPI_FILE_EXTENSIONS,
    OpenApiDocumentDependenciesField,
    OpenApiDocumentField,
    OpenApiSourceDependenciesField,
    OpenApiSourceField,
)
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
from pants.base.specs import FileLiteralSpec, RawSpecs
from pants.engine.fs import Digest
from pants.engine.internals.graph import (
    determine_explicitly_provided_dependencies,
    hydrate_sources,
    resolve_targets,
)
from pants.engine.internals.selectors import concurrently
from pants.engine.intrinsics import get_digest_contents
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import (
    DependenciesRequest,
    FieldSet,
    HydrateSourcesRequest,
    InferDependenciesRequest,
    InferredDependencies,
)
from pants.engine.unions import UnionRule
from pants.util.frozendict import FrozenDict


@dataclass(frozen=True)
class ParseOpenApiSources:
    sources_digest: Digest
    paths: tuple[str, ...]


@dataclass(frozen=True)
class OpenApiDependencies:
    dependencies: FrozenDict[str, frozenset[str]]


@rule
async def parse_openapi_sources(request: ParseOpenApiSources) -> OpenApiDependencies:
    digest_contents = await get_digest_contents(request.sources_digest)
    dependencies: dict[str, frozenset[str]] = {}

    for digest_content in digest_contents:
        spec = None

        if digest_content.path.endswith(".json"):
            with contextlib.suppress(json.JSONDecodeError):
                spec = json.loads(digest_content.content)
        elif digest_content.path.endswith(".yaml") or digest_content.path.endswith(".yml"):
            with contextlib.suppress(yaml.YAMLError):
                spec = yaml.safe_load(digest_content.content)

        if not spec or not isinstance(spec, dict):
            dependencies[digest_content.path] = frozenset()
            continue

        dependencies[digest_content.path] = _find_local_refs(digest_content.path, spec)

    return OpenApiDependencies(dependencies=FrozenDict(dependencies))


def _find_local_refs(path: str, d: Mapping[str, Any]) -> frozenset[str]:
    local_refs: set[str] = set()

    for k, v in d.items():
        if isinstance(v, dict):
            local_refs.update(_find_local_refs(path, v))
        elif k == "$ref" and isinstance(v, str):
            # https://swagger.io/specification/#reference-object
            # https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03
            v = v.split("#", 1)[0]

            if any(v.endswith(ext) for ext in OPENAPI_FILE_EXTENSIONS) and "://" not in v:
                # Resolution is performed relative to the referring document.
                normalized = os.path.normpath(os.path.join(os.path.dirname(path), v))

                if not normalized.startswith("../"):
                    local_refs.add(normalized)

    return frozenset(local_refs)


# -----------------------------------------------------------------------------------------------
# `openapi_document` dependency inference
# -----------------------------------------------------------------------------------------------


@dataclass(frozen=True)
class OpenApiDocumentDependenciesInferenceFieldSet(FieldSet):
    required_fields = (OpenApiDocumentField, OpenApiDocumentDependenciesField)

    sources: OpenApiDocumentField
    dependencies: OpenApiDocumentDependenciesField


class InferOpenApiDocumentDependenciesRequest(InferDependenciesRequest):
    infer_from = OpenApiDocumentDependenciesInferenceFieldSet


@rule
async def infer_openapi_document_dependencies(
    request: InferOpenApiDocumentDependenciesRequest,
) -> InferredDependencies:
    explicitly_provided_deps, hydrated_sources = await concurrently(
        determine_explicitly_provided_dependencies(
            **implicitly(DependenciesRequest(request.field_set.dependencies))
        ),
        hydrate_sources(HydrateSourcesRequest(request.field_set.sources), **implicitly()),
    )
    candidate_targets = await resolve_targets(
        **implicitly(
            RawSpecs(
                file_literals=(FileLiteralSpec(*hydrated_sources.snapshot.files),),
                description_of_origin="the `openapi_document` dependency inference",
            )
        )
    )

    addresses = frozenset(
        [target.address for target in candidate_targets if target.has_field(OpenApiSourceField)]
    )
    dependencies = explicitly_provided_deps.remaining_after_disambiguation(
        addresses.union(explicitly_provided_deps.includes),
        owners_must_be_ancestors=False,
    )

    return InferredDependencies(dependencies)


# -----------------------------------------------------------------------------------------------
# `openapi_source` dependency inference
# -----------------------------------------------------------------------------------------------


@dataclass(frozen=True)
class OpenApiSourceDependenciesInferenceFieldSet(FieldSet):
    required_fields = (OpenApiSourceField, OpenApiSourceDependenciesField)

    sources: OpenApiSourceField
    dependencies: OpenApiSourceDependenciesField


class InferOpenApiSourceDependenciesRequest(InferDependenciesRequest):
    infer_from = OpenApiSourceDependenciesInferenceFieldSet


@rule
async def infer_openapi_module_dependencies(
    request: InferOpenApiSourceDependenciesRequest,
) -> InferredDependencies:
    explicitly_provided_deps, hydrated_sources = await concurrently(
        determine_explicitly_provided_dependencies(
            **implicitly(DependenciesRequest(request.field_set.dependencies))
        ),
        hydrate_sources(HydrateSourcesRequest(request.field_set.sources), **implicitly()),
    )
    result = await parse_openapi_sources(
        ParseOpenApiSources(
            sources_digest=hydrated_sources.snapshot.digest,
            paths=hydrated_sources.snapshot.files,
        )
    )

    paths: set[str] = set()

    for source_file in hydrated_sources.snapshot.files:
        paths.update(result.dependencies[source_file])

    candidate_targets = await resolve_targets(
        **implicitly(
            RawSpecs(
                file_literals=tuple(FileLiteralSpec(path) for path in paths),
                unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
                description_of_origin="the `openapi_source` dependency inference",
            )
        )
    )

    addresses = frozenset(
        [target.address for target in candidate_targets if target.has_field(OpenApiSourceField)]
    )
    dependencies = explicitly_provided_deps.remaining_after_disambiguation(
        addresses.union(explicitly_provided_deps.includes),
        owners_must_be_ancestors=False,
    )

    return InferredDependencies(dependencies)


def rules():
    return [
        *collect_rules(),
        UnionRule(InferDependenciesRequest, InferOpenApiDocumentDependenciesRequest),
        UnionRule(InferDependenciesRequest, InferOpenApiSourceDependenciesRequest),
    ]

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	2✔
4
5	import contextlib	2✔
6	import json	2✔
7	import os.path	2✔
8	from collections.abc import Mapping	2✔
9	from dataclasses import dataclass	2✔
10	from typing import Any	2✔
11
12	import yaml	2✔
13
14	from pants.backend.openapi.target_types import (	2✔
15	OPENAPI_FILE_EXTENSIONS,
16	OpenApiDocumentDependenciesField,
17	OpenApiDocumentField,
18	OpenApiSourceDependenciesField,
19	OpenApiSourceField,
20	)
21	from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior	2✔
22	from pants.base.specs import FileLiteralSpec, RawSpecs	2✔
23	from pants.engine.fs import Digest	2✔
24	from pants.engine.internals.graph import (	2✔
25	determine_explicitly_provided_dependencies,
26	hydrate_sources,
27	resolve_targets,
28	)
29	from pants.engine.internals.selectors import concurrently	2✔
30	from pants.engine.intrinsics import get_digest_contents	2✔
31	from pants.engine.rules import collect_rules, implicitly, rule	2✔
32	from pants.engine.target import (	2✔
33	DependenciesRequest,
34	FieldSet,
35	HydrateSourcesRequest,
36	InferDependenciesRequest,
37	InferredDependencies,
38	)
39	from pants.engine.unions import UnionRule	2✔
40	from pants.util.frozendict import FrozenDict	2✔
41
42
43	@dataclass(frozen=True)	2✔
44	class ParseOpenApiSources:	2✔
45	sources_digest: Digest	2✔
46	paths: tuple[str, ...]	2✔
47
48
49	@dataclass(frozen=True)	2✔
50	class OpenApiDependencies:	2✔
51	dependencies: FrozenDict[str, frozenset[str]]	2✔
52
53
54	@rule	2✔
55	async def parse_openapi_sources(request: ParseOpenApiSources) -> OpenApiDependencies:	2✔
56	digest_contents = await get_digest_contents(request.sources_digest)	1✔
57	dependencies: dict[str, frozenset[str]] = {}	1✔
58
59	for digest_content in digest_contents:	1✔
60	spec = None	1✔
61
62	if digest_content.path.endswith(".json"):	1✔
63	with contextlib.suppress(json.JSONDecodeError):	1✔
64	spec = json.loads(digest_content.content)	1✔
65	elif digest_content.path.endswith(".yaml") or digest_content.path.endswith(".yml"):	1✔
66	with contextlib.suppress(yaml.YAMLError):	1✔
67	spec = yaml.safe_load(digest_content.content)	1✔
68
69	if not spec or not isinstance(spec, dict):	1✔
70	dependencies[digest_content.path] = frozenset()	1✔
71	continue	1✔
72
73	dependencies[digest_content.path] = _find_local_refs(digest_content.path, spec)	1✔
74
75	return OpenApiDependencies(dependencies=FrozenDict(dependencies))	1✔
76
77
78	def _find_local_refs(path: str, d: Mapping[str, Any]) -> frozenset[str]:	2✔
79	local_refs: set[str] = set()	1✔
80
81	for k, v in d.items():	1✔
82	if isinstance(v, dict):	1✔
UNCOV 83	local_refs.update(_find_local_refs(path, v))	×
84	elif k == "$ref" and isinstance(v, str):	1✔
85	# https://swagger.io/specification/#reference-object
86	# https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03
87	v = v.split("#", 1)[0]	1✔
88
89	if any(v.endswith(ext) for ext in OPENAPI_FILE_EXTENSIONS) and "://" not in v:	1✔
90	# Resolution is performed relative to the referring document.
91	normalized = os.path.normpath(os.path.join(os.path.dirname(path), v))	1✔
92
93	if not normalized.startswith("../"):	1✔
94	local_refs.add(normalized)	1✔
95
96	return frozenset(local_refs)	1✔
97
98
99	# -----------------------------------------------------------------------------------------------
100	# `openapi_document` dependency inference
101	# -----------------------------------------------------------------------------------------------
102
103
104	@dataclass(frozen=True)	2✔
105	class OpenApiDocumentDependenciesInferenceFieldSet(FieldSet):	2✔
106	required_fields = (OpenApiDocumentField, OpenApiDocumentDependenciesField)	2✔
107
108	sources: OpenApiDocumentField	2✔
109	dependencies: OpenApiDocumentDependenciesField	2✔
110
111
112	class InferOpenApiDocumentDependenciesRequest(InferDependenciesRequest):	2✔
113	infer_from = OpenApiDocumentDependenciesInferenceFieldSet	2✔
114
115
116	@rule	2✔
117	async def infer_openapi_document_dependencies(	2✔
118	request: InferOpenApiDocumentDependenciesRequest,
119	) -> InferredDependencies:
UNCOV 120	explicitly_provided_deps, hydrated_sources = await concurrently(	×
121	determine_explicitly_provided_dependencies(
122	**implicitly(DependenciesRequest(request.field_set.dependencies))
123	),
124	hydrate_sources(HydrateSourcesRequest(request.field_set.sources), **implicitly()),
125	)
UNCOV 126	candidate_targets = await resolve_targets(	×
127	**implicitly(
128	RawSpecs(
129	file_literals=(FileLiteralSpec(*hydrated_sources.snapshot.files),),
130	description_of_origin="the `openapi_document` dependency inference",
131	)
132	)
133	)
134
UNCOV 135	addresses = frozenset(	×
136	[target.address for target in candidate_targets if target.has_field(OpenApiSourceField)]
137	)
UNCOV 138	dependencies = explicitly_provided_deps.remaining_after_disambiguation(	×
139	addresses.union(explicitly_provided_deps.includes),
140	owners_must_be_ancestors=False,
141	)
142
UNCOV 143	return InferredDependencies(dependencies)	×
144
145
146	# -----------------------------------------------------------------------------------------------
147	# `openapi_source` dependency inference
148	# -----------------------------------------------------------------------------------------------
149
150
151	@dataclass(frozen=True)	2✔
152	class OpenApiSourceDependenciesInferenceFieldSet(FieldSet):	2✔
153	required_fields = (OpenApiSourceField, OpenApiSourceDependenciesField)	2✔
154
155	sources: OpenApiSourceField	2✔
156	dependencies: OpenApiSourceDependenciesField	2✔
157
158
159	class InferOpenApiSourceDependenciesRequest(InferDependenciesRequest):	2✔
160	infer_from = OpenApiSourceDependenciesInferenceFieldSet	2✔
161
162
163	@rule	2✔
164	async def infer_openapi_module_dependencies(	2✔
165	request: InferOpenApiSourceDependenciesRequest,
166	) -> InferredDependencies:
UNCOV 167	explicitly_provided_deps, hydrated_sources = await concurrently(	×
168	determine_explicitly_provided_dependencies(
169	**implicitly(DependenciesRequest(request.field_set.dependencies))
170	),
171	hydrate_sources(HydrateSourcesRequest(request.field_set.sources), **implicitly()),
172	)
UNCOV 173	result = await parse_openapi_sources(	×
174	ParseOpenApiSources(
175	sources_digest=hydrated_sources.snapshot.digest,
176	paths=hydrated_sources.snapshot.files,
177	)
178	)
179
UNCOV 180	paths: set[str] = set()	×
181
UNCOV 182	for source_file in hydrated_sources.snapshot.files:	×
UNCOV 183	paths.update(result.dependencies[source_file])	×
184
UNCOV 185	candidate_targets = await resolve_targets(	×
186	**implicitly(
187	RawSpecs(
188	file_literals=tuple(FileLiteralSpec(path) for path in paths),
189	unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
190	description_of_origin="the `openapi_source` dependency inference",
191	)
192	)
193	)
194
UNCOV 195	addresses = frozenset(	×
196	[target.address for target in candidate_targets if target.has_field(OpenApiSourceField)]
197	)
UNCOV 198	dependencies = explicitly_provided_deps.remaining_after_disambiguation(	×
199	addresses.union(explicitly_provided_deps.includes),
200	owners_must_be_ancestors=False,
201	)
202
UNCOV 203	return InferredDependencies(dependencies)	×
204
205
206	def rules():	2✔
207	return [	2✔
208	*collect_rules(),
209	UnionRule(InferDependenciesRequest, InferOpenApiDocumentDependenciesRequest),
210	UnionRule(InferDependenciesRequest, InferOpenApiSourceDependenciesRequest),
211	]

pantsbuild / pants / 22285099215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous