• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 26342152999

23 May 2026 07:59PM UTC coverage: 91.165% (-1.6%) from 92.792%
26342152999

push

github

web-flow
Run Linux ARM CI on Depot runners (#23363)

RunsOn is deprecating their v2 stack, and rather than migrate
to v3 we should use the resources kindly donated by Depot.

GitHub also now has Linux ARM runners, should we need them.

87305 of 95766 relevant lines covered (91.16%)

3.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/src/python/pants/backend/codegen/avro/java/rules.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
from __future__ import annotations
×
4

5
import os
×
6
from collections.abc import Iterable
×
7
from dataclasses import dataclass
×
8
from pathlib import PurePath
×
9

10
from pants.backend.codegen.avro.java.subsystem import AvroSubsystem
×
11
from pants.backend.codegen.avro.target_types import (
×
12
    AvroDependenciesField,
13
    AvroSourceField,
14
    AvroSourcesGeneratorTarget,
15
    AvroSourceTarget,
16
)
17
from pants.backend.java.target_types import JavaSourceField
×
18
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
×
19
from pants.build_graph.address import Address
×
20
from pants.core.goals.resolves import ExportableTool
×
21
from pants.engine.fs import (
×
22
    AddPrefix,
23
    CreateDigest,
24
    Digest,
25
    DigestSubset,
26
    Directory,
27
    GlobExpansionConjunction,
28
    MergeDigests,
29
    PathGlobs,
30
    RemovePrefix,
31
)
32
from pants.engine.internals.graph import hydrate_sources
×
33
from pants.engine.internals.selectors import concurrently
×
34
from pants.engine.intrinsics import (
×
35
    create_digest,
36
    digest_subset_to_digest,
37
    digest_to_snapshot,
38
    merge_digests,
39
    remove_prefix,
40
)
41
from pants.engine.process import execute_process_or_raise
×
42
from pants.engine.rules import collect_rules, implicitly, rule
×
43
from pants.engine.target import (
×
44
    FieldSet,
45
    GeneratedSources,
46
    GenerateSourcesRequest,
47
    HydrateSourcesRequest,
48
    InferDependenciesRequest,
49
    InferredDependencies,
50
)
51
from pants.engine.unions import UnionRule
×
52
from pants.jvm import jdk_rules
×
53
from pants.jvm.dependency_inference import artifact_mapper
×
54
from pants.jvm.dependency_inference.artifact_mapper import (
×
55
    AllJvmArtifactTargets,
56
    UnversionedCoordinate,
57
    find_jvm_artifacts_or_raise,
58
)
59
from pants.jvm.jdk_rules import InternalJdk, JvmProcess
×
60
from pants.jvm.resolve import jvm_tool
×
61
from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool
×
62
from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
×
63
from pants.jvm.subsystems import JvmSubsystem
×
64
from pants.jvm.target_types import JvmResolveField, PrefixedJvmJdkField, PrefixedJvmResolveField
×
65
from pants.source.source_root import SourceRootRequest, get_source_root
×
66
from pants.util.docutil import bin_name
×
67
from pants.util.logging import LogLevel
×
68

69

70
class GenerateJavaFromAvroRequest(GenerateSourcesRequest):
×
71
    input = AvroSourceField
×
72
    output = JavaSourceField
×
73

74

75
@dataclass(frozen=True)
×
76
class CompileAvroSourceRequest:
×
77
    digest: Digest
×
78
    path: str
×
79

80

81
@dataclass(frozen=True)
×
82
class CompiledAvroSource:
×
83
    output_digest: Digest
×
84

85

86
@rule
×
87
async def compile_avro_source(
×
88
    request: CompileAvroSourceRequest,
89
    jdk: InternalJdk,
90
    avro_tools: AvroSubsystem,
91
) -> CompiledAvroSource:
92
    output_dir = "_generated_files"
×
93
    toolcp_relpath = "__toolcp"
×
94

95
    lockfile_request = GenerateJvmLockfileFromTool.create(avro_tools)
×
96
    tool_classpath, subsetted_input_digest, empty_output_dir = await concurrently(
×
97
        materialize_classpath_for_tool(ToolClasspathRequest(lockfile=lockfile_request)),
98
        digest_subset_to_digest(
99
            DigestSubset(
100
                request.digest,
101
                PathGlobs(
102
                    [request.path],
103
                    glob_match_error_behavior=GlobMatchErrorBehavior.error,
104
                    conjunction=GlobExpansionConjunction.all_match,
105
                    description_of_origin="the Avro source file name",
106
                ),
107
            )
108
        ),
109
        create_digest(CreateDigest([Directory(output_dir)])),
110
    )
111

112
    input_digest = await merge_digests(
×
113
        MergeDigests(
114
            [
115
                subsetted_input_digest,
116
                empty_output_dir,
117
            ]
118
        )
119
    )
120

121
    extra_immutable_input_digests = {
×
122
        toolcp_relpath: tool_classpath.digest,
123
    }
124

125
    def make_avro_process(
×
126
        args: Iterable[str],
127
        *,
128
        overridden_input_digest: Digest | None = None,
129
        overridden_output_dir: str | None = None,
130
    ) -> JvmProcess:
131
        return JvmProcess(
×
132
            jdk=jdk,
133
            argv=(
134
                "org.apache.avro.tool.Main",
135
                *args,
136
            ),
137
            classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
138
            input_digest=(
139
                overridden_input_digest if overridden_input_digest is not None else input_digest
140
            ),
141
            extra_jvm_options=avro_tools.jvm_options,
142
            extra_immutable_input_digests=extra_immutable_input_digests,
143
            extra_nailgun_keys=extra_immutable_input_digests,
144
            description="Generating Java sources from Avro source.",
145
            level=LogLevel.DEBUG,
146
            output_directories=(overridden_output_dir if overridden_output_dir else output_dir,),
147
        )
148

149
    path = PurePath(request.path)
×
150
    if path.suffix == ".avsc":
×
151
        result = await execute_process_or_raise(
×
152
            **implicitly(
153
                {make_avro_process(["compile", "schema", request.path, output_dir]): JvmProcess}
154
            )
155
        )
156
    elif path.suffix == ".avpr":
×
157
        result = await execute_process_or_raise(
×
158
            **implicitly(
159
                {make_avro_process(["compile", "protocol", request.path, output_dir]): JvmProcess}
160
            )
161
        )
162
    elif path.suffix == ".avdl":
×
163
        idl_output_dir = "__idl"
×
164
        avpr_path = os.path.join(idl_output_dir, str(path.with_suffix(".avpr")))
×
165
        idl_output_dir_digest = await create_digest(
×
166
            CreateDigest([Directory(os.path.dirname(avpr_path))])
167
        )
168
        idl_input_digest = await merge_digests(MergeDigests([input_digest, idl_output_dir_digest]))
×
169
        idl_result = await execute_process_or_raise(
×
170
            **implicitly(
171
                {
172
                    make_avro_process(
173
                        ["idl", request.path, avpr_path],
174
                        overridden_input_digest=idl_input_digest,
175
                        overridden_output_dir=idl_output_dir,
176
                    ): JvmProcess
177
                }
178
            )
179
        )
180
        generated_files_dir = await create_digest(CreateDigest([Directory(output_dir)]))
×
181
        protocol_input_digest = await merge_digests(
×
182
            MergeDigests([idl_result.output_digest, generated_files_dir])
183
        )
184
        result = await execute_process_or_raise(
×
185
            **implicitly(
186
                {
187
                    make_avro_process(
188
                        ["compile", "protocol", avpr_path, output_dir],
189
                        overridden_input_digest=protocol_input_digest,
190
                    ): JvmProcess
191
                }
192
            )
193
        )
194
    else:
195
        raise AssertionError(
×
196
            f"Avro backend does not support files with extension `{path.suffix}`: {path}"
197
        )
198

199
    normalized_digest = await remove_prefix(RemovePrefix(result.output_digest, output_dir))
×
200
    return CompiledAvroSource(normalized_digest)
×
201

202

203
@rule(desc="Generate Java from Avro", level=LogLevel.DEBUG)
×
204
async def generate_java_from_avro(
×
205
    request: GenerateJavaFromAvroRequest,
206
) -> GeneratedSources:
207
    sources = await hydrate_sources(
×
208
        HydrateSourcesRequest(request.protocol_target[AvroSourceField]), **implicitly()
209
    )
210

211
    compile_results = await concurrently(
×
212
        compile_avro_source(CompileAvroSourceRequest(sources.snapshot.digest, path), **implicitly())
213
        for path in sources.snapshot.files
214
    )
215

216
    merged_output_digest, source_root = await concurrently(
×
217
        merge_digests(MergeDigests([r.output_digest for r in compile_results])),
218
        get_source_root(SourceRootRequest.for_target(request.protocol_target)),
219
    )
220

221
    source_root_restored = (
×
222
        await digest_to_snapshot(**implicitly(AddPrefix(merged_output_digest, source_root.path)))
223
        if source_root.path != "."
224
        else await digest_to_snapshot(merged_output_digest)
225
    )
226
    return GeneratedSources(source_root_restored)
×
227

228

229
@dataclass(frozen=True)
×
230
class AvroRuntimeDependencyInferenceFieldSet(FieldSet):
×
231
    required_fields = (
×
232
        AvroDependenciesField,
233
        JvmResolveField,
234
    )
235

236
    dependencies: AvroDependenciesField
×
237
    resolve: JvmResolveField
×
238

239

240
class InferAvroRuntimeDependencyRequest(InferDependenciesRequest):
×
241
    infer_from = AvroRuntimeDependencyInferenceFieldSet
×
242

243

244
@dataclass(frozen=True)
×
245
class ApacheAvroRuntimeForResolveRequest:
×
246
    resolve_name: str
×
247

248

249
@dataclass(frozen=True)
×
250
class ApacheAvroRuntimeForResolve:
×
251
    addresses: frozenset[Address]
×
252

253

254
_AVRO_GROUP = "org.apache.avro"
×
255

256

257
@rule
×
258
async def resolve_apache_avro_runtime_for_resolve(
×
259
    request: ApacheAvroRuntimeForResolveRequest,
260
    jvm_artifact_targets: AllJvmArtifactTargets,
261
    jvm: JvmSubsystem,
262
) -> ApacheAvroRuntimeForResolve:
263
    addresses = find_jvm_artifacts_or_raise(
×
264
        required_coordinates=[
265
            UnversionedCoordinate(
266
                group=_AVRO_GROUP,
267
                artifact="avro",
268
            ),
269
            UnversionedCoordinate(
270
                group=_AVRO_GROUP,
271
                artifact="avro-ipc",
272
            ),
273
        ],
274
        resolve=request.resolve_name,
275
        jvm_artifact_targets=jvm_artifact_targets,
276
        jvm=jvm,
277
        subsystem="the Apache Avro runtime",
278
        target_type="avro_sources",
279
    )
280
    return ApacheAvroRuntimeForResolve(addresses)
×
281

282

283
@rule
×
284
async def infer_apache_avro_java_dependencies(
×
285
    request: InferAvroRuntimeDependencyRequest, jvm: JvmSubsystem
286
) -> InferredDependencies:
287
    resolve = request.field_set.resolve.normalized_value(jvm)
×
288

289
    dependencies_info = await resolve_apache_avro_runtime_for_resolve(
×
290
        ApacheAvroRuntimeForResolveRequest(resolve), **implicitly()
291
    )
292
    return InferredDependencies(dependencies_info.addresses)
×
293

294

295
class MissingApacheAvroRuntimeInResolveError(ValueError):
×
296
    def __init__(self, resolve_name: str) -> None:
×
297
        super().__init__(
×
298
            f"The JVM resolve `{resolve_name}` does not contain a requirement for the Apache Avro runtime. "
299
            "Since at least one JVM target type in this repository consumes a `avro_sources` target "
300
            "in this resolve, the resolve must contain a `jvm_artifact` target for the Apache Avro runtime.\n\n"
301
            "Please add the following `jvm_artifact` targets somewhere in the repository and re-run "
302
            f"`{bin_name()} generate-lockfiles --resolve={resolve_name}`:\n"
303
            "jvm_artifact(\n"
304
            f'  name="{_AVRO_GROUP}_avro",\n'
305
            f'  group="{_AVRO_GROUP}",\n'
306
            f'  artifact="avro",\n'
307
            '  version="<your chosen version>",\n'
308
            f'  resolve="{resolve_name}",\n'
309
            ")\n\n"
310
            "jvm_artifact(\n"
311
            f'  name="{_AVRO_GROUP}_avro",\n'
312
            f'  group="{_AVRO_GROUP}",\n'
313
            f'  artifact="avro-ipc",\n'
314
            '  version="<your chosen version>",\n'
315
            f'  resolve="{resolve_name}",\n'
316
            ")"
317
        )
318

319

320
def rules():
×
321
    return (
×
322
        *collect_rules(),
323
        *jvm_tool.rules(),
324
        *jdk_rules.rules(),
325
        UnionRule(GenerateSourcesRequest, GenerateJavaFromAvroRequest),
326
        UnionRule(ExportableTool, AvroSubsystem),
327
        UnionRule(InferDependenciesRequest, InferAvroRuntimeDependencyRequest),
328
        AvroSourceTarget.register_plugin_field(PrefixedJvmJdkField),
329
        AvroSourcesGeneratorTarget.register_plugin_field(PrefixedJvmJdkField),
330
        AvroSourceTarget.register_plugin_field(PrefixedJvmResolveField),
331
        AvroSourcesGeneratorTarget.register_plugin_field(PrefixedJvmResolveField),
332
        # Needed to avoid rule graph errors (for dependency inference):
333
        *artifact_mapper.rules(),
334
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc