• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 19529437518

20 Nov 2025 07:44AM UTC coverage: 78.884% (-1.4%) from 80.302%
19529437518

push

github

web-flow
nfpm.native_libs: Add RPM package depends from packaged pex_binaries (#22899)

## PR Series Overview

This is the second in a series of PRs that introduces a new backend:
`pants.backend.npm.native_libs`
Initially, the backend will be available as:
`pants.backend.experimental.nfpm.native_libs`

I proposed this new backend (originally named `bindeps`) in discussion
#22396.

This backend will inspect ELF bin/lib files (like `lib*.so`) in packaged
contents (for this PR series, only in `pex_binary` targets) to identify
package dependency metadata and inject that metadata on the relevant
`nfpm_deb_package` or `nfpm_rpm_package` targets. Effectively, it will
provide an approximation of these native packager features:
- `rpm`: `rpmdeps` + `elfdeps`
- `deb`: `dh_shlibdeps` + `dpkg-shlibdeps` (These substitute
`${shlibs:Depends}` in debian control files have)

### Goal: Host-agnostic package builds

This pants backend is designed to be host-agnostic, like
[nFPM](https://nfpm.goreleaser.com/).

Native packaging tools are often restricted to a single release of a
single distro. Unlike native package builders, this new pants backend
does not use any of those distro-specific or distro-release-specific
utilities or local package databases. This new backend should be able to
run (help with building deb and rpm packages) anywhere that pants can
run (MacOS, rpm linux distros, deb linux distros, other linux distros,
docker, ...).

### Previous PRs in series

- #22873

## PR Overview

This PR adds rules in `nfpm.native_libs` to add package dependency
metadata to `nfpm_rpm_package`. The 2 new rules are:

- `inject_native_libs_dependencies_in_package_fields`:

    - An implementation of the polymorphic rule `inject_nfpm_package_fields`.
      This rule is low priority (`priority = 2`) so that in-repo plugins can
      override/augment what it injects. (See #22864)

    - Rule logic overview:
        - find any pex_binaries that will be packaged in an `nfpm_rpm_package`
   ... (continued)

96 of 118 new or added lines in 3 files covered. (81.36%)

910 existing lines in 53 files now uncovered.

73897 of 93678 relevant lines covered (78.88%)

3.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.22
/src/python/pants/backend/codegen/protobuf/python/rules_integration_test.py
1
# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
2✔
5

6
from textwrap import dedent
2✔
7

8
import pytest
2✔
9

10
from pants.backend.codegen.protobuf.python import additional_fields
2✔
11
from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import PythonProtobufMypyPlugin
2✔
12
from pants.backend.codegen.protobuf.python.python_protobuf_subsystem import (
2✔
13
    rules as protobuf_subsystem_rules,
14
)
15
from pants.backend.codegen.protobuf.python.rules import GeneratePythonFromProtobufRequest
2✔
16
from pants.backend.codegen.protobuf.python.rules import rules as protobuf_rules
2✔
17
from pants.backend.codegen.protobuf.target_types import (
2✔
18
    ProtobufSourceField,
19
    ProtobufSourcesGeneratorTarget,
20
)
21
from pants.backend.codegen.protobuf.target_types import rules as target_types_rules
2✔
22
from pants.backend.python.dependency_inference import module_mapper
2✔
23
from pants.core.util_rules import stripped_source_files
2✔
24
from pants.engine.addresses import Address
2✔
25
from pants.engine.target import GeneratedSources, HydratedSources, HydrateSourcesRequest
2✔
26
from pants.source.source_root import NoSourceRootError
2✔
27
from pants.testutil.python_interpreter_selection import all_major_minor_python_versions
2✔
28
from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error
2✔
29
from pants.util.resources import read_sibling_resource
2✔
30

31
GRPC_PROTO_STANZA = """
2✔
32
syntax = "proto3";
33

34
package dir1;
35

36
// The greeter service definition.
37
service Greeter {
38
  // Sends a greeting
39
  rpc SayHello (HelloRequest) returns (HelloReply) {}
40
}
41

42
// The request message containing the user's name.
43
message HelloRequest {
44
  string name = 1;
45
}
46

47
// The response message containing the greetings
48
message HelloReply {
49
  string message = 1;
50
}
51
"""
52

53

54
@pytest.fixture
2✔
55
def rule_runner() -> RuleRunner:
2✔
56
    return RuleRunner(
2✔
57
        rules=[
58
            *protobuf_rules(),
59
            *protobuf_subsystem_rules(),
60
            *additional_fields.rules(),
61
            *stripped_source_files.rules(),
62
            *target_types_rules(),
63
            *module_mapper.rules(),
64
            QueryRule(HydratedSources, [HydrateSourcesRequest]),
65
            QueryRule(GeneratedSources, [GeneratePythonFromProtobufRequest]),
66
        ],
67
        target_types=[ProtobufSourcesGeneratorTarget],
68
    )
69

70

71
def assert_files_generated(
2✔
72
    rule_runner: RuleRunner,
73
    address: Address,
74
    *,
75
    expected_files: list[str],
76
    source_roots: list[str],
77
    mypy: bool = False,
78
    extra_args: list[str] | None = None,
79
) -> None:
80
    args = [
2✔
81
        f"--source-root-patterns={repr(source_roots)}",
82
        "--no-python-protobuf-infer-runtime-dependency",
83
        *(extra_args or ()),
84
    ]
85
    if mypy:
2✔
UNCOV
86
        args.append("--python-protobuf-mypy-plugin")
×
87
    rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
2✔
88
    tgt = rule_runner.get_target(address)
2✔
89
    protocol_sources = rule_runner.request(
2✔
90
        HydratedSources, [HydrateSourcesRequest(tgt[ProtobufSourceField])]
91
    )
92
    generated_sources = rule_runner.request(
2✔
93
        GeneratedSources,
94
        [GeneratePythonFromProtobufRequest(protocol_sources.snapshot, tgt)],
95
    )
96
    assert set(generated_sources.snapshot.files) == set(expected_files)
2✔
97

98

99
def test_generates_python(rule_runner: RuleRunner) -> None:
2✔
100
    # This tests a few things:
101
    #  * We generate the correct file names.
102
    #  * Protobuf files can import other protobuf files, and those can import others
103
    #    (transitive dependencies). We'll only generate the requested target, though.
104
    #  * We can handle multiple source roots, which need to be preserved in the final output.
UNCOV
105
    rule_runner.write_files(
×
106
        {
107
            "src/protobuf/dir1/f.proto": dedent(
108
                """\
109
                syntax = "proto3";
110

111
                package dir1;
112

113
                message Person {
114
                  string name = 1;
115
                  int32 id = 2;
116
                  string email = 3;
117
                }
118
                """
119
            ),
120
            "src/protobuf/dir1/f2.proto": dedent(
121
                """\
122
                syntax = "proto3";
123

124
                package dir1;
125
                """
126
            ),
127
            "src/protobuf/dir1/BUILD": "protobuf_sources()",
128
            "src/protobuf/dir2/f.proto": dedent(
129
                """\
130
                syntax = "proto3";
131

132
                package dir2;
133

134
                import "dir1/f.proto";
135
                """
136
            ),
137
            "src/protobuf/dir2/BUILD": dedent(
138
                """\
139
                protobuf_sources(dependencies=['src/protobuf/dir1'],
140
                python_source_root='src/python')
141
                """
142
            ),
143
            # Test another source root.
144
            "tests/protobuf/test_protos/f.proto": dedent(
145
                """\
146
                syntax = "proto3";
147

148
                package test_protos;
149

150
                import "dir2/f.proto";
151
                """
152
            ),
153
            "tests/protobuf/test_protos/BUILD": (
154
                "protobuf_sources(dependencies=['src/protobuf/dir2'])"
155
            ),
156
        }
157
    )
158

UNCOV
159
    def assert_gen(addr: Address, expected: str) -> None:
×
UNCOV
160
        assert_files_generated(
×
161
            rule_runner,
162
            addr,
163
            source_roots=["src/python", "/src/protobuf", "/tests/protobuf"],
164
            expected_files=[expected],
165
        )
166

UNCOV
167
    assert_gen(
×
168
        Address("src/protobuf/dir1", relative_file_path="f.proto"), "src/protobuf/dir1/f_pb2.py"
169
    )
UNCOV
170
    assert_gen(
×
171
        Address("src/protobuf/dir1", relative_file_path="f2.proto"), "src/protobuf/dir1/f2_pb2.py"
172
    )
UNCOV
173
    assert_gen(
×
174
        Address("src/protobuf/dir2", relative_file_path="f.proto"), "src/python/dir2/f_pb2.py"
175
    )
UNCOV
176
    assert_gen(
×
177
        Address("tests/protobuf/test_protos", relative_file_path="f.proto"),
178
        "tests/protobuf/test_protos/f_pb2.py",
179
    )
180

181

182
def test_top_level_proto_root(rule_runner: RuleRunner) -> None:
2✔
UNCOV
183
    rule_runner.write_files(
×
184
        {
185
            "protos/f.proto": dedent(
186
                """\
187
                syntax = "proto3";
188

189
                package protos;
190
                """
191
            ),
192
            "protos/BUILD": "protobuf_sources()",
193
        }
194
    )
UNCOV
195
    assert_files_generated(
×
196
        rule_runner,
197
        Address("protos", relative_file_path="f.proto"),
198
        source_roots=["/"],
199
        expected_files=["protos/f_pb2.py"],
200
    )
201

202

203
def test_top_level_python_source_root(rule_runner: RuleRunner) -> None:
2✔
UNCOV
204
    rule_runner.write_files(
×
205
        {
206
            "src/proto/protos/f.proto": dedent(
207
                """\
208
                syntax = "proto3";
209

210
                package protos;
211
                """
212
            ),
213
            "src/proto/protos/BUILD": "protobuf_sources(python_source_root='.')",
214
        }
215
    )
UNCOV
216
    assert_files_generated(
×
217
        rule_runner,
218
        Address("src/proto/protos", relative_file_path="f.proto"),
219
        source_roots=["/", "src/proto"],
220
        expected_files=["protos/f_pb2.py"],
221
    )
222

223

224
def test_bad_python_source_root(rule_runner: RuleRunner) -> None:
2✔
UNCOV
225
    rule_runner.write_files(
×
226
        {
227
            "src/protobuf/dir1/f.proto": dedent(
228
                """\
229
                syntax = "proto3";
230

231
                package dir1;
232
                """
233
            ),
234
            "src/protobuf/dir1/BUILD": "protobuf_sources(python_source_root='notasourceroot')",
235
        }
236
    )
UNCOV
237
    with engine_error(NoSourceRootError):
×
UNCOV
238
        assert_files_generated(
×
239
            rule_runner,
240
            Address("src/protobuf/dir1", relative_file_path="f.proto"),
241
            source_roots=["src/protobuf"],
242
            expected_files=[],
243
        )
244

245

246
@pytest.mark.platform_specific_behavior
2✔
247
@pytest.mark.parametrize(
2✔
248
    "major_minor_interpreter",
249
    all_major_minor_python_versions(PythonProtobufMypyPlugin.default_interpreter_constraints),
250
)
251
def test_generate_type_stubs(rule_runner: RuleRunner, major_minor_interpreter: str) -> None:
2✔
252
    rule_runner.write_files(
2✔
253
        {
254
            "src/protobuf/dir1/f.proto": dedent(
255
                """\
256
                syntax = "proto3";
257

258
                package dir1;
259

260
                message Person {
261
                  string name = 1;
262
                  int32 id = 2;
263
                  string email = 3;
264
                }
265
                """
266
            ),
267
            "src/protobuf/dir1/BUILD": "protobuf_sources()",
268
        }
269
    )
270
    assert_files_generated(
2✔
271
        rule_runner,
272
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
273
        source_roots=["src/protobuf"],
274
        extra_args=[
275
            "--python-protobuf-generate-type-stubs",
276
            f"--mypy-protobuf-interpreter-constraints=['=={major_minor_interpreter}.*']",
277
        ],
278
        expected_files=["src/protobuf/dir1/f_pb2.py", "src/protobuf/dir1/f_pb2.pyi"],
279
    )
280

281

282
@pytest.mark.platform_specific_behavior
2✔
283
@pytest.mark.parametrize(
2✔
284
    "major_minor_interpreter",
285
    all_major_minor_python_versions(PythonProtobufMypyPlugin.default_interpreter_constraints),
286
)
287
def test_mypy_plugin(rule_runner: RuleRunner, major_minor_interpreter: str) -> None:
2✔
288
    rule_runner.write_files(
2✔
289
        {
290
            "src/protobuf/dir1/f.proto": dedent(
291
                """\
292
                syntax = "proto3";
293

294
                package dir1;
295

296
                message Person {
297
                  string name = 1;
298
                  int32 id = 2;
299
                  string email = 3;
300
                }
301
                """
302
            ),
303
            "src/protobuf/dir1/BUILD": "protobuf_sources()",
304
        }
305
    )
306
    assert_files_generated(
2✔
307
        rule_runner,
308
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
309
        source_roots=["src/protobuf"],
310
        extra_args=[
311
            "--python-protobuf-mypy-plugin",
312
            f"--mypy-protobuf-interpreter-constraints=['=={major_minor_interpreter}.*']",
313
        ],
314
        expected_files=["src/protobuf/dir1/f_pb2.py", "src/protobuf/dir1/f_pb2.pyi"],
315
    )
316

317

318
def test_grpc(rule_runner: RuleRunner) -> None:
2✔
UNCOV
319
    rule_runner.write_files(
×
320
        {
321
            "src/protobuf/dir1/f.proto": dedent(GRPC_PROTO_STANZA),
322
            "src/protobuf/dir1/BUILD": "protobuf_sources(grpc=True)",
323
        }
324
    )
UNCOV
325
    assert_files_generated(
×
326
        rule_runner,
327
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
328
        source_roots=["src/protobuf"],
329
        expected_files=["src/protobuf/dir1/f_pb2.py", "src/protobuf/dir1/f_pb2_grpc.py"],
330
    )
331

332

333
def test_grpc_mypy_plugin(rule_runner: RuleRunner) -> None:
2✔
UNCOV
334
    rule_runner.write_files(
×
335
        {
336
            "src/protobuf/dir1/f.proto": dedent(GRPC_PROTO_STANZA),
337
            "src/protobuf/dir1/BUILD": "protobuf_sources(grpc=True)",
338
        }
339
    )
UNCOV
340
    assert_files_generated(
×
341
        rule_runner,
342
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
343
        source_roots=["src/protobuf"],
344
        mypy=True,
345
        expected_files=[
346
            "src/protobuf/dir1/f_pb2.py",
347
            "src/protobuf/dir1/f_pb2.pyi",
348
            "src/protobuf/dir1/f_pb2_grpc.py",
349
            "src/protobuf/dir1/f_pb2_grpc.pyi",
350
        ],
351
    )
352

353

354
def test_grpc_pre_v2_mypy_plugin(rule_runner: RuleRunner) -> None:
2✔
UNCOV
355
    rule_runner.write_files(
×
356
        {
357
            "src/protobuf/dir1/f.proto": dedent(GRPC_PROTO_STANZA),
358
            "src/protobuf/dir1/BUILD": "protobuf_sources(grpc=True)",
359
            "mypy-protobuf.lock": read_sibling_resource(
360
                __name__, "test_grpc_pre_v2_mypy_plugin.lock"
361
            ),
362
        }
363
    )
UNCOV
364
    assert_files_generated(
×
365
        rule_runner,
366
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
367
        source_roots=["src/protobuf"],
368
        extra_args=[
369
            "--python-protobuf-mypy-plugin",
370
            "--python-resolves={'mypy-protobuf':'mypy-protobuf.lock'}",
371
            "--mypy-protobuf-install-from-resolve=mypy-protobuf",
372
        ],
373
        expected_files=[
374
            "src/protobuf/dir1/f_pb2.py",
375
            "src/protobuf/dir1/f_pb2.pyi",
376
            "src/protobuf/dir1/f_pb2_grpc.py",
377
        ],
378
    )
379

380

381
def test_grpclib_plugin(rule_runner: RuleRunner) -> None:
2✔
UNCOV
382
    rule_runner.write_files(
×
383
        {
384
            "src/protobuf/dir1/f.proto": dedent(GRPC_PROTO_STANZA),
385
            "src/protobuf/dir1/BUILD": "protobuf_sources(grpc=True)",
386
        }
387
    )
UNCOV
388
    assert_files_generated(
×
389
        rule_runner,
390
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
391
        source_roots=["src/protobuf"],
392
        extra_args=[
393
            "--python-protobuf-grpclib-plugin",
394
            "--no-python-protobuf-grpcio-plugin",
395
        ],
396
        expected_files=[
397
            "src/protobuf/dir1/f_pb2.py",
398
            "src/protobuf/dir1/f_grpc.py",
399
        ],
400
    )
401

402

403
def test_all_plugins(rule_runner: RuleRunner) -> None:
2✔
UNCOV
404
    rule_runner.write_files(
×
405
        {
406
            "src/protobuf/dir1/f.proto": dedent(GRPC_PROTO_STANZA),
407
            "src/protobuf/dir1/BUILD": "protobuf_sources(grpc=True)",
408
        }
409
    )
UNCOV
410
    assert_files_generated(
×
411
        rule_runner,
412
        Address("src/protobuf/dir1", relative_file_path="f.proto"),
413
        source_roots=["src/protobuf"],
414
        extra_args=[
415
            "--python-protobuf-grpclib-plugin",
416
            "--python-protobuf-grpcio-plugin",
417
            "--python-protobuf-mypy-plugin",
418
        ],
419
        expected_files=[
420
            "src/protobuf/dir1/f_pb2.py",
421
            "src/protobuf/dir1/f_pb2.pyi",
422
            "src/protobuf/dir1/f_pb2_grpc.py",
423
            "src/protobuf/dir1/f_pb2_grpc.pyi",
424
            "src/protobuf/dir1/f_grpc.py",
425
        ],
426
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc