• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 19529437518

20 Nov 2025 07:44AM UTC coverage: 78.884% (-1.4%) from 80.302%
19529437518

push

github

web-flow
nfpm.native_libs: Add RPM package depends from packaged pex_binaries (#22899)

## PR Series Overview

This is the second in a series of PRs that introduces a new backend:
`pants.backend.npm.native_libs`
Initially, the backend will be available as:
`pants.backend.experimental.nfpm.native_libs`

I proposed this new backend (originally named `bindeps`) in discussion
#22396.

This backend will inspect ELF bin/lib files (like `lib*.so`) in packaged
contents (for this PR series, only in `pex_binary` targets) to identify
package dependency metadata and inject that metadata on the relevant
`nfpm_deb_package` or `nfpm_rpm_package` targets. Effectively, it will
provide an approximation of these native packager features:
- `rpm`: `rpmdeps` + `elfdeps`
- `deb`: `dh_shlibdeps` + `dpkg-shlibdeps` (These substitute
`${shlibs:Depends}` in debian control files have)

### Goal: Host-agnostic package builds

This pants backend is designed to be host-agnostic, like
[nFPM](https://nfpm.goreleaser.com/).

Native packaging tools are often restricted to a single release of a
single distro. Unlike native package builders, this new pants backend
does not use any of those distro-specific or distro-release-specific
utilities or local package databases. This new backend should be able to
run (help with building deb and rpm packages) anywhere that pants can
run (MacOS, rpm linux distros, deb linux distros, other linux distros,
docker, ...).

### Previous PRs in series

- #22873

## PR Overview

This PR adds rules in `nfpm.native_libs` to add package dependency
metadata to `nfpm_rpm_package`. The 2 new rules are:

- `inject_native_libs_dependencies_in_package_fields`:

    - An implementation of the polymorphic rule `inject_nfpm_package_fields`.
      This rule is low priority (`priority = 2`) so that in-repo plugins can
      override/augment what it injects. (See #22864)

    - Rule logic overview:
        - find any pex_binaries that will be packaged in an `nfpm_rpm_package`
   ... (continued)

96 of 118 new or added lines in 3 files covered. (81.36%)

910 existing lines in 53 files now uncovered.

73897 of 93678 relevant lines covered (78.88%)

3.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67
/src/python/pants/backend/python/dependency_inference/default_module_mapping.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
# NB: The project names must follow the naming scheme at
5
#  https://www.python.org/dev/peps/pep-0503/#normalized-names.
6

7
import re
11✔
8
from collections.abc import Callable
11✔
9
from enum import Enum
11✔
10
from functools import partial
11✔
11
from re import Match
11✔
12

13

14
class PackageSeparator(Enum):
11✔
15
    DOT = "."
11✔
16
    UNDERSCORE = "_"
11✔
17
    NONE = ""
11✔
18

19

20
def all_hyphen_to_separator(m: Match[str], separator: PackageSeparator) -> str:
11✔
21
    """Convert all hyphens to a package separator e.g. azure-foo-bar -> azure.foo.bar or
22
    azure_foo_bar.
23

24
    >>> all_hyphen_to_separator(re.match(r"^azure-.+", "azure-foo-bar"), PackageSeparator.DOT)
25
    'azure.foo.bar'
26
    >>> all_hyphen_to_separator(re.match(r"^azure-.+", "azure-foo-bar"), PackageSeparator.UNDERSCORE)
27
    'azure_foo_bar'
28
    >>> all_hyphen_to_separator(re.match(r"^azure-.+", "azure-foo-bar"), PackageSeparator.NONE)
29
    'azurefoobar'
30
    """
UNCOV
31
    return m.string.replace("-", separator.value)
×
32

33

34
def first_group_hyphen_to_separator(m: Match[str], separator: PackageSeparator) -> str:
11✔
35
    """Convert the first group(regex match group) of hyphens to underscores. Only returns the first
36
    group and must contain at least one group.
37

38
    >>> first_group_hyphen_to_separator(re.match(r"^django-((.+(-.+)?))", "django-admin-cursor-paginator"), separator=PackageSeparator.UNDERSCORE)
39
    'admin_cursor_paginator'
40
    >>> first_group_hyphen_to_separator(re.match(r"^django-((.+(-.+)?))", "django-admin-cursor-paginator"), separator=PackageSeparator.DOT)
41
    'admin.cursor.paginator'
42
    >>> first_group_hyphen_to_separator(re.match(r"^django-((.+(-.+)?))", "django-admin-cursor-paginator"), separator=PackageSeparator.NONE)
43
    'admincursorpaginator'
44
    """
UNCOV
45
    if m.re.groups == 0 or not m.groups():
×
UNCOV
46
        raise ValueError(f"expected at least one group in the pattern{m.re.pattern} but got none.")
×
UNCOV
47
    return str(m.groups()[0]).replace("-", separator.value)
×
48

49

50
def two_groups_hyphens_two_replacements_with_suffix(
11✔
51
    m: Match[str],
52
    first_group_replacement: PackageSeparator = PackageSeparator.DOT,
53
    second_group_replacement: PackageSeparator = PackageSeparator.NONE,
54
    custom_suffix: str = "",
55
) -> str:
56
    """take two groups, and by default, the first will have '-' replaced with '.', the second will
57
    have '-' replaced with '' e.g. google-cloud-foo-bar -> group1(google.cloud.)group2(foobar)
58

59
    >>> two_groups_hyphens_two_replacements_with_suffix(re.match(r"^(google-cloud-)([^.]+)", "google-cloud-foo-bar"))
60
    'google.cloud.foobar'
61
    >>> two_groups_hyphens_two_replacements_with_suffix(re.match(r"^(google-cloud-)([^.]+)", "google-cloud-foo-bar"), first_group_replacement=PackageSeparator.UNDERSCORE, second_group_replacement=PackageSeparator.DOT)
62
    'google_cloud_foo.bar'
63
    """
UNCOV
64
    if m.re.groups < 2 or not m.groups():
×
UNCOV
65
        raise ValueError(f"expected at least two groups in the pattern{m.re.pattern}.")
×
UNCOV
66
    prefix = m.string[m.start(1) : m.end(1)].replace("-", first_group_replacement.value)
×
UNCOV
67
    suffix = m.string[m.start(2) : m.end(2)].replace("-", second_group_replacement.value)
×
UNCOV
68
    return f"{prefix}{suffix}{custom_suffix}"
×
69

70

71
# common replacement methods
72
all_hyphen_to_dot = partial(all_hyphen_to_separator, separator=PackageSeparator.DOT)
11✔
73
all_hyphen_to_underscore = partial(all_hyphen_to_separator, separator=PackageSeparator.UNDERSCORE)
11✔
74
first_group_hyphen_to_dot = partial(first_group_hyphen_to_separator, separator=PackageSeparator.DOT)
11✔
75
first_group_hyphen_to_underscore = partial(
11✔
76
    first_group_hyphen_to_separator, separator=PackageSeparator.UNDERSCORE
77
)
78

79
"""
11✔
80
A mapping of Patterns and their replacements. will be used with `re.sub`.
81
The match is either a string or a function`(str) -> str`; that takes a re.Match and returns
82
the replacement. see re.sub for more information
83

84
then if an import in the python code is google.cloud.foo, then the package of
85
google-cloud-foo will be used.
86
"""
87
DEFAULT_MODULE_PATTERN_MAPPING: dict[re.Pattern, list[Callable[[Match[str]], str]]] = {
11✔
88
    re.compile(r"""^azure-.+"""): [all_hyphen_to_dot],
89
    re.compile(r"""^django-((.+(-.+)?))"""): [first_group_hyphen_to_underscore],
90
    # See https://github.com/googleapis/google-cloud-python#libraries for all Google cloud
91
    # libraries. We only add libraries in GA, not beta.
92
    re.compile(r"""^(google-cloud-)([^.]+)"""): [
93
        partial(two_groups_hyphens_two_replacements_with_suffix, custom_suffix=custom_suffix)
94
        for custom_suffix in ("", "_v1", "_v2", "_v3")
95
    ],
96
    re.compile(r"""^(opentelemetry-instrumentation-)([^.]+)"""): [
97
        partial(
98
            two_groups_hyphens_two_replacements_with_suffix,
99
            second_group_replacement=PackageSeparator.UNDERSCORE,
100
        ),
101
    ],
102
    re.compile(r"""^oslo-.+"""): [all_hyphen_to_underscore],
103
    re.compile(r"""^python-(.+)"""): [first_group_hyphen_to_underscore],
104
    re.compile(r"""^apache-(airflow-providers-.+)"""): [first_group_hyphen_to_dot],
105
}
106

107
DEFAULT_MODULE_MAPPING: dict[str, tuple[str, ...]] = {
11✔
108
    "absl-py": ("absl",),
109
    "acryl-datahub": ("datahub",),
110
    "ansicolors": ("colors",),
111
    "antlr4-python3-runtime": ("antlr4",),
112
    "apache-airflow": ("airflow",),
113
    "atlassian-python-api": ("atlassian",),
114
    "attrs": ("attr", "attrs"),
115
    "auth0-python": ("auth0",),
116
    "beautifulsoup4": ("bs4",),
117
    "biopython": ("Bio", "BioSQL"),
118
    "bitvector": ("BitVector",),
119
    "cattrs": ("cattr", "cattrs"),
120
    "cloud-sql-python-connector": ("google.cloud.sql.connector",),
121
    "confluent-kafka": ("confluent_kafka",),
122
    "coolprop": ("CoolProp",),
123
    "databricks-sdk": ("databricks.sdk",),
124
    "databricks-sql-connector": (
125
        "databricks.sql",
126
        "databricks.sqlalchemy",
127
    ),
128
    "delta-spark": ("delta",),
129
    "discord-py": ("discord",),
130
    "django-activity-stream": ("actstream",),
131
    "django-cors-headers": ("corsheaders",),
132
    "django-countries": ("django_countries",),
133
    "django-filter": ("django_filters",),
134
    "django-fsm": ("django_fsm",),
135
    "django-notifications-hq": ("notifications",),
136
    "django-oauth-toolkit": ("oauth2_provider",),
137
    "django-object-actions": ("django_object_actions",),
138
    "django-postgres-extra": ("psqlextra",),
139
    "django-redis": ("django_redis",),
140
    "django-scim2": ("django_scim",),
141
    "django-two-factor-auth": ("two_factor",),
142
    "djangorestframework": ("rest_framework",),
143
    "djangorestframework-api-key": ("rest_framework_api_key",),
144
    "djangorestframework-dataclasses": ("rest_framework_dataclasses",),
145
    "djangorestframework-jwt": ("rest_framework_jwt",),
146
    "djangorestframework-queryfields": ("drf_queryfields",),
147
    "djangorestframework-simplejwt": ("rest_framework_simplejwt",),
148
    "dnspython": ("dns",),
149
    "drf-api-tracking": ("rest_framework_tracking",),
150
    "elastic-apm": ("elasticapm",),
151
    "enum34": ("enum",),
152
    "factory-boy": ("factory",),
153
    "faiss-cpu": ("faiss",),
154
    "faiss-gpu": ("faiss",),
155
    "fluent-logger": ("fluent",),
156
    "fonttools": ("fontTools",),
157
    "gitpython": ("git",),
158
    "google-api-python-client": ("googleapiclient",),
159
    "google-auth": (
160
        "google.auth",
161
        "google.oauth2",
162
    ),
163
    "graphql-core": ("graphql",),
164
    "grpcio": ("grpc",),
165
    "grpcio-channelz": ("grpcio_channelz",),
166
    "grpcio-health-checking": ("grpc_health",),
167
    "grpcio-reflection": ("grpc_reflection",),
168
    "grpcio-status": ("grpc_status",),
169
    "grpcio-testing": ("grpc_testing",),
170
    "hdrhistogram": ("hdrh",),
171
    "honeycomb-opentelemetry": ("honeycomb.opentelemetry",),
172
    "ipython": ("IPython",),
173
    "jack-client": ("jack",),
174
    "kafka-python": ("kafka",),
175
    "lark-parser": ("lark",),
176
    "launchdarkly-server-sdk": ("ldclient",),
177
    "mail-parser": ("mailparser",),
178
    "matplotlib": ("matplotlib", "mpl_toolkits"),
179
    "matrix-nio": ("nio",),
180
    "mysql-connector-python": ("mysql.connector",),
181
    "netcdf4": ("netCDF4",),
182
    "o365": ("O365",),
183
    "opencv-python": ("cv2",),
184
    "opencv-python-headless": ("cv2",),
185
    "opensearch-py": ("opensearchpy",),
186
    # opentelemetry
187
    "opentelemetry-api": (
188
        "opentelemetry._logs",
189
        "opentelemetry.attributes",
190
        "opentelemetry.baggage",
191
        "opentelemetry.context",
192
        "opentelemetry.environment_variables",
193
        "opentelemetry.metrics",
194
        "opentelemetry.propagate",
195
        "opentelemetry.propagators",
196
        "opentelemetry.trace",
197
        "opentelemetry.util",
198
        "opentelemetry.version",
199
    ),
200
    "opentelemetry-exporter-otlp": ("opentelemetry.exporter.otlp",),
201
    "opentelemetry-exporter-otlp-proto-grpc": ("opentelemetry.exporter.otlp.proto.grpc",),
202
    "opentelemetry-exporter-otlp-proto-http": ("opentelemetry.exporter.otlp.proto.http",),
203
    "opentelemetry-instrumentation-kafka-python": ("opentelemetry.instrumentation.kafka",),
204
    "opentelemetry-proto": ("opentelemetry.proto",),
205
    "opentelemetry-sdk": ("opentelemetry.sdk",),
206
    "opentelemetry-semantic-conventions": ("opentelemetry.semconv",),
207
    "opentelemetry-test-utils": ("opentelemetry.test",),
208
    "paho-mqtt": ("paho",),
209
    "phonenumberslite": ("phonenumbers",),
210
    "pillow": ("PIL",),
211
    "pip-tools": ("piptools",),
212
    "progressbar2": ("progressbar",),
213
    "protobuf": ("google.protobuf",),
214
    "psycopg2-binary": ("psycopg2",),
215
    "py-healthcheck": ("healthcheck",),
216
    "pycrypto": ("Crypto",),
217
    "pycryptodome": ("Crypto",),
218
    "pyerfa": ("erfa",),
219
    "pygithub": ("github",),
220
    "pygobject": ("gi",),
221
    "pyhamcrest": ("hamcrest",),
222
    "pyjwt": ("jwt",),
223
    "pykube-ng": ("pykube",),
224
    "pymongo": ("bson", "gridfs", "pymongo"),
225
    "pymupdf": ("fitz", "pymupdf"),
226
    "pynacl": ("nacl",),
227
    "pyopenssl": ("OpenSSL",),
228
    "pypdf2": ("PyPDF2",),
229
    "pypi-kenlm": ("kenlm",),
230
    "pyshp": ("shapefile",),
231
    "pysocks": ("socks",),
232
    "pytest": ("pytest", "_pytest"),
233
    "pytest-runner": ("ptr",),
234
    "python-sat": ("pysat",),
235
    "python-json-logger": ("pythonjsonlogger",),
236
    "python-levenshtein": ("Levenshtein",),
237
    "python-lsp-jsonrpc": ("pylsp_jsonrpc",),
238
    "pywinrm": ("winrm",),
239
    "pywavelets": ("pywt",),
240
    "pyyaml": ("yaml",),
241
    "randomwords": ("random_words",),
242
    "robotraconteur": ("RobotRaconteur",),
243
    "scikit-image": ("skimage",),
244
    "scikit-learn": ("sklearn",),
245
    "scikit-video": ("skvideo",),
246
    "sisl": ("sisl", "sisl_toolbox"),
247
    "setuptools": ("easy_install", "pkg_resources", "setuptools"),
248
    "snowflake-connector-python": ("snowflake.connector",),
249
    "snowflake-snowpark-python": ("snowflake.snowpark",),
250
    "snowflake-sqlalchemy": ("snowflake.sqlalchemy",),
251
    "sseclient-py": ("sseclient",),
252
    "strawberry-graphql": ("strawberry",),
253
    "streamlit-aggrid": ("st_aggrid",),
254
    "unleashclient": ("UnleashClient",),
255
    "websocket-client": ("websocket",),
256
}
257

258
DEFAULT_TYPE_STUB_MODULE_PATTERN_MAPPING: dict[re.Pattern, list[Callable[[Match[str]], str]]] = {
11✔
259
    re.compile(r"""^stubs[_-](.+)"""): [first_group_hyphen_to_underscore],
260
    re.compile(r"""^types[_-](.+)"""): [first_group_hyphen_to_underscore],
261
    re.compile(r"""^(.+)[_-]stubs"""): [first_group_hyphen_to_underscore],
262
    re.compile(r"""^(.+)[_-]types"""): [first_group_hyphen_to_underscore],
263
}
264

265
DEFAULT_TYPE_STUB_MODULE_MAPPING: dict[str, tuple[str, ...]] = {
11✔
266
    "djangorestframework-types": ("rest_framework",),
267
    "lark-stubs": ("lark",),
268
    "types-beautifulsoup4": ("bs4",),
269
    "types-enum34": ("enum34",),
270
    "types-grpcio": ("grpc",),
271
    "types-grpcio-channelz": ("grpcio_channelz",),
272
    "types-grpcio-health-checking": ("grpc_health",),
273
    "types-grpcio-reflection": ("grpc_reflection",),
274
    "types-grpcio-status": ("grpc_status",),
275
    "types-pillow": ("PIL",),
276
    "types-protobuf": ("google.protobuf",),
277
    "types-pycrypto": ("Crypto",),
278
    "types-pyopenssl": ("OpenSSL",),
279
    "types-pyyaml": ("yaml",),
280
    "types-python-dateutil": ("dateutil",),
281
    "types-setuptools": ("easy_install", "pkg_resources", "setuptools"),
282
}
283

284
if __name__ == "__main__":
11✔
285
    import doctest
×
286

287
    doctest.testmod()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc