24637157883

Committed 19 Apr 2026 07:23PM UTC coverage: 52.377% (-40.5%) from 92.924%

Build # 24637157883

Build Type

Pull #23274

github

Committed by

web-flow

Commit Message

Merge b54f275c2 into 0283af69e

Pull Request Pull Request #23274: rust: upgrade to v1.95.0

Coverage Stats

31658 of 60443 relevant lines covered (52.38%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/url_handlers/s3/register.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import logging
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
from urllib.parse import urlsplit

from pants.backend.url_handlers.s3.subsystem import S3AuthSigning, S3Subsystem
from pants.core.util_rules.env_vars import environment_vars_subset
from pants.engine.download_file import URLDownloadHandler
from pants.engine.env_vars import EnvironmentVarsRequest
from pants.engine.environment import ChosenLocalEnvironmentName, EnvironmentName
from pants.engine.fs import Digest, NativeDownloadFile
from pants.engine.internals.native_engine import EMPTY_FILE_DIGEST, FileDigest
from pants.engine.intrinsics import download_file
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.unions import UnionRule
from pants.option.global_options import GlobalOptions
from pants.util.strutil import softwrap

CONTENT_TYPE = "binary/octet-stream"

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class AWSCredentials:
    creds: Any
    default_region: str | None


@rule
async def access_aws_credentials(
    local_environment_name: ChosenLocalEnvironmentName,
) -> AWSCredentials:
    try:
        from botocore import credentials
        from botocore import session as boto_session
    except ImportError:
        logger.warning(
            softwrap(
                """
                In order to resolve s3:// URLs, Pants must load AWS credentials. To do so, `botocore`
                must be importable in Pants' environment.

                To do that add an entry to `[GLOBAL].plugins` of a pip-resolvable package to download from PyPI.
                (E.g. `botocore == 1.29.39`). Note that the `botocore` package from PyPI at the time
                of writing is >70MB, so an alternate package providing the `botocore` modules may be
                advisable (such as [`botocore-a-la-carte`](https://pypi.org/project/botocore-a-la-carte/)).
                """
            )
        )
        raise

    env_vars = await environment_vars_subset(
        EnvironmentVarsRequest(
            [
                "AWS_PROFILE",
                "AWS_REGION",
                "AWS_ACCESS_KEY_ID",
                "AWS_SECRET_ACCESS_KEY",
                "AWS_SESSION_TOKEN",
            ]
        ),
        **implicitly(
            {
                local_environment_name.val: EnvironmentName,
            }
        ),
    )

    session = boto_session.Session()

    aws_profile = env_vars.get("AWS_PROFILE")
    if aws_profile:
        session.set_config_variable("profile", aws_profile)

    aws_region = env_vars.get("AWS_REGION")
    if aws_region:
        session.set_config_variable("region", aws_region)

    aws_access_key = env_vars.get("AWS_ACCESS_KEY_ID")
    aws_secret_key = env_vars.get("AWS_SECRET_ACCESS_KEY")
    aws_session_token = env_vars.get("AWS_SESSION_TOKEN")
    if aws_access_key and aws_secret_key:
        session.set_credentials(
            credentials.Credentials(
                access_key=aws_access_key,
                secret_key=aws_secret_key,
                token=aws_session_token,
            )
        )

    creds = credentials.create_credential_resolver(session).load_credentials()
    default_region = session.get_config_variable("region")

    return AWSCredentials(creds=creds, default_region=default_region)


@dataclass(frozen=True)
class S3DownloadFile:
    region: str
    bucket: str
    key: str
    query: str
    expected_digest: FileDigest


@rule
async def download_from_s3(
    request: S3DownloadFile,
    aws_credentials: AWSCredentials,
    global_options: GlobalOptions,
    s3_subsystem: S3Subsystem,
) -> Digest:
    from botocore import auth, compat, exceptions  # pants: no-infer-dep

    virtual_hosted_url = f"https://{request.bucket}.s3.amazonaws.com/{request.key}"
    if request.region:
        virtual_hosted_url = (
            f"https://{request.bucket}.s3.{request.region}.amazonaws.com/{request.key}"
        )
    if request.query:
        virtual_hosted_url += f"?{request.query}"

    headers = compat.HTTPHeaders()
    signer = None
    http_request = None

    if s3_subsystem.auth_signing == S3AuthSigning.SIGV4:
        # sigv4 uses the virtual_hosted_url for the auth request
        http_request = SimpleNamespace(
            url=virtual_hosted_url,
            headers=headers,
            method="GET",
            auth_path=None,
            data=None,
            params={},
            context={},
            body={},
        )

        # Add x-amz-content-SHA256 as per boto code
        # ref link - https://github.com/boto/botocore/blob/547b20801770c8ea4255ee9c3b809fea6b9f6bc4/botocore/auth.py#L52C1-L54C2
        headers.add_header(
            "X-Amz-Content-SHA256",
            EMPTY_FILE_DIGEST.fingerprint,
        )

        # A region is required to sign the request with sigv4. If we don't know where the bucket is,
        # default to the region from the credentials
        signing_region = request.region or aws_credentials.default_region
        if not signing_region:
            raise Exception(
                "An aws region is required to sign requests with sigv4. Please specify a region in the url or configure the default region in aws config or environment variables."
            )

        signer = auth.SigV4Auth(aws_credentials.creds, "s3", signing_region)

    else:
        assert s3_subsystem.auth_signing == S3AuthSigning.HMACV1
        # NB: The URL for HmacV1 auth is expected to be in path-style
        path_style_url = "https://s3"
        if request.region:
            path_style_url += f".{request.region}"
        path_style_url += f".amazonaws.com/{request.bucket}/{request.key}"
        if request.query:
            path_style_url += f"?{request.query}"

        http_request = SimpleNamespace(
            url=path_style_url,
            headers=headers,
            method="GET",
            auth_path=None,
        )
        signer = auth.HmacV1Auth(aws_credentials.creds)

    # NB: The added Auth header doesn't need to be valid when accessing a public bucket. When
    # hand-testing, you MUST test against a private bucket to ensure it works for private buckets too.
    try:
        signer.add_auth(http_request)
    except exceptions.NoCredentialsError:
        pass  # The user can still access public S3 buckets without credentials

    return await download_file(
        NativeDownloadFile(
            url=virtual_hosted_url,
            expected_digest=request.expected_digest,
            auth_headers=dict(http_request.headers),
            retry_delay_duration=global_options.file_downloads_retry_delay,
            max_attempts=global_options.file_downloads_max_attempts,
        )
    )


class DownloadS3SchemeURL(URLDownloadHandler):
    match_scheme = "s3"


@rule
async def download_file_from_s3_scheme(request: DownloadS3SchemeURL) -> Digest:
    split = urlsplit(request.url)
    return await download_from_s3(
        S3DownloadFile(
            region="",
            bucket=split.netloc,
            key=split.path[1:],
            query=split.query,
            expected_digest=request.expected_digest,
        ),
        **implicitly(),
    )


class DownloadS3AuthorityVirtualHostedStyleURL(URLDownloadHandler):
    match_authority = "*.s3*amazonaws.com"


@rule
async def download_file_from_virtual_hosted_s3_authority(
    request: DownloadS3AuthorityVirtualHostedStyleURL,
) -> Digest:
    split = urlsplit(request.url)
    bucket, aws_netloc = split.netloc.split(".", 1)
    return await download_from_s3(
        S3DownloadFile(
            region=aws_netloc.split(".")[1] if aws_netloc.count(".") == 3 else "",
            bucket=bucket,
            key=split.path[1:],
            query=split.query,
            expected_digest=request.expected_digest,
        ),
        **implicitly(),
    )


class DownloadS3AuthorityPathStyleURL(URLDownloadHandler):
    match_authority = "s3.*amazonaws.com"


@rule
async def download_file_from_path_s3_authority(request: DownloadS3AuthorityPathStyleURL) -> Digest:
    split = urlsplit(request.url)
    _, bucket, key = split.path.split("/", 2)
    return await download_from_s3(
        S3DownloadFile(
            region=split.netloc.split(".")[1] if split.netloc.count(".") == 3 else "",
            bucket=bucket,
            key=key,
            query=split.query,
            expected_digest=request.expected_digest,
        ),
        **implicitly(),
    )


def rules():
    return [
        UnionRule(URLDownloadHandler, DownloadS3SchemeURL),
        UnionRule(URLDownloadHandler, DownloadS3AuthorityVirtualHostedStyleURL),
        UnionRule(URLDownloadHandler, DownloadS3AuthorityPathStyleURL),
        *collect_rules(),
    ]

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	×
4
5	import logging	×
6	from dataclasses import dataclass	×
7	from types import SimpleNamespace	×
8	from typing import Any	×
9	from urllib.parse import urlsplit	×
10
11	from pants.backend.url_handlers.s3.subsystem import S3AuthSigning, S3Subsystem	×
12	from pants.core.util_rules.env_vars import environment_vars_subset	×
13	from pants.engine.download_file import URLDownloadHandler	×
14	from pants.engine.env_vars import EnvironmentVarsRequest	×
15	from pants.engine.environment import ChosenLocalEnvironmentName, EnvironmentName	×
16	from pants.engine.fs import Digest, NativeDownloadFile	×
17	from pants.engine.internals.native_engine import EMPTY_FILE_DIGEST, FileDigest	×
18	from pants.engine.intrinsics import download_file	×
19	from pants.engine.rules import collect_rules, implicitly, rule	×
20	from pants.engine.unions import UnionRule	×
21	from pants.option.global_options import GlobalOptions	×
22	from pants.util.strutil import softwrap	×
23
24	CONTENT_TYPE = "binary/octet-stream"	×
25
26	logger = logging.getLogger(__name__)	×
27
28
29	@dataclass(frozen=True)	×
30	class AWSCredentials:	×
31	creds: Any	×
32	default_region: str \| None	×
33
34
35	@rule	×
36	async def access_aws_credentials(	×
37	local_environment_name: ChosenLocalEnvironmentName,
38	) -> AWSCredentials:
39	try:	×
40	from botocore import credentials	×
41	from botocore import session as boto_session	×
42	except ImportError:	×
43	logger.warning(	×
44	softwrap(
45	"""
46	In order to resolve s3:// URLs, Pants must load AWS credentials. To do so, `botocore`
47	must be importable in Pants' environment.
48
49	To do that add an entry to `[GLOBAL].plugins` of a pip-resolvable package to download from PyPI.
50	(E.g. `botocore == 1.29.39`). Note that the `botocore` package from PyPI at the time
51	of writing is >70MB, so an alternate package providing the `botocore` modules may be
52	advisable (such as [`botocore-a-la-carte`](https://pypi.org/project/botocore-a-la-carte/)).
53	"""
54	)
55	)
56	raise	×
57
58	env_vars = await environment_vars_subset(	×
59	EnvironmentVarsRequest(
60	[
61	"AWS_PROFILE",
62	"AWS_REGION",
63	"AWS_ACCESS_KEY_ID",
64	"AWS_SECRET_ACCESS_KEY",
65	"AWS_SESSION_TOKEN",
66	]
67	),
68	**implicitly(
69	{
70	local_environment_name.val: EnvironmentName,
71	}
72	),
73	)
74
75	session = boto_session.Session()	×
76
77	aws_profile = env_vars.get("AWS_PROFILE")	×
78	if aws_profile:	×
79	session.set_config_variable("profile", aws_profile)	×
80
81	aws_region = env_vars.get("AWS_REGION")	×
82	if aws_region:	×
83	session.set_config_variable("region", aws_region)	×
84
85	aws_access_key = env_vars.get("AWS_ACCESS_KEY_ID")	×
86	aws_secret_key = env_vars.get("AWS_SECRET_ACCESS_KEY")	×
87	aws_session_token = env_vars.get("AWS_SESSION_TOKEN")	×
88	if aws_access_key and aws_secret_key:	×
89	session.set_credentials(	×
90	credentials.Credentials(
91	access_key=aws_access_key,
92	secret_key=aws_secret_key,
93	token=aws_session_token,
94	)
95	)
96
97	creds = credentials.create_credential_resolver(session).load_credentials()	×
98	default_region = session.get_config_variable("region")	×
99
100	return AWSCredentials(creds=creds, default_region=default_region)	×
101
102
103	@dataclass(frozen=True)	×
104	class S3DownloadFile:	×
105	region: str	×
106	bucket: str	×
107	key: str	×
108	query: str	×
109	expected_digest: FileDigest	×
110
111
112	@rule	×
113	async def download_from_s3(	×
114	request: S3DownloadFile,
115	aws_credentials: AWSCredentials,
116	global_options: GlobalOptions,
117	s3_subsystem: S3Subsystem,
118	) -> Digest:
119	from botocore import auth, compat, exceptions # pants: no-infer-dep	×
120
121	virtual_hosted_url = f"https://{request.bucket}.s3.amazonaws.com/{request.key}"	×
122	if request.region:	×
123	virtual_hosted_url = (	×
124	f"https://{request.bucket}.s3.{request.region}.amazonaws.com/{request.key}"
125	)
126	if request.query:	×
127	virtual_hosted_url += f"?{request.query}"	×
128
129	headers = compat.HTTPHeaders()	×
130	signer = None	×
131	http_request = None	×
132
133	if s3_subsystem.auth_signing == S3AuthSigning.SIGV4:	×
134	# sigv4 uses the virtual_hosted_url for the auth request
135	http_request = SimpleNamespace(	×
136	url=virtual_hosted_url,
137	headers=headers,
138	method="GET",
139	auth_path=None,
140	data=None,
141	params={},
142	context={},
143	body={},
144	)
145
146	# Add x-amz-content-SHA256 as per boto code
147	# ref link - https://github.com/boto/botocore/blob/547b20801770c8ea4255ee9c3b809fea6b9f6bc4/botocore/auth.py#L52C1-L54C2
148	headers.add_header(	×
149	"X-Amz-Content-SHA256",
150	EMPTY_FILE_DIGEST.fingerprint,
151	)
152
153	# A region is required to sign the request with sigv4. If we don't know where the bucket is,
154	# default to the region from the credentials
155	signing_region = request.region or aws_credentials.default_region	×
156	if not signing_region:	×
157	raise Exception(	×
158	"An aws region is required to sign requests with sigv4. Please specify a region in the url or configure the default region in aws config or environment variables."
159	)
160
161	signer = auth.SigV4Auth(aws_credentials.creds, "s3", signing_region)	×
162
163	else:
164	assert s3_subsystem.auth_signing == S3AuthSigning.HMACV1	×
165	# NB: The URL for HmacV1 auth is expected to be in path-style
166	path_style_url = "https://s3"	×
167	if request.region:	×
168	path_style_url += f".{request.region}"	×
169	path_style_url += f".amazonaws.com/{request.bucket}/{request.key}"	×
170	if request.query:	×
171	path_style_url += f"?{request.query}"	×
172
173	http_request = SimpleNamespace(	×
174	url=path_style_url,
175	headers=headers,
176	method="GET",
177	auth_path=None,
178	)
179	signer = auth.HmacV1Auth(aws_credentials.creds)	×
180
181	# NB: The added Auth header doesn't need to be valid when accessing a public bucket. When
182	# hand-testing, you MUST test against a private bucket to ensure it works for private buckets too.
183	try:	×
184	signer.add_auth(http_request)	×
185	except exceptions.NoCredentialsError:	×
186	pass # The user can still access public S3 buckets without credentials	×
187
188	return await download_file(	×
189	NativeDownloadFile(
190	url=virtual_hosted_url,
191	expected_digest=request.expected_digest,
192	auth_headers=dict(http_request.headers),
193	retry_delay_duration=global_options.file_downloads_retry_delay,
194	max_attempts=global_options.file_downloads_max_attempts,
195	)
196	)
197
198
199	class DownloadS3SchemeURL(URLDownloadHandler):	×
200	match_scheme = "s3"	×
201
202
203	@rule	×
204	async def download_file_from_s3_scheme(request: DownloadS3SchemeURL) -> Digest:	×
205	split = urlsplit(request.url)	×
206	return await download_from_s3(	×
207	S3DownloadFile(
208	region="",
209	bucket=split.netloc,
210	key=split.path[1:],
211	query=split.query,
212	expected_digest=request.expected_digest,
213	),
214	**implicitly(),
215	)
216
217
218	class DownloadS3AuthorityVirtualHostedStyleURL(URLDownloadHandler):	×
219	match_authority = ".s3amazonaws.com"	×
220
221
222	@rule	×
223	async def download_file_from_virtual_hosted_s3_authority(	×
224	request: DownloadS3AuthorityVirtualHostedStyleURL,
225	) -> Digest:
226	split = urlsplit(request.url)	×
227	bucket, aws_netloc = split.netloc.split(".", 1)	×
228	return await download_from_s3(	×
229	S3DownloadFile(
230	region=aws_netloc.split(".")[1] if aws_netloc.count(".") == 3 else "",
231	bucket=bucket,
232	key=split.path[1:],
233	query=split.query,
234	expected_digest=request.expected_digest,
235	),
236	**implicitly(),
237	)
238
239
240	class DownloadS3AuthorityPathStyleURL(URLDownloadHandler):	×
241	match_authority = "s3.*amazonaws.com"	×
242
243
244	@rule	×
245	async def download_file_from_path_s3_authority(request: DownloadS3AuthorityPathStyleURL) -> Digest:	×
246	split = urlsplit(request.url)	×
247	_, bucket, key = split.path.split("/", 2)	×
248	return await download_from_s3(	×
249	S3DownloadFile(
250	region=split.netloc.split(".")[1] if split.netloc.count(".") == 3 else "",
251	bucket=bucket,
252	key=key,
253	query=split.query,
254	expected_digest=request.expected_digest,
255	),
256	**implicitly(),
257	)
258
259
260	def rules():	×
261	return [	×
262	UnionRule(URLDownloadHandler, DownloadS3SchemeURL),
263	UnionRule(URLDownloadHandler, DownloadS3AuthorityVirtualHostedStyleURL),
264	UnionRule(URLDownloadHandler, DownloadS3AuthorityPathStyleURL),
265	*collect_rules(),
266	]

pantsbuild / pants / 24637157883

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous