19114618549

Committed 05 Nov 2025 07:59PM UTC coverage: 94.093% (+0.03%) from 94.065%

Build # 19114618549

Build Type

Pull #1069

github

Committed by

peterkle

Commit Message

add more tests

Pull Request Pull Request #1069: RDISCROWD-8392 upgrade to boto3

Run Details

214 of 223 new or added lines in 7 files covered. (95.96%)

152 existing lines in 8 files now uncovered.

17920 of 19045 relevant lines covered (94.09%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.98

/pybossa/cloud_store_api/base_s3_client.py

from urllib.parse import urlsplit, urlunsplit
from botocore.exceptions import ClientError
from botocore.config import Config
import boto3
from pybossa.cloud_store_api.base_conn import BaseConnection


class BaseS3Client(BaseConnection):
    """
    Base class for S3 clients that provides common boto3 initialization
    and request modification patterns.

    This class extends BaseConnection to maintain compatibility with existing
    code while providing shared functionality for S3 client implementations.
    """

    def __init__(
        self,
        aws_access_key_id=None,
        aws_secret_access_key=None,
        aws_session_token=None,
        profile_name=None,
        endpoint_url=None,
        region_name=None,
        s3_ssl_no_verify=False,
        host_suffix="",
        **kwargs
    ):
        self.host_suffix = host_suffix or ""
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key

        # Initialize http_connection_kwargs for compatibility with legacy tests
        self.http_connection_kwargs = {}
        if s3_ssl_no_verify:
            import ssl
            self.http_connection_kwargs['context'] = ssl._create_unverified_context()

        # Create boto3 session
        session = (
            boto3.session.Session(profile_name=profile_name)
            if profile_name
            else boto3.session.Session()
        )

        # Configure path-style addressing (emulates OrdinaryCallingFormat)
        config = Config(
            region_name=region_name,
            s3={"addressing_style": "path"},
        )

        # Handle SSL verification
        verify = False if s3_ssl_no_verify else None  # None = default verify behavior

        self.client = session.client(
            "s3",
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key,
            aws_session_token=aws_session_token,
            endpoint_url=endpoint_url,
            config=config,
            verify=verify,
        )

        # Register hooks if needed - subclasses can override this logic
        if self._should_register_hooks():
            self.client.meta.events.register(
                "before-sign.s3",
                self._before_sign_hook,
            )

    def _should_register_hooks(self):
        """
        Determine when hooks should be registered.
        Subclasses can override this to customize hook registration logic.
        """
        return bool(self.host_suffix)

    def _before_sign_hook(self, request, **kwargs):
        """
        Base hook that handles host_suffix path modification.
        Subclasses can override or extend this method for additional functionality.
        """
        if self.host_suffix:
            self._apply_host_suffix(request)

    def _apply_host_suffix(self, request):
        """Apply host_suffix to the request URL path."""
        parts = urlsplit(request.url)
        # Ensure we don't double-prefix
        new_path = (self.host_suffix.rstrip("/") + "/" +
                    parts.path.lstrip("/")).replace("//", "/")
        request.url = urlunsplit(
            (parts.scheme, parts.netloc, new_path, parts.query, parts.fragment))

    def get_path(self, path):
        """
        Return the path with host_suffix prepended, for compatibility with legacy tests.
        This emulates the behavior that was expected from the old boto2 implementation.
        """
        if not self.host_suffix:
            return path
        
        # Normalize the path to ensure proper formatting
        if not path.startswith('/'):
            path = '/' + path
        
        # Combine host_suffix and path, avoiding double slashes
        combined = (self.host_suffix.rstrip("/") + "/" + path.lstrip("/")).replace("//", "/")
        
        # Ensure trailing slash if the original path was just '/'
        if path == '/' and not combined.endswith('/'):
            combined += '/'
            
        return combined

    # Override BaseConnection's delete_key to provide tolerant delete behavior
    def delete_key(self, bucket, path, **kwargs):
        """
        Delete an object, treating 200 and 204 as success.
        This overrides BaseConnection's delete_key to provide more tolerant behavior.
        """
        try:
            resp = self.client.delete_object(Bucket=bucket, Key=path, **kwargs)
            status = resp.get("ResponseMetadata", {}).get("HTTPStatusCode", 0)
            if status not in (200, 204):
                raise ClientError(
                    {
                        "Error": {"Code": str(status), "Message": "Unexpected status"},
                        "ResponseMetadata": {"HTTPStatusCode": status},
                    },
                    operation_name="DeleteObject",
                )
            return True
        except ClientError:
            # Propagate any other errors
            raise

    # Additional convenience methods for boto3 compatibility
    def get_object(self, bucket, key, **kwargs):
        """Get object using boto3 client interface."""
        return self.client.get_object(Bucket=bucket, Key=key, **kwargs)

    def put_object(self, bucket, key, body, **kwargs):
        """Put object using boto3 client interface."""
        return self.client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs)

    def list_objects(self, bucket, prefix="", **kwargs):
        """List objects using boto3 client interface."""
        return self.client.list_objects_v2(Bucket=bucket, Prefix=prefix, **kwargs)

    def upload_file(self, filename, bucket, key, **kwargs):
        """Upload file using boto3 client interface."""
        return self.client.upload_file(filename, bucket, key, ExtraArgs=kwargs or {})

    def raw(self):
        """Access the underlying boto3 client for advanced operations."""
        return self.client

1	from urllib.parse import urlsplit, urlunsplit	1✔
2	from botocore.exceptions import ClientError	1✔
3	from botocore.config import Config	1✔
4	import boto3	1✔
5	from pybossa.cloud_store_api.base_conn import BaseConnection	1✔
6
7
8	class BaseS3Client(BaseConnection):	1✔
9	"""
10	Base class for S3 clients that provides common boto3 initialization
11	and request modification patterns.
12
13	This class extends BaseConnection to maintain compatibility with existing
14	code while providing shared functionality for S3 client implementations.
15	"""
16
17	def __init__(	1✔
18	self,
19	aws_access_key_id=None,
20	aws_secret_access_key=None,
21	aws_session_token=None,
22	profile_name=None,
23	endpoint_url=None,
24	region_name=None,
25	s3_ssl_no_verify=False,
26	host_suffix="",
27	**kwargs
28	):
29	self.host_suffix = host_suffix or ""	1✔
30	self.aws_access_key_id = aws_access_key_id	1✔
31	self.aws_secret_access_key = aws_secret_access_key	1✔
32
33	# Initialize http_connection_kwargs for compatibility with legacy tests
34	self.http_connection_kwargs = {}	1✔
35	if s3_ssl_no_verify:	1✔
36	import ssl	1✔
37	self.http_connection_kwargs['context'] = ssl._create_unverified_context()	1✔
38
39	# Create boto3 session
40	session = (	1✔
41	boto3.session.Session(profile_name=profile_name)
42	if profile_name
43	else boto3.session.Session()
44	)
45
46	# Configure path-style addressing (emulates OrdinaryCallingFormat)
47	config = Config(	1✔
48	region_name=region_name,
49	s3={"addressing_style": "path"},
50	)
51
52	# Handle SSL verification
53	verify = False if s3_ssl_no_verify else None # None = default verify behavior	1✔
54
55	self.client = session.client(	1✔
56	"s3",
57	aws_access_key_id=aws_access_key_id,
58	aws_secret_access_key=aws_secret_access_key,
59	aws_session_token=aws_session_token,
60	endpoint_url=endpoint_url,
61	config=config,
62	verify=verify,
63	)
64
65	# Register hooks if needed - subclasses can override this logic
66	if self._should_register_hooks():	1✔
67	self.client.meta.events.register(	1✔
68	"before-sign.s3",
69	self._before_sign_hook,
70	)
71
72	def _should_register_hooks(self):	1✔
73	"""
74	Determine when hooks should be registered.
75	Subclasses can override this to customize hook registration logic.
76	"""
NEW 77	return bool(self.host_suffix)	×
78
79	def _before_sign_hook(self, request, **kwargs):	1✔
80	"""
81	Base hook that handles host_suffix path modification.
82	Subclasses can override or extend this method for additional functionality.
83	"""
84	if self.host_suffix:	1✔
85	self._apply_host_suffix(request)	1✔
86
87	def _apply_host_suffix(self, request):	1✔
88	"""Apply host_suffix to the request URL path."""
89	parts = urlsplit(request.url)	1✔
90	# Ensure we don't double-prefix
91	new_path = (self.host_suffix.rstrip("/") + "/" +	1✔
92	parts.path.lstrip("/")).replace("//", "/")
93	request.url = urlunsplit(	1✔
94	(parts.scheme, parts.netloc, new_path, parts.query, parts.fragment))
95
96	def get_path(self, path):	1✔
97	"""
98	Return the path with host_suffix prepended, for compatibility with legacy tests.
99	This emulates the behavior that was expected from the old boto2 implementation.
100	"""
101	if not self.host_suffix:	1✔
NEW 102	return path	×
103
104	# Normalize the path to ensure proper formatting
105	if not path.startswith('/'):	1✔
NEW 106	path = '/' + path	×
107
108	# Combine host_suffix and path, avoiding double slashes
109	combined = (self.host_suffix.rstrip("/") + "/" + path.lstrip("/")).replace("//", "/")	1✔
110
111	# Ensure trailing slash if the original path was just '/'
112	if path == '/' and not combined.endswith('/'):	1✔
NEW 113	combined += '/'	×
114
115	return combined	1✔
116
117	# Override BaseConnection's delete_key to provide tolerant delete behavior
118	def delete_key(self, bucket, path, **kwargs):	1✔
119	"""
120	Delete an object, treating 200 and 204 as success.
121	This overrides BaseConnection's delete_key to provide more tolerant behavior.
122	"""
123	try:	1✔
124	resp = self.client.delete_object(Bucket=bucket, Key=path, **kwargs)	1✔
125	status = resp.get("ResponseMetadata", {}).get("HTTPStatusCode", 0)	1✔
126	if status not in (200, 204):	1✔
127	raise ClientError(	1✔
128	{
129	"Error": {"Code": str(status), "Message": "Unexpected status"},
130	"ResponseMetadata": {"HTTPStatusCode": status},
131	},
132	operation_name="DeleteObject",
133	)
134	return True	1✔
135	except ClientError:	1✔
136	# Propagate any other errors
137	raise	1✔
138
139	# Additional convenience methods for boto3 compatibility
140	def get_object(self, bucket, key, **kwargs):	1✔
141	"""Get object using boto3 client interface."""
142	return self.client.get_object(Bucket=bucket, Key=key, **kwargs)	1✔
143
144	def put_object(self, bucket, key, body, **kwargs):	1✔
145	"""Put object using boto3 client interface."""
146	return self.client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs)	1✔
147
148	def list_objects(self, bucket, prefix="", **kwargs):	1✔
149	"""List objects using boto3 client interface."""
150	return self.client.list_objects_v2(Bucket=bucket, Prefix=prefix, **kwargs)	1✔
151
152	def upload_file(self, filename, bucket, key, **kwargs):	1✔
153	"""Upload file using boto3 client interface."""
154	return self.client.upload_file(filename, bucket, key, ExtraArgs=kwargs or {})	1✔
155
156	def raw(self):	1✔
157	"""Access the underlying boto3 client for advanced operations."""
158	return self.client	1✔

bloomberg / pybossa / 19114618549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous