#11948

pending completion

Build # #11948

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/datajoint/datajoint-python/commit/<a class=hub.com/datajoint/datajoint-python/commit/506806fcfa73a86e68771347f16cdd19c13bce54">506806fcf<a href="https://github.com/datajoint/datajoint-python/commit/506806fcfa73a86e68771347f16cdd19c13bce54">">Merge </a><a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/<a class="double-link" href="https://github.com/datajoint/datajoint-python/commit/98201fa81590f45189bfdc52757f4adf3d0296e9">98201fa81</a>">98201fa81</a><a href="https://github.com/datajoint/datajoint-python/commit/506806fcfa73a86e68771347f16cdd19c13bce54"> into 413e2f90d">413e2f90d</a>

Run Details

801 of 801 new or added lines in 25 files covered. (100.0%)

3061 of 3386 relevant lines covered (90.4%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.08

/datajoint/s3.py

"""
AWS S3 operations
"""
from io import BytesIO
import minio  # https://docs.minio.io/docs/python-client-api-reference
import urllib3
import warnings
import uuid
import logging
from pathlib import Path
from . import errors

logger = logging.getLogger(__name__)


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """

    def __init__(
        self,
        endpoint,
        bucket,
        access_key,
        secret_key,
        *,
        secure=False,
        proxy_server=None,
        **_
    ):
        # from https://docs.min.io/docs/python-client-api-reference
        self.client = minio.Minio(
            endpoint,
            access_key=access_key,
            secret_key=secret_key,
            secure=secure,
            http_client=(
                urllib3.ProxyManager(
                    proxy_server,
                    timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
                    cert_reqs="CERT_REQUIRED",
                    retries=urllib3.Retry(
                        total=5,
                        backoff_factor=0.2,
                        status_forcelist=[500, 502, 503, 504],
                    ),
                )
                if proxy_server
                else None
            ),
        )
        self.bucket = bucket
        if not self.client.bucket_exists(bucket):
            raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)

    def put(self, name, buffer):
        logger.debug("put: {}:{}".format(self.bucket, name))
        return self.client.put_object(
            self.bucket, str(name), BytesIO(buffer), length=len(buffer)
        )

    def fput(self, local_file, name, metadata=None):
        logger.debug("fput: {} -> {}:{}".format(self.bucket, local_file, name))
        return self.client.fput_object(
            self.bucket, str(name), str(local_file), metadata=metadata
        )

    def get(self, name):
        logger.debug("get: {}:{}".format(self.bucket, name))
        try:
            return self.client.get_object(self.bucket, str(name)).data
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                raise errors.MissingExternalFile("Missing s3 key %s" % name)
            else:
                raise e

    def fget(self, name, local_filepath):
        """get file from object name to local filepath"""
        logger.debug("fget: {}:{}".format(self.bucket, name))
        name = str(name)
        stat = self.client.stat_object(self.bucket, name)
        meta = {k.lower().lstrip("x-amz-meta"): v for k, v in stat.metadata.items()}
        data = self.client.get_object(self.bucket, name)
        local_filepath = Path(local_filepath)
        local_filepath.parent.mkdir(parents=True, exist_ok=True)
        with local_filepath.open("wb") as f:
            for d in data.stream(1 << 16):
                f.write(d)
        if "contents_hash" in meta:
            return uuid.UUID(meta["contents_hash"])

    def exists(self, name):
        logger.debug("exists: {}:{}".format(self.bucket, name))
        try:
            self.client.stat_object(self.bucket, str(name))
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                return False
            else:
                raise e
        return True

    def get_size(self, name):
        logger.debug("get_size: {}:{}".format(self.bucket, name))
        try:
            return self.client.stat_object(self.bucket, str(name)).size
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                raise errors.MissingExternalFile
            raise e

    def remove_object(self, name):
        logger.debug("remove_object: {}:{}".format(self.bucket, name))
        try:
            self.client.remove_object(self.bucket, str(name))
        except minio.error.MinioException:
            raise errors.DataJointError("Failed to delete %s from s3 storage" % name)

1	"""
2	AWS S3 operations
3	"""
4	from io import BytesIO	1✔
5	import minio # https://docs.minio.io/docs/python-client-api-reference	1✔
6	import urllib3	1✔
7	import warnings	1✔
8	import uuid	1✔
9	import logging	1✔
10	from pathlib import Path	1✔
11	from . import errors	1✔
12
13	logger = logging.getLogger(__name__)	1✔
14
15
16	class Folder:	1✔
17	"""
18	A Folder instance manipulates a flat folder of objects within an S3-compatible object store
19	"""
20
21	def __init__(	1✔
22	self,
23	endpoint,
24	bucket,
25	access_key,
26	secret_key,
27	*,
28	secure=False,
29	proxy_server=None,
30	**_
31	):
32	# from https://docs.min.io/docs/python-client-api-reference
33	self.client = minio.Minio(	1✔
34	endpoint,
35	access_key=access_key,
36	secret_key=secret_key,
37	secure=secure,
38	http_client=(
39	urllib3.ProxyManager(
40	proxy_server,
41	timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
42	cert_reqs="CERT_REQUIRED",
43	retries=urllib3.Retry(
44	total=5,
45	backoff_factor=0.2,
46	status_forcelist=[500, 502, 503, 504],
47	),
48	)
49	if proxy_server
50	else None
51	),
52	)
53	self.bucket = bucket	1✔
54	if not self.client.bucket_exists(bucket):	1✔
55	raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)	×
56
57	def put(self, name, buffer):	1✔
58	logger.debug("put: {}:{}".format(self.bucket, name))	1✔
59	return self.client.put_object(	1✔
60	self.bucket, str(name), BytesIO(buffer), length=len(buffer)
61	)
62
63	def fput(self, local_file, name, metadata=None):	1✔
64	logger.debug("fput: {} -> {}:{}".format(self.bucket, local_file, name))	1✔
65	return self.client.fput_object(	1✔
66	self.bucket, str(name), str(local_file), metadata=metadata
67	)
68
69	def get(self, name):	1✔
70	logger.debug("get: {}:{}".format(self.bucket, name))	1✔
71	try:	1✔
72	return self.client.get_object(self.bucket, str(name)).data	1✔
73	except minio.error.S3Error as e:	1✔
74	if e.code == "NoSuchKey":	1✔
75	raise errors.MissingExternalFile("Missing s3 key %s" % name)	1✔
76	else:
77	raise e	×
78
79	def fget(self, name, local_filepath):	1✔
80	"""get file from object name to local filepath"""
81	logger.debug("fget: {}:{}".format(self.bucket, name))	1✔
82	name = str(name)	1✔
83	stat = self.client.stat_object(self.bucket, name)	1✔
84	meta = {k.lower().lstrip("x-amz-meta"): v for k, v in stat.metadata.items()}	1✔
85	data = self.client.get_object(self.bucket, name)	1✔
86	local_filepath = Path(local_filepath)	1✔
87	local_filepath.parent.mkdir(parents=True, exist_ok=True)	1✔
88	with local_filepath.open("wb") as f:	1✔
89	for d in data.stream(1 << 16):	1✔
90	f.write(d)	1✔
91	if "contents_hash" in meta:	1✔
92	return uuid.UUID(meta["contents_hash"])	1✔
93
94	def exists(self, name):	1✔
95	logger.debug("exists: {}:{}".format(self.bucket, name))	1✔
96	try:	1✔
97	self.client.stat_object(self.bucket, str(name))	1✔
98	except minio.error.S3Error as e:	1✔
99	if e.code == "NoSuchKey":	1✔
100	return False	1✔
101	else:
102	raise e	×
103	return True	×
104
105	def get_size(self, name):	1✔
106	logger.debug("get_size: {}:{}".format(self.bucket, name))	×
107	try:	×
108	return self.client.stat_object(self.bucket, str(name)).size	×
109	except minio.error.S3Error as e:	×
110	if e.code == "NoSuchKey":	×
111	raise errors.MissingExternalFile	×
112	raise e	×
113
114	def remove_object(self, name):	1✔
115	logger.debug("remove_object: {}:{}".format(self.bucket, name))	1✔
116	try:	1✔
117	self.client.remove_object(self.bucket, str(name))	1✔
118	except minio.error.MinioException:	1✔
119	raise errors.DataJointError("Failed to delete %s from s3 storage" % name)	1✔

datajoint / datajoint-python / #11948

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous