#12880

pending completion

Build # #12880

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #1067 from CBroz1/master

Add support for insert CSV

Run Details

4 of 4 new or added lines in 1 file covered. (100.0%)

3102 of 3424 relevant lines covered (90.6%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.81

/datajoint/s3.py

"""
AWS S3 operations
"""
from io import BytesIO
import minio  # https://docs.minio.io/docs/python-client-api-reference
import urllib3
import uuid
import logging
from pathlib import Path
from . import errors

logger = logging.getLogger(__name__.split(".")[0])


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """

    def __init__(
        self,
        endpoint,
        bucket,
        access_key,
        secret_key,
        *,
        secure=False,
        proxy_server=None,
        **_
    ):
        # from https://docs.min.io/docs/python-client-api-reference
        self.client = minio.Minio(
            endpoint,
            access_key=access_key,
            secret_key=secret_key,
            secure=secure,
            http_client=(
                urllib3.ProxyManager(
                    proxy_server,
                    timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
                    cert_reqs="CERT_REQUIRED",
                    retries=urllib3.Retry(
                        total=5,
                        backoff_factor=0.2,
                        status_forcelist=[500, 502, 503, 504],
                    ),
                )
                if proxy_server
                else None
            ),
        )
        self.bucket = bucket
        if not self.client.bucket_exists(bucket):
            raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)

    def put(self, name, buffer):
        logger.debug("put: {}:{}".format(self.bucket, name))
        return self.client.put_object(
            self.bucket, str(name), BytesIO(buffer), length=len(buffer)
        )

    def fput(self, local_file, name, metadata=None):
        logger.debug("fput: {} -> {}:{}".format(self.bucket, local_file, name))
        return self.client.fput_object(
            self.bucket, str(name), str(local_file), metadata=metadata
        )

    def get(self, name):
        logger.debug("get: {}:{}".format(self.bucket, name))
        try:
            return self.client.get_object(self.bucket, str(name)).data
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                raise errors.MissingExternalFile("Missing s3 key %s" % name)
            else:
                raise e

    def fget(self, name, local_filepath):
        """get file from object name to local filepath"""
        logger.debug("fget: {}:{}".format(self.bucket, name))
        name = str(name)
        stat = self.client.stat_object(self.bucket, name)
        meta = {k.lower().lstrip("x-amz-meta"): v for k, v in stat.metadata.items()}
        data = self.client.get_object(self.bucket, name)
        local_filepath = Path(local_filepath)
        local_filepath.parent.mkdir(parents=True, exist_ok=True)
        with local_filepath.open("wb") as f:
            for d in data.stream(1 << 16):
                f.write(d)
        if "contents_hash" in meta:
            return uuid.UUID(meta["contents_hash"])

    def exists(self, name):
        logger.debug("exists: {}:{}".format(self.bucket, name))
        try:
            self.client.stat_object(self.bucket, str(name))
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                return False
            else:
                raise e
        return True

    def get_size(self, name):
        logger.debug("get_size: {}:{}".format(self.bucket, name))
        try:
            return self.client.stat_object(self.bucket, str(name)).size
        except minio.error.S3Error as e:
            if e.code == "NoSuchKey":
                raise errors.MissingExternalFile
            raise e

    def remove_object(self, name):
        logger.debug("remove_object: {}:{}".format(self.bucket, name))
        try:
            self.client.remove_object(self.bucket, str(name))
        except minio.error.MinioException:
            raise errors.DataJointError("Failed to delete %s from s3 storage" % name)

1	"""
2	AWS S3 operations
3	"""
4	from io import BytesIO	1✔
5	import minio # https://docs.minio.io/docs/python-client-api-reference	1✔
6	import urllib3	1✔
7	import uuid	1✔
8	import logging	1✔
9	from pathlib import Path	1✔
10	from . import errors	1✔
11
12	logger = logging.getLogger(__name__.split(".")[0])	1✔
13
14
15	class Folder:	1✔
16	"""
17	A Folder instance manipulates a flat folder of objects within an S3-compatible object store
18	"""
19
20	def __init__(	1✔
21	self,
22	endpoint,
23	bucket,
24	access_key,
25	secret_key,
26	*,
27	secure=False,
28	proxy_server=None,
29	**_
30	):
31	# from https://docs.min.io/docs/python-client-api-reference
32	self.client = minio.Minio(	1✔
33	endpoint,
34	access_key=access_key,
35	secret_key=secret_key,
36	secure=secure,
37	http_client=(
38	urllib3.ProxyManager(
39	proxy_server,
40	timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
41	cert_reqs="CERT_REQUIRED",
42	retries=urllib3.Retry(
43	total=5,
44	backoff_factor=0.2,
45	status_forcelist=[500, 502, 503, 504],
46	),
47	)
48	if proxy_server
49	else None
50	),
51	)
52	self.bucket = bucket	1✔
53	if not self.client.bucket_exists(bucket):	1✔
54	raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)	×
55
56	def put(self, name, buffer):	1✔
57	logger.debug("put: {}:{}".format(self.bucket, name))	1✔
58	return self.client.put_object(	1✔
59	self.bucket, str(name), BytesIO(buffer), length=len(buffer)
60	)
61
62	def fput(self, local_file, name, metadata=None):	1✔
63	logger.debug("fput: {} -> {}:{}".format(self.bucket, local_file, name))	1✔
64	return self.client.fput_object(	1✔
65	self.bucket, str(name), str(local_file), metadata=metadata
66	)
67
68	def get(self, name):	1✔
69	logger.debug("get: {}:{}".format(self.bucket, name))	1✔
70	try:	1✔
71	return self.client.get_object(self.bucket, str(name)).data	1✔
72	except minio.error.S3Error as e:	1✔
73	if e.code == "NoSuchKey":	1✔
74	raise errors.MissingExternalFile("Missing s3 key %s" % name)	1✔
75	else:
76	raise e	×
77
78	def fget(self, name, local_filepath):	1✔
79	"""get file from object name to local filepath"""
80	logger.debug("fget: {}:{}".format(self.bucket, name))	1✔
81	name = str(name)	1✔
82	stat = self.client.stat_object(self.bucket, name)	1✔
83	meta = {k.lower().lstrip("x-amz-meta"): v for k, v in stat.metadata.items()}	1✔
84	data = self.client.get_object(self.bucket, name)	1✔
85	local_filepath = Path(local_filepath)	1✔
86	local_filepath.parent.mkdir(parents=True, exist_ok=True)	1✔
87	with local_filepath.open("wb") as f:	1✔
88	for d in data.stream(1 << 16):	1✔
89	f.write(d)	1✔
90	if "contents_hash" in meta:	1✔
91	return uuid.UUID(meta["contents_hash"])	1✔
92
93	def exists(self, name):	1✔
94	logger.debug("exists: {}:{}".format(self.bucket, name))	1✔
95	try:	1✔
96	self.client.stat_object(self.bucket, str(name))	1✔
97	except minio.error.S3Error as e:	1✔
98	if e.code == "NoSuchKey":	1✔
99	return False	1✔
100	else:
101	raise e	×
102	return True	×
103
104	def get_size(self, name):	1✔
105	logger.debug("get_size: {}:{}".format(self.bucket, name))	×
106	try:	×
107	return self.client.stat_object(self.bucket, str(name)).size	×
108	except minio.error.S3Error as e:	×
109	if e.code == "NoSuchKey":	×
110	raise errors.MissingExternalFile	×
111	raise e	×
112
113	def remove_object(self, name):	1✔
114	logger.debug("remove_object: {}:{}".format(self.bucket, name))	1✔
115	try:	1✔
116	self.client.remove_object(self.bucket, str(name))	1✔
117	except minio.error.MinioException:	1✔
118	raise errors.DataJointError("Failed to delete %s from s3 storage" % name)	1✔

datajoint / datajoint-python / #12880

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous