1556244288

Build Type

github

Committed by GitHub

Commit Message

Merge 668dcccdf into 98b5ddca3

Pull Request Pull Request #289: Use black to autoformat Python

Run Details

21 of 281 branches covered (7.47%)

Branch coverage included in aggregate %.

220 of 315 new or added lines in 10 files covered. (69.84%)

6 existing lines in 2 files now uncovered.

1410 of 3181 relevant lines covered (44.33%)

2.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.2

/deepcell_label/exporters.py

"""Classes to export a DeepCell Label project as a .npz or .trk file."""
import boto3
import io
import pathlib
import tempfile
import tarfile
import json
from urllib.parse import urlparse

import numpy as np

from deepcell_label.config import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY


class Exporter:
    """
    Interface to export work from a DeepCell Label project.
    """

    def __init__(self, project):
        self.project = project
        self.path = self.format_path()

    def format_path(self):
        """
        Converts the path to have a valid extension and
        adds the Project's token to create a unique filename.
        """
        path = urlparse(self.project.path).path
        path = path.strip("/")
        path = pathlib.Path(path)
        if self.project.is_track:
            path = path.with_suffix(".trk")
        else:
            path = path.with_suffix(".npz")
        return str(path)

    def export(self):
        """
        Exports an image stack from a DeepCell Label project,
        including raw image stack, labeled image stack, and optional label metadata dicts.
        """
        _export = self.get_export()
        filestream = _export()
        return filestream

    def get_export(self):
        """
        Returns:
            function: exports a DeepCell Label project into a BytesIO buffer
        """
        if self.project.is_zstack:
            _export = self.export_npz
        elif self.project.is_track:
            _export = self.export_trk
        else:
            raise ValueError("Cannot export file: {}".format(self.path))
        return _export

    def export_npz(self):
        """
        Creates a npz file based on the image stacks edited in a DeepCell Label project.

        Args:
            project (deepcell_label.models.Project):
                DeepCell Label project containing image data to save

        Returns:
            BytesIO: data buffer containing .npz data
        """
        # save file to BytesIO object
        store_npz = io.BytesIO()

        # X and y are array names by convention
        np.savez(store_npz, X=self.project.raw_array, y=self.project.label_array)
        store_npz.seek(0)

        return store_npz

    def export_trk(self):
        # clear any empty tracks before saving file
        tracks = self.project.labels.cell_info[0]
        empty_tracks = []
        for key in tracks:
            if not tracks[key]["frames"]:
                empty_tracks.append(tracks[key]["label"])
        for track in empty_tracks:
            del tracks[track]

        # Save image data to create file object in memory
        trk_file_obj = io.BytesIO()
        with tarfile.open(fileobj=trk_file_obj, mode="w") as trks:
            with tempfile.NamedTemporaryFile("w") as lineage_file:
                json.dump(tracks, lineage_file, indent=1)
                lineage_file.flush()
                trks.add(lineage_file.name, "lineage.json")

            with tempfile.NamedTemporaryFile() as raw_file:
                np.save(raw_file, self.project.raw_array)
                raw_file.flush()
                trks.add(raw_file.name, "raw.npy")

            with tempfile.NamedTemporaryFile() as tracked_file:
                np.save(tracked_file, self.project.label_array)
                tracked_file.flush()
                trks.add(tracked_file.name, "tracked.npy")

        trk_file_obj.seek(0)
        return trk_file_obj


class S3Exporter(Exporter):
    """
    Implementation of Exporter interface to upload files to S3 buckets.
    """

    def export(self, bucket):
        filestream = super().export()
        # store npz file object in bucket/path
        s3 = self._get_s3_client()
        s3.upload_fileobj(filestream, bucket, self.path)
        return filestream

    def _get_s3_client(self):
        return boto3.client(
            "s3",
            aws_access_key_id=AWS_ACCESS_KEY_ID,
            aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        )

1	"""Classes to export a DeepCell Label project as a .npz or .trk file."""
2	import boto3	4✔
3	import io	4✔
4	import pathlib	4✔
5	import tempfile	4✔
6	import tarfile	4✔
7	import json	4✔
8	from urllib.parse import urlparse	4✔
9
10	import numpy as np	4✔
11
12	from deepcell_label.config import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY	4✔
13
14
15	class Exporter:	4✔
16	"""
17	Interface to export work from a DeepCell Label project.
18	"""
19
20	def __init__(self, project):	4✔
21	self.project = project	4✔
22	self.path = self.format_path()	4✔
23
24	def format_path(self):	4✔
25	"""
26	Converts the path to have a valid extension and
27	adds the Project's token to create a unique filename.
28	"""
29	path = urlparse(self.project.path).path	4✔
30	path = path.strip("/")	4✔
31	path = pathlib.Path(path)	4✔
32	if self.project.is_track:	4✔
33	path = path.with_suffix(".trk")	4✔
34	else:
35	path = path.with_suffix(".npz")	4✔
36	return str(path)	4✔
37
38	def export(self):	4✔
39	"""
40	Exports an image stack from a DeepCell Label project,
41	including raw image stack, labeled image stack, and optional label metadata dicts.
42	"""
43	_export = self.get_export()	4✔
44	filestream = _export()	4✔
45	return filestream	4✔
46
47	def get_export(self):	4✔
48	"""
49	Returns:
50	function: exports a DeepCell Label project into a BytesIO buffer
51	"""
52	if self.project.is_zstack:	4✔
53	_export = self.export_npz	4✔
54	elif self.project.is_track:	4✔
55	_export = self.export_trk	4✔
56	else:
NEW 57	raise ValueError("Cannot export file: {}".format(self.path))	×
58	return _export	4✔
59
60	def export_npz(self):	4✔
61	"""
62	Creates a npz file based on the image stacks edited in a DeepCell Label project.
63
64	Args:
65	project (deepcell_label.models.Project):
66	DeepCell Label project containing image data to save
67
68	Returns:
69	BytesIO: data buffer containing .npz data
70	"""
71	# save file to BytesIO object
72	store_npz = io.BytesIO()	4✔
73
74	# X and y are array names by convention
75	np.savez(store_npz, X=self.project.raw_array, y=self.project.label_array)	4✔
76	store_npz.seek(0)	4✔
77
78	return store_npz	4✔
79
80	def export_trk(self):	4✔
81	# clear any empty tracks before saving file
82	tracks = self.project.labels.cell_info[0]	4✔
83	empty_tracks = []	4✔
84	for key in tracks:	4✔
NEW 85	if not tracks[key]["frames"]:	×
NEW 86	empty_tracks.append(tracks[key]["label"])	×
87	for track in empty_tracks:	4✔
88	del tracks[track]	×
89
90	# Save image data to create file object in memory
91	trk_file_obj = io.BytesIO()	4✔
92	with tarfile.open(fileobj=trk_file_obj, mode="w") as trks:	4✔
93	with tempfile.NamedTemporaryFile("w") as lineage_file:	4✔
94	json.dump(tracks, lineage_file, indent=1)	4✔
95	lineage_file.flush()	4✔
96	trks.add(lineage_file.name, "lineage.json")	4✔
97
98	with tempfile.NamedTemporaryFile() as raw_file:	4✔
99	np.save(raw_file, self.project.raw_array)	4✔
100	raw_file.flush()	4✔
101	trks.add(raw_file.name, "raw.npy")	4✔
102
103	with tempfile.NamedTemporaryFile() as tracked_file:	4✔
104	np.save(tracked_file, self.project.label_array)	4✔
105	tracked_file.flush()	4✔
106	trks.add(tracked_file.name, "tracked.npy")	4✔
107
108	trk_file_obj.seek(0)	4✔
109	return trk_file_obj	4✔
110
111
112	class S3Exporter(Exporter):	4✔
113	"""
114	Implementation of Exporter interface to upload files to S3 buckets.
115	"""
116
117	def export(self, bucket):	4✔
118	filestream = super().export()	4✔
119	# store npz file object in bucket/path
120	s3 = self._get_s3_client()	4✔
121	s3.upload_fileobj(filestream, bucket, self.path)	4✔
122	return filestream	4✔
123
124	def _get_s3_client(self):	4✔
125	return boto3.client(	4✔
126	"s3",
127	aws_access_key_id=AWS_ACCESS_KEY_ID,
128	aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
129	)

vanvalenlab / deepcell-label / 1556244288

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous