2498140206

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/vanvalenlab/deepcell-label/commit/<a class=hub.com/vanvalenlab/deepcell-label/commit/5664f4970592732952a156ae68a48b057fa9b460">5664f4970<a href="https://github.com/vanvalenlab/deepcell-label/commit/5664f4970592732952a156ae68a48b057fa9b460">">Merge </a><a class="double-link" href="https://github.com/vanvalenlab/deepcell-label/commit/<a class="double-link" href="https://github.com/vanvalenlab/deepcell-label/commit/cee9dd50cc0fd8e2e8c90ee38e83e4dd70bfaeb0">cee9dd50c</a>">cee9dd50c</a><a href="https://github.com/vanvalenlab/deepcell-label/commit/5664f4970592732952a156ae68a48b057fa9b460"> into 680a958de">680a958de</a>

Pull Request Pull Request #333: Update layout, instructions, exporting and remove visualizers

Run Details

310 of 609 branches covered (50.9%)

Branch coverage included in aggregate %.

188 of 228 new or added lines in 41 files covered. (82.46%)

2309 of 3449 relevant lines covered (66.95%)

508.44 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.84

/deepcell_label/export.py

"""
Converts data from the client into a ZIP to export from DeepCell Label.
"""

import io
import itertools
import json
import zipfile

import numpy as np
import tifffile


class Export:
    def __init__(self, labels_zip):
        self.labels_zip = labels_zip
        self.export_zip = io.BytesIO()

        self.load_dimensions()
        self.load_labeled()
        self.load_raw()
        self.load_cells()

        self.labeled, self.cells = rewrite_labeled(self.labeled, self.cells)

        self.write_export_zip()
        self.export_zip.seek(0)

    def load_dimensions(self):
        """Loads dimensions of the raw and labeled arrays from dimensions.json."""
        with zipfile.ZipFile(self.labels_zip) as zf:
            with zf.open('dimensions.json') as f:
                dimensions = json.load(f)
                self.height = dimensions['height']
                self.width = dimensions['width']
                self.duration = dimensions['duration']
                self.num_channels = dimensions['numChannels']
                self.num_features = dimensions['numFeatures']

    def load_labeled(self):
        """Loads the labeled array from the labeled.dat file."""
        with zipfile.ZipFile(self.labels_zip) as zf:
            with zf.open('labeled.dat') as f:
                labeled = np.frombuffer(f.read(), np.int32)
                self.labeled = np.reshape(
                    labeled,
                    (self.num_features, self.duration, self.height, self.width),
                )

    def load_raw(self):
        """Loads the raw array from the raw.dat file."""
        with zipfile.ZipFile(self.labels_zip) as zf:
            with zf.open('raw.dat') as f:
                raw = np.frombuffer(f.read(), np.uint8)
                self.raw = np.reshape(
                    raw, (self.num_channels, self.duration, self.height, self.width)
                )

    def load_cells(self):
        """Loads cell labels from cells.json."""
        with zipfile.ZipFile(self.labels_zip) as zf:
            with zf.open('cells.json') as f:
                self.cells = json.load(f)

    def write_export_zip(self):
        """Writes an export zip with OME TIFF files instead of raw.dat and labeled.dat."""
        # Rewrite all other files in input zip to export zip
        with zipfile.ZipFile(self.export_zip, 'w') as export_zf, zipfile.ZipFile(
            self.labels_zip
        ) as input_zf:
            for item in input_zf.infolist():
                # Writes all other files (divisions.json, spots.csv, etc.) to export zip
                if item.filename not in [
                    'dimensions.json',
                    'labeled.dat',
                    'raw.dat',
                    'cells.json',
                ]:
                    buffer = input_zf.read(item.filename)
                    export_zf.writestr(item, buffer)
            # Write updated cells
            export_zf.writestr('cells.json', json.dumps(self.cells))
            # Write OME TIFF for labeled
            labeled_ome_tiff = io.BytesIO()
            tifffile.imwrite(
                labeled_ome_tiff,
                self.labeled,
                ome=True,
                photometric='minisblack',
                compression='zlib',
                metadata={'axes': 'CZYX'},
            )
            labeled_ome_tiff.seek(0)
            export_zf.writestr('y.ome.tiff', labeled_ome_tiff.read())
            # Write OME TIFF for raw
            raw_ome_tiff = io.BytesIO()
            tifffile.imwrite(
                raw_ome_tiff,
                self.raw,
                ome=True,
                photometric='minisblack',
                compression='zlib',
                metadata={'axes': 'CZYX'},
            )
            raw_ome_tiff.seek(0)
            export_zf.writestr('X.ome.tiff', raw_ome_tiff.read())


def rewrite_labeled(labeled, cells):
    """
    Rewrites the labeled to use values from cell labels.

    Args:
        labeled: numpy array of shape (num_features, duration, height, width)
        cells: list of cells labels like { "cell": 1, "value": 1, "t": 0}

    Returns:
        (numpy array of shape (num_features, duration, height, width), cells with updated values)
    """
    new_labeled = np.zeros(labeled.shape)
    # TODO: add 'c' to cell labels to independently reassign cells between features & rewrite each feature separately
    (f, duration, height, width) = labeled.shape
    new_cells = []
    for t in range(duration):
        cells_at_t = list(filter(lambda c: c['t'] == t, cells))
        values = itertools.groupby(cells_at_t, lambda c: c['value'])
        overlap_values = []

        # Rewrite non-overlapping values with cells
        for value, group in values:
            group = list(group)
            if len(group) == 1:
                cell = group[0]['cell']
                frame = labeled[:, t, :, :]
                new_labeled[:, t, :, :][frame == value] = cell
                new_cells.append({'cell': cell, 'value': cell, 't': t})
            else:
                overlap_values.append([value, group])

        # Rewrite overlapping values with values higher than all cells
        new_overlap_value = max(cells_at_t, key=lambda c: c['cell'])['cell'] + 1
        for overlap_value, overlap_cells in overlap_values:
            for cell in overlap_cells:
                frame = labeled[:, t, :, :]
                new_labeled[:, t, :, :][frame == overlap_value] = new_overlap_value
                new_cells.append(
                    {'cell': cell['cell'], 'value': new_overlap_value, 't': t}
                )
            new_overlap_value += 1
    return new_labeled, new_cells

1	"""
2	Converts data from the client into a ZIP to export from DeepCell Label.
3	"""
4
5	import io	1✔
6	import itertools	1✔
7	import json	1✔
8	import zipfile	1✔
9
10	import numpy as np	1✔
11	import tifffile	1✔
12
13
14	class Export:	1✔
15	def __init__(self, labels_zip):	1✔
16	self.labels_zip = labels_zip	×
17	self.export_zip = io.BytesIO()	×
18
19	self.load_dimensions()	×
20	self.load_labeled()	×
21	self.load_raw()	×
NEW 22	self.load_cells()	×
23
NEW 24	self.labeled, self.cells = rewrite_labeled(self.labeled, self.cells)	×
25
26	self.write_export_zip()	×
27	self.export_zip.seek(0)	×
28
29	def load_dimensions(self):	1✔
30	"""Loads dimensions of the raw and labeled arrays from dimensions.json."""
31	with zipfile.ZipFile(self.labels_zip) as zf:	×
32	with zf.open('dimensions.json') as f:	×
33	dimensions = json.load(f)	×
34	self.height = dimensions['height']	×
35	self.width = dimensions['width']	×
36	self.duration = dimensions['duration']	×
37	self.num_channels = dimensions['numChannels']	×
38	self.num_features = dimensions['numFeatures']	×
39
40	def load_labeled(self):	1✔
41	"""Loads the labeled array from the labeled.dat file."""
42	with zipfile.ZipFile(self.labels_zip) as zf:	×
43	with zf.open('labeled.dat') as f:	×
44	labeled = np.frombuffer(f.read(), np.int32)	×
45	self.labeled = np.reshape(	×
46	labeled,
47	(self.num_features, self.duration, self.height, self.width),
48	)
49
50	def load_raw(self):	1✔
51	"""Loads the raw array from the raw.dat file."""
52	with zipfile.ZipFile(self.labels_zip) as zf:	×
53	with zf.open('raw.dat') as f:	×
54	raw = np.frombuffer(f.read(), np.uint8)	×
55	self.raw = np.reshape(	×
56	raw, (self.num_channels, self.duration, self.height, self.width)
57	)
58
59	def load_cells(self):	1✔
60	"""Loads cell labels from cells.json."""
NEW 61	with zipfile.ZipFile(self.labels_zip) as zf:	×
NEW 62	with zf.open('cells.json') as f:	×
NEW 63	self.cells = json.load(f)	×
64
65	def write_export_zip(self):	1✔
66	"""Writes an export zip with OME TIFF files instead of raw.dat and labeled.dat."""
67	# Rewrite all other files in input zip to export zip
68	with zipfile.ZipFile(self.export_zip, 'w') as export_zf, zipfile.ZipFile(	×
69	self.labels_zip
70	) as input_zf:
71	for item in input_zf.infolist():	×
72	# Writes all other files (divisions.json, spots.csv, etc.) to export zip
NEW 73	if item.filename not in [	×
74	'dimensions.json',
75	'labeled.dat',
76	'raw.dat',
77	'cells.json',
78	]:
79	buffer = input_zf.read(item.filename)	×
80	export_zf.writestr(item, buffer)	×
81	# Write updated cells
NEW 82	export_zf.writestr('cells.json', json.dumps(self.cells))	×
83	# Write OME TIFF for labeled
84	labeled_ome_tiff = io.BytesIO()	×
85	tifffile.imwrite(	×
86	labeled_ome_tiff,
87	self.labeled,
88	ome=True,
89	photometric='minisblack',
90	compression='zlib',
91	metadata={'axes': 'CZYX'},
92	)
93	labeled_ome_tiff.seek(0)	×
94	export_zf.writestr('y.ome.tiff', labeled_ome_tiff.read())	×
95	# Write OME TIFF for raw
96	raw_ome_tiff = io.BytesIO()	×
97	tifffile.imwrite(	×
98	raw_ome_tiff,
99	self.raw,
100	ome=True,
101	photometric='minisblack',
102	compression='zlib',
103	metadata={'axes': 'CZYX'},
104	)
105	raw_ome_tiff.seek(0)	×
106	export_zf.writestr('X.ome.tiff', raw_ome_tiff.read())	×
107
108
109	def rewrite_labeled(labeled, cells):	1✔
110	"""
111	Rewrites the labeled to use values from cell labels.
112
113	Args:
114	labeled: numpy array of shape (num_features, duration, height, width)
115	cells: list of cells labels like { "cell": 1, "value": 1, "t": 0}
116
117	Returns:
118	(numpy array of shape (num_features, duration, height, width), cells with updated values)
119	"""
120	new_labeled = np.zeros(labeled.shape)	1✔
121	# TODO: add 'c' to cell labels to independently reassign cells between features & rewrite each feature separately
122	(f, duration, height, width) = labeled.shape	1✔
123	new_cells = []	1✔
124	for t in range(duration):	1✔
125	cells_at_t = list(filter(lambda c: c['t'] == t, cells))	1✔
126	values = itertools.groupby(cells_at_t, lambda c: c['value'])	1✔
127	overlap_values = []	1✔
128
129	# Rewrite non-overlapping values with cells
130	for value, group in values:	1✔
131	group = list(group)	1✔
132	if len(group) == 1:	1✔
133	cell = group[0]['cell']	1✔
134	frame = labeled[:, t, :, :]	1✔
135	new_labeled[:, t, :, :][frame == value] = cell	1✔
136	new_cells.append({'cell': cell, 'value': cell, 't': t})	1✔
137	else:
138	overlap_values.append([value, group])	1✔
139
140	# Rewrite overlapping values with values higher than all cells
141	new_overlap_value = max(cells_at_t, key=lambda c: c['cell'])['cell'] + 1	1✔
142	for overlap_value, overlap_cells in overlap_values:	1✔
143	for cell in overlap_cells:	1✔
144	frame = labeled[:, t, :, :]	1✔
145	new_labeled[:, t, :, :][frame == overlap_value] = new_overlap_value	1✔
146	new_cells.append(	1✔
147	{'cell': cell['cell'], 'value': new_overlap_value, 't': t}
148	)
149	new_overlap_value += 1	1✔
150	return new_labeled, new_cells	1✔

vanvalenlab / deepcell-label / 2498140206

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous