2593410123

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/vanvalenlab/deepcell-label/commit/<a class=hub.com/vanvalenlab/deepcell-label/commit/<a class="double-link" href="https://git"><a class=hub.com/vanvalenlab/deepcell-label/commit/<a class="double-link" href="https://git"><a class=hub.com/vanvalenlab/deepcell-label/commit/<a class="double-link" href="https://git"><a class=hub.com/vanvalenlab/deepcell-label/commit/<a class="double-link" href="https://git"><a class=hub.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590">3ab64029a">&lt;a href=&quot;https://github.com/vanvalenlab/deepcell-label/commit/</a><a class="double-link" href="https://github.com/vanvalenlab/deepcell-label/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/vanvalenlab/deepcell-label/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/vanvalenlab/deepcell-label/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/vanvalenlab/deepcell-label/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/vanvalenlab/deepcell-label/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590">3ab64029a</a><a href="https://github.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/vanvalenlab/deepcell-label/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/vanvalenlab/deepcell-label/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/vanvalenlab/deepcell-label/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/vanvalenlab/deepcell-label/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590&quot;&gt;3ab64029a&lt;/a&gt;&lt;a href=&quot;https://github.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/vanvalenlab/deepcell-label/commit/3ab64029a7dbabca8a24e4fd7e04d138205f5590&amp;quot;&amp;... (continued)

Pull Request Pull Request #319: Handle batches when loading TIFFs and zips of TIFFs

Run Details

313 of 606 branches covered (51.65%)

Branch coverage included in aggregate %.

36 of 52 new or added lines in 1 file covered. (69.23%)

2358 of 3518 relevant lines covered (67.03%)

323.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.41

/deepcell_label/loaders.py

"""
Class to load data into a DeepCell Label project file
Loads both raw image data and labels
"""

import io
import itertools
import json
import re
import tarfile
import tempfile
import zipfile

import magic
import numpy as np
from PIL import Image
from tifffile import TiffFile, TiffWriter

from deepcell_label.utils import convert_lineage, reshape


class Loader:
    """
    Loads and writes data into a DeepCell Label project zip.
    """

    def __init__(self, image_file=None, label_file=None, axes=None):
        """
        Args:
            image_file: file zip object containing a png, zip, tiff, or npz file
            label_file: file like object containing a zip
            axes: dimension order of the image data
        """
        self.X = None
        self.y = None
        self.spots = None
        self.divisions = None
        self.cells = None

        self.image_file = image_file
        self.label_file = label_file if label_file else image_file
        self.axes = axes

        with tempfile.TemporaryFile() as project_file:
            with zipfile.ZipFile(project_file, 'w', zipfile.ZIP_DEFLATED) as zip:
                self.zip = zip
                self.load()
                self.write()
            project_file.seek(0)
            self.data = project_file.read()

    def load(self):
        """Loads data from input files."""
        self.X = load_images(self.image_file, self.axes)
        self.y = load_segmentation(self.label_file)
        self.spots = load_spots(self.label_file)
        self.divisions = load_divisions(self.label_file)
        self.cells = load_cells(self.label_file)

        if self.y is None:
            shape = (*self.X.shape[:-1], 1)
            self.y = np.zeros(shape)

    def write(self):
        """Writes loaded data to zip."""
        self.write_images()
        self.write_segmentation()
        self.write_spots()
        self.write_divisions()
        self.write_cells()

    def write_images(self):
        """
        Writes raw images to X.ome.tiff in the output zip.

        Raises:
            ValueError: no image data has been loaded to write
        """
        X = self.X
        if X is not None:
            # Rescale data
            max, min = np.max(X), np.min(X)
            X = (X - min) / (max - min if max - min > 0 else 1) * 255
            X = X.astype(np.uint8)
            # Move channel axis
            X = np.moveaxis(X, -1, 1)
            images = io.BytesIO()
            with TiffWriter(images, ome=True) as tif:
                tif.write(X, compression='zlib', metadata={'axes': 'ZCYX'})
            images.seek(0)
            self.zip.writestr('X.ome.tiff', images.read())
        # else:
        #     raise ValueError('No images found in files')

    def write_segmentation(self):
        """Writes segmentation to y.ome.tiff in the output zip."""
        y = self.y
        if y.shape[:-1] != self.X.shape[:-1]:
            raise ValueError(
                'Segmentation shape %s is incompatible with image shape %s'
                % (y.shape, self.X.shape)
            )
        # TODO: check if float vs int matters
        y = y.astype(np.int32)
        # Move channel axis
        y = np.moveaxis(y, -1, 1)

        segmentation = io.BytesIO()
        with TiffWriter(segmentation, ome=True) as tif:
            tif.write(y, compression='zlib', metadata={'axes': 'ZCYX'})
        segmentation.seek(0)
        self.zip.writestr('y.ome.tiff', segmentation.read())

    def write_spots(self):
        """Writes spots to spots.csv in the output zip."""
        if self.spots is not None:
            buffer = io.BytesIO()
            buffer.write(self.spots)
            buffer.seek(0)
            self.zip.writestr('spots.csv', buffer.read())

    def write_divisions(self):
        """Writes divisions to divisions.json in the output zip."""
        self.zip.writestr('divisions.json', json.dumps(self.divisions))

    def write_cells(self):
        """Writes cells to cells.json in the output zip."""
        if self.cells is None:
            cells = []
            for t in range(self.y.shape[0]):
                for value in np.unique(self.y[t]):
                    if value != 0:
                        cells.append(
                            {'cell': int(value), 'value': int(value), 't': int(t)}
                        )
            self.cells = cells
        self.zip.writestr('cells.json', json.dumps(self.cells))


def load_images(image_file, axes=None):
    """
    Loads image data from image file.

    Args:
        image_file: zip, npy, tiff, or png file object containing image data

    Returns:
        numpy array or None if no image data found
    """
    X = load_zip(image_file)
    if X is None:
        X = load_npy(image_file)
    if X is None:
        X = load_tiff(image_file, axes)
    if X is None:
        X = load_png(image_file)
    if X is None:
        X = load_trk(image_file, filename='raw.npy')
    return X


def load_segmentation(f):
    """
    Loads segmentation array from label file.

    Args:
        label_file: file with zipped npy or tiff containing segmentation data

    Returns:
        numpy array or None if no segmentation data found
    """
    f.seek(0)
    if zipfile.is_zipfile(f):
        zf = zipfile.ZipFile(f, 'r')
        y = load_zip_numpy(zf, name='y')
        if y is None:
            y = load_zip_tiffs(zf, filename='y.ome.tiff')
        return y
    if tarfile.is_tarfile(f.name):
        return load_trk(f, filename='tracked.npy')


def load_spots(f):
    """
    Load spots data from label file.

    Args:
        zf: file with zipped csv containing spots data

    Returns:
        bytes read from csv in zip or None if no csv in zip
    """
    f.seek(0)
    if zipfile.is_zipfile(f):
        zf = zipfile.ZipFile(f, 'r')
        return load_zip_csv(zf)


def load_divisions(f):
    """
    Load divisions from divisions.json in project archive

    Loading from lineage.json from .trk file is supported, but deprecated.

    Args:
        zf: zip file with divisions.json
            or tarfile with lineage.json

    Returns:
        dict or None if divisions.json not found
    """
    f.seek(0)
    divisions = None
    if zipfile.is_zipfile(f):
        zf = zipfile.ZipFile(f, 'r')
        divisions = load_zip_json(zf, filename='divisions.json')
        lineage = load_zip_json(zf, filename='lineage.json')
        if lineage:
            divisions = convert_lineage(lineage)
    elif tarfile.is_tarfile(f.name):
        lineage = load_trk(f, filename='lineage.json')
        divisions = convert_lineage(lineage)
    if divisions is None:
        return []
    return divisions


def load_cells(f):
    """
    Load cells from label file.

    Args:
        zf: zip file with cells json

    Returns:
        dict or None if no json in zip
    """
    f.seek(0)
    if zipfile.is_zipfile(f):
        zf = zipfile.ZipFile(f, 'r')
        return load_zip_json(zf, filename='cells.json')


def load_zip_numpy(zf, name='X'):
    """
    Loads a numpy array from the zip file
    If loading an NPZ with multiple arrays, name selects which one to load

    Args:
        zf: a ZipFile with a npy or npz file
        name (str): name of the array to load

    Returns:
        numpy array or None if no png in zip
    """
    for filename in zf.namelist():
        if filename == f'{name}.npy':
            with zf.open(filename) as f:
                return np.load(f)
        if filename.endswith('.npz'):
            with zf.open(filename) as f:
                npz = np.load(f)
                return npz[name] if name in npz.files else npz[npz.files[0]]


def load_zip_tiffs(zf, filename):
    """
    Returns an array with all tiff image data in the zip file

    Args:
        zf: a ZipFile containing tiffs to load

    Returns:
        numpy array or None if no tiffs in zip
    """
    if filename in zf.namelist():
        with zf.open(filename) as f:
            if 'TIFF image data' in magic.from_buffer(f.read(2048)):
                f.seek(0)
                tiff = TiffFile(f)
                # TODO: check when there are multiple series
                axes = tiff.series[0].axes
                array = reshape(tiff.asarray(), axes, 'ZYXC')
                return array
            else:
                print(f'{filename} is not a tiff file.')
    else:
        print(f'{filename} not found in zip.')
    print('Loading all tiffs in zip.')
    tiffs = {}
    for name in zf.namelist():
        with zf.open(name) as f:
            if 'TIFF image data' in magic.from_buffer(f.read(2048)):
                f.seek(0)
                tiff = TiffFile(f).asarray()
                tiffs[name] = tiff
    if len(tiffs) > 0:
        regex = r'(.*)batch_(\d*)_feature_(\d*)\.tif'

        def get_batch(filename):
            match = re.match(regex, filename)
            if match:
                return int(match.group(2))

        def get_feature(filename):
            match = re.match(regex, filename)
            if match:
                return int(match.group(3))

        filenames = list(tiffs.keys())
        all_have_batch = all(map(lambda x: x is not None, map(get_batch, filenames)))
        if all_have_batch:  # Use batches as Z dimension
            batches = {}
            for batch, batch_group in itertools.groupby(filenames, get_batch):
                # Stack features on last axis
                features = [
                    tiffs[filename]
                    for filename in sorted(list(batch_group), key=get_feature)
                ]
                batches[batch] = np.stack(features, axis=-1)
            # Stack batches on first axis
            batches = map(lambda x: x[1][0], sorted(batches.items()))
            array = np.stack(batches, axis=0)
            return array
        else:  # Use each tiff as a channel and stack on the last axis
            y = np.stack(tiffs.values(), axis=-1)
            # Add Z axis
            if y.ndim == 3:
                y = y[np.newaxis, ...]
            return y


def load_zip_png(zf):
    """
    Returns the image data array for the first PNG image in the zip file

    Args:
        zf: a ZipFile with a PNG

    Returns:
        numpy array or None if no png in zip
    """
    for name in zf.namelist():
        with zf.open(name) as f:
            if 'PNG image data' in magic.from_buffer(f.read(2048)):
                f.seek(0)
                png = Image.open(f)
                return np.array(png)


def load_zip_csv(zf):
    """
    Returns the binary data for the first CSV file in the zip file, if it exists.

    Args:
        zf: a ZipFile with a CSV

    Returns:
        bytes or None if not a csv file
    """
    for name in zf.namelist():
        if name.endswith('.csv'):
            with zf.open(name) as f:
                return f.read()


def load_zip_json(zf, filename=None):
    """
    Returns a dicstion json file in the zip file, if it exists.

    Args:
        zf: a ZipFile with a CSV

    Returns:
        bytes or None if not a csv file
    """
    if filename in zf.namelist():
        with zf.open(filename) as f:
            try:
                f.seek(0)
                return json.load(f)
            except json.JSONDecodeError as e:
                print(f'Warning: Could not load {filename} as JSON. {e.msg}')
                return
    print(f'Warning: JSON file {filename} not found.')


def load_zip(f):
    """
    Loads image data from a zip file by loading from the npz, tiff, or png files in the archive

    Args:
        f: file object

    Returns:
        numpy array or None if not a zip file
    """
    f.seek(0)
    if zipfile.is_zipfile(f):
        zf = zipfile.ZipFile(f, 'r')
        X = load_zip_numpy(zf)
        if X is None:
            X = load_zip_tiffs(zf, filename='X.ome.tiff')
        if X is None:
            X = load_zip_png(zf)
        return X


def load_npy(f):
    """
    Loads image data from a npy file

    Args:
        f: file object

    Returns:
        numpy array or None if not a npy file
    """
    f.seek(0)
    if 'NumPy data file' in magic.from_buffer(f.read(2048)):
        f.seek(0)
        npy = np.load(f)
        return npy


def load_tiff(f, axes=None):
    """
    Loads image data from a tiff file

    Args:
        f: file object

    Returns:
        numpy array or None if not a tiff file

    Raises:
        ValueError: tiff has less than 2 or more than 4 dimensions
    """
    f.seek(0)
    if 'TIFF image data' in magic.from_buffer(f.read(2048)):
        f.seek(0)
        tiff = TiffFile(io.BytesIO(f.read()))
        # Load array
        if tiff.is_imagej:
            X = tiff.asarray()
            # TODO: use axes to know which axes to add and permute
            # TODO: handle tiffs with multiple series
            axes = tiff.series[0].axes
            if len(axes) != len(X.shape):
                print(
                    f'Warning: TIFF has shape {X.shape} and axes {axes} in ImageJ metadata'
                )
        elif tiff.is_ome:
            # TODO: use DimensionOrder from OME-TIFF metadata to know which axes to add and permute
            X = tiff.asarray(squeeze=False)
        else:
            X = tiff.asarray(squeeze=False)
        # Standardize dimensions
        if X.ndim == 0:
            raise ValueError('Loaded image has no data')
        elif X.ndim == 1:
            raise ValueError('Loaded tiff is 1 dimensional')
        elif X.ndim == 2:
            # Add Z and C axes
            return X[np.newaxis, ..., np.newaxis]
        elif X.ndim == 3:
            if axes == 'BXY':
                return X[..., np.newaxis]
            elif axes == 'XYC':
                return X[np.newaxis, ...]
            elif axes == 'CXY':
                X = np.moveaxis(X, 0, -1)
                return X[np.newaxis, ...]
            elif axes == 'XYB':
                X = np.moveaxis(X, -1, 0)
                return X[..., np.newaxis]
            else:  # Treat smallest axis as channels
                print(
                    f'Warning: tiff with shape {X.shape} has 3 dimensions '
                    f'with unsupported axes {axes}. '
                    'Using smallest axis as channels.'
                )
                smallest_axis = np.argmin(X.shape)
                X = np.moveaxis(X, smallest_axis, -1)
                return X[np.newaxis, ...]
        elif X.ndim == 4:
            if axes == 'BXYC':
                return X
            elif axes == 'CXYB':
                X = np.moveaxis(X, (0, -1), (-1, 0))
                return X
            else:
                print(
                    f'Warning: tiff with shape {X.shape} has 4 dimensions, '
                    f'but axes is {axes}. Assuming BXYC.'
                )
                return X
        else:
            raise ValueError(
                f'Loaded tiff with shape {X.shape} has more than 4 dimensions.'
            )


def load_png(f):
    """
    Loads image data from a png file

    Args:
        f: file object

    Returns:
        numpy array or None if not a png file
    """
    f.seek(0)
    if 'PNG image data' in magic.from_buffer(f.read(2048)):
        f.seek(0)
        image = Image.open(f, formats=['PNG'])
        # Add channel dimension at end to single channel images
        if image.mode == 'L':  # uint8
            X = np.array(image)
            X = np.expand_dims(X, -1)
        # TODO: support higher bit raw images
        # Currently all images are converted to uint8
        elif image.mode == 'I' or image.mode == 'F':  # int32 and float32
            # Rescale data
            max, min = np.max(image), np.min(image)
            X = (image - min) / (max - min if max - min > 0 else 1) * 255
            X = X.astype(np.uint8)
            X = np.expand_dims(X, -1)
        else:  # P, RGB, RGBA, CMYK,YCbCr
            # Create three RGB channels
            # Handles RGB, RGBA, P modes
            X = np.array(image.convert('RGB'))
        # Add T axis at start
        X = np.expand_dims(X, 0)
        return X


def load_trk(f, filename='raw.npy'):
    """
    Loads image data from a .trk file containing raw.npy, tracked.npy, and lineage.json

    Args:
        f: file object containing a .trk file
        filename: name of the file within the .trk to load

    Returns:
        numpy array (for raw.npy or tracked.npy) or dictionary (for lineage.json)
    """
    f.seek(0)
    if tarfile.is_tarfile(f.name):
        with tarfile.open(fileobj=f) as trks:
            if filename == 'raw.npy' or filename == 'tracked.npy':
                # numpy can't read these from disk...
                with io.BytesIO() as array_file:
                    array_file.write(trks.extractfile(filename).read())
                    array_file.seek(0)
                    return np.load(array_file)
            if filename == 'lineage.json':
                return json.loads(
                    trks.extractfile(trks.getmember('lineage.json')).read().decode()
                )

1	"""
2	Class to load data into a DeepCell Label project file
3	Loads both raw image data and labels
4	"""
5
6	import io	1✔
7	import itertools	1✔
8	import json	1✔
9	import re	1✔
10	import tarfile	1✔
11	import tempfile	1✔
12	import zipfile	1✔
13
14	import magic	1✔
15	import numpy as np	1✔
16	from PIL import Image	1✔
17	from tifffile import TiffFile, TiffWriter	1✔
18
19	from deepcell_label.utils import convert_lineage, reshape	1✔
20
21
22	class Loader:	1✔
23	"""
24	Loads and writes data into a DeepCell Label project zip.
25	"""
26
27	def __init__(self, image_file=None, label_file=None, axes=None):	1✔
28	"""
29	Args:
30	image_file: file zip object containing a png, zip, tiff, or npz file
31	label_file: file like object containing a zip
32	axes: dimension order of the image data
33	"""
34	self.X = None	1✔
35	self.y = None	1✔
36	self.spots = None	1✔
37	self.divisions = None	1✔
38	self.cells = None	1✔
39
40	self.image_file = image_file	1✔
41	self.label_file = label_file if label_file else image_file	1✔
42	self.axes = axes	1✔
43
44	with tempfile.TemporaryFile() as project_file:	1✔
45	with zipfile.ZipFile(project_file, 'w', zipfile.ZIP_DEFLATED) as zip:	1✔
46	self.zip = zip	1✔
47	self.load()	1✔
48	self.write()	1✔
49	project_file.seek(0)	1✔
50	self.data = project_file.read()	1✔
51
52	def load(self):	1✔
53	"""Loads data from input files."""
54	self.X = load_images(self.image_file, self.axes)	1✔
55	self.y = load_segmentation(self.label_file)	1✔
56	self.spots = load_spots(self.label_file)	1✔
57	self.divisions = load_divisions(self.label_file)	1✔
58	self.cells = load_cells(self.label_file)	1✔
59
60	if self.y is None:	1✔
61	shape = (*self.X.shape[:-1], 1)	1✔
62	self.y = np.zeros(shape)	1✔
63
64	def write(self):	1✔
65	"""Writes loaded data to zip."""
66	self.write_images()	1✔
67	self.write_segmentation()	1✔
68	self.write_spots()	1✔
69	self.write_divisions()	1✔
70	self.write_cells()	1✔
71
72	def write_images(self):	1✔
73	"""
74	Writes raw images to X.ome.tiff in the output zip.
75
76	Raises:
77	ValueError: no image data has been loaded to write
78	"""
79	X = self.X	1✔
80	if X is not None:	1✔
81	# Rescale data
82	max, min = np.max(X), np.min(X)	1✔
83	X = (X - min) / (max - min if max - min > 0 else 1) * 255	1✔
84	X = X.astype(np.uint8)	1✔
85	# Move channel axis
86	X = np.moveaxis(X, -1, 1)	1✔
87	images = io.BytesIO()	1✔
88	with TiffWriter(images, ome=True) as tif:	1✔
89	tif.write(X, compression='zlib', metadata={'axes': 'ZCYX'})	1✔
90	images.seek(0)	1✔
91	self.zip.writestr('X.ome.tiff', images.read())	1✔
92	# else:
93	# raise ValueError('No images found in files')
94
95	def write_segmentation(self):	1✔
96	"""Writes segmentation to y.ome.tiff in the output zip."""
97	y = self.y	1✔
98	if y.shape[:-1] != self.X.shape[:-1]:	1✔
99	raise ValueError(	×
100	'Segmentation shape %s is incompatible with image shape %s'
101	% (y.shape, self.X.shape)
102	)
103	# TODO: check if float vs int matters
104	y = y.astype(np.int32)	1✔
105	# Move channel axis
106	y = np.moveaxis(y, -1, 1)	1✔
107
108	segmentation = io.BytesIO()	1✔
109	with TiffWriter(segmentation, ome=True) as tif:	1✔
110	tif.write(y, compression='zlib', metadata={'axes': 'ZCYX'})	1✔
111	segmentation.seek(0)	1✔
112	self.zip.writestr('y.ome.tiff', segmentation.read())	1✔
113
114	def write_spots(self):	1✔
115	"""Writes spots to spots.csv in the output zip."""
116	if self.spots is not None:	1✔
117	buffer = io.BytesIO()	×
118	buffer.write(self.spots)	×
119	buffer.seek(0)	×
120	self.zip.writestr('spots.csv', buffer.read())	×
121
122	def write_divisions(self):	1✔
123	"""Writes divisions to divisions.json in the output zip."""
124	self.zip.writestr('divisions.json', json.dumps(self.divisions))	1✔
125
126	def write_cells(self):	1✔
127	"""Writes cells to cells.json in the output zip."""
128	if self.cells is None:	1✔
129	cells = []	1✔
130	for t in range(self.y.shape[0]):	1✔
131	for value in np.unique(self.y[t]):	1✔
132	if value != 0:	1✔
133	cells.append(	1✔
134	{'cell': int(value), 'value': int(value), 't': int(t)}
135	)
136	self.cells = cells	1✔
137	self.zip.writestr('cells.json', json.dumps(self.cells))	1✔
138
139
140	def load_images(image_file, axes=None):	1✔
141	"""
142	Loads image data from image file.
143
144	Args:
145	image_file: zip, npy, tiff, or png file object containing image data
146
147	Returns:
148	numpy array or None if no image data found
149	"""
150	X = load_zip(image_file)	1✔
151	if X is None:	1✔
152	X = load_npy(image_file)	1✔
153	if X is None:	1✔
154	X = load_tiff(image_file, axes)	1✔
155	if X is None:	1✔
156	X = load_png(image_file)	1✔
157	if X is None:	1✔
158	X = load_trk(image_file, filename='raw.npy')	1✔
159	return X	1✔
160
161
162	def load_segmentation(f):	1✔
163	"""
164	Loads segmentation array from label file.
165
166	Args:
167	label_file: file with zipped npy or tiff containing segmentation data
168
169	Returns:
170	numpy array or None if no segmentation data found
171	"""
172	f.seek(0)	1✔
173	if zipfile.is_zipfile(f):	1✔
174	zf = zipfile.ZipFile(f, 'r')	1✔
175	y = load_zip_numpy(zf, name='y')	1✔
176	if y is None:	1✔
177	y = load_zip_tiffs(zf, filename='y.ome.tiff')	1✔
178	return y	1✔
179	if tarfile.is_tarfile(f.name):	1✔
180	return load_trk(f, filename='tracked.npy')	×
181
182
183	def load_spots(f):	1✔
184	"""
185	Load spots data from label file.
186
187	Args:
188	zf: file with zipped csv containing spots data
189
190	Returns:
191	bytes read from csv in zip or None if no csv in zip
192	"""
193	f.seek(0)	1✔
194	if zipfile.is_zipfile(f):	1✔
195	zf = zipfile.ZipFile(f, 'r')	1✔
196	return load_zip_csv(zf)	1✔
197
198
199	def load_divisions(f):	1✔
200	"""
201	Load divisions from divisions.json in project archive
202
203	Loading from lineage.json from .trk file is supported, but deprecated.
204
205	Args:
206	zf: zip file with divisions.json
207	or tarfile with lineage.json
208
209	Returns:
210	dict or None if divisions.json not found
211	"""
212	f.seek(0)	1✔
213	divisions = None	1✔
214	if zipfile.is_zipfile(f):	1✔
215	zf = zipfile.ZipFile(f, 'r')	1✔
216	divisions = load_zip_json(zf, filename='divisions.json')	1✔
217	lineage = load_zip_json(zf, filename='lineage.json')	1✔
218	if lineage:	1✔
219	divisions = convert_lineage(lineage)	×
220	elif tarfile.is_tarfile(f.name):	1✔
221	lineage = load_trk(f, filename='lineage.json')	×
222	divisions = convert_lineage(lineage)	×
223	if divisions is None:	1✔
224	return []	1✔
225	return divisions	×
226
227
228	def load_cells(f):	1✔
229	"""
230	Load cells from label file.
231
232	Args:
233	zf: zip file with cells json
234
235	Returns:
236	dict or None if no json in zip
237	"""
238	f.seek(0)	1✔
239	if zipfile.is_zipfile(f):	1✔
240	zf = zipfile.ZipFile(f, 'r')	1✔
241	return load_zip_json(zf, filename='cells.json')	1✔
242
243
244	def load_zip_numpy(zf, name='X'):	1✔
245	"""
246	Loads a numpy array from the zip file
247	If loading an NPZ with multiple arrays, name selects which one to load
248
249	Args:
250	zf: a ZipFile with a npy or npz file
251	name (str): name of the array to load
252
253	Returns:
254	numpy array or None if no png in zip
255	"""
256	for filename in zf.namelist():	1✔
257	if filename == f'{name}.npy':	1✔
258	with zf.open(filename) as f:	1✔
259	return np.load(f)	1✔
260	if filename.endswith('.npz'):	1✔
261	with zf.open(filename) as f:	×
262	npz = np.load(f)	×
263	return npz[name] if name in npz.files else npz[npz.files[0]]	×
264
265
266	def load_zip_tiffs(zf, filename):	1✔
267	"""
268	Returns an array with all tiff image data in the zip file
269
270	Args:
271	zf: a ZipFile containing tiffs to load
272
273	Returns:
274	numpy array or None if no tiffs in zip
275	"""
276	if filename in zf.namelist():	1✔
277	with zf.open(filename) as f:	×
278	if 'TIFF image data' in magic.from_buffer(f.read(2048)):	×
279	f.seek(0)	×
280	tiff = TiffFile(f)	×
281	# TODO: check when there are multiple series
282	axes = tiff.series[0].axes	×
283	array = reshape(tiff.asarray(), axes, 'ZYXC')	×
284	return array	×
285	else:
286	print(f'{filename} is not a tiff file.')	×
287	else:
288	print(f'{filename} not found in zip.')	1✔
289	print('Loading all tiffs in zip.')	1✔
290	tiffs = {}	1✔
291	for name in zf.namelist():	1✔
292	with zf.open(name) as f:	1✔
293	if 'TIFF image data' in magic.from_buffer(f.read(2048)):	1✔
294	f.seek(0)	1✔
295	tiff = TiffFile(f).asarray()	1✔
296	tiffs[name] = tiff	1✔
297	if len(tiffs) > 0:	1✔
298	regex = r'(.)batch_(\d)_feature_(\d*)\.tif'	1✔
299
300	def get_batch(filename):	1✔
301	match = re.match(regex, filename)	1✔
302	if match:	1✔
303	return int(match.group(2))	1✔
304
305	def get_feature(filename):	1✔
306	match = re.match(regex, filename)	1✔
307	if match:	1✔
308	return int(match.group(3))	1✔
309
310	filenames = list(tiffs.keys())	1✔
311	all_have_batch = all(map(lambda x: x is not None, map(get_batch, filenames)))	1✔
312	if all_have_batch: # Use batches as Z dimension	1✔
313	batches = {}	1✔
314	for batch, batch_group in itertools.groupby(filenames, get_batch):	1✔
315	# Stack features on last axis
316	features = [	1✔
317	tiffs[filename]
318	for filename in sorted(list(batch_group), key=get_feature)
319	]
320	batches[batch] = np.stack(features, axis=-1)	1✔
321	# Stack batches on first axis
322	batches = map(lambda x: x[1][0], sorted(batches.items()))	1✔
323	array = np.stack(batches, axis=0)	1✔
324	return array	1✔
325	else: # Use each tiff as a channel and stack on the last axis
326	y = np.stack(tiffs.values(), axis=-1)	1✔
327	# Add Z axis
328	if y.ndim == 3:	1✔
329	y = y[np.newaxis, ...]	1✔
330	return y	1✔
331
332
333	def load_zip_png(zf):	1✔
334	"""
335	Returns the image data array for the first PNG image in the zip file
336
337	Args:
338	zf: a ZipFile with a PNG
339
340	Returns:
341	numpy array or None if no png in zip
342	"""
343	for name in zf.namelist():	×
344	with zf.open(name) as f:	×
345	if 'PNG image data' in magic.from_buffer(f.read(2048)):	×
346	f.seek(0)	×
347	png = Image.open(f)	×
348	return np.array(png)	×
349
350
351	def load_zip_csv(zf):	1✔
352	"""
353	Returns the binary data for the first CSV file in the zip file, if it exists.
354
355	Args:
356	zf: a ZipFile with a CSV
357
358	Returns:
359	bytes or None if not a csv file
360	"""
361	for name in zf.namelist():	1✔
362	if name.endswith('.csv'):	1✔
363	with zf.open(name) as f:	×
364	return f.read()	×
365
366
367	def load_zip_json(zf, filename=None):	1✔
368	"""
369	Returns a dicstion json file in the zip file, if it exists.
370
371	Args:
372	zf: a ZipFile with a CSV
373
374	Returns:
375	bytes or None if not a csv file
376	"""
377	if filename in zf.namelist():	1✔
378	with zf.open(filename) as f:	×
379	try:	×
380	f.seek(0)	×
381	return json.load(f)	×
382	except json.JSONDecodeError as e:	×
383	print(f'Warning: Could not load {filename} as JSON. {e.msg}')	×
384	return	×
385	print(f'Warning: JSON file {filename} not found.')	1✔
386
387
388	def load_zip(f):	1✔
389	"""
390	Loads image data from a zip file by loading from the npz, tiff, or png files in the archive
391
392	Args:
393	f: file object
394
395	Returns:
396	numpy array or None if not a zip file
397	"""
398	f.seek(0)	1✔
399	if zipfile.is_zipfile(f):	1✔
400	zf = zipfile.ZipFile(f, 'r')	1✔
401	X = load_zip_numpy(zf)	1✔
402	if X is None:	1✔
403	X = load_zip_tiffs(zf, filename='X.ome.tiff')	1✔
404	if X is None:	1✔
405	X = load_zip_png(zf)	×
406	return X	1✔
407
408
409	def load_npy(f):	1✔
410	"""
411	Loads image data from a npy file
412
413	Args:
414	f: file object
415
416	Returns:
417	numpy array or None if not a npy file
418	"""
419	f.seek(0)	1✔
420	if 'NumPy data file' in magic.from_buffer(f.read(2048)):	1✔
421	f.seek(0)	×
422	npy = np.load(f)	×
423	return npy	×
424
425
426	def load_tiff(f, axes=None):	1✔
427	"""
428	Loads image data from a tiff file
429
430	Args:
431	f: file object
432
433	Returns:
434	numpy array or None if not a tiff file
435
436	Raises:
437	ValueError: tiff has less than 2 or more than 4 dimensions
438	"""
439	f.seek(0)	1✔
440	if 'TIFF image data' in magic.from_buffer(f.read(2048)):	1✔
441	f.seek(0)	1✔
442	tiff = TiffFile(io.BytesIO(f.read()))	1✔
443	# Load array
444	if tiff.is_imagej:	1✔
445	X = tiff.asarray()	×
446	# TODO: use axes to know which axes to add and permute
447	# TODO: handle tiffs with multiple series
448	axes = tiff.series[0].axes	×
449	if len(axes) != len(X.shape):	×
450	print(	×
451	f'Warning: TIFF has shape {X.shape} and axes {axes} in ImageJ metadata'
452	)
453	elif tiff.is_ome:	1✔
454	# TODO: use DimensionOrder from OME-TIFF metadata to know which axes to add and permute
455	X = tiff.asarray(squeeze=False)	×
456	else:
457	X = tiff.asarray(squeeze=False)	1✔
458	# Standardize dimensions
459	if X.ndim == 0:	1✔
NEW 460	raise ValueError('Loaded image has no data')	×
461	elif X.ndim == 1:	1✔
NEW 462	raise ValueError('Loaded tiff is 1 dimensional')	×
463	elif X.ndim == 2:	1✔
464	# Add Z and C axes
465	return X[np.newaxis, ..., np.newaxis]	×
466	elif X.ndim == 3:	1✔
NEW 467	if axes == 'BXY':	×
NEW 468	return X[..., np.newaxis]	×
NEW 469	elif axes == 'XYC':	×
470	return X[np.newaxis, ...]	×
NEW 471	elif axes == 'CXY':	×
472	X = np.moveaxis(X, 0, -1)	×
473	return X[np.newaxis, ...]	×
NEW 474	elif axes == 'XYB':	×
NEW 475	X = np.moveaxis(X, -1, 0)	×
476	return X[..., np.newaxis]	×
477	else: # Treat smallest axis as channels
NEW 478	print(	×
479	f'Warning: tiff with shape {X.shape} has 3 dimensions '
480	f'with unsupported axes {axes}. '
481	'Using smallest axis as channels.'
482	)
NEW 483	smallest_axis = np.argmin(X.shape)	×
NEW 484	X = np.moveaxis(X, smallest_axis, -1)	×
NEW 485	return X[np.newaxis, ...]	×
486	elif X.ndim == 4:	1✔
487	if axes == 'BXYC':	1✔
NEW 488	return X	×
489	elif axes == 'CXYB':	1✔
NEW 490	X = np.moveaxis(X, (0, -1), (-1, 0))	×
NEW 491	return X	×
492	else:
493	print(	1✔
494	f'Warning: tiff with shape {X.shape} has 4 dimensions, '
495	f'but axes is {axes}. Assuming BXYC.'
496	)
497	return X	1✔
498	else:
NEW 499	raise ValueError(	×
500	f'Loaded tiff with shape {X.shape} has more than 4 dimensions.'
501	)
502
503
504	def load_png(f):	1✔
505	"""
506	Loads image data from a png file
507
508	Args:
509	f: file object
510
511	Returns:
512	numpy array or None if not a png file
513	"""
514	f.seek(0)	1✔
515	if 'PNG image data' in magic.from_buffer(f.read(2048)):	1✔
516	f.seek(0)	1✔
517	image = Image.open(f, formats=['PNG'])	1✔
518	# Add channel dimension at end to single channel images
519	if image.mode == 'L': # uint8	1✔
520	X = np.array(image)	1✔
521	X = np.expand_dims(X, -1)	1✔
522	# TODO: support higher bit raw images
523	# Currently all images are converted to uint8
524	elif image.mode == 'I' or image.mode == 'F': # int32 and float32	×
525	# Rescale data
526	max, min = np.max(image), np.min(image)	×
527	X = (image - min) / (max - min if max - min > 0 else 1) * 255	×
528	X = X.astype(np.uint8)	×
529	X = np.expand_dims(X, -1)	×
530	else: # P, RGB, RGBA, CMYK,YCbCr
531	# Create three RGB channels
532	# Handles RGB, RGBA, P modes
533	X = np.array(image.convert('RGB'))	×
534	# Add T axis at start
535	X = np.expand_dims(X, 0)	1✔
536	return X	1✔
537
538
539	def load_trk(f, filename='raw.npy'):	1✔
540	"""
541	Loads image data from a .trk file containing raw.npy, tracked.npy, and lineage.json
542
543	Args:
544	f: file object containing a .trk file
545	filename: name of the file within the .trk to load
546
547	Returns:
548	numpy array (for raw.npy or tracked.npy) or dictionary (for lineage.json)
549	"""
550	f.seek(0)	1✔
551	if tarfile.is_tarfile(f.name):	1✔
552	with tarfile.open(fileobj=f) as trks:	×
553	if filename == 'raw.npy' or filename == 'tracked.npy':	×
554	# numpy can't read these from disk...
555	with io.BytesIO() as array_file:	×
556	array_file.write(trks.extractfile(filename).read())	×
557	array_file.seek(0)	×
558	return np.load(array_file)	×
559	if filename == 'lineage.json':	×
560	return json.loads(	×
561	trks.extractfile(trks.getmember('lineage.json')).read().decode()
562	)

vanvalenlab / deepcell-label / 2593410123

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous