7536

pending completion

Build # 7536

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Adding TMI (#825)

* Added TMI

* swapped an instance of imsave

* pillow requirements

* docs build fix

* import sorting, remove unused imports

* add missing tmi import

* removed files

* use tmi v0.0.1

* tmi merging updates

tmi merging updates - notebooks

* isort

* test flexibile tmi

* travis test

* charset-normalizer normalize this

* add pip to travis install

* use tmi v0.1.0

Run Details

85 of 85 new or added lines in 18 files covered. (100.0%)

2990 of 3149 relevant lines covered (94.95%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.56

/ark/utils/plot_utils.py

import os
import pathlib
import shutil
from operator import contains
from typing import List, Union

import matplotlib.cm as cm
import matplotlib.colors as colors
import matplotlib.pyplot as plt
import natsort
import numpy as np
import pandas as pd
import xarray as xr
from mpl_toolkits.axes_grid1 import make_axes_locatable
from skimage.exposure import rescale_intensity
from skimage.segmentation import find_boundaries
from tmi import io_utils, load_utils, misc_utils
from tmi.settings import EXTENSION_TYPES


def plot_neighborhood_cluster_result(img_xr, fovs, k, save_dir=None, cmap_name='tab20',
                                     fov_col='fovs', figsize=(10, 10)):
    """Takes an xarray containing labeled images and displays them.
    Args:
        img_xr (xarray.DataArray):
            xarray containing labeled cell objects.
        fovs (list):
            list of fovs to display.
        k (int):
            number of clusters (neighborhoods)
        save_dir (str):
            If provided, the image will be saved to this location.
        cmap_name (str):
            Cmap to use for the image that will be displayed.
        fov_col (str):
            column with the fovs names in `img_xr`.
        figsize (tuple):
            Size of the image that will be displayed.
    """

    # verify the fovs are valid
    misc_utils.verify_in_list(fov_names=fovs, unique_fovs=img_xr.fovs.values)

    # define the colormap, add black for empty slide
    mycols = cm.get_cmap(cmap_name, k).colors
    mycols = np.vstack(([0, 0, 0, 1], mycols))
    cmap = colors.ListedColormap(mycols)
    bounds = [i-0.5 for i in np.linspace(0, k+1, k+2)]
    norm = colors.BoundaryNorm(bounds, cmap.N)

    for fov in fovs:
        # define the figure
        plt.figure(figsize=figsize)

        # define the axis
        ax = plt.gca()

        # make the title
        plt.title(fov)

        # show the image on the figure
        im = plt.imshow(img_xr[img_xr[fov_col] == fov].values.squeeze(),
                        cmap=cmap, norm=norm)

        # remove the axes
        plt.axis('off')

        # remove the gridlines
        plt.grid(visible=None)

        # ensure the colorbar matches up with the margins on the right
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)

        # draw the colorbar
        tick_names = ['Cluster'+str(x) for x in range(1, k+1)]
        tick_names = ['Empty'] + tick_names
        cbar = plt.colorbar(im, cax=cax, ticks=np.arange(len(tick_names)))
        cbar.set_ticks(cbar.ax.get_yticks())
        cbar.ax.set_yticklabels(tick_names)

        # save if specified
        if save_dir:
            misc_utils.save_figure(save_dir, f'{fov}.png')


# TODO: possibly need to merge this with plot_neighborhood_cluster_result
def plot_pixel_cell_cluster_overlay(img_xr, fovs, cluster_id_to_name_path, metacluster_colors,
                                    save_dir=None, fov_col='fovs', figsize=(10, 10)):
    """Overlays the pixel and cell clusters on an image

    Args:
        img_xr (xarray.DataArray):
            xarray containing labeled pixel or cell clusters
        fovs (list):
            list of fovs to display
        cluster_id_to_name_path (str):
            a path to a CSV identifying the pixel/cell cluster to manually-defined name mapping
            this is output by the remapping visualization found in `metacluster_remap_gui`
        metacluster_colors (dict):
            maps each metacluster id to a color
        save_dir (str):
            If provided, the image will be saved to this location.
        fov_col (str):
            column with the fovs names in `img_xr`.
        figsize (tuple):
            Size of the image that will be displayed.
    """

    # verify the fovs are valid
    misc_utils.verify_in_list(fov_names=fovs, unique_fovs=img_xr.fovs.values)

    # verify cluster_id_to_name_path exists
    io_utils.validate_paths(cluster_id_to_name_path)

    # read the cluster to name mapping
    cluster_id_to_name = pd.read_csv(cluster_id_to_name_path)

    # this mapping file needs to contain the following columns:
    # 'cluster', 'metacluster', and 'mc_name'
    # NOTE: check for 'cluster' ensures this file was generated by the interactive visualization
    misc_utils.verify_same_elements(
        cluster_mapping_cols=cluster_id_to_name.columns.values,
        required_cols=['cluster', 'metacluster', 'mc_name']
    )

    # subset on just metacluster and mc_name
    metacluster_id_to_name = cluster_id_to_name[['metacluster', 'mc_name']].copy()

    # take only the unique pairs
    metacluster_id_to_name = metacluster_id_to_name.drop_duplicates()

    # sort by metacluster id ascending, this will help when making the colormap
    metacluster_id_to_name = metacluster_id_to_name.sort_values(by='metacluster')

    # assert the metacluster index in colors matches with the ids in metacluster_id_to_name
    misc_utils.verify_same_elements(
        metacluster_colors_ids=list(metacluster_colors.keys()),
        metacluster_mapping_ids=metacluster_id_to_name['metacluster'].values
    )

    # use metacluster_colors to add the colors to metacluster_id_to_name
    metacluster_id_to_name['color'] = metacluster_id_to_name['metacluster'].map(
        metacluster_colors
    )

    # need to add black to denote a pixel with no clusters
    mc_colors = [(0.0, 0.0, 0.0)] + list(metacluster_id_to_name['color'].values)

    # map each metacluster_id_to_name to its index + 1
    # NOTE: explicitly needed to ensure correct colormap colors are drawn and colorbar
    # is indexed correctly when plotted
    metacluster_to_index = {}
    for index, row in metacluster_id_to_name.reset_index(drop=True).iterrows():
        metacluster_to_index[row['metacluster']] = index + 1

    # generate the colormap
    cmap = colors.ListedColormap(mc_colors)
    norm = colors.BoundaryNorm(
        np.linspace(0, len(mc_colors), len(mc_colors) + 1) - 0.5,
        len(mc_colors)
    )

    for fov in fovs:
        # retrieve the image associated with the FOV
        fov_img = img_xr[img_xr[fov_col] == fov].values

        # assign any metacluster id not in metacluster_id_to_name to 0 (not including 0 itself)
        # done as a precaution, should not usually happen
        acceptable_cluster_ids = [0] + list(metacluster_id_to_name['metacluster'])
        fov_img[~np.isin(fov_img, acceptable_cluster_ids)] = 0

        # explicitly relabel each value in fov_img with its index in mc_colors
        # to ensure proper indexing into colormap
        for mc, mc_index in metacluster_to_index.items():
            fov_img[fov_img == mc] = mc_index

        # define the figure
        fig = plt.figure(figsize=figsize)

        # make the title
        plt.title(fov)

        # display the image
        overlay = plt.imshow(
            fov_img.squeeze(),
            cmap=cmap,
            norm=norm,
            origin='upper'
        )

        # remove the axes
        plt.axis('off')

        # remove the gridlines
        plt.grid(b=None)

        # define the colorbar with annotations
        cax = fig.add_axes([0.9, 0.1, 0.01, 0.8])
        cbar = plt.colorbar(
            overlay,
            ticks=np.arange(len(mc_colors)),
            cax=cax,
            orientation='vertical'
        )
        cbar.ax.set_yticklabels(['Empty'] + list(metacluster_id_to_name['mc_name'].values))

        # save if specified
        if save_dir:
            misc_utils.save_figure(save_dir, f'{fov}.png')


def tif_overlay_preprocess(segmentation_labels, plotting_tif):
    """Validates plotting_tif and preprocesses it accordingly
    Args:
        segmentation_labels (numpy.ndarray):
            2D numpy array of labeled cell objects
        plotting_tif (numpy.ndarray):
            2D or 3D numpy array of imaging signal
    Returns:
        numpy.ndarray:
            The preprocessed image
    """

    if len(plotting_tif.shape) == 2:
        if plotting_tif.shape != segmentation_labels.shape:
            raise ValueError("plotting_tif and segmentation_labels array dimensions not equal.")
        else:
            # convert RGB image with the blue channel containing the plotting tif data
            formatted_tif = np.zeros((plotting_tif.shape[0], plotting_tif.shape[1], 3),
                                     dtype=plotting_tif.dtype)
            formatted_tif[..., 2] = plotting_tif
    elif len(plotting_tif.shape) == 3:
        # can only support up to 3 channels
        if plotting_tif.shape[2] > 3:
            raise ValueError("max 3 channels of overlay supported, got {}".
                             format(plotting_tif.shape))

        # set first n channels (in reverse order) of formatted_tif to plotting_tif
        # (n = num channels in plotting_tif)
        formatted_tif = np.zeros((plotting_tif.shape[0], plotting_tif.shape[1], 3),
                                 dtype=plotting_tif.dtype)
        formatted_tif[..., :plotting_tif.shape[2]] = plotting_tif
        formatted_tif = np.flip(formatted_tif, axis=2)
    else:
        raise ValueError("plotting tif must be 2D or 3D array, got {}".
                         format(plotting_tif.shape))

    return formatted_tif


def create_overlay(fov, segmentation_dir, data_dir,
                   img_overlay_chans, seg_overlay_comp, alternate_segmentation=None):
    """Take in labeled contour data, along with optional mibi tif and second contour,
    and overlay them for comparison"
    Generates the outline(s) of the mask(s) as well as intensity from plotting tif. Predicted
    contours are colored red, while alternate contours are colored white.

    Args:
        fov (str):
            The name of the fov to overlay
        segmentation_dir (str):
            The path to the directory containing the segmentation data
        data_dir (str):
            The path to the directory containing the nuclear and whole cell image data
        img_overlay_chans (list):
            List of channels the user will overlay
        seg_overlay_comp (str):
            The segmented compartment the user will overlay
        alternate_segmentation (numpy.ndarray):
            2D numpy array of labeled cell objects
    Returns:
        numpy.ndarray:
            The image with the channel overlay
    """

    # load the specified fov data in
    plotting_tif = load_utils.load_imgs_from_dir(
        data_dir=data_dir,
        files=[fov + '.tiff'],
        xr_dim_name='channels',
        xr_channel_names=['nuclear_channel', 'membrane_channel']
    )

    # verify that the provided image channels exist in plotting_tif
    misc_utils.verify_in_list(
        provided_channels=img_overlay_chans,
        img_channels=plotting_tif.channels.values
    )

    # subset the plotting tif with the provided image overlay channels
    plotting_tif = plotting_tif.loc[fov, :, :, img_overlay_chans].values

    # read the segmentation data in
    segmentation_labels_cell = load_utils.load_imgs_from_dir(data_dir=segmentation_dir,
                                                             files=[fov + '_whole_cell.tiff'],
                                                             xr_dim_name='compartments',
                                                             xr_channel_names=['whole_cell'],
                                                             trim_suffix='_whole_cell',
                                                             match_substring='_whole_cell')
    segmentation_labels_nuc = load_utils.load_imgs_from_dir(data_dir=segmentation_dir,
                                                            files=[fov + '_nuclear.tiff'],
                                                            xr_dim_name='compartments',
                                                            xr_channel_names=['nuclear'],
                                                            trim_suffix='_nuclear',
                                                            match_substring='_nuclear')

    segmentation_labels = xr.DataArray(np.concatenate((segmentation_labels_cell.values,
                                                      segmentation_labels_nuc.values),
                                                      axis=-1),
                                       coords=[segmentation_labels_cell.fovs,
                                               segmentation_labels_cell.rows,
                                               segmentation_labels_cell.cols,
                                               ['whole_cell', 'nuclear']],
                                       dims=segmentation_labels_cell.dims)

    # verify that the provided segmentation channels exist in segmentation_labels
    misc_utils.verify_in_list(
        provided_compartments=seg_overlay_comp,
        seg_compartments=segmentation_labels.compartments.values
    )

    # subset segmentation labels with the provided segmentation overlay channels
    segmentation_labels = segmentation_labels.loc[fov, :, :, seg_overlay_comp].values

    # overlay the segmentation labels over the image
    plotting_tif = tif_overlay_preprocess(segmentation_labels, plotting_tif)

    # define borders of cells in mask
    predicted_contour_mask = find_boundaries(segmentation_labels,
                                             connectivity=1, mode='inner').astype(np.uint8)
    predicted_contour_mask[predicted_contour_mask > 0] = 255

    # rescale each channel to go from 0 to 255
    rescaled = np.zeros(plotting_tif.shape, dtype='uint8')

    for idx in range(plotting_tif.shape[2]):
        if np.max(plotting_tif[:, :, idx]) == 0:
            # don't need to rescale this channel
            pass
        else:
            percentiles = np.percentile(plotting_tif[:, :, idx][plotting_tif[:, :, idx] > 0],
                                        [5, 95])
            rescaled_intensity = rescale_intensity(plotting_tif[:, :, idx],
                                                   in_range=(percentiles[0], percentiles[1]),
                                                   out_range='uint8')
            rescaled[:, :, idx] = rescaled_intensity

    # overlay first contour on all three RGB, to have it show up as white border
    rescaled[predicted_contour_mask > 0, :] = 255

    # overlay second contour as red outline if present
    if alternate_segmentation is not None:

        if segmentation_labels.shape != alternate_segmentation.shape:
            raise ValueError("segmentation_labels and alternate_"
                             "segmentation array dimensions not equal.")

        # define borders of cell in mask
        alternate_contour_mask = find_boundaries(alternate_segmentation, connectivity=1,
                                                 mode='inner').astype(np.uint8)
        rescaled[alternate_contour_mask > 0, 0] = 255
        rescaled[alternate_contour_mask > 0, 1:] = 0

    return rescaled


def set_minimum_color_for_colormap(cmap, default=(0, 0, 0, 1)):
    """ Changes minimum value in provided colormap to black (#000000) or provided color

    This is useful for instances where zero-valued regions of an image should be
    distinct from positive regions (i.e transparent or non-colormap member color)

    Args:
        cmap (matplotlib.colors.Colormap):
            matplotlib color map
        default (Iterable):
            RGBA color values for minimum color. Default is black, (0, 0, 0, 1).

    Returns:
        matplotlib.colors.Colormap:
            corrected colormap
    """
    cmapN = cmap.N
    corrected = cmap(np.arange(cmapN))
    corrected[0, :] = list(default)
    return colors.ListedColormap(corrected)


def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.Path],
                      img_data_path: Union[str, pathlib.Path],
                      mask_output_dir: Union[str, pathlib.Path],
                      mapping: Union[str, pathlib.Path, pd.DataFrame],
                      seg_dir: Union[str, pathlib.Path],
                      mask_suffix: str = "_mask",
                      seg_suffix_name: str = "_whole_cell.tiff",
                      img_sub_folder: str = ""):
    """Creates a mantis project directory so that it can be opened by the mantis viewer.
    Copies fovs, segmentation files, masks, and mapping csv's into a new directory structure.
    Here is how the contents of the mantis project folder will look like.

    ```{code-block} sh
    mantis_project
    ├── fov0
    │   ├── cell_segmentation.tiff
    │   ├── chan0.tiff
    │   ├── chan1.tiff
    │   ├── chan2.tiff
    │   ├── ...
    │   ├── population_mask.csv
    │   └── population_mask.tiff
    └── fov1
    │   ├── cell_segmentation.tiff
    │   ├── chan0.tiff
    │   ├── chan1.tiff
    │   ├── chan2.tiff
    │   ├── ...
    │   ├── population_mask.csv
    │   └── population_mask.tiff
    └── ...
    ```

    Args:
        fovs (List[str]):
            A list of FOVs to create a Mantis Project for.
        mantis_project_path (Union[str, pathlib.Path]):
            The folder where the mantis project will be created.
        img_data_path (Union[str, pathlib.Path]):
            The location of the all the fovs you wish to create a project from.
        mask_output_dir (Union[str, pathlib.Path]):
            The folder containing all the masks of the fovs.
        mapping (Union[str, pathlib.Path, pd.DataFrame]):
            The location of the mapping file, or the mapping Pandas DataFrame itself.
        seg_dir (Union[str, pathlib.Path]):
            The location of the segmentation directory for the fovs.
        mask_suffix (str, optional):
            The suffix used to find the mask tiffs. Defaults to "_mask".
        seg_suffix_name (str, optional):
            The suffix of the segmentation file and it's file extension.
            Defaults to "_whole_cell.tiff".
        img_sub_folder (str, optional):
            The subfolder where the channels exist within the `img_data_path`.
            Defaults to "normalized".
    """

    if not os.path.exists(mantis_project_path):
        os.makedirs(mantis_project_path)

    # create key from cluster number to cluster name
    if type(mapping) in {pathlib.Path, str}:
        map_df = pd.read_csv(mapping)
    elif type(mapping) is pd.DataFrame:
        map_df = mapping
    else:
        ValueError("Mapping must either be a path to an already saved mapping csv, \
                   or a DataFrame that is already loaded in.")

    map_df = map_df.loc[:, ['metacluster', 'mc_name']]
    # remove duplicates from df
    map_df = map_df.drop_duplicates()
    map_df = map_df.sort_values(by=['metacluster'])

    # rename for mantis names
    map_df = map_df.rename({'metacluster': 'region_id', 'mc_name': 'region_name'}, axis=1)

    # get names of fovs with masks
    mask_names_loaded = (io_utils.list_files(mask_output_dir, mask_suffix))
    mask_names_delimited = io_utils.extract_delimited_names(mask_names_loaded,
                                                            delimiter=mask_suffix)
    mask_names_sorted = natsort.natsorted(mask_names_delimited)

    # use `fovs`, a subset of the FOVs in `total_fov_names` which
    # is a list of FOVs in `img_data_path`
    fovs = natsort.natsorted(fovs)
    misc_utils.verify_in_list(fovs=fovs, img_data_fovs=mask_names_delimited)

    # Filter out the masks that do not have an associated FOV.
    mask_names = filter(lambda mn: any(contains(mn, f) for f in fovs), mask_names_sorted)

    # create a folder with image data, pixel masks, and segmentation mask
    for fov, mn in zip(fovs, mask_names):
        # set up paths
        img_source_dir = os.path.join(img_data_path, fov, img_sub_folder)
        output_dir = os.path.join(mantis_project_path, fov)

        # copy image data if not already copied in from previous round of clustering
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

            # copy all channels into new folder
            chans = io_utils.list_files(img_source_dir, substrs=EXTENSION_TYPES["IMAGE"])
            for chan in chans:
                shutil.copy(os.path.join(img_source_dir, chan), os.path.join(output_dir, chan))

        # copy mask into new folder
        mask_name: str = mn + mask_suffix + ".tiff"
        shutil.copy(os.path.join(mask_output_dir, mask_name),
                    os.path.join(output_dir, 'population{}.tiff'.format(mask_suffix)))

        # copy the segmentation files into the output directory
        seg_name: str = fov + seg_suffix_name
        shutil.copy(os.path.join(seg_dir, seg_name),
                    os.path.join(output_dir, 'cell_segmentation.tiff'))

        # copy mapping into directory
        map_df.to_csv(os.path.join(output_dir, 'population{}.csv'.format(mask_suffix)),
                      index=False)

1	import os	1✔
2	import pathlib	1✔
3	import shutil	1✔
4	from operator import contains	1✔
5	from typing import List, Union	1✔
6
7	import matplotlib.cm as cm	1✔
8	import matplotlib.colors as colors	1✔
9	import matplotlib.pyplot as plt	1✔
10	import natsort	1✔
11	import numpy as np	1✔
12	import pandas as pd	1✔
13	import xarray as xr	1✔
14	from mpl_toolkits.axes_grid1 import make_axes_locatable	1✔
15	from skimage.exposure import rescale_intensity	1✔
16	from skimage.segmentation import find_boundaries	1✔
17	from tmi import io_utils, load_utils, misc_utils	1✔
18	from tmi.settings import EXTENSION_TYPES	1✔
19
20
21	def plot_neighborhood_cluster_result(img_xr, fovs, k, save_dir=None, cmap_name='tab20',	1✔
22	fov_col='fovs', figsize=(10, 10)):
23	"""Takes an xarray containing labeled images and displays them.
24	Args:
25	img_xr (xarray.DataArray):
26	xarray containing labeled cell objects.
27	fovs (list):
28	list of fovs to display.
29	k (int):
30	number of clusters (neighborhoods)
31	save_dir (str):
32	If provided, the image will be saved to this location.
33	cmap_name (str):
34	Cmap to use for the image that will be displayed.
35	fov_col (str):
36	column with the fovs names in `img_xr`.
37	figsize (tuple):
38	Size of the image that will be displayed.
39	"""
40
41	# verify the fovs are valid
42	misc_utils.verify_in_list(fov_names=fovs, unique_fovs=img_xr.fovs.values)	×
43
44	# define the colormap, add black for empty slide
45	mycols = cm.get_cmap(cmap_name, k).colors	×
46	mycols = np.vstack(([0, 0, 0, 1], mycols))	×
47	cmap = colors.ListedColormap(mycols)	×
48	bounds = [i-0.5 for i in np.linspace(0, k+1, k+2)]	×
49	norm = colors.BoundaryNorm(bounds, cmap.N)	×
50
51	for fov in fovs:	×
52	# define the figure
53	plt.figure(figsize=figsize)	×
54
55	# define the axis
56	ax = plt.gca()	×
57
58	# make the title
59	plt.title(fov)	×
60
61	# show the image on the figure
62	im = plt.imshow(img_xr[img_xr[fov_col] == fov].values.squeeze(),	×
63	cmap=cmap, norm=norm)
64
65	# remove the axes
66	plt.axis('off')	×
67
68	# remove the gridlines
69	plt.grid(visible=None)	×
70
71	# ensure the colorbar matches up with the margins on the right
72	divider = make_axes_locatable(ax)	×
73	cax = divider.append_axes("right", size="5%", pad=0.05)	×
74
75	# draw the colorbar
76	tick_names = ['Cluster'+str(x) for x in range(1, k+1)]	×
77	tick_names = ['Empty'] + tick_names	×
78	cbar = plt.colorbar(im, cax=cax, ticks=np.arange(len(tick_names)))	×
79	cbar.set_ticks(cbar.ax.get_yticks())	×
80	cbar.ax.set_yticklabels(tick_names)	×
81
82	# save if specified
83	if save_dir:	×
84	misc_utils.save_figure(save_dir, f'{fov}.png')	×
85
86
87	# TODO: possibly need to merge this with plot_neighborhood_cluster_result
88	def plot_pixel_cell_cluster_overlay(img_xr, fovs, cluster_id_to_name_path, metacluster_colors,	1✔
89	save_dir=None, fov_col='fovs', figsize=(10, 10)):
90	"""Overlays the pixel and cell clusters on an image
91
92	Args:
93	img_xr (xarray.DataArray):
94	xarray containing labeled pixel or cell clusters
95	fovs (list):
96	list of fovs to display
97	cluster_id_to_name_path (str):
98	a path to a CSV identifying the pixel/cell cluster to manually-defined name mapping
99	this is output by the remapping visualization found in `metacluster_remap_gui`
100	metacluster_colors (dict):
101	maps each metacluster id to a color
102	save_dir (str):
103	If provided, the image will be saved to this location.
104	fov_col (str):
105	column with the fovs names in `img_xr`.
106	figsize (tuple):
107	Size of the image that will be displayed.
108	"""
109
110	# verify the fovs are valid
111	misc_utils.verify_in_list(fov_names=fovs, unique_fovs=img_xr.fovs.values)	1✔
112
113	# verify cluster_id_to_name_path exists
114	io_utils.validate_paths(cluster_id_to_name_path)	1✔
115
116	# read the cluster to name mapping
117	cluster_id_to_name = pd.read_csv(cluster_id_to_name_path)	1✔
118
119	# this mapping file needs to contain the following columns:
120	# 'cluster', 'metacluster', and 'mc_name'
121	# NOTE: check for 'cluster' ensures this file was generated by the interactive visualization
122	misc_utils.verify_same_elements(	1✔
123	cluster_mapping_cols=cluster_id_to_name.columns.values,
124	required_cols=['cluster', 'metacluster', 'mc_name']
125	)
126
127	# subset on just metacluster and mc_name
128	metacluster_id_to_name = cluster_id_to_name[['metacluster', 'mc_name']].copy()	1✔
129
130	# take only the unique pairs
131	metacluster_id_to_name = metacluster_id_to_name.drop_duplicates()	1✔
132
133	# sort by metacluster id ascending, this will help when making the colormap
134	metacluster_id_to_name = metacluster_id_to_name.sort_values(by='metacluster')	1✔
135
136	# assert the metacluster index in colors matches with the ids in metacluster_id_to_name
137	misc_utils.verify_same_elements(	1✔
138	metacluster_colors_ids=list(metacluster_colors.keys()),
139	metacluster_mapping_ids=metacluster_id_to_name['metacluster'].values
140	)
141
142	# use metacluster_colors to add the colors to metacluster_id_to_name
143	metacluster_id_to_name['color'] = metacluster_id_to_name['metacluster'].map(	1✔
144	metacluster_colors
145	)
146
147	# need to add black to denote a pixel with no clusters
148	mc_colors = [(0.0, 0.0, 0.0)] + list(metacluster_id_to_name['color'].values)	1✔
149
150	# map each metacluster_id_to_name to its index + 1
151	# NOTE: explicitly needed to ensure correct colormap colors are drawn and colorbar
152	# is indexed correctly when plotted
153	metacluster_to_index = {}	1✔
154	for index, row in metacluster_id_to_name.reset_index(drop=True).iterrows():	1✔
155	metacluster_to_index[row['metacluster']] = index + 1	1✔
156
157	# generate the colormap
158	cmap = colors.ListedColormap(mc_colors)	1✔
159	norm = colors.BoundaryNorm(	1✔
160	np.linspace(0, len(mc_colors), len(mc_colors) + 1) - 0.5,
161	len(mc_colors)
162	)
163
164	for fov in fovs:	1✔
165	# retrieve the image associated with the FOV
166	fov_img = img_xr[img_xr[fov_col] == fov].values	1✔
167
168	# assign any metacluster id not in metacluster_id_to_name to 0 (not including 0 itself)
169	# done as a precaution, should not usually happen
170	acceptable_cluster_ids = [0] + list(metacluster_id_to_name['metacluster'])	1✔
171	fov_img[~np.isin(fov_img, acceptable_cluster_ids)] = 0	1✔
172
173	# explicitly relabel each value in fov_img with its index in mc_colors
174	# to ensure proper indexing into colormap
175	for mc, mc_index in metacluster_to_index.items():	1✔
176	fov_img[fov_img == mc] = mc_index	1✔
177
178	# define the figure
179	fig = plt.figure(figsize=figsize)	1✔
180
181	# make the title
182	plt.title(fov)	1✔
183
184	# display the image
185	overlay = plt.imshow(	1✔
186	fov_img.squeeze(),
187	cmap=cmap,
188	norm=norm,
189	origin='upper'
190	)
191
192	# remove the axes
193	plt.axis('off')	1✔
194
195	# remove the gridlines
196	plt.grid(b=None)	1✔
197
198	# define the colorbar with annotations
199	cax = fig.add_axes([0.9, 0.1, 0.01, 0.8])	1✔
200	cbar = plt.colorbar(	1✔
201	overlay,
202	ticks=np.arange(len(mc_colors)),
203	cax=cax,
204	orientation='vertical'
205	)
206	cbar.ax.set_yticklabels(['Empty'] + list(metacluster_id_to_name['mc_name'].values))	1✔
207
208	# save if specified
209	if save_dir:	1✔
210	misc_utils.save_figure(save_dir, f'{fov}.png')	1✔
211
212
213	def tif_overlay_preprocess(segmentation_labels, plotting_tif):	1✔
214	"""Validates plotting_tif and preprocesses it accordingly
215	Args:
216	segmentation_labels (numpy.ndarray):
217	2D numpy array of labeled cell objects
218	plotting_tif (numpy.ndarray):
219	2D or 3D numpy array of imaging signal
220	Returns:
221	numpy.ndarray:
222	The preprocessed image
223	"""
224
225	if len(plotting_tif.shape) == 2:	1✔
226	if plotting_tif.shape != segmentation_labels.shape:	1✔
227	raise ValueError("plotting_tif and segmentation_labels array dimensions not equal.")	1✔
228	else:
229	# convert RGB image with the blue channel containing the plotting tif data
230	formatted_tif = np.zeros((plotting_tif.shape[0], plotting_tif.shape[1], 3),	1✔
231	dtype=plotting_tif.dtype)
232	formatted_tif[..., 2] = plotting_tif	1✔
233	elif len(plotting_tif.shape) == 3:	1✔
234	# can only support up to 3 channels
235	if plotting_tif.shape[2] > 3:	1✔
236	raise ValueError("max 3 channels of overlay supported, got {}".	1✔
237	format(plotting_tif.shape))
238
239	# set first n channels (in reverse order) of formatted_tif to plotting_tif
240	# (n = num channels in plotting_tif)
241	formatted_tif = np.zeros((plotting_tif.shape[0], plotting_tif.shape[1], 3),	1✔
242	dtype=plotting_tif.dtype)
243	formatted_tif[..., :plotting_tif.shape[2]] = plotting_tif	1✔
244	formatted_tif = np.flip(formatted_tif, axis=2)	1✔
245	else:
246	raise ValueError("plotting tif must be 2D or 3D array, got {}".	1✔
247	format(plotting_tif.shape))
248
249	return formatted_tif	1✔
250
251
252	def create_overlay(fov, segmentation_dir, data_dir,	1✔
253	img_overlay_chans, seg_overlay_comp, alternate_segmentation=None):
254	"""Take in labeled contour data, along with optional mibi tif and second contour,
255	and overlay them for comparison"
256	Generates the outline(s) of the mask(s) as well as intensity from plotting tif. Predicted
257	contours are colored red, while alternate contours are colored white.
258
259	Args:
260	fov (str):
261	The name of the fov to overlay
262	segmentation_dir (str):
263	The path to the directory containing the segmentation data
264	data_dir (str):
265	The path to the directory containing the nuclear and whole cell image data
266	img_overlay_chans (list):
267	List of channels the user will overlay
268	seg_overlay_comp (str):
269	The segmented compartment the user will overlay
270	alternate_segmentation (numpy.ndarray):
271	2D numpy array of labeled cell objects
272	Returns:
273	numpy.ndarray:
274	The image with the channel overlay
275	"""
276
277	# load the specified fov data in
278	plotting_tif = load_utils.load_imgs_from_dir(	1✔
279	data_dir=data_dir,
280	files=[fov + '.tiff'],
281	xr_dim_name='channels',
282	xr_channel_names=['nuclear_channel', 'membrane_channel']
283	)
284
285	# verify that the provided image channels exist in plotting_tif
286	misc_utils.verify_in_list(	1✔
287	provided_channels=img_overlay_chans,
288	img_channels=plotting_tif.channels.values
289	)
290
291	# subset the plotting tif with the provided image overlay channels
292	plotting_tif = plotting_tif.loc[fov, :, :, img_overlay_chans].values	1✔
293
294	# read the segmentation data in
295	segmentation_labels_cell = load_utils.load_imgs_from_dir(data_dir=segmentation_dir,	1✔
296	files=[fov + '_whole_cell.tiff'],
297	xr_dim_name='compartments',
298	xr_channel_names=['whole_cell'],
299	trim_suffix='_whole_cell',
300	match_substring='_whole_cell')
301	segmentation_labels_nuc = load_utils.load_imgs_from_dir(data_dir=segmentation_dir,	1✔
302	files=[fov + '_nuclear.tiff'],
303	xr_dim_name='compartments',
304	xr_channel_names=['nuclear'],
305	trim_suffix='_nuclear',
306	match_substring='_nuclear')
307
308	segmentation_labels = xr.DataArray(np.concatenate((segmentation_labels_cell.values,	1✔
309	segmentation_labels_nuc.values),
310	axis=-1),
311	coords=[segmentation_labels_cell.fovs,
312	segmentation_labels_cell.rows,
313	segmentation_labels_cell.cols,
314	['whole_cell', 'nuclear']],
315	dims=segmentation_labels_cell.dims)
316
317	# verify that the provided segmentation channels exist in segmentation_labels
318	misc_utils.verify_in_list(	1✔
319	provided_compartments=seg_overlay_comp,
320	seg_compartments=segmentation_labels.compartments.values
321	)
322
323	# subset segmentation labels with the provided segmentation overlay channels
324	segmentation_labels = segmentation_labels.loc[fov, :, :, seg_overlay_comp].values	1✔
325
326	# overlay the segmentation labels over the image
327	plotting_tif = tif_overlay_preprocess(segmentation_labels, plotting_tif)	1✔
328
329	# define borders of cells in mask
330	predicted_contour_mask = find_boundaries(segmentation_labels,	1✔
331	connectivity=1, mode='inner').astype(np.uint8)
332	predicted_contour_mask[predicted_contour_mask > 0] = 255	1✔
333
334	# rescale each channel to go from 0 to 255
335	rescaled = np.zeros(plotting_tif.shape, dtype='uint8')	1✔
336
337	for idx in range(plotting_tif.shape[2]):	1✔
338	if np.max(plotting_tif[:, :, idx]) == 0:	1✔
339	# don't need to rescale this channel
340	pass	1✔
341	else:
342	percentiles = np.percentile(plotting_tif[:, :, idx][plotting_tif[:, :, idx] > 0],	1✔
343	[5, 95])
344	rescaled_intensity = rescale_intensity(plotting_tif[:, :, idx],	1✔
345	in_range=(percentiles[0], percentiles[1]),
346	out_range='uint8')
347	rescaled[:, :, idx] = rescaled_intensity	1✔
348
349	# overlay first contour on all three RGB, to have it show up as white border
350	rescaled[predicted_contour_mask > 0, :] = 255	1✔
351
352	# overlay second contour as red outline if present
353	if alternate_segmentation is not None:	1✔
354
355	if segmentation_labels.shape != alternate_segmentation.shape:	1✔
356	raise ValueError("segmentation_labels and alternate_"	1✔
357	"segmentation array dimensions not equal.")
358
359	# define borders of cell in mask
360	alternate_contour_mask = find_boundaries(alternate_segmentation, connectivity=1,	1✔
361	mode='inner').astype(np.uint8)
362	rescaled[alternate_contour_mask > 0, 0] = 255	1✔
363	rescaled[alternate_contour_mask > 0, 1:] = 0	1✔
364
365	return rescaled	1✔
366
367
368	def set_minimum_color_for_colormap(cmap, default=(0, 0, 0, 1)):	1✔
369	""" Changes minimum value in provided colormap to black (#000000) or provided color
370
371	This is useful for instances where zero-valued regions of an image should be
372	distinct from positive regions (i.e transparent or non-colormap member color)
373
374	Args:
375	cmap (matplotlib.colors.Colormap):
376	matplotlib color map
377	default (Iterable):
378	RGBA color values for minimum color. Default is black, (0, 0, 0, 1).
379
380	Returns:
381	matplotlib.colors.Colormap:
382	corrected colormap
383	"""
384	cmapN = cmap.N	1✔
385	corrected = cmap(np.arange(cmapN))	1✔
386	corrected[0, :] = list(default)	1✔
387	return colors.ListedColormap(corrected)	1✔
388
389
390	def create_mantis_dir(fovs: List[str], mantis_project_path: Union[str, pathlib.Path],	1✔
391	img_data_path: Union[str, pathlib.Path],
392	mask_output_dir: Union[str, pathlib.Path],
393	mapping: Union[str, pathlib.Path, pd.DataFrame],
394	seg_dir: Union[str, pathlib.Path],
395	mask_suffix: str = "_mask",
396	seg_suffix_name: str = "_whole_cell.tiff",
397	img_sub_folder: str = ""):
398	"""Creates a mantis project directory so that it can be opened by the mantis viewer.
399	Copies fovs, segmentation files, masks, and mapping csv's into a new directory structure.
400	Here is how the contents of the mantis project folder will look like.
401
402	```{code-block} sh
403	mantis_project
404	├── fov0
405	│ ├── cell_segmentation.tiff
406	│ ├── chan0.tiff
407	│ ├── chan1.tiff
408	│ ├── chan2.tiff
409	│ ├── ...
410	│ ├── population_mask.csv
411	│ └── population_mask.tiff
412	└── fov1
413	│ ├── cell_segmentation.tiff
414	│ ├── chan0.tiff
415	│ ├── chan1.tiff
416	│ ├── chan2.tiff
417	│ ├── ...
418	│ ├── population_mask.csv
419	│ └── population_mask.tiff
420	└── ...
421	```
422
423	Args:
424	fovs (List[str]):
425	A list of FOVs to create a Mantis Project for.
426	mantis_project_path (Union[str, pathlib.Path]):
427	The folder where the mantis project will be created.
428	img_data_path (Union[str, pathlib.Path]):
429	The location of the all the fovs you wish to create a project from.
430	mask_output_dir (Union[str, pathlib.Path]):
431	The folder containing all the masks of the fovs.
432	mapping (Union[str, pathlib.Path, pd.DataFrame]):
433	The location of the mapping file, or the mapping Pandas DataFrame itself.
434	seg_dir (Union[str, pathlib.Path]):
435	The location of the segmentation directory for the fovs.
436	mask_suffix (str, optional):
437	The suffix used to find the mask tiffs. Defaults to "_mask".
438	seg_suffix_name (str, optional):
439	The suffix of the segmentation file and it's file extension.
440	Defaults to "_whole_cell.tiff".
441	img_sub_folder (str, optional):
442	The subfolder where the channels exist within the `img_data_path`.
443	Defaults to "normalized".
444	"""
445
446	if not os.path.exists(mantis_project_path):	1✔
447	os.makedirs(mantis_project_path)	1✔
448
449	# create key from cluster number to cluster name
450	if type(mapping) in {pathlib.Path, str}:	1✔
451	map_df = pd.read_csv(mapping)	1✔
452	elif type(mapping) is pd.DataFrame:	1✔
453	map_df = mapping	1✔
454	else:
455	ValueError("Mapping must either be a path to an already saved mapping csv, \	×
456	or a DataFrame that is already loaded in.")
457
458	map_df = map_df.loc[:, ['metacluster', 'mc_name']]	1✔
459	# remove duplicates from df
460	map_df = map_df.drop_duplicates()	1✔
461	map_df = map_df.sort_values(by=['metacluster'])	1✔
462
463	# rename for mantis names
464	map_df = map_df.rename({'metacluster': 'region_id', 'mc_name': 'region_name'}, axis=1)	1✔
465
466	# get names of fovs with masks
467	mask_names_loaded = (io_utils.list_files(mask_output_dir, mask_suffix))	1✔
468	mask_names_delimited = io_utils.extract_delimited_names(mask_names_loaded,	1✔
469	delimiter=mask_suffix)
470	mask_names_sorted = natsort.natsorted(mask_names_delimited)	1✔
471
472	# use `fovs`, a subset of the FOVs in `total_fov_names` which
473	# is a list of FOVs in `img_data_path`
474	fovs = natsort.natsorted(fovs)	1✔
475	misc_utils.verify_in_list(fovs=fovs, img_data_fovs=mask_names_delimited)	1✔
476
477	# Filter out the masks that do not have an associated FOV.
478	mask_names = filter(lambda mn: any(contains(mn, f) for f in fovs), mask_names_sorted)	1✔
479
480	# create a folder with image data, pixel masks, and segmentation mask
481	for fov, mn in zip(fovs, mask_names):	1✔
482	# set up paths
483	img_source_dir = os.path.join(img_data_path, fov, img_sub_folder)	1✔
484	output_dir = os.path.join(mantis_project_path, fov)	1✔
485
486	# copy image data if not already copied in from previous round of clustering
487	if not os.path.exists(output_dir):	1✔
488	os.makedirs(output_dir)	1✔
489
490	# copy all channels into new folder
491	chans = io_utils.list_files(img_source_dir, substrs=EXTENSION_TYPES["IMAGE"])	1✔
492	for chan in chans:	1✔
493	shutil.copy(os.path.join(img_source_dir, chan), os.path.join(output_dir, chan))	1✔
494
495	# copy mask into new folder
496	mask_name: str = mn + mask_suffix + ".tiff"	1✔
497	shutil.copy(os.path.join(mask_output_dir, mask_name),	1✔
498	os.path.join(output_dir, 'population{}.tiff'.format(mask_suffix)))
499
500	# copy the segmentation files into the output directory
501	seg_name: str = fov + seg_suffix_name	1✔
502	shutil.copy(os.path.join(seg_dir, seg_name),	1✔
503	os.path.join(output_dir, 'cell_segmentation.tiff'))
504
505	# copy mapping into directory
506	map_df.to_csv(os.path.join(output_dir, 'population{}.csv'.format(mask_suffix)),	1✔
507	index=False)

angelolab / ark-analysis / 7536

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous