7941848753

Committed 17 Feb 2024 12:59PM UTC coverage: 86.194% (-0.4%) from 86.58%

Build # 7941848753

Build Type

push

github

Committed by

MattClarkson

Commit Message

Merge branch '222-n-rendering-layers-in-overlay'

Run Details

3 of 8 new or added lines in 1 file covered. (37.5%)

71 existing lines in 3 files now uncovered.

1798 of 2086 relevant lines covered (86.19%)

10.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/sksurgeryvtk/widgets/vtk_rendering_generator.py

# -*- coding: utf-8 -*-

"""
Module to provide a basic VTK render window for test data generation.
"""

# pylint: disable=too-many-instance-attributes,
# pylint: disable=no-name-in-module, too-many-arguments

import os
import numpy as np
import cv2
from PySide6 import QtWidgets
import sksurgerycore.utilities.file_utilities as fu
import sksurgerycore.configuration.configuration_manager as config
import sksurgeryvtk.widgets.vtk_overlay_window as vo
import sksurgeryvtk.models.surface_model_loader as sl
import sksurgeryvtk.camera.vtk_camera_model as cm
import sksurgeryvtk.utils.matrix_utils as mu


class VTKRenderingGenerator(QtWidgets.QWidget):
    """
    Class contains a VTKOverlayWindow and a few extra functions to
    facilitate rendering loops for generating test data.

    :param models_file: JSON file describing VTK models, in SNAPPY format
    :param background_image: RGB image to render in background
    :param intrinsic_file: [3x3] matrix in text file, in numpy format
    :param camera_to_world: list of [rx,ry,rz,tx,ty,tz] in degrees/millimetres
    :param left_to_right: list of [rx,ry,rz,tx,ty,tz] in degrees/millimetres
    :param offscreen: if true, renders offscreen
    :param zbuffer: if true, causes VTK to render just the z-buffer
    :param gaussian_sigma: if non-zero, adds blurring to the rendered image
    :param gaussian_window_size: window size of OpenCV Gaussian kernel
    :param clipping_range: VTK clipping range (near, far)
    :param init_widget: If True we will call self.Initialize and self.Start
        as part of the init function. Set to false if you're on Linux.
    """
    def __init__(self,
                 models_file,
                 background_image,
                 intrinsic_file,
                 camera_to_world=None,
                 left_to_right=None,
                 offscreen=False,
                 zbuffer=False,
                 gaussian_sigma=0.0,
                 gaussian_window_size=11,
                 clipping_range=(1, 1000),
                 init_widget=True
                 ):
        super().__init__()
        self.gaussian_sigma = gaussian_sigma
        self.gaussian_window_size = gaussian_window_size

        self.img = cv2.imread(background_image)

        self.configuration_manager = config.ConfigurationManager(models_file)
        self.configuration_data = self.configuration_manager.get_copy()

        file = fu.get_absolute_path_of_file(models_file)
        dirname = os.path.dirname(file)

        self.model_loader = sl.SurfaceModelLoader(self.configuration_data,
                                                  dirname
                                                  )

        self.window_container = QtWidgets.QWidget(self)
        self.layout = QtWidgets.QVBoxLayout(self.window_container)
        self.setLayout(self.layout)
        self.layout.setSpacing(0)
        self.layout.setContentsMargins(0, 0, 0, 0)

        self.overlay = vo.VTKOverlayWindow(offscreen=offscreen,
                                           zbuffer=zbuffer,
                                           init_widget=init_widget)
        self.overlay.set_video_image(self.img)
        self.overlay.add_vtk_models(self.model_loader.get_surface_models())
        self.clip_near = clipping_range[0]
        self.clip_far = clipping_range[1]

        self.intrinsics = np.loadtxt(intrinsic_file, dtype=float)
        self.setup_intrinsics()

        self.left_camera_to_world = np.eye(4)
        self.camera_to_world = np.eye(4)
        self.left_to_right = np.eye(4)
        self.setup_camera_extrinsics(camera_to_world, left_to_right)

        self.layout.addWidget(self.overlay)
        # self.overlay.show()
        self.overlay.Initialize()
        # self.overlay.Start()

    def closeEvent(self, QCloseEvent):
        super().closeEvent(QCloseEvent)
        self.overlay.Finalize()

    def set_clipping_range(self, minimum, maximum):
        """
        Sets the clipping range on the foreground camera.

        :param minimum: minimum in millimetres
        :param maximum: maximum in millimetres
        """
        self.clip_near = minimum
        self.clip_far = maximum
        self.overlay.get_foreground_camera().SetClippingRange(minimum, maximum)

    def set_smoothing(self, sigma, window_size):
        """
        Sets the Gaussian blur.

        :param sigma: standard deviation of Gaussian function.
        :param window_size: sets the window size of Gaussian kernel (pixels).
        """
        self.gaussian_sigma = sigma
        self.gaussian_window_size = window_size

    def setup_intrinsics(self):
        """
        Set the intrinsics of the foreground vtkCamera.
        """
        f_x = self.intrinsics[0, 0]
        c_x = self.intrinsics[0, 2]
        f_y = self.intrinsics[1, 1]
        c_y = self.intrinsics[1, 2]
        width, height = self.img.shape[1], self.img.shape[0]

        cm.set_camera_intrinsics(self.overlay.get_foreground_renderer(),
                                 self.overlay.get_foreground_camera(),
                                 width,
                                 height,
                                 f_x,
                                 f_y,
                                 c_x,
                                 c_y,
                                 self.clip_near,
                                 self.clip_far)

    def setup_camera_extrinsics(self,
                                camera_to_world,
                                left_to_right=None
                                ):
        """
        Decomposes parameter strings into 6DOF
        parameters, and sets up camera-to-world and left_to_right for stereo.

        :param camera_to_world: list of [rx,ry,rz,tx,ty,tz] in degrees/mm
        :param left_to_right: list of [rx,ry,rz,tx,ty,tz] in degrees/mm
        """
        if camera_to_world is not None:
            self.left_camera_to_world = mu.create_matrix_from_list(
                camera_to_world)
        if left_to_right is not None:
            self.left_to_right = mu.create_matrix_from_list(left_to_right)
        self.camera_to_world = cm.compute_right_camera_pose(
            self.left_camera_to_world,
            self.left_to_right)
        self.overlay.set_camera_pose(self.camera_to_world)

    def set_all_model_to_world(self, model_to_world):
        """
        Decomposes the model_to_world string into rx,ry,rx,tx,ty,rz,
        constructs a 4x4 matrix, and applies it to all models.

        :param model_to_world: [4x4] numpy ndarray, rigid transform
        """
        if model_to_world is not None:
            m2w = mu.create_matrix_from_list(model_to_world)
            vtk_matrix = mu.create_vtk_matrix_from_numpy(m2w)
            for model in self.model_loader.get_surface_models():
                model.set_user_matrix(vtk_matrix)

    def set_model_to_worlds(self, dict_of_transforms):
        """
        Given a dictionary of transforms, will iterate by name,
        and apply the transform to the named object.
        :param dict_of_transforms: {name, [rx, ry, rz, tx, ty, tz]}
        """
        if dict_of_transforms is not None:
            for name in dict_of_transforms:
                if name in self.model_loader.get_surface_model_names():
                    model = self.model_loader.get_surface_model(name)
                    m2w = mu.create_matrix_from_list(dict_of_transforms[name])
                    vtk_matrix = mu.create_vtk_matrix_from_numpy(m2w)
                    model.set_user_matrix(vtk_matrix)
                else:
                    raise ValueError("'" + name + "' is not in set of models.")

    def get_image(self):
        """
        Returns the rendered image, with post processing like smoothing.
        :return: numpy ndarray representing rendered image (RGB)
        """
        self.overlay.Render()
        self.repaint()
        img = self.overlay.convert_scene_to_numpy_array()
        smoothed = img
        if self.gaussian_sigma > 0:
            smoothed = cv2.GaussianBlur(img,
                                        (self.gaussian_window_size,
                                         self.gaussian_window_size),
                                        self.gaussian_sigma)
        return smoothed

    def get_masks(self):
        """
        If we want to render masks for test data for DL models for instance,
        we typically want distinct masks per model object. This method
        returns a dictionary of new images corresponding to each named model.

        If model is shaded, the shading is turned off to get masks,
        the masks are acquired, and the shading is applied again.

        Note: You should ensure self.gaussian_sigma == 0 (the default),
        and in the .json file.
        """
        result = {}
        # Check shading of models.
        models = self.model_loader.get_surface_models()
        shaded_models = [] # To track which ones were shaded.
        for model in models:
            if not model.no_shading:
                # Shading is active, deactivate it
                model.set_no_shading(True)
                shaded_models.append(True)
            else:
                shaded_models.append(False)

        # Should not be shaded.
        img = self.get_image()
        for index, model in enumerate(models):
            name = model.get_name()
            colour = (np.asarray(model.get_colour()) * 255).astype(np.uint8)
            mask = cv2.inRange(img, colour, colour)
            result[name] = mask
            # Use shaded_models list to reactivate shading to original setting.
            if shaded_models[index]:
                model.set_no_shading(False)

        return result

1	# -- coding: utf-8 --
2
3	"""	12✔
4	Module to provide a basic VTK render window for test data generation.
5	"""
6
7	# pylint: disable=too-many-instance-attributes,
8	# pylint: disable=no-name-in-module, too-many-arguments
9
10	import os	12✔
11	import numpy as np	12✔
12	import cv2	12✔
13	from PySide6 import QtWidgets	12✔
14	import sksurgerycore.utilities.file_utilities as fu	12✔
15	import sksurgerycore.configuration.configuration_manager as config	12✔
16	import sksurgeryvtk.widgets.vtk_overlay_window as vo	12✔
17	import sksurgeryvtk.models.surface_model_loader as sl	12✔
18	import sksurgeryvtk.camera.vtk_camera_model as cm	12✔
19	import sksurgeryvtk.utils.matrix_utils as mu	12✔
20
21
22	class VTKRenderingGenerator(QtWidgets.QWidget):	12✔
23	"""
24	Class contains a VTKOverlayWindow and a few extra functions to
25	facilitate rendering loops for generating test data.
26
27	:param models_file: JSON file describing VTK models, in SNAPPY format
28	:param background_image: RGB image to render in background
29	:param intrinsic_file: [3x3] matrix in text file, in numpy format
30	:param camera_to_world: list of [rx,ry,rz,tx,ty,tz] in degrees/millimetres
31	:param left_to_right: list of [rx,ry,rz,tx,ty,tz] in degrees/millimetres
32	:param offscreen: if true, renders offscreen
33	:param zbuffer: if true, causes VTK to render just the z-buffer
34	:param gaussian_sigma: if non-zero, adds blurring to the rendered image
35	:param gaussian_window_size: window size of OpenCV Gaussian kernel
36	:param clipping_range: VTK clipping range (near, far)
37	:param init_widget: If True we will call self.Initialize and self.Start
38	as part of the init function. Set to false if you're on Linux.
39	"""
40	def __init__(self,	12✔
41	models_file,
42	background_image,
43	intrinsic_file,
44	camera_to_world=None,
45	left_to_right=None,
46	offscreen=False,
47	zbuffer=False,
48	gaussian_sigma=0.0,
49	gaussian_window_size=11,
50	clipping_range=(1, 1000),
51	init_widget=True
52	):
53	super().__init__()	12✔
54	self.gaussian_sigma = gaussian_sigma	12✔
55	self.gaussian_window_size = gaussian_window_size	12✔
56
57	self.img = cv2.imread(background_image)	12✔
58
59	self.configuration_manager = config.ConfigurationManager(models_file)	12✔
60	self.configuration_data = self.configuration_manager.get_copy()	12✔
61
62	file = fu.get_absolute_path_of_file(models_file)	12✔
63	dirname = os.path.dirname(file)	12✔
64
65	self.model_loader = sl.SurfaceModelLoader(self.configuration_data,	12✔
66	dirname
67	)
68
69	self.window_container = QtWidgets.QWidget(self)	12✔
70	self.layout = QtWidgets.QVBoxLayout(self.window_container)	12✔
71	self.setLayout(self.layout)	12✔
72	self.layout.setSpacing(0)	12✔
73	self.layout.setContentsMargins(0, 0, 0, 0)	12✔
74
75	self.overlay = vo.VTKOverlayWindow(offscreen=offscreen,	12✔
76	zbuffer=zbuffer,
77	init_widget=init_widget)
78	self.overlay.set_video_image(self.img)	12✔
79	self.overlay.add_vtk_models(self.model_loader.get_surface_models())	12✔
80	self.clip_near = clipping_range[0]	12✔
81	self.clip_far = clipping_range[1]	12✔
82
83	self.intrinsics = np.loadtxt(intrinsic_file, dtype=float)	12✔
84	self.setup_intrinsics()	12✔
85
86	self.left_camera_to_world = np.eye(4)	12✔
87	self.camera_to_world = np.eye(4)	12✔
88	self.left_to_right = np.eye(4)	12✔
89	self.setup_camera_extrinsics(camera_to_world, left_to_right)	12✔
90
91	self.layout.addWidget(self.overlay)	12✔
92	# self.overlay.show()
93	self.overlay.Initialize()	12✔
94	# self.overlay.Start()
95
96	def closeEvent(self, QCloseEvent):	12✔
97	super().closeEvent(QCloseEvent)	12✔
98	self.overlay.Finalize()	12✔
99
100	def set_clipping_range(self, minimum, maximum):	12✔
101	"""
102	Sets the clipping range on the foreground camera.
103
104	:param minimum: minimum in millimetres
105	:param maximum: maximum in millimetres
106	"""
107	self.clip_near = minimum	12✔
108	self.clip_far = maximum	12✔
109	self.overlay.get_foreground_camera().SetClippingRange(minimum, maximum)	12✔
110
111	def set_smoothing(self, sigma, window_size):	12✔
112	"""
113	Sets the Gaussian blur.
114
115	:param sigma: standard deviation of Gaussian function.
116	:param window_size: sets the window size of Gaussian kernel (pixels).
117	"""
UNCOV 118	self.gaussian_sigma = sigma	8✔
UNCOV 119	self.gaussian_window_size = window_size	8✔
120
121	def setup_intrinsics(self):	12✔
122	"""
123	Set the intrinsics of the foreground vtkCamera.
124	"""
125	f_x = self.intrinsics[0, 0]	12✔
126	c_x = self.intrinsics[0, 2]	12✔
127	f_y = self.intrinsics[1, 1]	12✔
128	c_y = self.intrinsics[1, 2]	12✔
129	width, height = self.img.shape[1], self.img.shape[0]	12✔
130
131	cm.set_camera_intrinsics(self.overlay.get_foreground_renderer(),	12✔
132	self.overlay.get_foreground_camera(),
133	width,
134	height,
135	f_x,
136	f_y,
137	c_x,
138	c_y,
139	self.clip_near,
140	self.clip_far)
141
142	def setup_camera_extrinsics(self,	12✔
143	camera_to_world,
144	left_to_right=None
145	):
146	"""
147	Decomposes parameter strings into 6DOF
148	parameters, and sets up camera-to-world and left_to_right for stereo.
149
150	:param camera_to_world: list of [rx,ry,rz,tx,ty,tz] in degrees/mm
151	:param left_to_right: list of [rx,ry,rz,tx,ty,tz] in degrees/mm
152	"""
153	if camera_to_world is not None:	12✔
UNCOV 154	self.left_camera_to_world = mu.create_matrix_from_list(	8✔
155	camera_to_world)
156	if left_to_right is not None:	12✔
UNCOV 157	self.left_to_right = mu.create_matrix_from_list(left_to_right)	8✔
158	self.camera_to_world = cm.compute_right_camera_pose(	12✔
159	self.left_camera_to_world,
160	self.left_to_right)
161	self.overlay.set_camera_pose(self.camera_to_world)	12✔
162
163	def set_all_model_to_world(self, model_to_world):	12✔
164	"""
165	Decomposes the model_to_world string into rx,ry,rx,tx,ty,rz,
166	constructs a 4x4 matrix, and applies it to all models.
167
168	:param model_to_world: [4x4] numpy ndarray, rigid transform
169	"""
UNCOV 170	if model_to_world is not None:	8✔
UNCOV 171	m2w = mu.create_matrix_from_list(model_to_world)	8✔
UNCOV 172	vtk_matrix = mu.create_vtk_matrix_from_numpy(m2w)	8✔
UNCOV 173	for model in self.model_loader.get_surface_models():	8✔
UNCOV 174	model.set_user_matrix(vtk_matrix)	8✔
175
176	def set_model_to_worlds(self, dict_of_transforms):	12✔
177	"""
178	Given a dictionary of transforms, will iterate by name,
179	and apply the transform to the named object.
180	:param dict_of_transforms: {name, [rx, ry, rz, tx, ty, tz]}
181	"""
UNCOV 182	if dict_of_transforms is not None:	8✔
UNCOV 183	for name in dict_of_transforms:	8✔
UNCOV 184	if name in self.model_loader.get_surface_model_names():	8✔
UNCOV 185	model = self.model_loader.get_surface_model(name)	8✔
UNCOV 186	m2w = mu.create_matrix_from_list(dict_of_transforms[name])	8✔
UNCOV 187	vtk_matrix = mu.create_vtk_matrix_from_numpy(m2w)	8✔
UNCOV 188	model.set_user_matrix(vtk_matrix)	8✔
189	else:
UNCOV 190	raise ValueError("'" + name + "' is not in set of models.")	8✔
191
192	def get_image(self):	12✔
193	"""
194	Returns the rendered image, with post processing like smoothing.
195	:return: numpy ndarray representing rendered image (RGB)
196	"""
197	self.overlay.Render()	12✔
198	self.repaint()	12✔
199	img = self.overlay.convert_scene_to_numpy_array()	12✔
200	smoothed = img	12✔
201	if self.gaussian_sigma > 0:	12✔
UNCOV 202	smoothed = cv2.GaussianBlur(img,	8✔
203	(self.gaussian_window_size,
204	self.gaussian_window_size),
205	self.gaussian_sigma)
206	return smoothed	12✔
207
208	def get_masks(self):	12✔
209	"""
210	If we want to render masks for test data for DL models for instance,
211	we typically want distinct masks per model object. This method
212	returns a dictionary of new images corresponding to each named model.
213
214	If model is shaded, the shading is turned off to get masks,
215	the masks are acquired, and the shading is applied again.
216
217	Note: You should ensure self.gaussian_sigma == 0 (the default),
218	and in the .json file.
219	"""
220	result = {}	12✔
221	# Check shading of models.
222	models = self.model_loader.get_surface_models()	12✔
223	shaded_models = [] # To track which ones were shaded.	12✔
224	for model in models:	12✔
225	if not model.no_shading:	12✔
226	# Shading is active, deactivate it
UNCOV 227	model.set_no_shading(True)	8✔
UNCOV 228	shaded_models.append(True)	8✔
229	else:
230	shaded_models.append(False)	12✔
231
232	# Should not be shaded.
233	img = self.get_image()	12✔
234	for index, model in enumerate(models):	12✔
235	name = model.get_name()	12✔
236	colour = (np.asarray(model.get_colour()) * 255).astype(np.uint8)	12✔
237	mask = cv2.inRange(img, colour, colour)	12✔
238	result[name] = mask	12✔
239	# Use shaded_models list to reactivate shading to original setting.
240	if shaded_models[index]:	12✔
UNCOV 241	model.set_no_shading(False)	8✔
242
243	return result	12✔

SciKit-Surgery / scikit-surgeryvtk / 7941848753

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous