Shared state changes inside a VTK interactor

[resammc]

Hello,

So, I'm trying to make some changes on VTK actors inside a render window (e.g., I would like to pick and drag them on the screen). That works just fine as long as I do everything inside the interactor methods. If I try to trigger a change by modifying a state variable, things start to not work in the expected way.

Here is an example.

import vtk
from trame.app import get_server
from trame.ui.vuetify3 import SinglePageLayout
from trame.decorators import TrameApp, change, controller
from trame.widgets import vuetify3 as vuetify

from trame_rca.utils import RcaViewAdapter, RcaRenderScheduler, RcaEncoder
from trame.widgets import rca

# VTK factory initialization
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch  # noqa
import vtkmodules.vtkRenderingOpenGL2  # noqa
from vtkmodules.vtkWebCore import vtkRemoteInteractionAdapter, vtkWebApplication


class CustomInteractorStyle(vtk.vtkInteractorStyleTrackballCamera):
    def __init__(self, parent=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.parent = parent
        self.sphereSource = parent.sphereSource
        self.sphereActor = parent.sphereActor
        self.server = parent.server
        self.left_down = False
        self.renderer = parent.renderer

        self.AddObserver("LeftButtonPressEvent", self.on_left_press)
        self.AddObserver("LeftButtonReleaseEvent", self.on_left_release)
        self.AddObserver("MouseMoveEvent", self.on_mouse_move) 

    def on_left_press(self, obj, event):        
        interactor = self.GetInteractor()
        x, y = interactor.GetEventPosition()
        picker = vtk.vtkPropPicker()
        picker.Pick(x, y, 0, self.renderer)
        pickedActor = picker.GetActor()
        self.active_point_index = None
        if pickedActor == self.sphereActor:
            self.active_point_index = 1
        self.left_down = True
        self.OnLeftButtonDown()
        return

    def on_left_release(self, obj, event):
        self.left_down = False
        self.active_point_index = None
        self.OnLeftButtonUp()
        return

    def on_mouse_move(self, obj, event):
        if self.left_down and self.active_point_index is not None:
            interactor = self.GetInteractor()
            x, y = interactor.GetEventPosition()
            self.parent.state.displayCoordinates = (x, y, 0)
            # self.parent.state.flush()
            return
        self.OnMouseMove()
        return

@TrameApp()
class MyApp:
    def __init__(self, server=None):
        self.add_renderer_and_renderWindow()
        self.server = get_server(server, client_type="vue3")
        self.view_scheduler = None
        self.view_handler = None
        self.ui = self._ui()
        self.sphereSource = None
    
    def _updateView(self):
        if self.view_handler:
            self.view_scheduler.schedule_render()
    
    def add_renderer_and_renderWindow(self):
        self.renderer = vtk.vtkRenderer()
        self.renderWindow = vtk.vtkRenderWindow()
        self.renderWindow.AddRenderer(self.renderer)
        self.renderWindowInteractor = vtk.vtkRenderWindowInteractor()
        self.renderWindowInteractor.SetRenderWindow(self.renderWindow)
        self.renderWindowInteractor.GetInteractorStyle().SetCurrentStyleToTrackballCamera()
        
    def add_sphere(self):
        self.sphereSource = vtk.vtkSphereSource()
        self.sphereMapper = vtk.vtkPolyDataMapper()
        self.sphereMapper.SetInputConnection(self.sphereSource.GetOutputPort())
        self.sphereActor = vtk.vtkActor()
        self.sphereActor.SetMapper(self.sphereMapper)
        self.renderer.AddActor(self.sphereActor)
        self.renderer.ResetCamera()

    @controller.add("on_server_ready")
    def init_rca(self, **kwargs):
        # RemoteControllerArea
        self.view_scheduler = RcaRenderScheduler(
            self.renderWindow, target_fps=30, rca_encoder=RcaEncoder.JPEG
        )
        self.view_handler = RcaViewAdapter(self.renderWindow, self.view_scheduler, "view")
        self.server.controller.rc_area_register(self.view_handler)
        self.add_sphere()
        self.custom_style = CustomInteractorStyle(self)
        self.custom_style.SetDefaultRenderer(self.renderer)
        self.renderWindowInteractor.SetInteractorStyle(self.custom_style)
        self.renderWindowInteractor.Initialize()

    @property
    def state(self):
        return self.server.state

    @property
    def ctrl(self):
        return self.server.controller

    def _ui(self):
        with SinglePageLayout(
            self.server, theme=("theme", "dark")
        ) as layout:
            with layout.toolbar:
                vuetify.VTextField(v_model=("textMessage", ""))
                with vuetify.VBtn(
                        icon=True, click=self.ctrl.view_update
                    ):
                        vuetify.VIcon("mdi-sync-circle")
            with layout.content:
                with vuetify.VContainer(
                    fluid=True,
                    classes="pa-0 fill-height",
                ):
                    self.view = rca.RemoteControlledArea(
                        name="view",
                        display="image",
                    )
                    self.ctrl.view_update = self._updateView
            return layout

    @change("displayCoordinates")
    def displayCoordinates_changed(self, displayCoordinates, **kwargs):
        self.state.textMessage = f"displayCoordinates changed to {displayCoordinates}"

def main():
    app = MyApp()
    app.server.start()


if __name__ == "__main__":
    main()

The idea is simple. There is a sphere on the screen. If you click and hold on it while moving the mouse cursor, the value of state.displayCoordinates should get updated which then updates the text in the text field (or at least it is what I'm expecting). In reality however, unless I add a self.parent.state.flush() line after the change, it will not take effect. Even self.parent.state.dirty("displayCoordinates") does not work.

Also maybe interesting: without self.parent.state.flush(), after clicking and moving the mouse has finished, if one clicks on the view_update icon at the top-right side, the change takes place and the coordinates will be displayed in the text field.

So, to summarize, it looks like the main problem is that changing the state variable does not take effect until I flush the whole state (which is expensive and slow in my main application).

My quesiton is, am I doing something wrong here or missing something?

Thank you in advance.

Resam

[alesgenova]

I'm having some issues running the example you provided, so I can't confirm it, but you could try the following:

with self.parent.state:
    self.parent.state.displayCoordinates = (x, y, 0)

[resammc]

Thanks. This actually works, but, I think, only because at the end of the "with" statement the state gets flushed (#617 (comment))

My problem with flush is that it's computationally expensive:
from the documentation:
flush()[: Force pushing modified state and execute any @state.change listener
and I have some heavy tasked @state.changes in my app.

would have been great if the dirty() could perform the operation

[jourdain]

The flush is not expensive in itself, it is really on your side that you need to be mindful of what you listen to and what computation you end up doing from it. If you don't have any listener on displayCoordinates, nothing slow should happen.

If you need to debounce your processing, it is up to you to do it, not trame. Trame does the debouncing at the network layer for you.

[resammc]

Of course you are right that I'm responsible for my code architecture and how it reacts to different changes. It's just not clear to me why even state.dirty() cannot trigger the listener and I must use flush (or why simply updating displayCoordinates does not automatically trigger it).

[jourdain]

https://trame.readthedocs.io/en/latest/core.state.html#trame_server.state.State.dirty

[resammc]

Thanks. I think I overlooked the last part of the sentence. So far, whenever I was using dirty() there was no need to call flush separately (and I was also not using the with structure); perhaps that added to my confusion.

[jourdain]

Dirty is only needed when you modify in-place the same object/array. By just assigning a new value, the dirty call get triggered automatically. For the state flush, it get executed at the end of the rpc and batch executed any change detected then. Using the with state:, it allow you to control the scope of when the current set of dirty properties are getting flushed. Also keep in mind that it is async, so you are just scheduling the state.change(s), which will truly run once the event loop get some free time.

[alesgenova]

If the updates coming from the interactor are too frequent, you could debounce or throttle the reaction to them.
Quick possible implementation:

import asyncio
from functools import wraps
from trame.app import asynchronous

def defer_task(func, wait):
    @wraps(func)
    def wrapper(*args, **kwargs):
        async def exec():
            await asyncio.sleep(wait)
            result = func(*args, **kwargs)
            if asyncio.iscoroutine(result):
                await result

        return asynchronous.create_task(exec())

    return wrapper


def debounce(func, wait):
    task = None

    @wraps(func)
    async def wrapper(*args, **kwargs):
        nonlocal task
        if task is not None:
            task.cancel()

        def execute(*args, **kwargs):
            nonlocal task
            result = func(*args, **kwargs)
            task = None
            return result

        task = defer_task(execute, wait)(*args, **kwargs)

    return wrapper


def throttle(func, wait):
    task = None
    last_args = None
    last_kwargs = None

    @wraps(func)
    async def wrapper(*args, **kwargs):
        nonlocal task
        nonlocal last_args
        nonlocal last_kwargs

        last_args = args
        last_kwargs = kwargs

        def execute():
            nonlocal task
            result = func(*last_args, **last_kwargs)
            task = None
            return result

        if task is None:
            task = defer_task(execute, wait)()

    return wrapper

# Usage
def fn():
    pass

debounced_fn = debounce(fn, 1)
throttled_fn = throttle(fn, 1)

# No matter how many times you invoke debounced_fn,
# it will only trigger fn once after 1 second has passed since the last call to debounced_fn
debounced_fn()

# No matter how many times you invoke throttled_fn,
# it will only trigger fn at most once per 1 second
throttled_fn()

[resammc]

Thanks a lot! This will definitely come in handy.