Concurrently executing a command and pushing event leads to dead lock
We have a DS that, when started, continuously push events with push_change_event in the context of an ancillary thread. When executing the Stop command, the ancillary thread is cleaned up, but sometime the thread is already executing push_change_event when the stop command is processed. My understanding is that the command takes a lock and then push_change_event try to take it as well, which gives:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/debionne/miniconda3/envs/pytango/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "dead_lock.py", line 26, in run
self._device.push_change_event("noise", self._device.read_noise())
PyTango.DevFailed: DevFailed[
DevError[
desc = Not able to acquire serialization (dev, class or process) monitor
origin = TangoMonitor::get_monitor
reason = API_CommandTimedOut
severity = ERR]
Setting the SerialModel to NO_SYNC fixes the issue but it's not a good option.
The following code shows the issue:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import time
import threading
import numpy
from tango import DevState
from tango.server import run
from tango.server import Device
from tango.server import attribute, command
class DeadLock(Device) :
noise = attribute(label="Noise",
dtype=((int,),),
max_dim_x=1024, max_dim_y=1024)
class PushingThread(threading.Thread):
def __init__(self, device):
self._stop = False
self._device = device
super(DeadLock.PushingThread, self).__init__()
self.timestamp = time.time()
def run(self):
while self._stop is False:
self._device.push_change_event("noise", self._device.read_noise())
self.timestamp=time.time()
def init_device(self):
Device.init_device(self)
self._lock = threading.Condition()
self._stop = False
self.set_change_event('noise', True, False)
self._pushing_event_thread = self.PushingThread(self)
self._pushing_event_thread.start()
def read_noise(self):
return numpy.random.random_integers(1000, size=(100, 100))
@command
def start(self):
self.set_state(DevState.ON)
@command
def stop(self):
with self._lock:
self._pushing_event_thread._stop=True
self._lock.notify()
self._pushing_event_thread.join()
self.set_state(DevState.OFF)
if __name__ == "__main__":
DeadLock.run_server()
Edited by Reynald Bourtembourg