Refactored backends to be more robust by wrapping the core logic into a try-except logic with sleep and retry

platypush/backend/__init__.py

@@ -7,6 +7,7 @@ import importlib
 import logging
 import sys
 import threading
+import time
 
 from threading import Thread
 
@@ -34,6 +35,7 @@ class Backend(Thread):
     """
 
     _default_response_timeout = 5
+    _backend_reload_timeout = 10
 
     def __init__(self, bus=None, **kwargs):
         """
@@ -211,9 +213,38 @@ class Backend(Thread):
         redis.send_message(msg, queue_name=queue_name)
 
 
+    def exec(self):
+        """ Backend thread logic. To be implemented in the derived classes """
+        raise RuntimeError('Backend class {} does not implement the exec() method'.
+                           format(self.__class__.__name__))
+
     def run(self):
-        """ Starts the backend thread. To be implemented in the derived classes """
+        """ Thread runner. It wraps the exec method in a retry block """
+
+        super().run()
         self.thread_id = threading.get_ident()
+        error = None
+
+        while not self.should_stop():
+            try:
+                self.exec()
+            except Exception as e:
+                error = e
+
+            if not self.should_stop():
+                if error:
+                    self.logger.error(('Backend {} terminated with an exception, ' +
+                                      'reloading in {} seconds').format(
+                                          self.__class__.__name__,
+                                          self._backend_reload_timeout))
+                    self.logger.exception(error)
+                else:
+                    self.logger.warning(('Backend {} unexpectedly terminated, ' +
+                                        'reloading in {} seconds').format(
+                                            self.__class__.__name__,
+                                            self._backend_reload_timeout))
+
+                time.sleep(self._backend_reload_timeout)
 
     def on_stop(self):
         """ Callback invoked when the process stops """

platypush/backend/assistant/google/__init__.py

@@ -110,8 +110,7 @@ class AssistantGoogleBackend(Backend):
             self.assistant.stop_conversation()
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         with Assistant(self.credentials, self.device_model_id) as assistant:
             self.assistant = assistant

platypush/backend/assistant/google/pushtotalk.py

@@ -176,9 +176,8 @@ class AssistantGooglePushtotalkBackend(Backend):
         """ Speech recognized handler """
         self.bus.post(SpeechRecognizedEvent(phrase=speech))
 
-    def run(self):
+    def exec(self):
         """ Backend executor """
-        super().run()
 
         with SampleAssistant(self.lang, self.device_model_id, self.device_id,
                              self.conversation_stream,

platypush/backend/assistant/snowboy/__init__.py

@@ -72,8 +72,7 @@ class AssistantSnowboyBackend(Backend):
             self.bus.post(HotwordDetectedEvent(hotword=self.hotword))
         return callback
 
-    def run(self):
+    def exec(self):
-        super().run()
         self.detector.start(self.hotword_detected())
 
 

platypush/backend/button/flic/__init__.py

@@ -112,8 +112,7 @@ class ButtonFlicBackend(Backend):
 
         return _f
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.client.handle_events()
 

platypush/backend/camera/pi.py

@@ -160,8 +160,7 @@ class CameraPiBackend(Backend):
             self.logger.warning('Failed to stop recording')
             self.logger.exception(e)
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         if not self.redis:
             self.redis = get_backend('redis')

platypush/backend/http/__init__.py

@@ -377,8 +377,7 @@ class HttpBackend(Backend):
         loop.run_forever()
 
 
-    def run(self):
+    def exec(self):
-        super().run()
         os.putenv('FLASK_APP', 'platypush')
         os.putenv('FLASK_ENV', 'production')
         self.logger.info('Initialized HTTP backend on port {}'.format(self.port))

platypush/backend/http/poll/__init__.py

@@ -72,8 +72,7 @@ class HttpPollBackend(Backend):
             self.requests.append(request)
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         while not self.should_stop():
             for request in self.requests:

platypush/backend/inotify/__init__.py

@@ -48,8 +48,7 @@ class InotifyBackend(Backend):
 
         self.inotify_watch = None
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.inotify_watch = inotify.adapters.Inotify()
         for path in self.watch_paths:

platypush/backend/joystick.py

@@ -29,8 +29,7 @@ class JoystickBackend(Backend):
 
         self.device = device
 
-    def run(self):
+    def exec(self):
-        super().run()
         self.logger.info('Initialized joystick backend on device {}'.format(self.device))
 
         while not self.should_stop():

platypush/backend/kafka/__init__.py

@@ -81,8 +81,7 @@ class KafkaBackend(Backend):
             self.logger.warning('Exception occurred while closing Kafka connection')
             self.logger.exception(e)
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.consumer = KafkaConsumer(self.topic, bootstrap_servers=self.server)
         self.logger.info('Initialized kafka backend - server: {}, topic: {}'

platypush/backend/local/__init__.py

@@ -61,8 +61,7 @@ class LocalBackend(Backend):
         return Message.build(msg) if len(msg) else None
 
 
-    def run(self):
+    def exec(self):
-        super().run()
         self.logger.info('Initialized local backend on {} and {}'.
                      format(self.request_fifo, self.response_fifo))
 

platypush/backend/midi.py

@@ -102,8 +102,7 @@ class MidiBackend(Backend):
         return callback
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.midi.open_port(self.port_number)
         self.logger.info('Initialized MIDI backend, listening for events on device {}'.

platypush/backend/mqtt.py

@@ -45,7 +45,7 @@ class MqttBackend(Backend):
         publisher.single(self.topic, str(msg), hostname=self.host, port=self.port)
 
 
-    def run(self):
+    def exec(self):
         def on_connect(client, userdata, flags, rc):
             client.subscribe(self.topic)
 
@@ -57,7 +57,6 @@ class MqttBackend(Backend):
             self.logger.info('Received message on the MQTT backend: {}'.format(msg))
             self.on_message(msg)
 
-        super().run()
         client = mqtt.Client()
         client.on_connect = on_connect
         client.on_message = on_message

platypush/backend/music/mpd/__init__.py

@@ -39,8 +39,7 @@ class MusicMpdBackend(Backend):
         self.poll_seconds = poll_seconds
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         last_status = {}
         last_state = None

platypush/backend/pushbullet/__init__.py

@@ -175,8 +175,7 @@ class PushbulletBackend(Backend):
     def on_stop(self):
         self.ws.close()
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self._init_socket()
         self.logger.info('Initialized Pushbullet backend - device_id: {}'

platypush/backend/redis.py

@@ -60,8 +60,7 @@ class RedisBackend(Backend):
         return msg
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.logger.info('Initialized Redis backend on queue {} with arguments {}'.
                      format(self.queue, self.redis_args))

platypush/backend/scard/__init__.py

@@ -50,8 +50,7 @@ class ScardBackend(Backend):
             self.cardtype = AnyCardType()
 
 
-    def run(self):
+    def exec(self):
-        super().run()
 
         self.logger.info('Initialized smart card reader backend - ATR filter: {}'.
                      format(self.ATRs))

platypush/backend/sensor/__init__.py

@@ -40,8 +40,7 @@ class SensorBackend(Backend):
         """ To be implemented in the derived classes """
         raise NotImplementedError('To be implemented in a derived class')
 
-    def run(self):
+    def exec(self):
-        super().run()
         self.logger.info('Initialized {} sensor backend'.format(self.__class__.__name__))
 
         while not self.should_stop():

platypush/backend/sensor/ir/zeroborg/__init__.py

@@ -32,9 +32,7 @@ class SensorIrZeroborgBackend(Backend):
         self.logger.info('Initialized Zeroborg infrared sensor backend')
 
 
-    def run(self):
+    def exec(self):
-        super().run()
-
         while True:
             try:
                 self.zb.GetIrMessage()

platypush/backend/sensor/leap.py

@@ -61,9 +61,7 @@ class SensorLeapBackend(Backend):
         self.position_tolerance = position_tolerance
 
 
-    def run(self):
+    def exec(self):
-        super().run()
-
         listener = LeapListener(position_ranges=self.position_ranges,
                                 position_tolerance=self.position_tolerance)
 

platypush/backend/tcp.py

@@ -90,9 +90,7 @@ class TcpBackend(Backend):
 
         threading.Thread(target=_f_wrapper).run()
 
-    def run(self):
+    def exec(self):
-        super().run()
-
         serv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         serv_sock.bind((self.bind_address, self.port))
 

platypush/backend/weather/forecast.py

@@ -27,8 +27,7 @@ class WeatherForecastBackend(Backend):
     def send_message(self, msg):
         pass
 
-    def run(self):
+    def exec(self):
-        super().run()
         weather = get_plugin('weather.forecast')
         self.logger.info('Initialized weather forecast backend')