[core] Better Redis connection fail handling logic.

If the connection to Redis goes down, it shouldn't take down the main
thread.

Instead, catch `RedisConnectionError`, and execute `poll` in a loop
until the connection is restored.
This commit is contained in:
Fabio Manganiello 2024-07-24 21:33:04 +02:00
parent 357d92b479
commit 70db33b4e2
Signed by untrusted user: blacklight
GPG key ID: D90FBA7F76362774

View file

@ -1,5 +1,6 @@
import logging import logging
import threading import threading
import time
from platypush.bus import Bus from platypush.bus import Bus
from platypush.message import Message from platypush.message import Message
@ -14,18 +15,24 @@ class RedisBus(Bus):
DEFAULT_REDIS_QUEUE: str = 'platypush/bus' DEFAULT_REDIS_QUEUE: str = 'platypush/bus'
def __init__(self, *args, on_message=None, redis_queue=None, **kwargs): def __init__(self, *_, on_message=None, redis_queue=None, **kwargs):
from platypush.utils import get_redis
super().__init__(on_message=on_message) super().__init__(on_message=on_message)
self.redis = get_redis(*args, **kwargs)
self.redis_args = kwargs self.redis_args = kwargs
self._redis = None
self.redis_queue = redis_queue or self.DEFAULT_REDIS_QUEUE self.redis_queue = redis_queue or self.DEFAULT_REDIS_QUEUE
self.on_message = on_message self.on_message = on_message
self.thread_id = threading.get_ident() self.thread_id = threading.get_ident()
self._pubsub = None self._pubsub = None
self._pubsub_lock = threading.RLock() self._pubsub_lock = threading.RLock()
@property
def redis(self):
from platypush.utils import get_redis
if not self._redis:
self._redis = get_redis(**self.redis_args)
return self._redis
@property @property
def pubsub(self): def pubsub(self):
with self._pubsub_lock: with self._pubsub_lock:
@ -37,14 +44,24 @@ class RedisBus(Bus):
""" """
Polls the Redis queue for new messages Polls the Redis queue for new messages
""" """
from platypush.message.event.application import ApplicationStartedEvent from redis.exceptions import ConnectionError as RedisConnectionError
from platypush.message.event.application import ApplicationStartedEvent
from platypush.utils import redis_pools
has_error = False
while not self.should_stop():
with self.pubsub as pubsub: with self.pubsub as pubsub:
try:
pubsub.subscribe(self.redis_queue) pubsub.subscribe(self.redis_queue)
self.post(ApplicationStartedEvent()) self.post(ApplicationStartedEvent())
try:
for msg in pubsub.listen(): for msg in pubsub.listen():
if has_error:
logger.info('Redis connection restored')
has_error = False
if msg.get('type') != 'message': if msg.get('type') != 'message':
continue continue
@ -58,8 +75,20 @@ class RedisBus(Bus):
self.on_message(parsed_msg) self.on_message(parsed_msg)
except Exception as e: except Exception as e:
logger.exception(e) logger.exception(e)
except RedisConnectionError as e:
if not (self.should_stop() or has_error):
logger.warning('Redis connection error: %s', e)
has_error = True
pubsub.close()
redis_pools.clear() # Clear the connection pool
self._redis = None
time.sleep(1)
finally: finally:
try:
pubsub.unsubscribe(self.redis_queue) pubsub.unsubscribe(self.redis_queue)
except RedisConnectionError:
pass
def post(self, msg): def post(self, msg):
""" """