blacklight/platypush
1
0
Fork 0
forked from platypush/platypush
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added --db CLI option and support for configuration over environment.

Browse source 
Closes: #280
This commit is contained in:
Fabio Manganiello 2024-04-05 02:54:45 +02:00
parent c8361aa475
commit 5f6fd4aa54
Signed by: blacklight
GPG key ID: D90FBA7F76362774
6 changed files with 208 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

186
platypush/app/_app.py
View file

@ -1,5 +1,6 @@
from contextlib import contextmanager
import logging
import pathlib
import os
import signal
import subprocess
@ -41,6 +42,7 @@ class Application:
self,
config_file: Optional[str] = None,
workdir: Optional[str] = None,
db: Optional[str] = None,
logsdir: Optional[str] = None,
cachedir: Optional[str] = None,
device_id: Optional[str] = None,
@ -56,30 +58,87 @@ class Application:
ctrl_sock: Optional[str] = None,
):
"""
:param config_file: Configuration file override (default: None).
:param workdir: Overrides the ``workdir`` setting in the configuration
file (default: None).
:param logsdir: Set logging directory. If not specified, the
``filename`` setting under the ``logging`` section of the
configuration file is used. If not set, logging will be sent to
stdout and stderr.
:param cachedir: Overrides the ``cachedir`` setting in the configuration
file (default: None).
:param device_id: Override the device ID used to identify this
instance. If not passed here, it is inferred from the configuration
(device_id field). If not present there either, it is inferred from
the hostname.
:param config_file: Configuration file. The order of precedence is:
- The ``config_file`` parameter (or the ``-c``/``--config`` command
line argument).
- The ``PLATYPUSH_CONFIG`` environment variable.
- ``./config.yaml``
- ``~/.config/platypush/config.yaml``
- ``/etc/platypush/config.yaml``
:param workdir: Working directory where the application will store its
data and integration plugins will store their data. The order of
precedence is:
- The ``workdir`` parameter (or the ``-w``/``--workdir`` command
line argument).
- The ``PLATYPUSH_WORKDIR`` environment variable.
- The ``workdir`` field in the configuration file.
- ``~/.local/share/platypush``
:param db: Main database engine for the application. Supports SQLAlchemy
engine strings. The order of precedence is:
- The ``db`` parameter (or the ``--main-db``/``--db`` command
line argument).
- The ``PLATYPUSH_DB`` environment variable.
- The ``db`` field in the configuration file.
- ``sqlite:///<workdir>/main.db``
:param logsdir: Logs directory where the application will store its logs.
The order of precedence is:
- The ``logsdir`` parameter (or the ``-l``/``--logsdir`` command
line argument).
- The ``PLATYPUSH_LOGSDIR`` environment variable.
- The ``logging`` -> ``filename`` field in the configuration
file (the ``format`` and ``level`` fields can be set as well
using Python logging configuration).
- stdout and stderr
:param cachedir: Directory where the application and the plugins will store
their cache data. The order of precedence is:
- The ``cachedir`` parameter (or the ``--cachedir`` command line
argument).
- The ``PLATYPUSH_CACHEDIR`` environment variable.
- The ``cachedir`` field in the configuration file.
- ``~/.cache/platypush``
:param device_id: Device ID used to identify this instance. The order
of precedence is:
- The ``device_id`` parameter (or the ``--device-id`` command
line argument).
- The ``PLATYPUSH_DEVICE_ID`` environment variable.
- The ``device_id`` field in the configuration file.
- The hostname of the machine.
:param pidfile: File where platypush will store its PID upon launch,
useful if you're planning to integrate the application within a
service or a launcher script (default: None).
service or a launcher script. Order of precedence:
- The ``pidfile`` parameter (or the ``-P``/``--pidfile`` command
line argument).
- The ``PLATYPUSH_PIDFILE`` environment variable.
- No PID file.
:param requests_to_process: Exit after processing the specified
number of requests (default: None, loop forever).
number of requests (default: None, loop forever). This is usually
useful for testing purposes.
:param no_capture_stdout: Set to true if you want to disable the
stdout capture by the logging system (default: False).
:param no_capture_stderr: Set to true if you want to disable the
stderr capture by the logging system (default: False).
:param redis_queue: Name of the (Redis) queue used for dispatching
messages (default: platypush/bus).
messages. Order of precedence:
- The ``redis_queue`` parameter (or the ``--redis-queue`` command
line argument).
- The ``PLATYPUSH_REDIS_QUEUE`` environment variable.
- ``platypush/bus``
:param verbose: Enable debug/verbose logging, overriding the stored
configuration (default: False).
:param start_redis: If set, it starts a managed Redis instance upon
@ -87,34 +146,57 @@ class Application:
server). This is particularly useful when running the application
inside of Docker containers, without relying on ``docker-compose``
to start multiple containers, and in tests (default: False).
:param redis_host: Host of the Redis server to be used. It overrides
the settings in the ``redis`` section of the configuration file.
:param redis_port: Port of the local Redis server. It overrides the
settings in the ``redis`` section of the configuration file.
:param redis_host: Host of the Redis server to be used. The order of
precedence is:
- The ``redis_host`` parameter (or the ``--redis-host`` command
line argument).
- The ``PLATYPUSH_REDIS_HOST`` environment variable.
- The ``redis`` -> ``host`` field in the configuration file.
- The ``backend.redis`` -> ``redis_args`` -> ``host`` field in
the configuration file.
- ``localhost``
:param redis_port: Port of the Redis server to be used. The order of
precedence is:
- The ``redis_port`` parameter (or the ``--redis-port`` command
line argument).
- The ``PLATYPUSH_REDIS_PORT`` environment variable.
- The ``redis`` -> ``port`` field in the configuration file.
- The ``backend.redis`` -> ``redis_args`` -> ``port`` field in
the configuration file.
- ``6379``
:param ctrl_sock: If set, it identifies a path to a UNIX domain socket
that the application can use to send control messages (e.g. STOP
and RESTART) to its parent.
"""
self.pidfile = pidfile
self.pidfile = pidfile or os.environ.get('PLATYPUSH_PIDFILE')
self.bus: Optional[Bus] = None
self.redis_queue = redis_queue or RedisBus.DEFAULT_REDIS_QUEUE
self.config_file = config_file
self._verbose = verbose
self._logsdir = (
os.path.abspath(os.path.expanduser(logsdir)) if logsdir else None
self.redis_queue = (
redis_queue
or os.environ.get('PLATYPUSH_REDIS_QUEUE')
or RedisBus.DEFAULT_REDIS_QUEUE
)
self.config_file = config_file or os.environ.get('PLATYPUSH_CONFIG')
self.verbose = verbose
self.db_engine = db or os.environ.get('PLATYPUSH_DB')
self.device_id = device_id or os.environ.get('PLATYPUSH_DEVICE_ID')
self.logsdir = self.expand_path(logsdir or os.environ.get('PLATYPUSH_LOGSDIR'))
self.workdir = self.expand_path(workdir or os.environ.get('PLATYPUSH_WORKDIR'))
self.cachedir = self.expand_path(
cachedir or os.environ.get('PLATYPUSH_CACHEDIR')
)
Config.init(
self.config_file,
device_id=device_id,
workdir=os.path.abspath(os.path.expanduser(workdir)) if workdir else None,
cachedir=os.path.abspath(os.path.expanduser(cachedir))
if cachedir
else None,
ctrl_sock=os.path.abspath(os.path.expanduser(ctrl_sock))
if ctrl_sock
else None,
device_id=self.device_id,
workdir=self.workdir,
db=self.db_engine,
cachedir=self.cachedir,
ctrl_sock=self.expand_path(ctrl_sock),
)
self.no_capture_stdout = no_capture_stdout
@ -125,24 +207,25 @@ class Application:
self.processed_requests = 0
self.cron_scheduler = None
self.start_redis = start_redis
self.redis_host = redis_host
self.redis_port = redis_port
self.redis_conf = {}
self.redis_host = redis_host or os.environ.get('PLATYPUSH_REDIS_HOST')
self.redis_port = redis_port or os.environ.get('PLATYPUSH_REDIS_PORT')
self._redis_conf = {
'host': self.redis_host or 'localhost',
'port': self.redis_port or self._default_redis_port,
}
self._redis_proc: Optional[subprocess.Popen] = None
self.cmd_stream = CommandStream(ctrl_sock)
self._init_bus()
self._init_logging()
@staticmethod
def expand_path(path: Optional[str]) -> Optional[str]:
return os.path.abspath(os.path.expanduser(path)) if path else None
def _init_bus(self):
self._redis_conf = get_redis_conf()
self._redis_conf['port'] = self.redis_port or self._redis_conf.get(
'port', self._default_redis_port
)
if self.redis_host:
self._redis_conf['host'] = self.redis_host
self._redis_conf = {**self._redis_conf, **get_redis_conf()}
Config.set('redis', self._redis_conf)
self.bus = RedisBus(
redis_queue=self.redis_queue,
@ -152,12 +235,18 @@ class Application:
def _init_logging(self):
logging_conf = Config.get('logging') or {}
if self._verbose:
if self.verbose:
logging_conf['level'] = logging.DEBUG
if self._logsdir:
logging_conf['filename'] = os.path.join(self._logsdir, 'platypush.log')
if self.logsdir:
logging_conf['filename'] = os.path.join(self.logsdir, 'platypush.log')
logging_conf.pop('stream', None)
if logging_conf.get('filename'):
pathlib.Path(os.path.dirname(logging_conf['filename'])).mkdir(
parents=True, exist_ok=True
)
Config.set('logging', logging_conf)
logging.basicConfig(**logging_conf)
@ -214,6 +303,7 @@ class Application:
workdir=opts.workdir,
cachedir=opts.cachedir,
logsdir=opts.logsdir,
db=opts.db_engine,
device_id=opts.device_id,
pidfile=opts.pidfile,
no_capture_stdout=opts.no_capture_stdout,

59
platypush/backend/redis/__init__.py
View file

@ -1,43 +1,41 @@
import json
from typing import Optional, Union
from typing import Any, Dict, Optional, Union
from redis import Redis
from platypush.backend import Backend
from platypush.context import get_plugin
from platypush.message import Message
from platypush.utils import get_redis_conf
class RedisBackend(Backend):
"""
Backend that reads messages from a configured Redis queue (default:
``platypush_bus_mq``) and posts them to the application bus. Very
useful when you have plugin whose code is executed in another process
``platypush/backend/redis``) and forwards them to the application bus.
Useful when you have plugin whose code is executed in another process
and can't post events or requests to the application bus.
"""
def __init__(self, *args, queue='platypush_bus_mq', redis_args=None, **kwargs):
def __init__(
self,
*args,
queue: str = 'platypush/backend/redis',
redis_args: Optional[Dict[str, Any]] = None,
**kwargs
):
"""
:param queue: Queue name to listen on (default: ``platypush_bus_mq``)
:type queue: str
:param queue: Name of the Redis queue to listen to (default:
``platypush/backend/redis``).
:param redis_args: Arguments that will be passed to the redis-py constructor (e.g. host, port, password), see
https://redis-py.readthedocs.io/en/latest/
:type redis_args: dict
:param redis_args: Arguments that will be passed to the redis-py
constructor (e.g. host, port, password). See
https://redis-py.readthedocs.io/en/latest/connections.html#redis.Redis
for supported parameters.
"""
super().__init__(*args, **kwargs)
if redis_args is None:
redis_args = {}
self.redis_args = redis_args or get_redis_conf()
self.queue = queue
if not redis_args:
redis_plugin = get_plugin('redis')
if redis_plugin and redis_plugin.kwargs:
redis_args = redis_plugin.kwargs
self.redis_args = redis_args
self.redis: Optional[Redis] = None
def send_message(
@ -47,20 +45,21 @@ class RedisBackend(Backend):
Send a message to a Redis queue.
:param msg: Message to send, as a ``Message`` object or a string.
:param queue_name: Queue name to send the message to (default: ``platypush_bus_mq``).
:param queue_name: Queue name to send the message to (default:
configured ``queue`` value).
"""
if not self.redis:
self.logger.warning('The Redis backend is not yet running.')
return
self.redis.rpush(queue_name or self.queue, str(msg))
def get_message(self, queue_name=None):
def get_message(self, queue_name: Optional[str] = None) -> Optional[Message]:
queue = queue_name or self.queue
assert self.redis, 'The Redis backend is not yet running.'
data = self.redis.blpop(queue, timeout=1)
if data is None:
return
return None
msg = data[1].decode()
try:
@ -68,15 +67,9 @@ class RedisBackend(Backend):
except Exception as e:
self.logger.debug(str(e))
try:
import ast
msg = Message.build(ast.literal_eval(msg))
msg = json.loads(msg)
except Exception as ee:
self.logger.debug(str(ee))
try:
msg = json.loads(msg)
except Exception as eee:
self.logger.exception(eee)
self.logger.exception(ee)
return msg

12
platypush/cli.py
View file

@ -32,6 +32,18 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace:
help='Custom working directory to be used for the application',
)
parser.add_argument(
'--main-db',
'--db',
dest='db_engine',
required=False,
default=None,
help='Custom database engine to be used for the application '
'(e.g. sqlite:///:memory: or sqlite:///path/to/db.sqlite). '
'If missing, it falls back to the value of the `main.db` setting in the configuration file. '
'If missing, it falls back to sqlite://<workdir>/main.db.',
)
parser.add_argument(
'--cachedir',
dest='cachedir',

22
platypush/config/__init__.py
View file

@ -97,6 +97,7 @@ class Config:
cfgfile: Optional[str] = None,
workdir: Optional[str] = None,
cachedir: Optional[str] = None,
db: Optional[str] = None,
):
"""
Constructor. Always use the class as a singleton (i.e. through
@ -106,6 +107,7 @@ class Config:
location in _cfgfile_locations).
:param workdir: Overrides the default working directory.
:param cachedir: Overrides the default cache directory.
:param db: Overrides the default database connection string.
"""
self.backends = {}
@ -124,7 +126,7 @@ class Config:
self._init_secrets()
self._init_dirs(workdir=workdir, cachedir=cachedir)
self._init_db()
self._init_db(db=db)
self._init_logging()
self._init_device_id()
self._init_environment()
@ -175,7 +177,14 @@ class Config:
self._config['logging'] = logging_config
def _init_db(self):
def _init_db(self, db: Optional[str] = None):
# If the db connection string is passed as an argument, use it
if db:
self._config['db'] = {
'engine': db,
}
return
# Initialize the default db connection string
db_engine = self._config.get('main.db', '')
if db_engine:
@ -474,6 +483,7 @@ class Config:
cfgfile: Optional[str] = None,
workdir: Optional[str] = None,
cachedir: Optional[str] = None,
db: Optional[str] = None,
force_reload: bool = False,
) -> "Config":
"""
@ -481,7 +491,7 @@ class Config:
"""
if force_reload or cls._instance is None:
cfg_args = [cfgfile] if cfgfile else []
cls._instance = Config(*cfg_args, workdir=workdir, cachedir=cachedir)
cls._instance = Config(*cfg_args, workdir=workdir, cachedir=cachedir, db=db)
return cls._instance
@classmethod
@ -546,6 +556,7 @@ class Config:
device_id: Optional[str] = None,
workdir: Optional[str] = None,
cachedir: Optional[str] = None,
db: Optional[str] = None,
ctrl_sock: Optional[str] = None,
**_,
):
@ -556,10 +567,11 @@ class Config:
:param device_id: Override the configured device_id.
:param workdir: Override the configured working directory.
:param cachedir: Override the configured cache directory.
:param db: Override the configured database connection string.
:param ctrl_sock: Override the configured control socket.
"""
cfg = cls._get_instance(
cfgfile, workdir=workdir, cachedir=cachedir, force_reload=True
cfgfile, workdir=workdir, cachedir=cachedir, db=db, force_reload=True
)
if device_id:
cfg.set('device_id', device_id)
@ -567,6 +579,8 @@ class Config:
cfg.set('workdir', workdir)
if cachedir:
cfg.set('cachedir', cachedir)
if db:
cfg.set('db', db)
if ctrl_sock:
cfg.set('ctrl_sock', ctrl_sock)

7
platypush/config/config.yaml
View file

@ -87,15 +87,20 @@
# # `workdir`. You can specify any other engine string here - the application has
# # been tested against SQLite, Postgres and MariaDB/MySQL >= 8.
# #
# # You can also specify a custom database engine at runtime (SQLAlchemy syntax
# # supported) through the `--db` argument otherwise.
# #
# # NOTE: If you want to use a DBMS other than SQLite, then you will also need to
# # ensure that a compatible Python driver is installed on the system where
# # Platypush is running. For example, Postgres will require the Python pg8000,
# # psycopg or another compatible driver.
#
# main.db:
# engine: sqlite:///home/user/.local/share/platypush/main.db
# engine: sqlite:////home/user/.local/share/platypush/main.db
# # OR, if you want to use e.g. Postgres with the pg8000 driver:
# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname
#
# # NOTE: The short syntax `main.db: <engine_string>` is also supported.
###
### ---------------------

22
platypush/plugins/redis/__init__.py
View file

@ -7,6 +7,9 @@ from platypush.utils import get_redis_conf
class RedisPlugin(Plugin):
"""
Plugin to send messages on Redis queues.
See https://redis-py.readthedocs.io/en/latest/connections.html#redis.Redis
for supported parameters.
"""
def __init__(self, *args, **kwargs):
@ -18,24 +21,19 @@ class RedisPlugin(Plugin):
return Redis(*self.args, **self.kwargs)
@action
def send_message(self, queue, msg, *args, **kwargs):
def send_message(self, queue: str, msg, *args, **kwargs):
"""
Send a message to a Redis queue.
:param queue: Queue name
:type queue: str
:param msg: Message to be sent
:type msg: str, bytes, list, dict, Message object
:param args: Args passed to the Redis constructor (see https://redis-py.readthedocs.io/en/latest/#redis.Redis)
:type args: list
:type msg: str, bytes, list, dict, :class:`platypush.message.Message`
:param args: Args passed to the Redis constructor (see
https://redis-py.readthedocs.io/en/latest/connections.html#redis.Redis)
:param kwargs: Kwargs passed to the Redis constructor (see
https://redis-py.readthedocs.io/en/latest/#redis.Redis)
:type kwargs: dict
https://redis-py.readthedocs.io/en/latest/connections.html#redis.Redis)
"""
if args or kwargs:
redis = Redis(*args, **kwargs)
else:
@ -48,7 +46,6 @@ class RedisPlugin(Plugin):
"""
:returns: The values specified in keys as a key/value dict (wraps MGET)
"""
return {
keys[i]: value.decode() if isinstance(value, bytes) else value
for (i, value) in enumerate(self._get_redis().mget(keys, *args))
@ -59,7 +56,6 @@ class RedisPlugin(Plugin):
"""
Set key/values based on mapping (wraps MSET)
"""
try:
return self._get_redis().mset(**kwargs)
except TypeError:
@ -80,7 +76,6 @@ class RedisPlugin(Plugin):
:param expiration: Expiration timeout (in seconds)
:type expiration: int
"""
return self._get_redis().expire(key, expiration)
@action
@ -90,7 +85,6 @@ class RedisPlugin(Plugin):
:param args: Keys to delete
"""
return self._get_redis().delete(*args)