From 2fda066e39af6ffd0fb9e504000fe24cf265c7f6 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 15 Jul 2018 20:12:23 +0200 Subject: [PATCH] - Support for Platypush main configuration db, where plugins and backends can store their data - Support for permanent cross-process storage of session variables through SQLite db - Support for db.select with table+filter instead of raw SQL query --- platypush/config/__init__.py | 5 +++ platypush/plugins/db/__init__.py | 55 +++++++++++++++++++++++++------- platypush/plugins/variable.py | 54 +++++++++++++++++++++++++++---- 3 files changed, 97 insertions(+), 17 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 32eaa5280..96e48ad90 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -67,6 +67,11 @@ class Config(object): self._config['workdir'] = self._workdir_location os.makedirs(self._config['workdir'], exist_ok=True) + self._config['db'] = self._config.get('main_db', { + 'engine': 'sqlite:///' + os.path.join( + os.environ['HOME'], '.local', 'share', 'platypush', 'main.db') + }) + logging_config = { 'level': logging.INFO, 'stream': sys.stdout, diff --git a/platypush/plugins/db/__init__.py b/platypush/plugins/db/__init__.py index 59776bd60..6b00c6c70 100644 --- a/platypush/plugins/db/__init__.py +++ b/platypush/plugins/db/__init__.py @@ -36,6 +36,14 @@ class DbPlugin(Plugin): else: return self.engine + def _build_condition(self, table, column, value): + if isinstance(value, str): + value = "'{}'".format(value) + elif not isinstance(value, int) and not isinstance(value, 'float'): + value = "'{}'".format(str(value)) + + return eval('table.c.{}=={}'.format(column, value)) + @action def execute(self, statement, engine=None, *args, **kwargs): """ @@ -64,19 +72,23 @@ class DbPlugin(Plugin): @action - def select(self, query, engine=None, *args, **kwargs): + def select(self, query=None, table=None, filter=None, engine=None, *args, **kwargs): """ Returns rows (as a list of hashes) given a query. :param query: SQL to be executed :type query: str + :param filter: Query WHERE filter expressed as a dictionary. This approach is preferred over specifying raw SQL in ``query`` as the latter approach may be prone to SQL injection, unless you need to build some complex SQL logic. + :type filter: dict + :param table: If you specified a filter instead of a raw query, you'll have to specify the target table + :type table: str :param engine: Engine to be used (default: default class engine) :type engine: str :param args: Extra arguments that will be passed to ``sqlalchemy.create_engine`` (see http://docs.sqlalchemy.org/en/latest/core/engines.html) :param kwargs: Extra kwargs that will be passed to ``sqlalchemy.create_engine`` (see http://docs.sqlalchemy.org/en/latest/core/engines.html) :returns: List of hashes representing the result rows. - Example: + Examples: Request:: @@ -90,25 +102,46 @@ class DbPlugin(Plugin): } } + or:: + + { + "type": "request", + "target": "your_host", + "action": "db.select", + "args": { + "engine": "sqlite:///:memory:", + "table": "table", + "filter": {"id": 1} + } + } + Response:: [ { "id": 1, "name": foo - }, - - { - "id": 2, - "name": bar } ] """ - engine = self._get_engine(engine, *args, **kwargs) + db = self._get_engine(engine, *args, **kwargs) - with engine.connect() as connection: + if table: + metadata = MetaData() + table = Table(table, metadata, autoload=True, autoload_with=db) + query = table.select() + + if filter: + for (k,v) in filter.items(): + query = query.where(self._build_condition(table, k, v)) + + if query is None: + raise RuntimeError('You need to specify either "query", or "table" and "filter"') + + with db.connect() as connection: result = connection.execute(query) + columns = result.keys() rows = [ { columns[i]: row[i] for i in range(0, len(columns)) } @@ -235,7 +268,7 @@ class DbPlugin(Plugin): update = table.update() for (k,v) in key.items(): - update = update.where(eval('table.c.{}=={}'.format(k, v))) + update = update.where(self._build_condition(table, k, v)) update = update.values(**values) engine.execute(update) @@ -282,7 +315,7 @@ class DbPlugin(Plugin): delete = table.delete() for (k,v) in record.items(): - delete = delete.where(eval('table.c.{}=={}'.format(k, v))) + delete = delete.where(self._build_condition(table, k, v)) engine.execute(delete) diff --git a/platypush/plugins/variable.py b/platypush/plugins/variable.py index efc883847..5d90cdfc2 100644 --- a/platypush/plugins/variable.py +++ b/platypush/plugins/variable.py @@ -1,3 +1,5 @@ +from platypush.config import Config +from platypush.context import get_plugin from platypush.plugins import Plugin, action @@ -7,9 +9,27 @@ class VariablePlugin(Plugin): accessed across your tasks. """ + _variable_table_name = 'variable' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._variables = {} + self.db_plugin = get_plugin('db') + + db = Config.get('db') + self.db_config = { + 'engine': db.get('engine'), + 'args': db.get('args', []), + 'kwargs': db.get('kwargs', {}) + } + + self._create_tables() + # self._variables = {} + + def _create_tables(self): + self.db_plugin.execute("""CREATE TABLE IF NOT EXISTS {}( + name varchar(255) not null primary key, + value text + )""".format(self._variable_table_name)) @action def get(self, name, default_value=None): @@ -24,7 +44,13 @@ class VariablePlugin(Plugin): :returns: A map in the format ``{"":""}`` """ - return {name: self._variables.get(name, default_value)} + rows = self.db_plugin.select(table=self._variable_table_name, + filter={'name': name}, + engine=self.db_config['engine'], + *self.db_config['args'], + **self.db_config['kwargs']).output + + return {name: rows[0]['value'] if rows else None} @action def set(self, **kwargs): @@ -34,10 +60,19 @@ class VariablePlugin(Plugin): :param kwargs: Key-value list of variables to set (e.g. ``foo='bar', answer=42``) """ - for (name, value) in kwargs.items(): - self._variables[name] = value + records = [ { 'name': k, 'value': v } + for (k,v) in kwargs.items() ] + + self.db_plugin.insert(table=self._variable_table_name, + records=records, key_columns=['name'], + engine=self.db_config['engine'], + on_duplicate_update=True, + *self.db_config['args'], + **self.db_config['kwargs']) + return kwargs + @action def unset(self, name): """ @@ -46,8 +81,15 @@ class VariablePlugin(Plugin): :param name: Name of the variable to remove :type name: str """ - if name in self._variables: - del self._variables[name] + + records = [ { 'name': name } ] + + self.db_plugin.delete(table=self._variable_table_name, + records=records, engine=self.db_config['engine'], + *self.db_config['args'], + **self.db_config['kwargs']) + + return True # vim:sw=4:ts=4:et: