From 5ba3fa1b5be69ddd27cad6540c57f0fcfd3a9722 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 8 Dec 2022 12:28:36 +0100 Subject: [PATCH 1/5] FIX: Parenthesized context managers are only available in Python >= 3.10 Since Parenthesized context managers are only supported on very recent versions of Python (thanks black for breaking back-compatibility), we should still use the old multiline syntax - it's not worth breaking compatibility with Python >= 3.6 and < 3.10 just to avoid typing a backslash. --- platypush/utils/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py index efd13f17d..1414587e3 100644 --- a/platypush/utils/__init__.py +++ b/platypush/utils/__init__.py @@ -412,10 +412,8 @@ def get_or_generate_jwt_rsa_key_pair(): pub_key_file = priv_key_file + '.pub' if os.path.isfile(priv_key_file) and os.path.isfile(pub_key_file): - with ( - open(pub_key_file, 'r') as f1, - open(priv_key_file, 'r') as f2 - ): + with open(pub_key_file, 'r') as f1, \ + open(priv_key_file, 'r') as f2: return ( rsa.PublicKey.load_pkcs1(f1.read().encode()), rsa.PrivateKey.load_pkcs1(f2.read().encode()), From 3b1147eaae6d92c1e26fcb881c77d430a969f389 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 8 Dec 2022 12:33:03 +0100 Subject: [PATCH 2/5] =?UTF-8?q?Bump=20version:=200.24.0=20=E2=86=92=200.24?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 +++++++ platypush/__init__.py | 2 +- setup.cfg | 3 ++- setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c22918ab..c1938d79e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. Given the high speed of development in the first phase, changes are being reported only starting from v0.20.2. +## [0.24.1] + +### Fixed + +- Removed a parenthesized context manager that broke compatibility with + Python < 3.10. + ## [0.24.0] ### Added diff --git a/platypush/__init__.py b/platypush/__init__.py index ea9e2d101..0456e892a 100644 --- a/platypush/__init__.py +++ b/platypush/__init__.py @@ -23,7 +23,7 @@ from .message.response import Response from .utils import set_thread_name, get_enabled_plugins __author__ = 'Fabio Manganiello ' -__version__ = '0.24.0' +__version__ = '0.24.1' logger = logging.getLogger('platypush') diff --git a/setup.cfg b/setup.cfg index 2013b49d0..5ea873348 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.24.0 +current_version = 0.24.1 commit = True tag = True @@ -11,3 +11,4 @@ max-line-length = 120 ignore = SIM105 W503 + diff --git a/setup.py b/setup.py index 86abd2ddc..ed852836b 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ backend = pkg_files('platypush/backend') setup( name="platypush", - version="0.24.0", + version="0.24.1", author="Fabio Manganiello", author_email="info@fabiomanganiello.com", description="Platypush service", From 219a0a99ca5c5949cf8a63d668ea42c809f15a57 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Fri, 9 Dec 2022 23:37:10 +0100 Subject: [PATCH 3/5] `main.db` should use the configured `workdir` when not specified. Closes: #234 Reviewed-On: https://git.platypush.tech/platypush/platypush/issues/234 --- CHANGELOG.md | 7 +++++++ platypush/config/__init__.py | 27 ++++++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1938d79e..362324d55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. Given the high speed of development in the first phase, changes are being reported only starting from v0.20.2. +## [Unreleased] + +## Fixed + +- The `main.db` configuration should use the configured `workdir` when no + values are specified. + ## [0.24.1] ### Fixed diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index cb5ab789b..addba07fa 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -10,6 +10,7 @@ import re import shutil import socket import sys +from urllib.parse import quote from typing import Optional import yaml @@ -72,7 +73,7 @@ class Config: if cfgfile is None: cfgfile = self._create_default_config() - self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + cfgfile = self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) self._config = self._read_config_file(self._cfgfile) if 'token' in self._config: @@ -80,6 +81,7 @@ class Config: if 'workdir' not in self._config: self._config['workdir'] = self._workdir_location + self._config['workdir'] = os.path.expanduser(self._config['workdir']) os.makedirs(self._config['workdir'], exist_ok=True) if 'scripts_dir' not in self._config: @@ -94,6 +96,7 @@ class Config: ) os.makedirs(self._config['dashboards_dir'], mode=0o755, exist_ok=True) + # Create a default (empty) __init__.py in the scripts folder init_py = os.path.join(self._config['scripts_dir'], '__init__.py') if not os.path.isfile(init_py): with open(init_py, 'w') as f: @@ -106,15 +109,21 @@ class Config: ) sys.path = [scripts_parent_dir] + sys.path - self._config['db'] = self._config.get( - 'main.db', - { - 'engine': 'sqlite:///' - + os.path.join( - os.path.expanduser('~'), '.local', 'share', 'platypush', 'main.db' + # Initialize the default db connection string + db_engine = self._config.get('main.db', '') + if db_engine: + if isinstance(db_engine, str): + db_engine = { + 'engine': db_engine, + } + else: + db_engine = { + 'engine': 'sqlite:///' + os.path.join( + quote(self._config['workdir']), 'main.db' ) - }, - ) + } + + self._config['db'] = db_engine logging_config = { 'level': logging.INFO, From 6713bf699453642fb34c35173972680320e57986 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 10 Dec 2022 14:52:10 +0100 Subject: [PATCH 4/5] Fixed `backend.zwave` event logic dispatch for recent versions of ZWaveJS. ZWaveJS has broken back-compatibility with zwavejs2mqtt when it comes to events format. Only a partial representation of the node and value objects is forwarded, and that's often not sufficient to infer the full state of the node with its values. The `_dispatch_event` logic has therefore been modified to accommodate both the implementation. This means that we have to go conservative in order to preserve back-compatibility and not over-complicate things, even if it (slightly) comes at the expense of performance. --- platypush/backend/zwave/mqtt/__init__.py | 154 +++++++++++++++-------- 1 file changed, 103 insertions(+), 51 deletions(-) diff --git a/platypush/backend/zwave/mqtt/__init__.py b/platypush/backend/zwave/mqtt/__init__.py index 7d38c8e18..ae5cb7d06 100644 --- a/platypush/backend/zwave/mqtt/__init__.py +++ b/platypush/backend/zwave/mqtt/__init__.py @@ -5,9 +5,17 @@ from typing import Optional, Type from platypush.backend.mqtt import MqttBackend from platypush.context import get_plugin -from platypush.message.event.zwave import ZwaveEvent, ZwaveNodeAddedEvent, ZwaveValueChangedEvent, \ - ZwaveNodeRemovedEvent, ZwaveNodeRenamedEvent, ZwaveNodeReadyEvent, ZwaveNodeEvent, ZwaveNodeAsleepEvent, \ - ZwaveNodeAwakeEvent +from platypush.message.event.zwave import ( + ZwaveEvent, + ZwaveNodeAddedEvent, + ZwaveValueChangedEvent, + ZwaveNodeRemovedEvent, + ZwaveNodeRenamedEvent, + ZwaveNodeReadyEvent, + ZwaveNodeEvent, + ZwaveNodeAsleepEvent, + ZwaveNodeAwakeEvent, +) class ZwaveMqttBackend(MqttBackend): @@ -41,6 +49,7 @@ class ZwaveMqttBackend(MqttBackend): """ from platypush.plugins.zwave.mqtt import ZwaveMqttPlugin + self.plugin: ZwaveMqttPlugin = get_plugin('zwave.mqtt') assert self.plugin, 'The zwave.mqtt plugin is not configured' @@ -61,75 +70,117 @@ class ZwaveMqttBackend(MqttBackend): 'password': self.plugin.password, } - listeners = [{ - **self.server_info, - 'topics': [ - self.plugin.events_topic + '/node/' + topic - for topic in ['node_ready', 'node_sleep', 'node_value_updated', 'node_metadata_updated', 'node_wakeup'] - ], - }] + listeners = [ + { + **self.server_info, + 'topics': [ + self.plugin.events_topic + '/node/' + topic + for topic in [ + 'node_ready', + 'node_sleep', + 'node_value_updated', + 'node_metadata_updated', + 'node_wakeup', + ] + ], + } + ] - super().__init__(*args, subscribe_default_topic=False, listeners=listeners, client_id=client_id, **kwargs) + super().__init__( + *args, + subscribe_default_topic=False, + listeners=listeners, + client_id=client_id, + **kwargs, + ) if not client_id: self.client_id += '-zwavejs-mqtt' - def _dispatch_event(self, event_type: Type[ZwaveEvent], node: Optional[dict] = None, value: Optional[dict] = None, - **kwargs): - if value and 'id' not in value: - value_id = f"{value['commandClass']}-{value.get('endpoint', 0)}-{value['property']}" - if 'propertyKey' in value: - value_id += '-' + str(value['propertyKey']) + def _dispatch_event( + self, + event_type: Type[ZwaveEvent], + node: dict, + value: Optional[dict] = None, + **kwargs, + ): + node_id = node.get('id') + assert node_id is not None, 'No node ID specified' - if value_id not in node.get('values', {}): - self.logger.warning(f'value_id {value_id} not found on node {node["id"]}') + # This is far from efficient (we are querying the latest version of the whole + # node for every event we receive), but this is the best we can do with recent + # versions of ZWaveJS that only transmit partial representations of the node and + # the value. The alternative would be to come up with a complex logic for merging + # cached and new values, with the risk of breaking back-compatibility with earlier + # implementations of zwavejs2mqtt. + node = kwargs['node'] = self.plugin.get_nodes(node_id).output # type: ignore + node_values = node.get('values', {}) + + if node and value: + # Infer the value_id structure if it's not provided on the event + value_id = value.get('id') + if value_id is None: + value_id = f"{value['commandClass']}-{value.get('endpoint', 0)}-{value['property']}" + if 'propertyKey' in value: + value_id += '-' + str(value['propertyKey']) + + # Prepend the node_id to value_id if it's not available in node['values'] + # (compatibility with more recent versions of ZwaveJS that don't provide + # the value_id on the events) + if value_id not in node_values: + value_id = f"{node_id}-{value_id}" + + if value_id not in node_values: + self.logger.warning(f'value_id {value_id} not found on node {node_id}') return - value = node['values'][value_id] + value = kwargs['value'] = node_values[value_id] - if value: - kwargs['value'] = self.plugin.value_to_dict(value) - - if node: - kwargs['node'] = self.plugin.node_to_dict(node) - node_id = kwargs['node']['node_id'] - - if event_type == ZwaveNodeEvent: - if node_id not in self._nodes: - event_type = ZwaveNodeAddedEvent - elif kwargs['node']['name'] != self._nodes[node_id]['name']: - event_type = ZwaveNodeRenamedEvent + if event_type == ZwaveNodeEvent: + # If this node_id wasn't cached before, then it's a new node + if node_id not in self._nodes: + event_type = ZwaveNodeAddedEvent + # If the name has changed, we have a rename event + elif node['name'] != self._nodes[node_id]['name']: + event_type = ZwaveNodeRenamedEvent if event_type == ZwaveNodeRemovedEvent: self._nodes.pop(node_id, None) else: - self._nodes[node_id] = kwargs['node'] + self._nodes[node_id] = node evt = event_type(**kwargs) self._events_queue.put(evt) - # zwavejs2mqtt currently treats some values (e.g. binary switches) in an inconsistent way, - # using two values - a read-only value called currentValue that gets updated on the - # node_value_updated topic, and a writable value called targetValue that doesn't get updated - # (see https://github.com/zwave-js/zwavejs2mqtt/blob/4a6a5c5f1274763fd3aced4cae2c72ea060716b5/docs/guide/migrating.md). - # To properly manage updates on writable values, propagate an event for both. - if event_type == ZwaveValueChangedEvent and kwargs.get('value', {}).get('property_id') == 'currentValue': - value = kwargs['value'].copy() - target_value_id = f'{kwargs["node"]["node_id"]}-{value["command_class"]}-{value.get("endpoint", 0)}' \ - f'-targetValue' - kwargs['value'] = kwargs['node'].get('values', {}).get(target_value_id) + if value and event_type == ZwaveValueChangedEvent: + value = value.copy() - if kwargs['value']: - kwargs['value']['data'] = value['data'] - kwargs['node']['values'][target_value_id] = kwargs['value'] - evt = event_type(**kwargs) - self._events_queue.put(evt) + # zwavejs2mqtt currently treats some values (e.g. binary switches) in an inconsistent way, + # using two values - a read-only value called currentValue that gets updated on the + # node_value_updated topic, and a writable value called targetValue that doesn't get updated + # (see https://github.com/zwave-js/zwavejs2mqtt/blob/4a6a5c5f1274763fd3aced4cae2c72ea060716b5 \ + # /docs/guide/migrating.md). + # To properly manage updates on writable values, propagate an event for both. + if value.get('property_id') == 'currentValue': + target_value_id = ( + f'{node_id}-{value["command_class"]}-{value.get("endpoint", 0)}' + f'-targetValue' + ) + target_value = node_values.get(target_value_id) + + if target_value: + kwargs['value']['data'] = value['data'] + kwargs['node']['values'][target_value_id] = kwargs['value'] + evt = event_type(**kwargs) + self._events_queue.put(evt) def on_mqtt_message(self): def handler(_, __, msg): if not msg.topic.startswith(self.events_topic): return - topic = msg.topic[len(self.events_topic) + 1:].split('/').pop() + topic = ( + msg.topic[(len(self.events_topic) + 1) :].split('/').pop() # noqa: E203 + ) data = msg.payload.decode() if not data: return @@ -141,7 +192,9 @@ class ZwaveMqttBackend(MqttBackend): try: if topic == 'node_value_updated': - self._dispatch_event(ZwaveValueChangedEvent, node=data[0], value=data[1]) + self._dispatch_event( + ZwaveValueChangedEvent, node=data[0], value=data[1] + ) elif topic == 'node_metadata_updated': self._dispatch_event(ZwaveNodeEvent, node=data[0]) elif topic == 'node_sleep': @@ -160,7 +213,6 @@ class ZwaveMqttBackend(MqttBackend): def run(self): super().run() self.logger.debug('Refreshing Z-Wave nodes') - # noinspection PyUnresolvedReferences self._nodes = self.plugin.get_nodes().output while not self.should_stop(): From 4c8190ac14729529fdd1a69ade8735e7f5d00d58 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 10 Dec 2022 15:37:05 +0100 Subject: [PATCH 5/5] =?UTF-8?q?Bump=20version:=200.24.1=20=E2=86=92=200.24?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 9 ++++++--- platypush/__init__.py | 2 +- setup.cfg | 3 +-- setup.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 362324d55..24ac59c2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,21 +4,24 @@ All notable changes to this project will be documented in this file. Given the high speed of development in the first phase, changes are being reported only starting from v0.20.2. -## [Unreleased] +## [0.24.2] - 2022-12-10 ## Fixed - The `main.db` configuration should use the configured `workdir` when no values are specified. -## [0.24.1] +- The `zwave.mqtt` is now compatible both with older (i.e. `zwavejs2mqtt`) and + newer (i.e. `ZwaveJS`) versions of the backend. + +## [0.24.1] - 2022-12-08 ### Fixed - Removed a parenthesized context manager that broke compatibility with Python < 3.10. -## [0.24.0] +## [0.24.0] - 2022-11-22 ### Added diff --git a/platypush/__init__.py b/platypush/__init__.py index 0456e892a..edc1f12d8 100644 --- a/platypush/__init__.py +++ b/platypush/__init__.py @@ -23,7 +23,7 @@ from .message.response import Response from .utils import set_thread_name, get_enabled_plugins __author__ = 'Fabio Manganiello ' -__version__ = '0.24.1' +__version__ = '0.24.2' logger = logging.getLogger('platypush') diff --git a/setup.cfg b/setup.cfg index 5ea873348..d7bb3fa60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.24.1 +current_version = 0.24.2 commit = True tag = True @@ -11,4 +11,3 @@ max-line-length = 120 ignore = SIM105 W503 - diff --git a/setup.py b/setup.py index ed852836b..16bacf3a0 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ backend = pkg_files('platypush/backend') setup( name="platypush", - version="0.24.1", + version="0.24.2", author="Fabio Manganiello", author_email="info@fabiomanganiello.com", description="Platypush service",