From f7c594cc3f42c4bfa044150363d8f47b9afbc05c Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 2 Apr 2022 22:47:23 +0200 Subject: [PATCH 01/12] get_bus() should return a default RedisBus() instance if the main bus is not registered --- platypush/context/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platypush/context/__init__.py b/platypush/context/__init__.py index 43f764d73..bcadf289a 100644 --- a/platypush/context/__init__.py +++ b/platypush/context/__init__.py @@ -133,8 +133,11 @@ def get_plugin(plugin_name, reload=False): def get_bus() -> Bus: global main_bus - assert main_bus, 'The bus is not registered' - return main_bus + if main_bus: + return main_bus + + from platypush.bus.redis import RedisBus + return RedisBus() def get_or_create_event_loop(): From 486801653a1dc0ba64316031b837d822305a65cc Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 3 Apr 2022 00:26:39 +0200 Subject: [PATCH 02/12] Added `.exception` action to logger plugin --- platypush/plugins/logger/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platypush/plugins/logger/__init__.py b/platypush/plugins/logger/__init__.py index efac544d6..73de5fb73 100644 --- a/platypush/plugins/logger/__init__.py +++ b/platypush/plugins/logger/__init__.py @@ -41,6 +41,13 @@ class LoggerPlugin(Plugin): """ self.logger.error(msg, *args, **kwargs) + @action + def exception(self, exception, *args, **kwargs): + """ + logger.exception wrapper + """ + self.logger.exception(exception, *args, **kwargs) + # vim:sw=4:ts=4:et: From 1b30bfc454c96be03562ffdb406b9b7b2843fa5e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Apr 2022 17:21:47 +0200 Subject: [PATCH 03/12] Added more pre-commit hooks --- .pre-commit-config.yaml | 21 +++++++++++++++++++-- setup.cfg | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2f163aa7..18c7457a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,31 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.1.0 hooks: # - id: trailing-whitespace # - id: end-of-file-fixer - id: check-yaml + - id: check-json + - id: check-xml + - id: check-symlinks - id: check-added-large-files - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs - rev: v1.1.1 + rev: v1.1.2 hooks: - id: markdown-toc + +- repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-comprehensions + - flake8-simplify + +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black diff --git a/setup.cfg b/setup.cfg index 862dce588..1dd336c25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,6 @@ tag = True [metadata] description-file = README.md + +[flake8] +max-line-length = 120 From ca25607262245d016024fc882fa7f4e519813261 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Apr 2022 20:55:10 +0200 Subject: [PATCH 04/12] Skip string and underscore normalization in black --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e7c6caf61 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +skip-string-normalization = true +skip-numeric-underscore-normalization = true + From 12887b61fe4d789024c24b3f6310e8880c9188a8 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 25 Apr 2022 14:02:13 +0200 Subject: [PATCH 05/12] Don't fail hard if the Linode API doesn't return a list of instances --- platypush/backend/linode/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platypush/backend/linode/__init__.py b/platypush/backend/linode/__init__.py index 756eb2757..216a9f621 100644 --- a/platypush/backend/linode/__init__.py +++ b/platypush/backend/linode/__init__.py @@ -26,7 +26,7 @@ class LinodeBackend(SensorBackend): self.instances = set(instances or []) def process_data(self, data: Dict[str, dict], new_data: Optional[Dict[str, dict]] = None, **kwargs): - instances = data['instances'] + instances = data.get('instances', {}) old_instances = (self.data or {}).get('instances', {}) if self.instances: From a80adc996ff9459614b2d54952568fcd6709fe6a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 25 Apr 2022 16:54:26 +0200 Subject: [PATCH 06/12] [WIP] Default config.yaml in case a configuration file is missing in the default locations --- MANIFEST.in | 1 + platypush/config/config.auto.yaml | 7 +++++++ platypush/config/config.default.yaml | 2 ++ 3 files changed, 10 insertions(+) create mode 100644 platypush/config/config.auto.yaml create mode 100644 platypush/config/config.default.yaml diff --git a/MANIFEST.in b/MANIFEST.in index e61e3251c..9163c3d77 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include platypush/backend/http/webapp/dist * include platypush/plugins/http/webpage/mercury-parser.js +include platypush/config/*.yaml global-include manifest.yaml diff --git a/platypush/config/config.auto.yaml b/platypush/config/config.auto.yaml new file mode 100644 index 000000000..466458baa --- /dev/null +++ b/platypush/config/config.auto.yaml @@ -0,0 +1,7 @@ +# Auto-generated configuration file. +# Do not edit manually - use the config.yaml file for manual modifications +# instead + +backend.http: + port: 8008 + websocket_port: 8009 diff --git a/platypush/config/config.default.yaml b/platypush/config/config.default.yaml new file mode 100644 index 000000000..1e36005fe --- /dev/null +++ b/platypush/config/config.default.yaml @@ -0,0 +1,2 @@ +include: + - config.auto.yaml From da73a5f1b9db2d4286ec3341d1c8e609b1dc703f Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Tue, 26 Apr 2022 19:30:26 +0200 Subject: [PATCH 07/12] Replaced deprecated json_output arg in NextCloud client with response.json_data --- platypush/plugins/nextcloud/__init__.py | 272 +++++++++++++++++++----- 1 file changed, 215 insertions(+), 57 deletions(-) diff --git a/platypush/plugins/nextcloud/__init__.py b/platypush/plugins/nextcloud/__init__.py index a2306cb3b..682c97d55 100644 --- a/platypush/plugins/nextcloud/__init__.py +++ b/platypush/plugins/nextcloud/__init__.py @@ -50,8 +50,13 @@ class NextcloudPlugin(Plugin): """ - def __init__(self, url: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, - **kwargs): + def __init__( + self, + url: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + **kwargs + ): """ :param url: URL to the index of your default NextCloud instance. :param username: Default NextCloud username. @@ -61,8 +66,13 @@ class NextcloudPlugin(Plugin): self.conf = ClientConfig(url=url, username=username, password=password) self._client = self._get_client(**self.conf.to_dict()) - def _get_client(self, url: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, - raise_on_empty: bool = False): + def _get_client( + self, + url: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + raise_on_empty: bool = False, + ): from nextcloud import NextCloud if not url: @@ -71,19 +81,25 @@ class NextcloudPlugin(Plugin): raise AssertionError('No url/username/password provided') return None - return NextCloud(endpoint=self.conf.url, user=self.conf.username, password=self.conf.password, - json_output=True) + return NextCloud( + endpoint=self.conf.url, + user=self.conf.username, + password=self.conf.password, + ) - return NextCloud(endpoint=url, user=username, password=password, json_output=True) + return NextCloud(endpoint=url, user=username, password=password) @staticmethod def _get_permissions(permissions: Optional[List[str]]) -> int: int_perm = 0 - for perm in (permissions or []): + for perm in permissions or []: perm = perm.upper() - assert hasattr(Permission, perm), 'Unknown permissions type: {}. Supported permissions: {}'.format( - perm, [p.name.lower() for p in Permission]) + assert hasattr( + Permission, perm + ), 'Unknown permissions type: {}. Supported permissions: {}'.format( + perm, [p.name.lower() for p in Permission] + ) if perm == 'ALL': int_perm = Permission.ALL.value @@ -96,8 +112,11 @@ class NextcloudPlugin(Plugin): @staticmethod def _get_share_type(share_type: str) -> int: share_type = share_type.upper() - assert hasattr(ShareType, share_type), 'Unknown share type: {}. Supported share types: {}'.format( - share_type, [s.name.lower() for s in ShareType]) + assert hasattr( + ShareType, share_type + ), 'Unknown share type: {}. Supported share types: {}'.format( + share_type, [s.name.lower() for s in ShareType] + ) return getattr(ShareType, share_type).value @@ -114,13 +133,23 @@ class NextcloudPlugin(Plugin): args=', '.join(args), sep=', ' if args and kwargs else '', kwargs=', '.join(['{}={}'.format(k, v) for k, v in kwargs.items()]), - error=response.meta.get('message', '[No message]') if hasattr(response, 'meta') else response.raw.reason) + error=response.meta.get('message', '[No message]') + if hasattr(response, 'meta') + else response.raw.reason, + ) - return response.data + return response.json_data @action - def get_activities(self, since: Optional[id] = None, limit: Optional[int] = None, object_type: Optional[str] = None, - object_id: Optional[int] = None, sort: str = 'desc', **server_args) -> List[str]: + def get_activities( + self, + since: Optional[id] = None, + limit: Optional[int] = None, + object_type: Optional[str] = None, + object_id: Optional[int] = None, + sort: str = 'desc', + **server_args + ) -> List[str]: """ Get the list of recent activities on an instance. @@ -132,9 +161,15 @@ class NextcloudPlugin(Plugin): :param server_args: Override the default server settings (see :meth:`._get_client` arguments). :return: The list of selected activities. """ - return self._execute(server_args, 'get_activities', since=since, limit=limit, object_type=object_type, - object_id=object_id, - sort=sort) + return self._execute( + server_args, + 'get_activities', + since=since, + limit=limit, + object_type=object_type, + object_id=object_id, + sort=sort, + ) @action def get_apps(self, **server_args) -> List[str]: @@ -216,8 +251,13 @@ class NextcloudPlugin(Plugin): return self._execute(server_args, 'get_group', group_id) @action - def get_groups(self, search: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, - **server_args) -> List[str]: + def get_groups( + self, + search: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + **server_args + ) -> List[str]: """ Search for groups. @@ -226,7 +266,9 @@ class NextcloudPlugin(Plugin): :param offset: Start offset. :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ - return self._execute(server_args, 'get_groups', search=search, limit=limit, offset=offset).get('groups', []) + return self._execute( + server_args, 'get_groups', search=search, limit=limit, offset=offset + ).get('groups', []) @action def create_group_folder(self, name: str, **server_args): @@ -268,7 +310,9 @@ class NextcloudPlugin(Plugin): return self._execute(server_args, 'get_group_folders') @action - def rename_group_folder(self, folder_id: Union[int, str], new_name: str, **server_args): + def rename_group_folder( + self, folder_id: Union[int, str], new_name: str, **server_args + ): """ Rename a group folder. @@ -279,7 +323,9 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'rename_group_folder', folder_id, new_name) @action - def grant_access_to_group_folder(self, folder_id: Union[int, str], group_id: str, **server_args): + def grant_access_to_group_folder( + self, folder_id: Union[int, str], group_id: str, **server_args + ): """ Grant access to a group folder to a given group. @@ -290,7 +336,9 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'grant_access_to_group_folder', folder_id, group_id) @action - def revoke_access_to_group_folder(self, folder_id: Union[int, str], group_id: str, **server_args): + def revoke_access_to_group_folder( + self, folder_id: Union[int, str], group_id: str, **server_args + ): """ Revoke access to a group folder to a given group. @@ -301,7 +349,9 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'revoke_access_to_group_folder', folder_id, group_id) @action - def set_group_folder_quota(self, folder_id: Union[int, str], quota: Optional[int], **server_args): + def set_group_folder_quota( + self, folder_id: Union[int, str], quota: Optional[int], **server_args + ): """ Set the quota of a group folder. @@ -309,11 +359,21 @@ class NextcloudPlugin(Plugin): :param quota: Quota in bytes - set None for unlimited. :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ - self._execute(server_args, 'set_quota_of_group_folder', folder_id, quota if quota is not None else -3) + self._execute( + server_args, + 'set_quota_of_group_folder', + folder_id, + quota if quota is not None else -3, + ) @action - def set_group_folder_permissions(self, folder_id: Union[int, str], group_id: str, permissions: List[str], - **server_args): + def set_group_folder_permissions( + self, + folder_id: Union[int, str], + group_id: str, + permissions: List[str], + **server_args + ): """ Set the permissions on a folder for a group. @@ -330,8 +390,13 @@ class NextcloudPlugin(Plugin): :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ - self._execute(server_args, 'set_permissions_to_group_folder', folder_id, group_id, - self._get_permissions(permissions)) + self._execute( + server_args, + 'set_permissions_to_group_folder', + folder_id, + group_id, + self._get_permissions(permissions), + ) @action def get_notifications(self, **server_args) -> list: @@ -372,8 +437,16 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'delete_notification', notification_id) @action - def create_share(self, path: str, share_type: str, share_with: Optional[str] = None, public_upload: bool = False, - password: Optional[str] = None, permissions: Optional[List[str]] = None, **server_args) -> dict: + def create_share( + self, + path: str, + share_type: str, + share_with: Optional[str] = None, + public_upload: bool = False, + password: Optional[str] = None, + permissions: Optional[List[str]] = None, + **server_args + ) -> dict: """ Share a file/folder with a user/group or a public link. @@ -442,9 +515,16 @@ class NextcloudPlugin(Plugin): """ share_type = self._get_share_type(share_type) permissions = self._get_permissions(permissions or ['read']) - return self._execute(server_args, 'create_share', path, share_type=share_type, share_with=share_with, - public_upload=public_upload, - password=password, permissions=permissions) + return self._execute( + server_args, + 'create_share', + path, + share_type=share_type, + share_with=share_with, + public_upload=public_upload, + password=password, + permissions=permissions, + ) @action def get_shares(self, **server_args) -> List[dict]: @@ -516,8 +596,15 @@ class NextcloudPlugin(Plugin): return self._execute(server_args, 'get_share_info', str(share_id)) @action - def update_share(self, share_id: int, public_upload: Optional[bool] = None, password: Optional[str] = None, - permissions: Optional[List[str]] = None, expire_date: Optional[str] = None, **server_args): + def update_share( + self, + share_id: int, + public_upload: Optional[bool] = None, + password: Optional[str] = None, + permissions: Optional[List[str]] = None, + expire_date: Optional[str] = None, + **server_args + ): """ Update the permissions of a shared resource. @@ -539,8 +626,15 @@ class NextcloudPlugin(Plugin): if permissions: permissions = self._get_permissions(permissions) - self._execute(server_args, 'update_share', share_id, public_upload=public_upload, password=password, - permissions=permissions, expire_date=expire_date) + self._execute( + server_args, + 'update_share', + share_id, + public_upload=public_upload, + password=password, + permissions=permissions, + expire_date=expire_date, + ) @action def create_user(self, user_id: str, password: str, **server_args): @@ -611,8 +705,13 @@ class NextcloudPlugin(Plugin): return self._execute(server_args, 'get_user', user_id) @action - def get_users(self, search: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, - **server_args) -> List[str]: + def get_users( + self, + search: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + **server_args + ) -> List[str]: """ Get the list of users matching some search criteria. @@ -621,7 +720,9 @@ class NextcloudPlugin(Plugin): :param offset: Search results offset (default: None). :return: List of the matched user IDs. """ - return self._execute(server_args, 'get_users', search=search, limit=limit, offset=offset) + return self._execute( + server_args, 'get_users', search=search, limit=limit, offset=offset + ) @action def delete_user(self, user_id: str, **server_args): @@ -733,8 +834,15 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'delete_path', user_id, path) @action - def upload_file(self, remote_path: str, local_path: Optional[str] = None, content: Optional[str] = None, - user_id: Optional[str] = None, timestamp: Optional[Union[datetime, int, str]] = None, **server_args): + def upload_file( + self, + remote_path: str, + local_path: Optional[str] = None, + content: Optional[str] = None, + user_id: Optional[str] = None, + timestamp: Optional[Union[datetime, int, str]] = None, + **server_args + ): """ Upload a file. @@ -753,17 +861,32 @@ class NextcloudPlugin(Plugin): if isinstance(timestamp, datetime): timestamp = int(timestamp.timestamp()) - assert (local_path or content) and not (local_path and content), 'Please specify either local_path or content' + assert (local_path or content) and not ( + local_path and content + ), 'Please specify either local_path or content' if local_path: method = 'upload_file' local_path = os.path.abspath(os.path.expanduser(local_path)) else: method = 'upload_file_contents' - return self._execute(server_args, method, user_id, local_path or content, remote_path, timestamp=timestamp) + return self._execute( + server_args, + method, + user_id, + local_path or content, + remote_path, + timestamp=timestamp, + ) @action - def download_file(self, remote_path: str, local_path: str, user_id: Optional[str] = None, **server_args): + def download_file( + self, + remote_path: str, + local_path: str, + user_id: Optional[str] = None, + **server_args + ): """ Download a file. @@ -783,8 +906,14 @@ class NextcloudPlugin(Plugin): os.chdir(cur_dir) @action - def list(self, path: str, user_id: Optional[str] = None, depth: int = 1, all_properties: bool = False, - **server_args) -> List[dict]: + def list( + self, + path: str, + user_id: Optional[str] = None, + depth: int = 1, + all_properties: bool = False, + **server_args + ) -> List[dict]: """ List the content of a folder on the NextCloud instance. @@ -795,10 +924,19 @@ class NextcloudPlugin(Plugin): :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ user_id = user_id or server_args.get('username', self.conf.username) - return self._execute(server_args, 'list_folders', user_id, path, depth=depth, all_properties=all_properties) + return self._execute( + server_args, + 'list_folders', + user_id, + path, + depth=depth, + all_properties=all_properties, + ) @action - def list_favorites(self, path: Optional[str] = None, user_id: Optional[str] = None, **server_args) -> List[dict]: + def list_favorites( + self, path: Optional[str] = None, user_id: Optional[str] = None, **server_args + ) -> List[dict]: """ List the favorite items for a user. @@ -810,7 +948,9 @@ class NextcloudPlugin(Plugin): return self._execute(server_args, 'list_folders', user_id, path) @action - def mark_favorite(self, path: Optional[str] = None, user_id: Optional[str] = None, **server_args): + def mark_favorite( + self, path: Optional[str] = None, user_id: Optional[str] = None, **server_args + ): """ Add a path to a user's favorites. @@ -822,7 +962,14 @@ class NextcloudPlugin(Plugin): self._execute(server_args, 'set_favorites', user_id, path) @action - def copy(self, path: str, destination: str, user_id: Optional[str] = None, overwrite: bool = False, **server_args): + def copy( + self, + path: str, + destination: str, + user_id: Optional[str] = None, + overwrite: bool = False, + **server_args + ): """ Copy a resource to another path. @@ -833,10 +980,19 @@ class NextcloudPlugin(Plugin): :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ user_id = user_id or server_args.get('username', self.conf.username) - self._execute(server_args, 'copy_path', user_id, path, destination, overwrite=overwrite) + self._execute( + server_args, 'copy_path', user_id, path, destination, overwrite=overwrite + ) @action - def move(self, path: str, destination: str, user_id: Optional[str] = None, overwrite: bool = False, **server_args): + def move( + self, + path: str, + destination: str, + user_id: Optional[str] = None, + overwrite: bool = False, + **server_args + ): """ Move a resource to another path. @@ -847,7 +1003,9 @@ class NextcloudPlugin(Plugin): :param server_args: Override the default server settings (see :meth:`._get_client` arguments). """ user_id = user_id or server_args.get('username', self.conf.username) - self._execute(server_args, 'move_path', user_id, path, destination, overwrite=overwrite) + self._execute( + server_args, 'move_path', user_id, path, destination, overwrite=overwrite + ) # vim:sw=4:ts=4:et: From 371fd7e46b1c4cbf2f7e3467c9a0e0d9bc7780b4 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 27 Apr 2022 13:57:42 +0200 Subject: [PATCH 08/12] Generate a default config.yaml if none is present instead of failing --- platypush/config/__init__.py | 178 ++++++++++++------ platypush/config/config.auto.yaml | 3 +- .../{config.default.yaml => config.yaml} | 0 3 files changed, 124 insertions(+), 57 deletions(-) rename platypush/config/{config.default.yaml => config.yaml} (100%) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 99e61e7b4..e9acfec45 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -1,4 +1,5 @@ import datetime +import glob import importlib import inspect import logging @@ -6,19 +7,25 @@ import os import pathlib import pkgutil import re +import shutil import socket import sys from typing import Optional import yaml -from platypush.utils import get_hash, is_functional_procedure, is_functional_hook, is_functional_cron +from platypush.utils import ( + get_hash, + is_functional_procedure, + is_functional_hook, + is_functional_cron, +) """ Config singleton instance """ _default_config_instance = None -class Config(object): +class Config: """ Configuration base class Usage: @@ -45,7 +52,9 @@ class Config(object): 'now': datetime.datetime.now, } - _workdir_location = os.path.join(os.path.expanduser('~'), '.local', 'share', 'platypush') + _workdir_location = os.path.join( + os.path.expanduser('~'), '.local', 'share', 'platypush' + ) _included_files = set() def __init__(self, cfgfile=None): @@ -61,14 +70,12 @@ class Config(object): cfgfile = self._get_default_cfgfile() if cfgfile is None: - raise RuntimeError('No config file specified and nothing found in {}' - .format(self._cfgfile_locations)) + cfgfile = self._create_default_config() self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) self._config = self._read_config_file(self._cfgfile) if 'token' in self._config: - self._config['token'] = self._config['token'] self._config['token_hash'] = get_hash(self._config['token']) if 'workdir' not in self._config: @@ -76,11 +83,15 @@ class Config(object): os.makedirs(self._config['workdir'], exist_ok=True) if 'scripts_dir' not in self._config: - self._config['scripts_dir'] = os.path.join(os.path.dirname(cfgfile), 'scripts') + self._config['scripts_dir'] = os.path.join( + os.path.dirname(cfgfile), 'scripts' + ) os.makedirs(self._config['scripts_dir'], mode=0o755, exist_ok=True) if 'dashboards_dir' not in self._config: - self._config['dashboards_dir'] = os.path.join(os.path.dirname(cfgfile), 'dashboards') + self._config['dashboards_dir'] = os.path.join( + os.path.dirname(cfgfile), 'dashboards' + ) os.makedirs(self._config['dashboards_dir'], mode=0o755, exist_ok=True) init_py = os.path.join(self._config['scripts_dir'], '__init__.py') @@ -90,13 +101,20 @@ class Config(object): # Include scripts_dir parent in sys.path so members can be imported in scripts # through the `scripts` package - scripts_parent_dir = str(pathlib.Path(self._config['scripts_dir']).absolute().parent) + scripts_parent_dir = str( + pathlib.Path(self._config['scripts_dir']).absolute().parent + ) sys.path = [scripts_parent_dir] + sys.path - self._config['db'] = self._config.get('main.db', { - 'engine': 'sqlite:///' + os.path.join( - os.path.expanduser('~'), '.local', 'share', 'platypush', 'main.db') - }) + self._config['db'] = self._config.get( + 'main.db', + { + 'engine': 'sqlite:///' + + os.path.join( + os.path.expanduser('~'), '.local', 'share', 'platypush', 'main.db' + ) + }, + ) logging_config = { 'level': logging.INFO, @@ -112,8 +130,11 @@ class Config(object): try: os.makedirs(logdir, exist_ok=True) except Exception as e: - print('Unable to create logs directory {}: {}'.format( - logdir, str(e))) + print( + 'Unable to create logs directory {}: {}'.format( + logdir, str(e) + ) + ) v = logfile del logging_config['stream'] @@ -150,9 +171,18 @@ class Config(object): self._init_components() self._init_dashboards(self._config['dashboards_dir']) + def _create_default_config(self): + cfg_mod_dir = os.path.dirname(os.path.abspath(__file__)) + cfgfile = self._cfgfile_locations[0] + cfgdir = pathlib.Path(cfgfile).parent + cfgdir.mkdir(parents=True, exist_ok=True) + for cfgfile in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')): + shutil.copy(cfgfile, str(cfgdir)) + + return cfgfile + def _read_config_file(self, cfgfile): - cfgfile_dir = os.path.dirname(os.path.abspath( - os.path.expanduser(cfgfile))) + cfgfile_dir = os.path.dirname(os.path.abspath(os.path.expanduser(cfgfile))) config = {} @@ -164,9 +194,11 @@ class Config(object): for section in file_config: if section == 'include': - include_files = file_config[section] \ - if isinstance(file_config[section], list) \ + include_files = ( + file_config[section] + if isinstance(file_config[section], list) else [file_config[section]] + ) for include_file in include_files: if not os.path.isabs(include_file): @@ -178,9 +210,13 @@ class Config(object): config[incl_section] = included_config[incl_section] elif section == 'scripts_dir': assert isinstance(file_config[section], str) - config['scripts_dir'] = os.path.abspath(os.path.expanduser(file_config[section])) - elif 'disabled' not in file_config[section] \ - or file_config[section]['disabled'] is False: + config['scripts_dir'] = os.path.abspath( + os.path.expanduser(file_config[section]) + ) + elif ( + 'disabled' not in file_config[section] + or file_config[section]['disabled'] is False + ): config[section] = file_config[section] return config @@ -189,27 +225,37 @@ class Config(object): try: module = importlib.import_module(modname) except Exception as e: - print('Unhandled exception while importing module {}: {}'.format(modname, str(e))) + print( + 'Unhandled exception while importing module {}: {}'.format( + modname, str(e) + ) + ) return prefix = modname + '.' if prefix is None else prefix - self.procedures.update(**{ - prefix + name: obj - for name, obj in inspect.getmembers(module) - if is_functional_procedure(obj) - }) + self.procedures.update( + **{ + prefix + name: obj + for name, obj in inspect.getmembers(module) + if is_functional_procedure(obj) + } + ) - self.event_hooks.update(**{ - prefix + name: obj - for name, obj in inspect.getmembers(module) - if is_functional_hook(obj) - }) + self.event_hooks.update( + **{ + prefix + name: obj + for name, obj in inspect.getmembers(module) + if is_functional_hook(obj) + } + ) - self.cronjobs.update(**{ - prefix + name: obj - for name, obj in inspect.getmembers(module) - if is_functional_cron(obj) - }) + self.cronjobs.update( + **{ + prefix + name: obj + for name, obj in inspect.getmembers(module) + if is_functional_cron(obj) + } + ) def _load_scripts(self): scripts_dir = self._config['scripts_dir'] @@ -218,14 +264,19 @@ class Config(object): scripts_modname = os.path.basename(scripts_dir) self._load_module(scripts_modname, prefix='') - for _, modname, _ in pkgutil.walk_packages(path=[scripts_dir], onerror=lambda x: None): + for _, modname, _ in pkgutil.walk_packages( + path=[scripts_dir], onerror=lambda _: None + ): self._load_module(modname) sys.path = sys_path def _init_components(self): for key in self._config.keys(): - if key.startswith('backend.') and '.'.join(key.split('.')[1:]) in self._backend_manifests: + if ( + key.startswith('backend.') + and '.'.join(key.split('.')[1:]) in self._backend_manifests + ): backend_name = '.'.join(key.split('.')[1:]) self.backends[backend_name] = self._config[key] elif key.startswith('event.hook.'): @@ -236,7 +287,7 @@ class Config(object): self.cronjobs[cron_name] = self._config[key] elif key.startswith('procedure.'): tokens = key.split('.') - _async = True if len(tokens) > 2 and tokens[1] == 'async' else False + _async = bool(len(tokens) > 2 and tokens[1] == 'async') procedure_name = '.'.join(tokens[2:] if len(tokens) > 2 else tokens[1:]) args = [] m = re.match(r'^([^(]+)\(([^)]+)\)\s*', procedure_name) @@ -265,7 +316,11 @@ class Config(object): self._init_manifests(plugins_dir) self._init_manifests(backends_dir) else: - manifests_map = self._plugin_manifests if base_dir.endswith('plugins') else self._backend_manifests + manifests_map = ( + self._plugin_manifests + if base_dir.endswith('plugins') + else self._backend_manifests + ) for mf in pathlib.Path(base_dir).rglob('manifest.yaml'): with open(mf, 'r') as f: manifest = yaml.safe_load(f)['manifest'] @@ -279,12 +334,11 @@ class Config(object): for (key, value) in self._default_constants.items(): self.constants[key] = value - @staticmethod - def get_dashboard(name: str, dashboards_dir: Optional[str] = None) -> Optional[str]: - global _default_config_instance - - # noinspection PyProtectedMember,PyProtectedMember,PyUnresolvedReferences - dashboards_dir = dashboards_dir or _default_config_instance._config['dashboards_dir'] + def _get_dashboard( + self, name: str, dashboards_dir: Optional[str] = None + ) -> Optional[str]: + dashboards_dir = dashboards_dir or self._config['dashboards_dir'] + assert dashboards_dir abspath = os.path.join(dashboards_dir, name + '.xml') if not os.path.isfile(abspath): return @@ -292,24 +346,37 @@ class Config(object): with open(abspath, 'r') as fp: return fp.read() - @classmethod - def get_dashboards(cls, dashboards_dir: Optional[str] = None) -> dict: - global _default_config_instance + def _get_dashboards(self, dashboards_dir: Optional[str] = None) -> dict: dashboards = {} - # noinspection PyProtectedMember,PyProtectedMember,PyUnresolvedReferences - dashboards_dir = dashboards_dir or _default_config_instance._config['dashboards_dir'] + dashboards_dir = dashboards_dir or self._config['dashboards_dir'] + assert dashboards_dir + for f in os.listdir(dashboards_dir): abspath = os.path.join(dashboards_dir, f) if not os.path.isfile(abspath) or not abspath.endswith('.xml'): continue name = f.split('.xml')[0] - dashboards[name] = cls.get_dashboard(name, dashboards_dir) + dashboards[name] = self._get_dashboard(name, dashboards_dir) return dashboards + @staticmethod + def get_dashboard(name: str, dashboards_dir: Optional[str] = None) -> Optional[str]: + global _default_config_instance + if _default_config_instance is None: + _default_config_instance = Config() + return _default_config_instance._get_dashboard(name, dashboards_dir) + + @classmethod + def get_dashboards(cls, dashboards_dir: Optional[str] = None) -> dict: + global _default_config_instance + if _default_config_instance is None: + _default_config_instance = Config() + return _default_config_instance._get_dashboards(dashboards_dir) + def _init_dashboards(self, dashboards_dir: str): - self.dashboards = self.get_dashboards(dashboards_dir) + self.dashboards = self._get_dashboards(dashboards_dir) @staticmethod def get_backends(): @@ -400,4 +467,5 @@ class Config(object): return _default_config_instance._config + # vim:sw=4:ts=4:et: diff --git a/platypush/config/config.auto.yaml b/platypush/config/config.auto.yaml index 466458baa..e9bbcd0ce 100644 --- a/platypush/config/config.auto.yaml +++ b/platypush/config/config.auto.yaml @@ -3,5 +3,4 @@ # instead backend.http: - port: 8008 - websocket_port: 8009 + enabled: True diff --git a/platypush/config/config.default.yaml b/platypush/config/config.yaml similarity index 100% rename from platypush/config/config.default.yaml rename to platypush/config/config.yaml From fee5fc4ae0d042cdb96a0087ef04465edfc77853 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 27 Apr 2022 14:52:41 +0200 Subject: [PATCH 09/12] HTTP backend dependencies moved from optional to required If Platypush is supposed to work also without a manually created `config.yaml`, and the HTTP backend is enabled by default in that configuration, then Flask and companions should be among the required dependencies. --- platypush/backend/http/__init__.py | 147 ++++++++++++++++++--------- platypush/backend/http/manifest.yaml | 3 - requirements.txt | 1 + setup.py | 42 +++++--- 4 files changed, 132 insertions(+), 61 deletions(-) diff --git a/platypush/backend/http/__init__.py b/platypush/backend/http/__init__.py index 2e3cb3960..21c649887 100644 --- a/platypush/backend/http/__init__.py +++ b/platypush/backend/http/__init__.py @@ -91,14 +91,16 @@ class HttpBackend(Backend): other music plugin enabled. --> - + + explicitly exposed as an HTTP resource through the backend + `resource_dirs` attribute. -->