diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f983579a7..b4801e3e44 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,13 +27,9 @@ Guidelines: you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons or the bus) then make sure to add tests and not to break the existing tests. -- If the feature requires an optional dependency then make sure to document it: - - - In the class docstring (see other plugins and backends for examples). - - In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as - an `extras_require` entry. - - In the plugin/backend class pydoc string. - - In the `manifest.yaml` - refer to the Wiki (how to write - [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins) - and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends)) - for examples on how to write an extension manifest file. +- If the feature requires an optional dependency then make sure to document it + in the `manifest.json` - refer to the Wiki (how to write + [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins) + and + [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends)) + for examples on how to write an extension manifest file. diff --git a/MANIFEST.in b/MANIFEST.in index 310df247fa..f09b5138c9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ recursive-include platypush/backend/http/webapp/dist * recursive-include platypush/install * include platypush/plugins/http/webpage/mercury-parser.js include platypush/config/*.yaml -global-include manifest.yaml +global-include manifest.json global-include components.json.gz diff --git a/README.md b/README.md index 1d03416b04..fe23ce3a72 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Platypush * [Install from sources](#install-from-sources) * [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions) + [Install via `extras` name](#install-via-extras-name) - + [Install via `manifest.yaml`](#install-via-manifestyaml) + + [Install via `manifest.json`](#install-via-manifestjson) + [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation) * [Virtual environment installation](#virtual-environment-installation) * [Docker installation](#docker-installation-1) @@ -216,16 +216,27 @@ ways to check the dependencies required by an extension: #### Install via `extras` name -All the extensions that require extra dependencies are listed in the -[`extras_require` section under -`setup.py`](https://git.platypush.tech/platypush/platypush/src/branch/master/setup.py#L84). +You can install extra dependencies via pip extras: -#### Install via `manifest.yaml` +```shell +pip install 'platypush[plugin1,plugin2,...]' +``` -All the plugins and backends have a `manifest.yaml` file in their source folder. +For example: + +```shell +pip install 'platypush[light.hue,music.mpd,rss]' +``` + +Will install Platypush with the dependencies for the `light.hue`, `music.mpd` +and `rss` plugins. + +#### Install via `manifest.json` + +All the plugins and backends have a `manifest.json` file in their source folder. Any extra dependencies are listed there -If you followed the `extras` or `manifest.yaml` way to discover the +If you followed the `extras` or `manifest.json` way to discover the dependencies, then you can install them in two ways: 1. `pip` installation: diff --git a/docs/source/_ext/add_dependencies.py b/docs/source/_ext/add_dependencies.py index 6f0cf5f493..9f231700bc 100644 --- a/docs/source/_ext/add_dependencies.py +++ b/docs/source/_ext/add_dependencies.py @@ -159,7 +159,7 @@ class IntegrationEnricher: base_path, *doc.split(os.sep)[:-1], *doc.split(os.sep)[-1].split('.'), - 'manifest.yaml', + 'manifest.json', ) if not os.path.isfile(manifest_file): diff --git a/platypush/common/__init__.py b/platypush/common/__init__.py index b4197e2b13..b773609ec5 100644 --- a/platypush/common/__init__.py +++ b/platypush/common/__init__.py @@ -31,7 +31,7 @@ def exec_wrapper(f: Callable[..., Any], *args, **kwargs): # pylint: disable=too-few-public-methods class ExtensionWithManifest: """ - This class models an extension with an associated manifest.yaml in the same + This class models an extension with an associated manifest.json in the same folder. """ @@ -40,11 +40,11 @@ class ExtensionWithManifest: def get_manifest(self) -> Manifest: manifest_file = os.path.join( - os.path.dirname(inspect.getfile(self.__class__)), 'manifest.yaml' + os.path.dirname(inspect.getfile(self.__class__)), 'manifest.json' ) assert os.path.isfile( manifest_file - ), f'The extension {self.__class__.__name__} has no associated manifest.yaml' + ), f'The extension {self.__class__.__name__} has no associated manifest.json' return Manifest.from_file(manifest_file) diff --git a/platypush/common/reflection/_model/integration.py b/platypush/common/reflection/_model/integration.py index 64d011600e..d1ae14e597 100644 --- a/platypush/common/reflection/_model/integration.py +++ b/platypush/common/reflection/_model/integration.py @@ -251,7 +251,7 @@ class Integration(Component, DocstringParser, Serializable): :return: Path of the manifest file for the integration. """ return os.path.join( - os.path.dirname(inspect.getfile(self.type)), "manifest.yaml" + os.path.dirname(inspect.getfile(self.type)), "manifest.json" ) @property diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 64478bbc53..ad64d5c46d 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -2,6 +2,7 @@ import datetime import glob import importlib import inspect +import json import logging import os import pathlib @@ -440,9 +441,13 @@ class Config: if base_dir.endswith('plugins') else self._backend_manifests ) - for mf in pathlib.Path(base_dir).rglob('manifest.yaml'): + + for mf in pathlib.Path(base_dir).rglob('manifest.json'): with open(mf, 'r') as f: - manifest = yaml.safe_load(f)['manifest'] + manifest = json.load(f).get('manifest') + if not manifest: + continue + comp_name = '.'.join(manifest['package'].split('.')[2:]) manifests_map[comp_name] = manifest diff --git a/platypush/plugins/application/__init__.py b/platypush/plugins/application/__init__.py index 641efa0e44..1222d2f237 100644 --- a/platypush/plugins/application/__init__.py +++ b/platypush/plugins/application/__init__.py @@ -81,5 +81,5 @@ class ApplicationPlugin(Plugin): ext = getter(extension) assert ext, f'Could not find extension {extension}' - manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml') + manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.json') return list(Manifest.from_file(manifest_file).install.to_install_commands()) diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 0a0ce78f17..2065d44a82 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -58,7 +58,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): years, some of its dependencies are quite old and may break more recent Python installations. Please refer to the comments in the `manifest file - `_. + `_. for more information on how to install the required dependencies, if the automated ways fail. """ diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 90ec2beea0..11c6bbce05 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -26,8 +26,6 @@ from typing import ( Union, ) -import yaml - from platypush.utils import get_src_root, is_root _available_package_manager = None @@ -538,7 +536,7 @@ class Manifest(ABC): return os.path.join( get_src_root(), *self.package.split('.')[1:], - 'manifest.yaml', + 'manifest.json', ) def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies: @@ -584,7 +582,7 @@ class Manifest(ABC): Parse a manifest filename into a ``Manifest`` class. """ with open(str(filename), 'r') as f: - manifest = yaml.safe_load(f).get('manifest', {}) + manifest = json.load(f).get('manifest', {}) assert 'type' in manifest, f'Manifest file {filename} has no type field' comp_type = ManifestType(manifest.pop('type')) @@ -657,9 +655,14 @@ class Manifests: and parse them into :class:`Manifest` objects. """ for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob( - 'manifest.yaml' + 'manifest.json' ): - yield Manifest.from_file(str(mf), pkg_manager=pkg_manager) + try: + yield Manifest.from_file(str(mf), pkg_manager=pkg_manager) + except Exception as e: + logger.debug( + 'Could not parse manifest file %s: %s', mf, e, exc_info=True + ) @staticmethod def by_config( @@ -681,12 +684,21 @@ class Manifests: for name in Config.get_backends().keys(): yield Manifest.from_file( - os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'), + os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.json'), pkg_manager=pkg_manager, ) for name in Config.get_plugins().keys(): yield Manifest.from_file( - os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'), + os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.json'), pkg_manager=pkg_manager, ) + + @staticmethod + def scan() -> Generator[Manifest, None, None]: + """ + Scan all the manifest files in the source tree and parse them into + :class:`Manifest` objects. + """ + for mf in pathlib.Path(get_src_root()).rglob('manifest.json'): + yield Manifest.from_file(str(mf)) diff --git a/setup.py b/setup.py index 630dc41a6b..3da3b1f38e 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ #!/usr/bin/env python +import json import os + from setuptools import setup, find_packages @@ -13,16 +15,53 @@ def readfile(fname): return f.read() -# noinspection PyShadowingBuiltins def pkg_files(dir): paths = [] - # noinspection PyShadowingNames - for path, _, files in os.walk(dir): + for p, _, files in os.walk(dir): for file in files: - paths.append(os.path.join('..', path, file)) + paths.append(os.path.join('..', p, file)) return paths +def scan_manifests(): + for root, _, files in os.walk('platypush'): + for file in files: + if file == 'manifest.json': + yield os.path.join(root, file) + + +def parse_deps(deps): + ret = [] + for dep in deps: + if dep.startswith('git+'): + repo_name = dep.split('/')[-1].split('.git')[0] + dep = f'{repo_name} @ {dep}' + + ret.append(dep) + + return ret + + +def parse_manifest(manifest_file): + with open(manifest_file) as f: + manifest = json.load(f).get('manifest') + if not manifest: + return None, None + + name = '.'.join(manifest['package'].split('.')[2:]) + return name, parse_deps(manifest.get('install', {}).get('pip', [])) + + +def parse_manifests(): + ret = {} + for manifest_file in scan_manifests(): + name, deps = parse_manifest(manifest_file) + if deps: + ret[name] = deps + + return ret + + plugins = pkg_files('platypush/plugins') backend = pkg_files('platypush/backend') @@ -87,206 +126,5 @@ setup( 'wheel', 'zeroconf>=0.27.0', ], - extras_require={ - # Support for Kafka backend and plugin - 'kafka': ['kafka-python'], - # Support for Pushbullet - 'pushbullet': [ - 'pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master' - ], - # This is only kept for back-compatibility purposes, as all the - # dependencies of the HTTP webserver are now core dependencies. - 'http': [], - # Support for MQTT backends - 'mqtt': ['paho-mqtt'], - # Support for RSS feeds parser - 'rss': ['feedparser', 'defusedxml'], - # Support for PDF generation - 'pdf': ['weasyprint'], - # Support for Philips Hue plugin - 'hue': ['phue'], - # Support for MPD/Mopidy music server plugin and backend - 'mpd': ['python-mpd2'], - # Support for Google text2speech plugin - 'google-tts': [ - 'oauth2client', - 'httplib2', - 'google-api-python-client', - 'google-auth', - 'google-cloud-texttospeech', - ], - # Support for OMXPlayer plugin - 'omxplayer': ['omxplayer-wrapper'], - # Support for YouTube - 'youtube': ['yt-dlp'], - # Support for torrents download - 'torrent': ['python-libtorrent'], - # Generic support for cameras - 'camera': ['numpy', 'Pillow'], - # Support for RaspberryPi camera - 'picamera': ['picamera', 'numpy', 'Pillow'], - # Support for inotify file monitors - 'inotify': ['inotify'], - # Support for Google Assistant - 'google-assistant': ['google-assistant-library', 'google-auth'], - # Support for the Google APIs - 'google': [ - 'oauth2client', - 'google-auth', - 'google-api-python-client', - 'httplib2', - ], - # Support for Last.FM scrobbler plugin - 'lastfm': ['pylast'], - # Support for real-time MIDI events - 'midi': ['rtmidi'], - # Support for RaspberryPi GPIO - 'rpi-gpio': ['RPi.GPIO'], - # Support for MCP3008 analog-to-digital converter plugin - 'mcp3008': ['adafruit-mcp3008'], - # Support for smart cards detection - 'scard': ['pyscard'], - # Support for serial port plugin - 'serial': ['pyserial'], - # Support for ICal calendars - 'ical': ['icalendar'], - # Support for joystick backend - 'joystick': ['inputs'], - # Support for Kodi plugin - 'kodi': ['kodi-json'], - # Support for Plex plugin - 'plex': ['plexapi'], - # Support for Chromecast plugin - 'chromecast': ['pychromecast'], - # Support for sound devices - 'sound': ['sounddevice', 'numpy'], - # Support for web media subtitles - 'subtitles': [ - 'webvtt-py', - 'python-opensubtitles @ https://github.com/agonzalezro/python-opensubtitles/tarball/master', - ], - # Support for mpv player plugin - 'mpv': ['python-mpv'], - # Support for NFC tags - 'nfc': ['nfcpy>=1.0', 'ndeflib'], - # Support for enviropHAT - 'envirophat': ['envirophat'], - # Support for GPS - 'gps': ['gps'], - # Support for BME280 environment sensor - 'bme280': ['pimoroni-bme280'], - # Support for LTR559 light/proximity sensor - 'ltr559': ['ltr559', 'smbus'], - # Support for VL53L1X laser ranger/distance sensor - 'vl53l1x': ['smbus2', 'vl53l1x'], - # Support for Dropbox integration - 'dropbox': ['dropbox'], - # Support for Leap Motion backend - 'leap': [ - 'leap-sdk @ https://github.com/BlackLight/leap-sdk-python3/tarball/master' - ], - # Support for Flic buttons - 'flic': [ - 'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master' - ], - # Support for Bluetooth devices - 'bluetooth': [ - 'bleak', - 'bluetooth-numbers', - 'TheengsDecoder', - 'pydbus', - 'pybluez @ https://github.com/pybluez/pybluez/tarball/master', - 'PyOBEX @ https://github.com/BlackLight/PyOBEX/tarball/master', - ], - # Support for TP-Link devices - 'tplink': ['pyHS100'], - # Support for PMW3901 2-Dimensional Optical Flow Sensor - 'pmw3901': ['pmw3901'], - # Support for MLX90640 thermal camera - 'mlx90640': ['Pillow'], - # Support for machine learning models and cameras over OpenCV - 'cv': ['opencv-python', 'numpy', 'Pillow'], - # Support for Node-RED integration - 'nodered': ['pynodered'], - # Support for Todoist integration - 'todoist': ['todoist-python'], - # Support for Trello integration - 'trello': ['py-trello'], - # Support for Google Pub/Sub - 'google-pubsub': ['google-cloud-pubsub', 'google-auth', 'httplib2'], - # Support for Google Translate - 'google-translate': ['google-cloud-translate', 'google-auth', 'httplib2'], - # Support for keyboard/mouse plugin - 'inputs': ['pyuserinput'], - # Support for Buienradar weather forecast - 'buienradar': ['buienradar'], - # Support for Telegram integration - 'telegram': ['python-telegram-bot'], - # Support for Arduino integration - 'arduino': ['pyserial', 'pyfirmata2'], - # Support for CUPS printers management - 'cups': ['pycups'], - # Support for Graphite integration - 'graphite': ['graphyte'], - # Support for CPU and memory monitoring and info - 'sys': ['py-cpuinfo'], - # Support for nmap integration - 'nmap': ['python-nmap'], - # Support for zigbee2mqtt - 'zigbee': ['paho-mqtt'], - # Support for Z-Wave - 'zwave': ['paho-mqtt'], - # Support for Mozilla DeepSpeech speech-to-text engine - 'deepspeech': ['deepspeech', 'numpy', 'sounddevice'], - # Support for PicoVoice hotword detection engine - 'picovoice-hotword': ['pvporcupine'], - # Support for PicoVoice speech-to-text engine - 'picovoice-speech': ['pvcheetah @ git+https://github.com/BlackLight/cheetah'], - # Support for OTP (One-Time Password) generation - 'otp': ['pyotp'], - # Support for Linode integration - 'linode': ['linode_api4'], - # Support for QR codes - 'qrcode': ['numpy', 'qrcode[pil]', 'Pillow', 'pyzbar'], - # Support for Tensorflow - 'tensorflow': ['numpy', 'tensorflow>=2.0', 'keras', 'pandas'], - # Support for Samsung TizenOS-based smart TVs - 'samsungtv': ['samsungtvws'], - # Support for SSH integration - 'ssh': ['paramiko'], - # Support for clipboard integration - 'clipboard': ['pyclip'], - # Support for luma.oled display drivers - 'luma-oled': ['luma.oled @ git+https://github.com/rm-hull/luma.oled'], - # Support for DBus integration - 'dbus': ['pydbus', 'defusedxml'], - # Support for Twilio integration - 'twilio': ['twilio'], - # Support for DHT11/DHT22/AM2302 temperature/humidity sensors - 'dht': [ - 'Adafruit_Python_DHT @ git+https://github.com/adafruit/Adafruit_Python_DHT' - ], - # Support for LCD display integration - 'lcd': ['RPi.GPIO', 'RPLCD'], - # Support for email integration - 'mail': ['imapclient', 'dnspython'], - # Support for NextCloud integration - 'nextcloud': ['nextcloud-api-wrapper'], - # Support for VLC integration - 'vlc': ['python-vlc'], - # Support for SmartThings integration - 'smartthings': ['pysmartthings', 'aiohttp'], - # Support for file.monitor backend - 'filemonitor': ['watchdog'], - # Support for Adafruit PCA9685 PWM controller - 'pca9685': ['adafruit-python-shell', 'adafruit-circuitpython-pca9685'], - # Support for ngrok integration - 'ngrok': ['pyngrok'], - # Support for IRC integration - 'irc': ['irc'], - # Support for the Matrix integration - 'matrix': ['matrix-nio'], - # Support for the XMPP integration - 'xmpp': ['aioxmpp', 'pytz'], - }, + extras_require=parse_manifests(), )