[#394] Dynamically generate setup extras.

Also, convert all code that relied on `manifest.yaml` to use
`manifest.json` instead.

Closes: #394

CONTRIBUTING.md

@@ -27,13 +27,9 @@ Guidelines:
   you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons
   or the bus) then make sure to add tests and not to break the existing tests.
 
-- If the feature requires an optional dependency then make sure to document it:
+- If the feature requires an optional dependency then make sure to document it
-
+  in the `manifest.json` - refer to the Wiki (how to write
-  - In the class docstring (see other plugins and backends for examples).
-  - In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as
-    an `extras_require` entry.
-  - In the plugin/backend class pydoc string.
-  - In the `manifest.yaml` - refer to the Wiki (how to write
   [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
-    and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
+  and
+  [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
   for examples on how to write an extension manifest file.

MANIFEST.in

@@ -2,5 +2,5 @@ recursive-include platypush/backend/http/webapp/dist *
 recursive-include platypush/install *
 include platypush/plugins/http/webpage/mercury-parser.js
 include platypush/config/*.yaml
-global-include manifest.yaml
+global-include manifest.json
 global-include components.json.gz

README.md

@@ -22,7 +22,7 @@ Platypush
   * [Install from sources](#install-from-sources)
   * [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions)
     + [Install via `extras` name](#install-via-extras-name)
-    + [Install via `manifest.yaml`](#install-via-manifestyaml)
+    + [Install via `manifest.json`](#install-via-manifestjson)
     + [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation)
   * [Virtual environment installation](#virtual-environment-installation)
   * [Docker installation](#docker-installation-1)
@@ -216,16 +216,27 @@ ways to check the dependencies required by an extension:
 
 #### Install via `extras` name
 
-All the extensions that require extra dependencies are listed in the
+You can install extra dependencies via pip extras:
-[`extras_require` section under
-`setup.py`](https://git.platypush.tech/platypush/platypush/src/branch/master/setup.py#L84).
 
-#### Install via `manifest.yaml`
+```shell
+pip install 'platypush[plugin1,plugin2,...]'
+```
 
-All the plugins and backends have a `manifest.yaml` file in their source folder.
+For example:
+
+```shell
+pip install 'platypush[light.hue,music.mpd,rss]'
+```
+
+Will install Platypush with the dependencies for the `light.hue`, `music.mpd`
+and `rss` plugins.
+
+#### Install via `manifest.json`
+
+All the plugins and backends have a `manifest.json` file in their source folder.
 Any extra dependencies are listed there
 
-If you followed the `extras` or `manifest.yaml` way to discover the
+If you followed the `extras` or `manifest.json` way to discover the
 dependencies, then you can install them in two ways:
 
 1. `pip` installation:

docs/source/_ext/add_dependencies.py

@@ -159,7 +159,7 @@ class IntegrationEnricher:
             base_path,
             *doc.split(os.sep)[:-1],
             *doc.split(os.sep)[-1].split('.'),
-            'manifest.yaml',
+            'manifest.json',
         )
 
         if not os.path.isfile(manifest_file):

platypush/common/__init__.py

@@ -31,7 +31,7 @@ def exec_wrapper(f: Callable[..., Any], *args, **kwargs):
 # pylint: disable=too-few-public-methods
 class ExtensionWithManifest:
     """
-    This class models an extension with an associated manifest.yaml in the same
+    This class models an extension with an associated manifest.json in the same
     folder.
     """
 
@@ -40,11 +40,11 @@ class ExtensionWithManifest:
 
     def get_manifest(self) -> Manifest:
         manifest_file = os.path.join(
-            os.path.dirname(inspect.getfile(self.__class__)), 'manifest.yaml'
+            os.path.dirname(inspect.getfile(self.__class__)), 'manifest.json'
         )
         assert os.path.isfile(
             manifest_file
-        ), f'The extension {self.__class__.__name__} has no associated manifest.yaml'
+        ), f'The extension {self.__class__.__name__} has no associated manifest.json'
 
         return Manifest.from_file(manifest_file)
 

platypush/common/reflection/_model/integration.py

@@ -251,7 +251,7 @@ class Integration(Component, DocstringParser, Serializable):
         :return: Path of the manifest file for the integration.
         """
         return os.path.join(
-            os.path.dirname(inspect.getfile(self.type)), "manifest.yaml"
+            os.path.dirname(inspect.getfile(self.type)), "manifest.json"
         )
 
     @property

platypush/config/__init__.py

@@ -2,6 +2,7 @@ import datetime
 import glob
 import importlib
 import inspect
+import json
 import logging
 import os
 import pathlib
@@ -440,9 +441,13 @@ class Config:
                 if base_dir.endswith('plugins')
                 else self._backend_manifests
             )
-            for mf in pathlib.Path(base_dir).rglob('manifest.yaml'):
+
+            for mf in pathlib.Path(base_dir).rglob('manifest.json'):
                 with open(mf, 'r') as f:
-                    manifest = yaml.safe_load(f)['manifest']
+                    manifest = json.load(f).get('manifest')
+                    if not manifest:
+                        continue
+
                     comp_name = '.'.join(manifest['package'].split('.')[2:])
                     manifests_map[comp_name] = manifest
 

platypush/plugins/application/__init__.py

@@ -81,5 +81,5 @@ class ApplicationPlugin(Plugin):
             ext = getter(extension)
 
         assert ext, f'Could not find extension {extension}'
-        manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
+        manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.json')
         return list(Manifest.from_file(manifest_file).install.to_install_commands())

platypush/plugins/assistant/google/__init__.py

@@ -58,7 +58,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
         years, some of its dependencies are quite old and may break more recent
         Python installations. Please refer to the comments in the `manifest
         file
-        <https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml>`_.
+        <https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.json>`_.
         for more information on how to install the required dependencies, if
         the automated ways fail.
     """

platypush/utils/manifest.py

@@ -26,8 +26,6 @@ from typing import (
     Union,
 )
 
-import yaml
-
 from platypush.utils import get_src_root, is_root
 
 _available_package_manager = None
@@ -538,7 +536,7 @@ class Manifest(ABC):
         return os.path.join(
             get_src_root(),
             *self.package.split('.')[1:],
-            'manifest.yaml',
+            'manifest.json',
         )
 
     def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
@@ -584,7 +582,7 @@ class Manifest(ABC):
         Parse a manifest filename into a ``Manifest`` class.
         """
         with open(str(filename), 'r') as f:
-            manifest = yaml.safe_load(f).get('manifest', {})
+            manifest = json.load(f).get('manifest', {})
 
         assert 'type' in manifest, f'Manifest file {filename} has no type field'
         comp_type = ManifestType(manifest.pop('type'))
@@ -657,9 +655,14 @@ class Manifests:
         and parse them into :class:`Manifest` objects.
         """
         for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob(
-            'manifest.yaml'
+            'manifest.json'
         ):
+            try:
                 yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
+            except Exception as e:
+                logger.debug(
+                    'Could not parse manifest file %s: %s', mf, e, exc_info=True
+                )
 
     @staticmethod
     def by_config(
@@ -681,12 +684,21 @@ class Manifests:
 
         for name in Config.get_backends().keys():
             yield Manifest.from_file(
-                os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'),
+                os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.json'),
                 pkg_manager=pkg_manager,
             )
 
         for name in Config.get_plugins().keys():
             yield Manifest.from_file(
-                os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
+                os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.json'),
                 pkg_manager=pkg_manager,
             )
+
+    @staticmethod
+    def scan() -> Generator[Manifest, None, None]:
+        """
+        Scan all the manifest files in the source tree and parse them into
+        :class:`Manifest` objects.
+        """
+        for mf in pathlib.Path(get_src_root()).rglob('manifest.json'):
+            yield Manifest.from_file(str(mf))

setup.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 
+import json
 import os
+
 from setuptools import setup, find_packages
 
 
@@ -13,16 +15,53 @@ def readfile(fname):
         return f.read()
 
 
-# noinspection PyShadowingBuiltins
 def pkg_files(dir):
     paths = []
-    # noinspection PyShadowingNames
+    for p, _, files in os.walk(dir):
-    for path, _, files in os.walk(dir):
         for file in files:
-            paths.append(os.path.join('..', path, file))
+            paths.append(os.path.join('..', p, file))
     return paths
 
 
+def scan_manifests():
+    for root, _, files in os.walk('platypush'):
+        for file in files:
+            if file == 'manifest.json':
+                yield os.path.join(root, file)
+
+
+def parse_deps(deps):
+    ret = []
+    for dep in deps:
+        if dep.startswith('git+'):
+            repo_name = dep.split('/')[-1].split('.git')[0]
+            dep = f'{repo_name} @ {dep}'
+
+        ret.append(dep)
+
+    return ret
+
+
+def parse_manifest(manifest_file):
+    with open(manifest_file) as f:
+        manifest = json.load(f).get('manifest')
+        if not manifest:
+            return None, None
+
+        name = '.'.join(manifest['package'].split('.')[2:])
+        return name, parse_deps(manifest.get('install', {}).get('pip', []))
+
+
+def parse_manifests():
+    ret = {}
+    for manifest_file in scan_manifests():
+        name, deps = parse_manifest(manifest_file)
+        if deps:
+            ret[name] = deps
+
+    return ret
+
+
 plugins = pkg_files('platypush/plugins')
 backend = pkg_files('platypush/backend')
 
@@ -87,206 +126,5 @@ setup(
         'wheel',
         'zeroconf>=0.27.0',
     ],
-    extras_require={
+    extras_require=parse_manifests(),
-        # Support for Kafka backend and plugin
-        'kafka': ['kafka-python'],
-        # Support for Pushbullet
-        'pushbullet': [
-            'pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'
-        ],
-        # This is only kept for back-compatibility purposes, as all the
-        # dependencies of the HTTP webserver are now core dependencies.
-        'http': [],
-        # Support for MQTT backends
-        'mqtt': ['paho-mqtt'],
-        # Support for RSS feeds parser
-        'rss': ['feedparser', 'defusedxml'],
-        # Support for PDF generation
-        'pdf': ['weasyprint'],
-        # Support for Philips Hue plugin
-        'hue': ['phue'],
-        # Support for MPD/Mopidy music server plugin and backend
-        'mpd': ['python-mpd2'],
-        # Support for Google text2speech plugin
-        'google-tts': [
-            'oauth2client',
-            'httplib2',
-            'google-api-python-client',
-            'google-auth',
-            'google-cloud-texttospeech',
-        ],
-        # Support for OMXPlayer plugin
-        'omxplayer': ['omxplayer-wrapper'],
-        # Support for YouTube
-        'youtube': ['yt-dlp'],
-        # Support for torrents download
-        'torrent': ['python-libtorrent'],
-        # Generic support for cameras
-        'camera': ['numpy', 'Pillow'],
-        # Support for RaspberryPi camera
-        'picamera': ['picamera', 'numpy', 'Pillow'],
-        # Support for inotify file monitors
-        'inotify': ['inotify'],
-        # Support for Google Assistant
-        'google-assistant': ['google-assistant-library', 'google-auth'],
-        # Support for the Google APIs
-        'google': [
-            'oauth2client',
-            'google-auth',
-            'google-api-python-client',
-            'httplib2',
-        ],
-        # Support for Last.FM scrobbler plugin
-        'lastfm': ['pylast'],
-        # Support for real-time MIDI events
-        'midi': ['rtmidi'],
-        # Support for RaspberryPi GPIO
-        'rpi-gpio': ['RPi.GPIO'],
-        # Support for MCP3008 analog-to-digital converter plugin
-        'mcp3008': ['adafruit-mcp3008'],
-        # Support for smart cards detection
-        'scard': ['pyscard'],
-        # Support for serial port plugin
-        'serial': ['pyserial'],
-        # Support for ICal calendars
-        'ical': ['icalendar'],
-        # Support for joystick backend
-        'joystick': ['inputs'],
-        # Support for Kodi plugin
-        'kodi': ['kodi-json'],
-        # Support for Plex plugin
-        'plex': ['plexapi'],
-        # Support for Chromecast plugin
-        'chromecast': ['pychromecast'],
-        # Support for sound devices
-        'sound': ['sounddevice', 'numpy'],
-        # Support for web media subtitles
-        'subtitles': [
-            'webvtt-py',
-            'python-opensubtitles @ https://github.com/agonzalezro/python-opensubtitles/tarball/master',
-        ],
-        # Support for mpv player plugin
-        'mpv': ['python-mpv'],
-        # Support for NFC tags
-        'nfc': ['nfcpy>=1.0', 'ndeflib'],
-        # Support for enviropHAT
-        'envirophat': ['envirophat'],
-        # Support for GPS
-        'gps': ['gps'],
-        # Support for BME280 environment sensor
-        'bme280': ['pimoroni-bme280'],
-        # Support for LTR559 light/proximity sensor
-        'ltr559': ['ltr559', 'smbus'],
-        # Support for VL53L1X laser ranger/distance sensor
-        'vl53l1x': ['smbus2', 'vl53l1x'],
-        # Support for Dropbox integration
-        'dropbox': ['dropbox'],
-        # Support for Leap Motion backend
-        'leap': [
-            'leap-sdk @ https://github.com/BlackLight/leap-sdk-python3/tarball/master'
-        ],
-        # Support for Flic buttons
-        'flic': [
-            'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
-        ],
-        # Support for Bluetooth devices
-        'bluetooth': [
-            'bleak',
-            'bluetooth-numbers',
-            'TheengsDecoder',
-            'pydbus',
-            'pybluez @ https://github.com/pybluez/pybluez/tarball/master',
-            'PyOBEX @ https://github.com/BlackLight/PyOBEX/tarball/master',
-        ],
-        # Support for TP-Link devices
-        'tplink': ['pyHS100'],
-        # Support for PMW3901 2-Dimensional Optical Flow Sensor
-        'pmw3901': ['pmw3901'],
-        # Support for MLX90640 thermal camera
-        'mlx90640': ['Pillow'],
-        # Support for machine learning models and cameras over OpenCV
-        'cv': ['opencv-python', 'numpy', 'Pillow'],
-        # Support for Node-RED integration
-        'nodered': ['pynodered'],
-        # Support for Todoist integration
-        'todoist': ['todoist-python'],
-        # Support for Trello integration
-        'trello': ['py-trello'],
-        # Support for Google Pub/Sub
-        'google-pubsub': ['google-cloud-pubsub', 'google-auth', 'httplib2'],
-        # Support for Google Translate
-        'google-translate': ['google-cloud-translate', 'google-auth', 'httplib2'],
-        # Support for keyboard/mouse plugin
-        'inputs': ['pyuserinput'],
-        # Support for Buienradar weather forecast
-        'buienradar': ['buienradar'],
-        # Support for Telegram integration
-        'telegram': ['python-telegram-bot'],
-        # Support for Arduino integration
-        'arduino': ['pyserial', 'pyfirmata2'],
-        # Support for CUPS printers management
-        'cups': ['pycups'],
-        # Support for Graphite integration
-        'graphite': ['graphyte'],
-        # Support for CPU and memory monitoring and info
-        'sys': ['py-cpuinfo'],
-        # Support for nmap integration
-        'nmap': ['python-nmap'],
-        # Support for zigbee2mqtt
-        'zigbee': ['paho-mqtt'],
-        # Support for Z-Wave
-        'zwave': ['paho-mqtt'],
-        # Support for Mozilla DeepSpeech speech-to-text engine
-        'deepspeech': ['deepspeech', 'numpy', 'sounddevice'],
-        # Support for PicoVoice hotword detection engine
-        'picovoice-hotword': ['pvporcupine'],
-        # Support for PicoVoice speech-to-text engine
-        'picovoice-speech': ['pvcheetah @ git+https://github.com/BlackLight/cheetah'],
-        # Support for OTP (One-Time Password) generation
-        'otp': ['pyotp'],
-        # Support for Linode integration
-        'linode': ['linode_api4'],
-        # Support for QR codes
-        'qrcode': ['numpy', 'qrcode[pil]', 'Pillow', 'pyzbar'],
-        # Support for Tensorflow
-        'tensorflow': ['numpy', 'tensorflow>=2.0', 'keras', 'pandas'],
-        # Support for Samsung TizenOS-based smart TVs
-        'samsungtv': ['samsungtvws'],
-        # Support for SSH integration
-        'ssh': ['paramiko'],
-        # Support for clipboard integration
-        'clipboard': ['pyclip'],
-        # Support for luma.oled display drivers
-        'luma-oled': ['luma.oled @ git+https://github.com/rm-hull/luma.oled'],
-        # Support for DBus integration
-        'dbus': ['pydbus', 'defusedxml'],
-        # Support for Twilio integration
-        'twilio': ['twilio'],
-        # Support for DHT11/DHT22/AM2302 temperature/humidity sensors
-        'dht': [
-            'Adafruit_Python_DHT @ git+https://github.com/adafruit/Adafruit_Python_DHT'
-        ],
-        # Support for LCD display integration
-        'lcd': ['RPi.GPIO', 'RPLCD'],
-        # Support for email integration
-        'mail': ['imapclient', 'dnspython'],
-        # Support for NextCloud integration
-        'nextcloud': ['nextcloud-api-wrapper'],
-        # Support for VLC integration
-        'vlc': ['python-vlc'],
-        # Support for SmartThings integration
-        'smartthings': ['pysmartthings', 'aiohttp'],
-        # Support for file.monitor backend
-        'filemonitor': ['watchdog'],
-        # Support for Adafruit PCA9685 PWM controller
-        'pca9685': ['adafruit-python-shell', 'adafruit-circuitpython-pca9685'],
-        # Support for ngrok integration
-        'ngrok': ['pyngrok'],
-        # Support for IRC integration
-        'irc': ['irc'],
-        # Support for the Matrix integration
-        'matrix': ['matrix-nio'],
-        # Support for the XMPP integration
-        'xmpp': ['aioxmpp', 'pytz'],
-    },
 )