forked from platypush/platypush
[#394] Dynamically generate setup extras.
Also, convert all code that relied on `manifest.yaml` to use `manifest.json` instead. Closes: #394
This commit is contained in:
parent
59c693d6a0
commit
f06233801b
GPG key ID:
D90FBA7F76362774
11 changed files with 103 additions and 241 deletions
|
|
@ -27,13 +27,9 @@ Guidelines:
|
|||
you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons
|
||||
or the bus) then make sure to add tests and not to break the existing tests.
|
||||
|
||||
- If the feature requires an optional dependency then make sure to document it:
|
||||
|
||||
- In the class docstring (see other plugins and backends for examples).
|
||||
- In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as
|
||||
an `extras_require` entry.
|
||||
- In the plugin/backend class pydoc string.
|
||||
- In the `manifest.yaml` - refer to the Wiki (how to write
|
||||
[plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
|
||||
and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
|
||||
for examples on how to write an extension manifest file.
|
||||
- If the feature requires an optional dependency then make sure to document it
|
||||
in the `manifest.json` - refer to the Wiki (how to write
|
||||
[plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
|
||||
and
|
||||
[backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
|
||||
for examples on how to write an extension manifest file.
|
||||
|
|
|
|||
|
|
@ -2,5 +2,5 @@ recursive-include platypush/backend/http/webapp/dist *
|
|||
recursive-include platypush/install *
|
||||
include platypush/plugins/http/webpage/mercury-parser.js
|
||||
include platypush/config/*.yaml
|
||||
global-include manifest.yaml
|
||||
global-include manifest.json
|
||||
global-include components.json.gz
|
||||
|
|
|
|||
25
README.md
25
README.md
|
|
@ -22,7 +22,7 @@ Platypush
|
|||
* [Install from sources](#install-from-sources)
|
||||
* [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions)
|
||||
+ [Install via `extras` name](#install-via-extras-name)
|
||||
+ [Install via `manifest.yaml`](#install-via-manifestyaml)
|
||||
+ [Install via `manifest.json`](#install-via-manifestjson)
|
||||
+ [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation)
|
||||
* [Virtual environment installation](#virtual-environment-installation)
|
||||
* [Docker installation](#docker-installation-1)
|
||||
|
|
@ -216,16 +216,27 @@ ways to check the dependencies required by an extension:
|
|||
|
||||
#### Install via `extras` name
|
||||
|
||||
All the extensions that require extra dependencies are listed in the
|
||||
[`extras_require` section under
|
||||
`setup.py`](https://git.platypush.tech/platypush/platypush/src/branch/master/setup.py#L84).
|
||||
You can install extra dependencies via pip extras:
|
||||
|
||||
#### Install via `manifest.yaml`
|
||||
```shell
|
||||
pip install 'platypush[plugin1,plugin2,...]'
|
||||
```
|
||||
|
||||
All the plugins and backends have a `manifest.yaml` file in their source folder.
|
||||
For example:
|
||||
|
||||
```shell
|
||||
pip install 'platypush[light.hue,music.mpd,rss]'
|
||||
```
|
||||
|
||||
Will install Platypush with the dependencies for the `light.hue`, `music.mpd`
|
||||
and `rss` plugins.
|
||||
|
||||
#### Install via `manifest.json`
|
||||
|
||||
All the plugins and backends have a `manifest.json` file in their source folder.
|
||||
Any extra dependencies are listed there
|
||||
|
||||
If you followed the `extras` or `manifest.yaml` way to discover the
|
||||
If you followed the `extras` or `manifest.json` way to discover the
|
||||
dependencies, then you can install them in two ways:
|
||||
|
||||
1. `pip` installation:
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ class IntegrationEnricher:
|
|||
base_path,
|
||||
*doc.split(os.sep)[:-1],
|
||||
*doc.split(os.sep)[-1].split('.'),
|
||||
'manifest.yaml',
|
||||
'manifest.json',
|
||||
)
|
||||
|
||||
if not os.path.isfile(manifest_file):
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ def exec_wrapper(f: Callable[..., Any], *args, **kwargs):
|
|||
# pylint: disable=too-few-public-methods
|
||||
class ExtensionWithManifest:
|
||||
"""
|
||||
This class models an extension with an associated manifest.yaml in the same
|
||||
This class models an extension with an associated manifest.json in the same
|
||||
folder.
|
||||
"""
|
||||
|
||||
|
|
@ -40,11 +40,11 @@ class ExtensionWithManifest:
|
|||
|
||||
def get_manifest(self) -> Manifest:
|
||||
manifest_file = os.path.join(
|
||||
os.path.dirname(inspect.getfile(self.__class__)), 'manifest.yaml'
|
||||
os.path.dirname(inspect.getfile(self.__class__)), 'manifest.json'
|
||||
)
|
||||
assert os.path.isfile(
|
||||
manifest_file
|
||||
), f'The extension {self.__class__.__name__} has no associated manifest.yaml'
|
||||
), f'The extension {self.__class__.__name__} has no associated manifest.json'
|
||||
|
||||
return Manifest.from_file(manifest_file)
|
||||
|
||||
|
|
|
|||
|
|
@ -251,7 +251,7 @@ class Integration(Component, DocstringParser, Serializable):
|
|||
:return: Path of the manifest file for the integration.
|
||||
"""
|
||||
return os.path.join(
|
||||
os.path.dirname(inspect.getfile(self.type)), "manifest.yaml"
|
||||
os.path.dirname(inspect.getfile(self.type)), "manifest.json"
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import datetime
|
|||
import glob
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
|
|
@ -440,9 +441,13 @@ class Config:
|
|||
if base_dir.endswith('plugins')
|
||||
else self._backend_manifests
|
||||
)
|
||||
for mf in pathlib.Path(base_dir).rglob('manifest.yaml'):
|
||||
|
||||
for mf in pathlib.Path(base_dir).rglob('manifest.json'):
|
||||
with open(mf, 'r') as f:
|
||||
manifest = yaml.safe_load(f)['manifest']
|
||||
manifest = json.load(f).get('manifest')
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
comp_name = '.'.join(manifest['package'].split('.')[2:])
|
||||
manifests_map[comp_name] = manifest
|
||||
|
||||
|
|
|
|||
|
|
@ -81,5 +81,5 @@ class ApplicationPlugin(Plugin):
|
|||
ext = getter(extension)
|
||||
|
||||
assert ext, f'Could not find extension {extension}'
|
||||
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
|
||||
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.json')
|
||||
return list(Manifest.from_file(manifest_file).install.to_install_commands())
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
|
|||
years, some of its dependencies are quite old and may break more recent
|
||||
Python installations. Please refer to the comments in the `manifest
|
||||
file
|
||||
<https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml>`_.
|
||||
<https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.json>`_.
|
||||
for more information on how to install the required dependencies, if
|
||||
the automated ways fail.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -26,8 +26,6 @@ from typing import (
|
|||
Union,
|
||||
)
|
||||
|
||||
import yaml
|
||||
|
||||
from platypush.utils import get_src_root, is_root
|
||||
|
||||
_available_package_manager = None
|
||||
|
|
@ -538,7 +536,7 @@ class Manifest(ABC):
|
|||
return os.path.join(
|
||||
get_src_root(),
|
||||
*self.package.split('.')[1:],
|
||||
'manifest.yaml',
|
||||
'manifest.json',
|
||||
)
|
||||
|
||||
def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
|
||||
|
|
@ -584,7 +582,7 @@ class Manifest(ABC):
|
|||
Parse a manifest filename into a ``Manifest`` class.
|
||||
"""
|
||||
with open(str(filename), 'r') as f:
|
||||
manifest = yaml.safe_load(f).get('manifest', {})
|
||||
manifest = json.load(f).get('manifest', {})
|
||||
|
||||
assert 'type' in manifest, f'Manifest file {filename} has no type field'
|
||||
comp_type = ManifestType(manifest.pop('type'))
|
||||
|
|
@ -657,9 +655,14 @@ class Manifests:
|
|||
and parse them into :class:`Manifest` objects.
|
||||
"""
|
||||
for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob(
|
||||
'manifest.yaml'
|
||||
'manifest.json'
|
||||
):
|
||||
yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
|
||||
try:
|
||||
yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
'Could not parse manifest file %s: %s', mf, e, exc_info=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def by_config(
|
||||
|
|
@ -681,12 +684,21 @@ class Manifests:
|
|||
|
||||
for name in Config.get_backends().keys():
|
||||
yield Manifest.from_file(
|
||||
os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'),
|
||||
os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.json'),
|
||||
pkg_manager=pkg_manager,
|
||||
)
|
||||
|
||||
for name in Config.get_plugins().keys():
|
||||
yield Manifest.from_file(
|
||||
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
|
||||
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.json'),
|
||||
pkg_manager=pkg_manager,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def scan() -> Generator[Manifest, None, None]:
|
||||
"""
|
||||
Scan all the manifest files in the source tree and parse them into
|
||||
:class:`Manifest` objects.
|
||||
"""
|
||||
for mf in pathlib.Path(get_src_root()).rglob('manifest.json'):
|
||||
yield Manifest.from_file(str(mf))
|
||||
|
|
|
|||
250
setup.py
250
setup.py
|
|
@ -1,6 +1,8 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
|
|
@ -13,16 +15,53 @@ def readfile(fname):
|
|||
return f.read()
|
||||
|
||||
|
||||
# noinspection PyShadowingBuiltins
|
||||
def pkg_files(dir):
|
||||
paths = []
|
||||
# noinspection PyShadowingNames
|
||||
for path, _, files in os.walk(dir):
|
||||
for p, _, files in os.walk(dir):
|
||||
for file in files:
|
||||
paths.append(os.path.join('..', path, file))
|
||||
paths.append(os.path.join('..', p, file))
|
||||
return paths
|
||||
|
||||
|
||||
def scan_manifests():
|
||||
for root, _, files in os.walk('platypush'):
|
||||
for file in files:
|
||||
if file == 'manifest.json':
|
||||
yield os.path.join(root, file)
|
||||
|
||||
|
||||
def parse_deps(deps):
|
||||
ret = []
|
||||
for dep in deps:
|
||||
if dep.startswith('git+'):
|
||||
repo_name = dep.split('/')[-1].split('.git')[0]
|
||||
dep = f'{repo_name} @ {dep}'
|
||||
|
||||
ret.append(dep)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def parse_manifest(manifest_file):
|
||||
with open(manifest_file) as f:
|
||||
manifest = json.load(f).get('manifest')
|
||||
if not manifest:
|
||||
return None, None
|
||||
|
||||
name = '.'.join(manifest['package'].split('.')[2:])
|
||||
return name, parse_deps(manifest.get('install', {}).get('pip', []))
|
||||
|
||||
|
||||
def parse_manifests():
|
||||
ret = {}
|
||||
for manifest_file in scan_manifests():
|
||||
name, deps = parse_manifest(manifest_file)
|
||||
if deps:
|
||||
ret[name] = deps
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
plugins = pkg_files('platypush/plugins')
|
||||
backend = pkg_files('platypush/backend')
|
||||
|
||||
|
|
@ -87,206 +126,5 @@ setup(
|
|||
'wheel',
|
||||
'zeroconf>=0.27.0',
|
||||
],
|
||||
extras_require={
|
||||
# Support for Kafka backend and plugin
|
||||
'kafka': ['kafka-python'],
|
||||
# Support for Pushbullet
|
||||
'pushbullet': [
|
||||
'pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'
|
||||
],
|
||||
# This is only kept for back-compatibility purposes, as all the
|
||||
# dependencies of the HTTP webserver are now core dependencies.
|
||||
'http': [],
|
||||
# Support for MQTT backends
|
||||
'mqtt': ['paho-mqtt'],
|
||||
# Support for RSS feeds parser
|
||||
'rss': ['feedparser', 'defusedxml'],
|
||||
# Support for PDF generation
|
||||
'pdf': ['weasyprint'],
|
||||
# Support for Philips Hue plugin
|
||||
'hue': ['phue'],
|
||||
# Support for MPD/Mopidy music server plugin and backend
|
||||
'mpd': ['python-mpd2'],
|
||||
# Support for Google text2speech plugin
|
||||
'google-tts': [
|
||||
'oauth2client',
|
||||
'httplib2',
|
||||
'google-api-python-client',
|
||||
'google-auth',
|
||||
'google-cloud-texttospeech',
|
||||
],
|
||||
# Support for OMXPlayer plugin
|
||||
'omxplayer': ['omxplayer-wrapper'],
|
||||
# Support for YouTube
|
||||
'youtube': ['yt-dlp'],
|
||||
# Support for torrents download
|
||||
'torrent': ['python-libtorrent'],
|
||||
# Generic support for cameras
|
||||
'camera': ['numpy', 'Pillow'],
|
||||
# Support for RaspberryPi camera
|
||||
'picamera': ['picamera', 'numpy', 'Pillow'],
|
||||
# Support for inotify file monitors
|
||||
'inotify': ['inotify'],
|
||||
# Support for Google Assistant
|
||||
'google-assistant': ['google-assistant-library', 'google-auth'],
|
||||
# Support for the Google APIs
|
||||
'google': [
|
||||
'oauth2client',
|
||||
'google-auth',
|
||||
'google-api-python-client',
|
||||
'httplib2',
|
||||
],
|
||||
# Support for Last.FM scrobbler plugin
|
||||
'lastfm': ['pylast'],
|
||||
# Support for real-time MIDI events
|
||||
'midi': ['rtmidi'],
|
||||
# Support for RaspberryPi GPIO
|
||||
'rpi-gpio': ['RPi.GPIO'],
|
||||
# Support for MCP3008 analog-to-digital converter plugin
|
||||
'mcp3008': ['adafruit-mcp3008'],
|
||||
# Support for smart cards detection
|
||||
'scard': ['pyscard'],
|
||||
# Support for serial port plugin
|
||||
'serial': ['pyserial'],
|
||||
# Support for ICal calendars
|
||||
'ical': ['icalendar'],
|
||||
# Support for joystick backend
|
||||
'joystick': ['inputs'],
|
||||
# Support for Kodi plugin
|
||||
'kodi': ['kodi-json'],
|
||||
# Support for Plex plugin
|
||||
'plex': ['plexapi'],
|
||||
# Support for Chromecast plugin
|
||||
'chromecast': ['pychromecast'],
|
||||
# Support for sound devices
|
||||
'sound': ['sounddevice', 'numpy'],
|
||||
# Support for web media subtitles
|
||||
'subtitles': [
|
||||
'webvtt-py',
|
||||
'python-opensubtitles @ https://github.com/agonzalezro/python-opensubtitles/tarball/master',
|
||||
],
|
||||
# Support for mpv player plugin
|
||||
'mpv': ['python-mpv'],
|
||||
# Support for NFC tags
|
||||
'nfc': ['nfcpy>=1.0', 'ndeflib'],
|
||||
# Support for enviropHAT
|
||||
'envirophat': ['envirophat'],
|
||||
# Support for GPS
|
||||
'gps': ['gps'],
|
||||
# Support for BME280 environment sensor
|
||||
'bme280': ['pimoroni-bme280'],
|
||||
# Support for LTR559 light/proximity sensor
|
||||
'ltr559': ['ltr559', 'smbus'],
|
||||
# Support for VL53L1X laser ranger/distance sensor
|
||||
'vl53l1x': ['smbus2', 'vl53l1x'],
|
||||
# Support for Dropbox integration
|
||||
'dropbox': ['dropbox'],
|
||||
# Support for Leap Motion backend
|
||||
'leap': [
|
||||
'leap-sdk @ https://github.com/BlackLight/leap-sdk-python3/tarball/master'
|
||||
],
|
||||
# Support for Flic buttons
|
||||
'flic': [
|
||||
'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
|
||||
],
|
||||
# Support for Bluetooth devices
|
||||
'bluetooth': [
|
||||
'bleak',
|
||||
'bluetooth-numbers',
|
||||
'TheengsDecoder',
|
||||
'pydbus',
|
||||
'pybluez @ https://github.com/pybluez/pybluez/tarball/master',
|
||||
'PyOBEX @ https://github.com/BlackLight/PyOBEX/tarball/master',
|
||||
],
|
||||
# Support for TP-Link devices
|
||||
'tplink': ['pyHS100'],
|
||||
# Support for PMW3901 2-Dimensional Optical Flow Sensor
|
||||
'pmw3901': ['pmw3901'],
|
||||
# Support for MLX90640 thermal camera
|
||||
'mlx90640': ['Pillow'],
|
||||
# Support for machine learning models and cameras over OpenCV
|
||||
'cv': ['opencv-python', 'numpy', 'Pillow'],
|
||||
# Support for Node-RED integration
|
||||
'nodered': ['pynodered'],
|
||||
# Support for Todoist integration
|
||||
'todoist': ['todoist-python'],
|
||||
# Support for Trello integration
|
||||
'trello': ['py-trello'],
|
||||
# Support for Google Pub/Sub
|
||||
'google-pubsub': ['google-cloud-pubsub', 'google-auth', 'httplib2'],
|
||||
# Support for Google Translate
|
||||
'google-translate': ['google-cloud-translate', 'google-auth', 'httplib2'],
|
||||
# Support for keyboard/mouse plugin
|
||||
'inputs': ['pyuserinput'],
|
||||
# Support for Buienradar weather forecast
|
||||
'buienradar': ['buienradar'],
|
||||
# Support for Telegram integration
|
||||
'telegram': ['python-telegram-bot'],
|
||||
# Support for Arduino integration
|
||||
'arduino': ['pyserial', 'pyfirmata2'],
|
||||
# Support for CUPS printers management
|
||||
'cups': ['pycups'],
|
||||
# Support for Graphite integration
|
||||
'graphite': ['graphyte'],
|
||||
# Support for CPU and memory monitoring and info
|
||||
'sys': ['py-cpuinfo'],
|
||||
# Support for nmap integration
|
||||
'nmap': ['python-nmap'],
|
||||
# Support for zigbee2mqtt
|
||||
'zigbee': ['paho-mqtt'],
|
||||
# Support for Z-Wave
|
||||
'zwave': ['paho-mqtt'],
|
||||
# Support for Mozilla DeepSpeech speech-to-text engine
|
||||
'deepspeech': ['deepspeech', 'numpy', 'sounddevice'],
|
||||
# Support for PicoVoice hotword detection engine
|
||||
'picovoice-hotword': ['pvporcupine'],
|
||||
# Support for PicoVoice speech-to-text engine
|
||||
'picovoice-speech': ['pvcheetah @ git+https://github.com/BlackLight/cheetah'],
|
||||
# Support for OTP (One-Time Password) generation
|
||||
'otp': ['pyotp'],
|
||||
# Support for Linode integration
|
||||
'linode': ['linode_api4'],
|
||||
# Support for QR codes
|
||||
'qrcode': ['numpy', 'qrcode[pil]', 'Pillow', 'pyzbar'],
|
||||
# Support for Tensorflow
|
||||
'tensorflow': ['numpy', 'tensorflow>=2.0', 'keras', 'pandas'],
|
||||
# Support for Samsung TizenOS-based smart TVs
|
||||
'samsungtv': ['samsungtvws'],
|
||||
# Support for SSH integration
|
||||
'ssh': ['paramiko'],
|
||||
# Support for clipboard integration
|
||||
'clipboard': ['pyclip'],
|
||||
# Support for luma.oled display drivers
|
||||
'luma-oled': ['luma.oled @ git+https://github.com/rm-hull/luma.oled'],
|
||||
# Support for DBus integration
|
||||
'dbus': ['pydbus', 'defusedxml'],
|
||||
# Support for Twilio integration
|
||||
'twilio': ['twilio'],
|
||||
# Support for DHT11/DHT22/AM2302 temperature/humidity sensors
|
||||
'dht': [
|
||||
'Adafruit_Python_DHT @ git+https://github.com/adafruit/Adafruit_Python_DHT'
|
||||
],
|
||||
# Support for LCD display integration
|
||||
'lcd': ['RPi.GPIO', 'RPLCD'],
|
||||
# Support for email integration
|
||||
'mail': ['imapclient', 'dnspython'],
|
||||
# Support for NextCloud integration
|
||||
'nextcloud': ['nextcloud-api-wrapper'],
|
||||
# Support for VLC integration
|
||||
'vlc': ['python-vlc'],
|
||||
# Support for SmartThings integration
|
||||
'smartthings': ['pysmartthings', 'aiohttp'],
|
||||
# Support for file.monitor backend
|
||||
'filemonitor': ['watchdog'],
|
||||
# Support for Adafruit PCA9685 PWM controller
|
||||
'pca9685': ['adafruit-python-shell', 'adafruit-circuitpython-pca9685'],
|
||||
# Support for ngrok integration
|
||||
'ngrok': ['pyngrok'],
|
||||
# Support for IRC integration
|
||||
'irc': ['irc'],
|
||||
# Support for the Matrix integration
|
||||
'matrix': ['matrix-nio'],
|
||||
# Support for the XMPP integration
|
||||
'xmpp': ['aioxmpp', 'pytz'],
|
||||
},
|
||||
extras_require=parse_manifests(),
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue