blacklight/platypush
1
0
Fork 0
forked from platypush/platypush
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#394] Dynamically generate setup extras.

Browse source 
Also, convert all code that relied on `manifest.yaml` to use
`manifest.json` instead.

Closes: #394
This commit is contained in:
Fabio Manganiello 2024-05-17 00:47:23 +02:00
parent 59c693d6a0
commit f06233801b
Signed by: blacklight
GPG key ID: D90FBA7F76362774
11 changed files with 103 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
CONTRIBUTING.md
View file

@ -27,13 +27,9 @@ Guidelines:
you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons
or the bus) then make sure to add tests and not to break the existing tests.
- If the feature requires an optional dependency then make sure to document it:
- In the class docstring (see other plugins and backends for examples).
- In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as
an `extras_require` entry.
- In the plugin/backend class pydoc string.
- In the `manifest.yaml` - refer to the Wiki (how to write
[plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
for examples on how to write an extension manifest file.
- If the feature requires an optional dependency then make sure to document it
in the `manifest.json` - refer to the Wiki (how to write
[plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
and
[backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
for examples on how to write an extension manifest file.

2
MANIFEST.in
View file

@ -2,5 +2,5 @@ recursive-include platypush/backend/http/webapp/dist *
recursive-include platypush/install *
include platypush/plugins/http/webpage/mercury-parser.js
include platypush/config/*.yaml
global-include manifest.yaml
global-include manifest.json
global-include components.json.gz

25
README.md
View file

@ -22,7 +22,7 @@ Platypush
* [Install from sources](#install-from-sources)
* [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions)
+ [Install via `extras` name](#install-via-extras-name)
+ [Install via `manifest.yaml`](#install-via-manifestyaml)
+ [Install via `manifest.json`](#install-via-manifestjson)
+ [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation)
* [Virtual environment installation](#virtual-environment-installation)
* [Docker installation](#docker-installation-1)
@ -216,16 +216,27 @@ ways to check the dependencies required by an extension:
#### Install via `extras` name
All the extensions that require extra dependencies are listed in the
[`extras_require` section under
`setup.py`](https://git.platypush.tech/platypush/platypush/src/branch/master/setup.py#L84).
You can install extra dependencies via pip extras:
#### Install via `manifest.yaml`
```shell
pip install 'platypush[plugin1,plugin2,...]'
```
All the plugins and backends have a `manifest.yaml` file in their source folder.
For example:
```shell
pip install 'platypush[light.hue,music.mpd,rss]'
```
Will install Platypush with the dependencies for the `light.hue`, `music.mpd`
and `rss` plugins.
#### Install via `manifest.json`
All the plugins and backends have a `manifest.json` file in their source folder.
Any extra dependencies are listed there
If you followed the `extras` or `manifest.yaml` way to discover the
If you followed the `extras` or `manifest.json` way to discover the
dependencies, then you can install them in two ways:
1. `pip` installation:

2
docs/source/_ext/add_dependencies.py
View file

@ -159,7 +159,7 @@ class IntegrationEnricher:
base_path,
*doc.split(os.sep)[:-1],
*doc.split(os.sep)[-1].split('.'),
'manifest.yaml',
'manifest.json',
)
if not os.path.isfile(manifest_file):

6
platypush/common/__init__.py
View file

@ -31,7 +31,7 @@ def exec_wrapper(f: Callable[..., Any], *args, **kwargs):
# pylint: disable=too-few-public-methods
class ExtensionWithManifest:
"""
This class models an extension with an associated manifest.yaml in the same
This class models an extension with an associated manifest.json in the same
folder.
"""
@ -40,11 +40,11 @@ class ExtensionWithManifest:
def get_manifest(self) -> Manifest:
manifest_file = os.path.join(
os.path.dirname(inspect.getfile(self.__class__)), 'manifest.yaml'
os.path.dirname(inspect.getfile(self.__class__)), 'manifest.json'
)
assert os.path.isfile(
manifest_file
), f'The extension {self.__class__.__name__} has no associated manifest.yaml'
), f'The extension {self.__class__.__name__} has no associated manifest.json'
return Manifest.from_file(manifest_file)

2
platypush/common/reflection/_model/integration.py
View file

@ -251,7 +251,7 @@ class Integration(Component, DocstringParser, Serializable):
:return: Path of the manifest file for the integration.
"""
return os.path.join(
os.path.dirname(inspect.getfile(self.type)), "manifest.yaml"
os.path.dirname(inspect.getfile(self.type)), "manifest.json"
)
@property

9
platypush/config/__init__.py
View file

@ -2,6 +2,7 @@ import datetime
import glob
import importlib
import inspect
import json
import logging
import os
import pathlib
@ -440,9 +441,13 @@ class Config:
if base_dir.endswith('plugins')
else self._backend_manifests
)
for mf in pathlib.Path(base_dir).rglob('manifest.yaml'):
for mf in pathlib.Path(base_dir).rglob('manifest.json'):
with open(mf, 'r') as f:
manifest = yaml.safe_load(f)['manifest']
manifest = json.load(f).get('manifest')
if not manifest:
continue
comp_name = '.'.join(manifest['package'].split('.')[2:])
manifests_map[comp_name] = manifest

2
platypush/plugins/application/__init__.py
View file

@ -81,5 +81,5 @@ class ApplicationPlugin(Plugin):
ext = getter(extension)
assert ext, f'Could not find extension {extension}'
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.json')
return list(Manifest.from_file(manifest_file).install.to_install_commands())

2
platypush/plugins/assistant/google/__init__.py
View file

@ -58,7 +58,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
years, some of its dependencies are quite old and may break more recent
Python installations. Please refer to the comments in the `manifest
file
<https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml>`_.
<https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.json>`_.
for more information on how to install the required dependencies, if
the automated ways fail.
"""

28
platypush/utils/manifest.py
View file

@ -26,8 +26,6 @@ from typing import (
Union,
)
import yaml
from platypush.utils import get_src_root, is_root
_available_package_manager = None
@ -538,7 +536,7 @@ class Manifest(ABC):
return os.path.join(
get_src_root(),
*self.package.split('.')[1:],
'manifest.yaml',
'manifest.json',
)
def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
@ -584,7 +582,7 @@ class Manifest(ABC):
Parse a manifest filename into a ``Manifest`` class.
"""
with open(str(filename), 'r') as f:
manifest = yaml.safe_load(f).get('manifest', {})
manifest = json.load(f).get('manifest', {})
assert 'type' in manifest, f'Manifest file {filename} has no type field'
comp_type = ManifestType(manifest.pop('type'))
@ -657,9 +655,14 @@ class Manifests:
and parse them into :class:`Manifest` objects.
"""
for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob(
'manifest.yaml'
'manifest.json'
):
yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
try:
yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
except Exception as e:
logger.debug(
'Could not parse manifest file %s: %s', mf, e, exc_info=True
)
@staticmethod
def by_config(
@ -681,12 +684,21 @@ class Manifests:
for name in Config.get_backends().keys():
yield Manifest.from_file(
os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'),
os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.json'),
pkg_manager=pkg_manager,
)
for name in Config.get_plugins().keys():
yield Manifest.from_file(
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.json'),
pkg_manager=pkg_manager,
)
@staticmethod
def scan() -> Generator[Manifest, None, None]:
"""
Scan all the manifest files in the source tree and parse them into
:class:`Manifest` objects.
"""
for mf in pathlib.Path(get_src_root()).rglob('manifest.json'):
yield Manifest.from_file(str(mf))

250
setup.py
View file

@ -1,6 +1,8 @@
#!/usr/bin/env python
import json
import os
from setuptools import setup, find_packages
@ -13,16 +15,53 @@ def readfile(fname):
return f.read()
# noinspection PyShadowingBuiltins
def pkg_files(dir):
paths = []
# noinspection PyShadowingNames
for path, _, files in os.walk(dir):
for p, _, files in os.walk(dir):
for file in files:
paths.append(os.path.join('..', path, file))
paths.append(os.path.join('..', p, file))
return paths
def scan_manifests():
for root, _, files in os.walk('platypush'):
for file in files:
if file == 'manifest.json':
yield os.path.join(root, file)
def parse_deps(deps):
ret = []
for dep in deps:
if dep.startswith('git+'):
repo_name = dep.split('/')[-1].split('.git')[0]
dep = f'{repo_name} @ {dep}'
ret.append(dep)
return ret
def parse_manifest(manifest_file):
with open(manifest_file) as f:
manifest = json.load(f).get('manifest')
if not manifest:
return None, None
name = '.'.join(manifest['package'].split('.')[2:])
return name, parse_deps(manifest.get('install', {}).get('pip', []))
def parse_manifests():
ret = {}
for manifest_file in scan_manifests():
name, deps = parse_manifest(manifest_file)
if deps:
ret[name] = deps
return ret
plugins = pkg_files('platypush/plugins')
backend = pkg_files('platypush/backend')
@ -87,206 +126,5 @@ setup(
'wheel',
'zeroconf>=0.27.0',
],
extras_require={
# Support for Kafka backend and plugin
'kafka': ['kafka-python'],
# Support for Pushbullet
'pushbullet': [
'pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'
],
# This is only kept for back-compatibility purposes, as all the
# dependencies of the HTTP webserver are now core dependencies.
'http': [],
# Support for MQTT backends
'mqtt': ['paho-mqtt'],
# Support for RSS feeds parser
'rss': ['feedparser', 'defusedxml'],
# Support for PDF generation
'pdf': ['weasyprint'],
# Support for Philips Hue plugin
'hue': ['phue'],
# Support for MPD/Mopidy music server plugin and backend
'mpd': ['python-mpd2'],
# Support for Google text2speech plugin
'google-tts': [
'oauth2client',
'httplib2',
'google-api-python-client',
'google-auth',
'google-cloud-texttospeech',
],
# Support for OMXPlayer plugin
'omxplayer': ['omxplayer-wrapper'],
# Support for YouTube
'youtube': ['yt-dlp'],
# Support for torrents download
'torrent': ['python-libtorrent'],
# Generic support for cameras
'camera': ['numpy', 'Pillow'],
# Support for RaspberryPi camera
'picamera': ['picamera', 'numpy', 'Pillow'],
# Support for inotify file monitors
'inotify': ['inotify'],
# Support for Google Assistant
'google-assistant': ['google-assistant-library', 'google-auth'],
# Support for the Google APIs
'google': [
'oauth2client',
'google-auth',
'google-api-python-client',
'httplib2',
],
# Support for Last.FM scrobbler plugin
'lastfm': ['pylast'],
# Support for real-time MIDI events
'midi': ['rtmidi'],
# Support for RaspberryPi GPIO
'rpi-gpio': ['RPi.GPIO'],
# Support for MCP3008 analog-to-digital converter plugin
'mcp3008': ['adafruit-mcp3008'],
# Support for smart cards detection
'scard': ['pyscard'],
# Support for serial port plugin
'serial': ['pyserial'],
# Support for ICal calendars
'ical': ['icalendar'],
# Support for joystick backend
'joystick': ['inputs'],
# Support for Kodi plugin
'kodi': ['kodi-json'],
# Support for Plex plugin
'plex': ['plexapi'],
# Support for Chromecast plugin
'chromecast': ['pychromecast'],
# Support for sound devices
'sound': ['sounddevice', 'numpy'],
# Support for web media subtitles
'subtitles': [
'webvtt-py',
'python-opensubtitles @ https://github.com/agonzalezro/python-opensubtitles/tarball/master',
],
# Support for mpv player plugin
'mpv': ['python-mpv'],
# Support for NFC tags
'nfc': ['nfcpy>=1.0', 'ndeflib'],
# Support for enviropHAT
'envirophat': ['envirophat'],
# Support for GPS
'gps': ['gps'],
# Support for BME280 environment sensor
'bme280': ['pimoroni-bme280'],
# Support for LTR559 light/proximity sensor
'ltr559': ['ltr559', 'smbus'],
# Support for VL53L1X laser ranger/distance sensor
'vl53l1x': ['smbus2', 'vl53l1x'],
# Support for Dropbox integration
'dropbox': ['dropbox'],
# Support for Leap Motion backend
'leap': [
'leap-sdk @ https://github.com/BlackLight/leap-sdk-python3/tarball/master'
],
# Support for Flic buttons
'flic': [
'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
],
# Support for Bluetooth devices
'bluetooth': [
'bleak',
'bluetooth-numbers',
'TheengsDecoder',
'pydbus',
'pybluez @ https://github.com/pybluez/pybluez/tarball/master',
'PyOBEX @ https://github.com/BlackLight/PyOBEX/tarball/master',
],
# Support for TP-Link devices
'tplink': ['pyHS100'],
# Support for PMW3901 2-Dimensional Optical Flow Sensor
'pmw3901': ['pmw3901'],
# Support for MLX90640 thermal camera
'mlx90640': ['Pillow'],
# Support for machine learning models and cameras over OpenCV
'cv': ['opencv-python', 'numpy', 'Pillow'],
# Support for Node-RED integration
'nodered': ['pynodered'],
# Support for Todoist integration
'todoist': ['todoist-python'],
# Support for Trello integration
'trello': ['py-trello'],
# Support for Google Pub/Sub
'google-pubsub': ['google-cloud-pubsub', 'google-auth', 'httplib2'],
# Support for Google Translate
'google-translate': ['google-cloud-translate', 'google-auth', 'httplib2'],
# Support for keyboard/mouse plugin
'inputs': ['pyuserinput'],
# Support for Buienradar weather forecast
'buienradar': ['buienradar'],
# Support for Telegram integration
'telegram': ['python-telegram-bot'],
# Support for Arduino integration
'arduino': ['pyserial', 'pyfirmata2'],
# Support for CUPS printers management
'cups': ['pycups'],
# Support for Graphite integration
'graphite': ['graphyte'],
# Support for CPU and memory monitoring and info
'sys': ['py-cpuinfo'],
# Support for nmap integration
'nmap': ['python-nmap'],
# Support for zigbee2mqtt
'zigbee': ['paho-mqtt'],
# Support for Z-Wave
'zwave': ['paho-mqtt'],
# Support for Mozilla DeepSpeech speech-to-text engine
'deepspeech': ['deepspeech', 'numpy', 'sounddevice'],
# Support for PicoVoice hotword detection engine
'picovoice-hotword': ['pvporcupine'],
# Support for PicoVoice speech-to-text engine
'picovoice-speech': ['pvcheetah @ git+https://github.com/BlackLight/cheetah'],
# Support for OTP (One-Time Password) generation
'otp': ['pyotp'],
# Support for Linode integration
'linode': ['linode_api4'],
# Support for QR codes
'qrcode': ['numpy', 'qrcode[pil]', 'Pillow', 'pyzbar'],
# Support for Tensorflow
'tensorflow': ['numpy', 'tensorflow>=2.0', 'keras', 'pandas'],
# Support for Samsung TizenOS-based smart TVs
'samsungtv': ['samsungtvws'],
# Support for SSH integration
'ssh': ['paramiko'],
# Support for clipboard integration
'clipboard': ['pyclip'],
# Support for luma.oled display drivers
'luma-oled': ['luma.oled @ git+https://github.com/rm-hull/luma.oled'],
# Support for DBus integration
'dbus': ['pydbus', 'defusedxml'],
# Support for Twilio integration
'twilio': ['twilio'],
# Support for DHT11/DHT22/AM2302 temperature/humidity sensors
'dht': [
'Adafruit_Python_DHT @ git+https://github.com/adafruit/Adafruit_Python_DHT'
],
# Support for LCD display integration
'lcd': ['RPi.GPIO', 'RPLCD'],
# Support for email integration
'mail': ['imapclient', 'dnspython'],
# Support for NextCloud integration
'nextcloud': ['nextcloud-api-wrapper'],
# Support for VLC integration
'vlc': ['python-vlc'],
# Support for SmartThings integration
'smartthings': ['pysmartthings', 'aiohttp'],
# Support for file.monitor backend
'filemonitor': ['watchdog'],
# Support for Adafruit PCA9685 PWM controller
'pca9685': ['adafruit-python-shell', 'adafruit-circuitpython-pca9685'],
# Support for ngrok integration
'ngrok': ['pyngrok'],
# Support for IRC integration
'irc': ['irc'],
# Support for the Matrix integration
'matrix': ['matrix-nio'],
# Support for the XMPP integration
'xmpp': ['aioxmpp', 'pytz'],
},
extras_require=parse_manifests(),
)