diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..376b7571f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +**/.git +**/node_modules +**/__pycache__ +**/venv +**/.mypy_cache +**/build diff --git a/.drone.yml b/.drone.yml index 6f8db1239..3ea4e5434 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null - git config --global --add safe.directory $PWD - git remote add github git@github.com:/BlackLight/platypush.git - - git pull github master + - git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github" - git push --all -v github - name: docs diff --git a/.gitignore b/.gitignore index 8befed612..20fcb3caf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ package.sh platypush/backend/http/static/resources/* docs/build .idea/ -config +/config platypush/backend/http/static/css/*/.sass-cache/ .vscode platypush/backend/http/static/js/lib/vue.js @@ -24,3 +24,4 @@ coverage.xml Session.vim /jsconfig.json /package.json +/Dockerfile diff --git a/MANIFEST.in b/MANIFEST.in index 9163c3d77..ac702393c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ recursive-include platypush/backend/http/webapp/dist * +recursive-include platypush/install * include platypush/plugins/http/webpage/mercury-parser.js include platypush/config/*.yaml global-include manifest.yaml diff --git a/bin/platyvenv b/bin/platyvenv deleted file mode 100755 index 1b0d27de7..000000000 --- a/bin/platyvenv +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash - -############################################################################## -# This script allows you to easily manage Platypush instances through Python # -# virtual environment. You can build environments from a config.yaml file # -# and automatically managed the required dependencies, as well as start, # -# stop and remove them # -# # -# @author: Fabio Manganiello # -# @licence: MIT # -############################################################################## - - -workdir="$HOME/.local/share/platypush/venv" - -function build { - cfgfile= - - while getopts ':c:' opt; do - case ${opt} in - c) - cfgfile=$OPTARG;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1;; - :) - echo "Option -$OPTARG requires the path to a Platypush configuration file" >&2 - exit 1;; - esac - done - - if [[ -z "$cfgfile" ]]; then - echo "Usage: $0 build -c " >&2 - exit 1 - fi - - echo "Parsing configuration file" - pip_cmd= - pkg_cmd= - includes=() - cmd_exec=() - - while read -r line; do - if echo "$line" | grep -E "^pip:\s*"; then - pip_cmd="$(echo "$line" | sed -r -e 's/^pip:\s*(.*)'/\\1/)" - elif echo "$line" | grep -E "^packages:\s*"; then - pkg_cmd="$(echo "$line" | sed -r -e 's/^packages:\s*(.*)'/\\1/)" - elif echo "$line" | grep -E "^exec:\s*"; then - cmd_exec+=("$(echo "$line" | sed -r -e 's/^exec:\s*(.*)'/\\1/)") - elif echo "$line" | grep -E "^include:\s*"; then - includes+=("$(echo "$line" | sed -r -e 's/^include:\s*(.*)'/\\1/)") - elif echo "$line" | grep -E "^device_id:\s*"; then - device_id="$(echo "$line" | sed -r -e 's/^device_id:\s*(.*)'/\\1/)" - fi - done <<< "$(python <" >&2 - exit 1 - fi - - env=$1 - envdir="${workdir}/${env}" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - cfgfile="${envdir}/etc/platypush/config.yaml" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - mkdir -p "${rundir}" - - if [[ -f "$pidfile" ]]; then - if pgrep -F "${pidfile}"; then - echo "Another instance (PID $(cat "${pidfile}")) is running, please stop that instance first" - exit 1 - fi - - echo "A PID file was found but the process does not seem to be running, starting anyway" - rm -f "$pidfile" - fi - - python3 -m venv "${envdir}" - cd "${envdir}" || exit 1 - source bin/activate - bin/platypush -c "$cfgfile" -P "$pidfile" & - start_time=$(date +'%s') - timeout=30 - - while :; do - [[ -f "$pidfile" ]] && break - now=$(date +'%s') - elapsed=$(( now-start_time )) - if (( elapsed >= timeout )); then - echo "Platypush instance '$env' did not start within $timeout seconds" >&2 - exit 1 - fi - - echo -n '.' - sleep 1 - done - - pid=$(cat "$pidfile") - echo - echo "Platypush environment $env started with PID $pid" - wait "${pid}" - echo "Platypush environment $env terminated" -} - -function stop { - if [[ -z "$1" ]]; then - echo "Usage: $0 stop " >&2 - exit 1 - fi - - env=$1 - envdir="${workdir}/${env}" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - if [[ ! -f "$pidfile" ]]; then - echo "No pidfile found for instance \"${env}\"" - exit 1 - fi - - pid=$(cat "$pidfile") - pids="$pid $(ps --no-headers -o pid= --ppid "$pid")" - # shellcheck disable=SC2086 - kill -9 ${pids} - rm -f "$pidfile" - echo "Instance '$env' with PID $pid stopped" -} - -function rme { - if [[ -z "$1" ]]; then - echo "Usage: $0 rm " >&2 - exit 1 - fi - - envdir="${workdir}/$1" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - if [[ -f "$pidfile" ]]; then - if pgrep -F "${pidfile}"; then - echo "Another instance (PID $(cat "$pidfile")) is running, please stop that instance first" - exit 1 - fi - - echo "A PID file was found but the process does not seem to be running, removing anyway" - fi - - echo "WARNING: This operation will permanently remove the Platypush environment $1" - echo -n "Are you sure you want to continue? (y/N) " - IFS= read -r answer - echo "$answer" | grep -E '^[yY]' >/dev/null || exit 0 - rm -rf "$envdir" - echo "$envdir removed" -} - -function usage { - echo "Usage: $0 [options]" >&2 - exit 1 -} - -if (( $# < 1 )); then - usage -fi - -action=$1 -shift -mkdir -p "${workdir}" - -# shellcheck disable=SC2048,SC2086 -case ${action} in - 'build') build $*;; - 'start') start $*;; - 'stop') stop $*;; - 'rm') rme $*;; - *) usage;; -esac diff --git a/examples/conf/config.yaml b/examples/conf/config.yaml deleted file mode 100644 index 5c9561a56..000000000 --- a/examples/conf/config.yaml +++ /dev/null @@ -1,379 +0,0 @@ -################################################################################# -# Sample platypush configuration file. -# Edit it and copy it to /etc/platypush/config.yaml for system installation or to -# ~/.config/platypush/config.yaml for user installation (recommended). -################################################################################# - -# -- -# include directive example -# -- -# -# You can split your configuration over multiple files -# and use the include directive to import them in your configuration. -# Relative paths are also supported, and computed using the config.yaml -# installation directory as base folder. Symlinks are also supported. -# -# Using multiple files is encouraged in the case of large configurations -# that can easily end up in a messy config.yaml file, as they help you -# keep your configuration more organized. -#include: -# - include/logging.yaml -# - include/media.yaml -# - include/sensors.yaml - -# platypush logs on stdout by default. You can use the logging section to specify -# an alternative file or change the logging level. -#logging: -# filename: ~/.local/log/platypush/platypush.log -# level: INFO - -# The device_id is used by many components of platypush and it should uniquely -# identify a device in your network. If nothing is specified then the hostname -# will be used. -#device_id: my_device - -## -- -## Plugin configuration examples -## -- -# -# Plugins configuration is very straightforward. Each plugin is mapped to -# a plugin class. The methods of the class with @action annotation will -# be exported as runnable actions, while the __init__ parameters are -# configuration attributes that you can initialize in your config.yaml. -# Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html -# -# In this example we'll configure the light.hue plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/light.hue.html -# for reference. You can easily install the required dependencies for the plugin through -# pip install 'platypush[hue]' -light.hue: - # IP address or hostname of the Hue bridge - bridge: 192.168.1.10 - # Groups that will be handled by default if nothing is specified on the request - groups: - - Living Room - -# Example configuration of music.mpd plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html -# You can easily install the dependencies through pip install 'platypush[mpd]' -music.mpd: - host: localhost - port: 6600 - -# Example configuration of media.chromecast plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html -# You can easily install the dependencies through pip install 'platypush[chromecast]' -media.chromecast: - chromecast: Living Room TV - -# Plugins with empty configuration can also be explicitly enabled by specifying -# enabled=True or disabled=False (it's a good practice if you want the -# corresponding web panel to be enabled, if available) -camera.pi: - enabled: True - -# Support for calendars - in this case Google and Facebook calendars -# Installing the dependencies: pip install 'platypush[ical,google]' -calendar: - calendars: - - type: platypush.plugins.google.calendar.GoogleCalendarPlugin - - type: platypush.plugins.calendar.ical.CalendarIcalPlugin - url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key - -## -- -## Backends configuration examples -## -- -# -# Backends are basically threads that run in the background and listen for something -# to happen and either trigger events or provide additional services on top of platypush. -# Just like plugins, backends are classes whose configuration matches one-to-one the -# supported parameters on the __init__ methods. You can check the documentation for the -# available backends here: https://docs.platypush.tech/en/latest/backends.html. -# Moreover, most of the backends will generate events that you can react to through custom -# event hooks. Check here for the events documentation: -# https://docs.platypush.tech/en/latest/events.html -# -# You may usually want to enable the HTTP backend, as it provides many useful features on -# top of platypush. Among those: -# -# - Expose the /execute endpoint, that allows you to send requests to platypush through a -# JSON-RPC interface. -# - Web panel, one of the key additiona features of platypush. Many plugins will expose web -# panel tabs for e.g. accessing and controlling lights, music, media and sensors. -# - Dashboard: platypush can be configured to show a custom dashboard on your screens with -# e.g. music platypush and weather info, news, upcoming calendar events and photo carousel. -# - Streaming support - the HTTP backend makes it possible to stream local media to other -# devices - e.g. Chromecasts and external browsers. -# -# To install the HTTP backend dependencies simply run 'pip install "platypush[http]"' -backend.http: - # Listening port - port: 8008 - - # Through resource_dirs you can specify external folders whose content can be accessed on - # the web server through a custom URL. In the case below we have a Dropbox folder containing - # our pictures and we mount it to the '/carousel' endpoint. - resource_dirs: - carousel: /mnt/hd/photos/carousel - -# The HTTP poll backend is a versatile backend that can monitor for HTTP-based resources and -# trigger events whenever new entries are available. In the example below we show how to use -# the backend to listen for changes on a set of RSS feeds. New content will be stored by default -# on a SQLite database under ~/.local/share/platypush/feeds/rss.db. -# Install the required dependencies through 'pip install "platypush[rss,db]"' -backend.http.poll: - requests: - - type: platypush.backend.http.request.rss.RssUpdates # HTTP poll type (RSS) - # Remote URL - url: http://www.theguardian.com/rss/world - # Custom title - title: The Guardian - World News - # How often we should check for changes - poll_seconds: 600 - # Maximum number of new entries to be processed - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://www.physorg.com/rss-feed - title: Phys.org - poll_seconds: 600 - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://feeds.feedburner.com/Techcrunch - title: Tech Crunch - poll_seconds: 600 - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://www.nytimes.com/services/xml/rss/nyt/HomePage.xml - title: The New York Times - poll_seconds: 300 - max_entries: 10 - -# MQTT backend. Installed required dependencies through 'pip install "platypush[mqtt]"' -backend.mqtt: - # Remote MQTT server IP or hostname - host: mqtt-server - # By default the backend will listen for messages on the platypush_bus_mq/device_id - # topic, but you can change the prefix using the topic attribute -# topic: MyBus - -# Raw TCP socket backend. It can run commands sent as JSON over telnet or netcat -#backend.tcp: -# port: 3333 - -## -- -## Assistant configuration examples -## -- -# -# Both Google Assistant and Alexa voice assistant interfaces are supported by platypush. -# You can easily make your custom voice assistant with a RaspberryPi and a USB microphone, -# or on your laptop. Note however that the Alexa integration is still experimental -# (mostly because of glitches and bugs on the avs package provided by Amazon), while the -# Google Assistant support should be more robust. The recommended way of triggering a -# hotword ('OK Google', 'Alexa' or any custom hotword you like) is through the snowboy -# backend (install it through 'pip install "platypush[hotword]"'). You can download custom -# voice model files (.umdl) from https://snowboy.kitt.ai. -backend.assistant.snowboy: - # Microphone audio gain - audio_gain: 1.1 - - models: - # "Computer" hotword model - computer: - # UMDL file path - voice_model_file: ~/.local/share/snowboy/models/computer.umdl - # Plugin to use (Google Assistant) - assistant_plugin: assistant.google.pushtotalk - # Language assistant (Italian) - assistant_language: it-IT - # Sound to play when the hotword is detected - detect_sound: ~/.local/share/sounds/hotword.wav - # Model sensitivity - sensitivity: 0.4 - # "OK Google" hotword model - ok_google: - voice_model_file: ~/.local/share/snowboy/models/OK Google.pmdl - assistant_plugin: assistant.google.pushtotalk - assistant_language: en-US - detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav - sensitivity: 0.4 - # "Alexa" voice model - alexa: - voice_model_file: ~/.local/share/snowboy/models/Alexa.pmdl - assistant_plugin: assistant.echo - assistant_language: en-US - detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav - sensitivity: 0.5 - -# Install Alexa dependencies with 'pip install "platypush[alexa]"' -assistant.echo: - audio_player: mplayer - -# Install Google Assistant dependencies with 'pip install "platypush[google-assistant-legacy]"' -assistant.google: - enabled: True - -backend.assistant.google: - enabled: True - -## -- -## Procedure examples -## -- -# -# Procedures are lists of actions that can be executed synchronously (default) or in parallel -# (procedure.async. prefix). Basic flow control operators (if/else/for) are also available. -# You can also access Python variables and evaluate Python expressions by using the ${} expressions. -# The 'context' special variable is a name->value dictionary containing the items returned from -# previous actions - for example if an action returned '{"status": "ok", "temperature":21.5}' then -# the following actions can access those variables through ${status} and ${temperature} respectively, -# and you can also add things like '- if ${temperature > 20.0}' or '- for ${temp in temperature_values}'. -# Alternatively, you can access those variable also through ${context.get('status')} or ${context.get('temperature')}. -# Other special variables that you can use in your procedures: -# -# - output: Will contain the parsed output of the previous action -# - errors: Will contain the errors of the previous action -# - event: If the procedure is executed within an event hook, it contains the event that triggered the hook -# -# An example procedure that can be called when you arrive home. You can run this procedure by sending a JSON -# message like this on whichever backend you like (HTTP, websocket, TCP, Redis, MQTT, Node-RED, Pushbullet...) -# {"type":"request", "action":"procedure.at_home"} -# You can for instance install Tasker+AutoLocation on your mobile and send this message whenever you enter -# your home area. -procedure.at_home: - # Set the db variable HOME to 1 - - action: variable.set - args: - HOME: 1 - - # Check the luminosity level from a connected LTR559 sensor - - action: gpio.sensor.ltr559.get_data - - # If it's below a certain threshold turn on the lights - - if ${int(light or 0) < 110}: - - action: light.hue.on - - # Say a welcome home message. Install dependencies through 'pip install "platypush[google-tts]"' - - action: tts.google.say - args: - text: Welcome home - - # Start the music - - action: music.mpd.play - -# Procedure that will be execute when you're outside of home -procedure.outside_home: - # Unset the db variable HOME - - action: variable.unset - args: - name: HOME - - # Stop the music - - action: music.mpd.stop - - # Turn off the lights - - action: light.hue.off - - # Start the camera streaming. Install the Pi Camera dependencies through - # 'pip install "platypush[picamera]"' - - action: camera.pi.start_streaming - args: - listen_port: 2222 - -# Procedures can also take optional arguments. The example below show a -# generic procedure to send a request to another platypush host over MQTT -# given target, action and args -procedure.send_request(target, action, args): - - action: mqtt.send_message - args: - topic: platypush_bus_mq/${target} - host: mqtt-server - port: 1883 - msg: - type: request - target: ${target} - action: ${action} - args: ${args} - -## -- -## Event hook examples -## -- -# -# Event hooks are procedures that are run when a certain condition is met. -# Check the documentation of the backends to see which events they can trigger. -# An event hook consists of two parts: an 'if' field that specifies on which -# event the hook will be triggered (type and attributes content), and a 'then' -# field that uses the same syntax as procedures to specify a list of actions to -# execute when the event is matched. -# -# The example below plays the music on mpd/mopidy when your voice assistant -# triggers a speech recognized event with "play the music" content. -event.hook.PlayMusicAssistantCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - # Note that basic regexes are supported, so the hook will be triggered - # both if you say "play the music" and "play music" - phrase: "play (the)? music" - then: - - action: music.mpd.play - -# This will turn on the lights when you say "turn on the lights" -event.hook.TurnOnLightsCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - phrase: "turn on (the)? lights?" - then: - - action: light.hue.on - -# This will play a song by a specified artist -event.hook.SearchSongVoiceCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - # Note that you can use the ${} operator in event matching to - # extract part of the matched string into context variables that - # can be accessed in your event hook. - phrase: "play ${title} by ${artist}" - then: - - action: music.mpd.clear - - action: music.mpd.search - args: - filter: - artist: ${artist} - title: ${title} - - # Play the first search result - - action: music.mpd.play - args: - resource: ${output[0]['file']} - -# This event will scrobble newly listened tracks on mpd/mopidy to last.fm -event.hook.ScrobbleNewTrack: - if: - type: platypush.message.event.music.NewPlayingTrackEvent - then: - - action: lastfm.scrobble - args: - artist: ${track['artist']} - title: ${track['title']} - - - action: lastfm.update_now_playing - args: - artist: ${track['artist']} - title: ${track['title']} - -## -- -## Cron examples -## -- -# -# Cronjobs allow you to execute procedures at periodic intervals. -# Standard UNIX cron syntax is supported, plus an optional 6th indicator -# at the end of the expression to run jobs with second granularity. -# The example below executes a script at intervals of 1 minute. -cron.TestCron: - cron_expression: '* * * * *' - actions: - - action: shell.exec - args: - cmd: ~/bin/myscript.sh - diff --git a/examples/config/config.yaml b/examples/config/config.yaml new file mode 120000 index 000000000..6e3e62269 --- /dev/null +++ b/examples/config/config.yaml @@ -0,0 +1 @@ +../../platypush/config/config.yaml \ No newline at end of file diff --git a/examples/conf/dashboard.xml b/examples/config/dashboard.xml similarity index 100% rename from examples/conf/dashboard.xml rename to examples/config/dashboard.xml diff --git a/examples/conf/hook.py b/examples/config/hook.py similarity index 88% rename from examples/conf/hook.py rename to examples/config/hook.py index 2ad765f4b..2f4081b81 100644 --- a/examples/conf/hook.py +++ b/examples/config/hook.py @@ -12,7 +12,10 @@ from platypush.utils import run from platypush.event.hook import hook # Event types that you want to react to -from platypush.message.event.assistant import ConversationStartEvent, SpeechRecognizedEvent +from platypush.message.event.assistant import ( + ConversationStartEvent, + SpeechRecognizedEvent, +) @hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}') @@ -23,10 +26,13 @@ def on_music_play_command(event, title=None, artist=None, **context): Note that in this specific case we can leverage the token-extraction feature of SpeechRecognizedEvent through ${} that operates on regex-like principles to extract any text that matches the pattern into context variables. """ - results = run('music.mpd.search', filter={ - 'artist': artist, - 'title': title, - }) + results = run( + 'music.mpd.search', + filter={ + 'artist': artist, + 'title': title, + }, + ) if results: run('music.mpd.play', results[0]['file']) diff --git a/examples/docker/Dockerfile b/examples/docker/Dockerfile deleted file mode 100644 index 374ffbdc5..000000000 --- a/examples/docker/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# Sample Dockerfile. Use platydock -c /path/to/custom/config.yaml -# to generate your custom Dockerfile. - - -FROM python:3.11-alpine - -RUN mkdir -p /install /app -COPY . /install -RUN apk add --update --no-cache redis -RUN apk add --update --no-cache --virtual build-base g++ rust linux-headers -RUN pip install -U pip -RUN cd /install && pip install . -RUN apk del build-base g++ rust linux-headers - -EXPOSE 8008 -VOLUME /app/config -VOLUME /app/workdir - -CMD python -m platypush --start-redis --config-file /app/config/config.yaml --workdir /app/workdir diff --git a/examples/nginx/nginx.sample.conf b/examples/nginx/nginx.sample.conf index 321ae42d9..b7cc465ab 100644 --- a/examples/nginx/nginx.sample.conf +++ b/examples/nginx/nginx.sample.conf @@ -1,12 +1,17 @@ # An nginx configuration that can be used to reverse proxy connections to your # Platypush' HTTP service. -server { - server_name my-platypush-host.domain.com; +upstream platypush { + # The address and port where the HTTP backend is listening + server 127.0.0.1:8008; +} - # Proxy standard HTTP connections to your Platypush IP +server { + server_name platypush.example.com; + + # Proxy standard HTTP connections location / { - proxy_pass http://my-platypush-host:8008/; + proxy_pass http://platypush; client_max_body_size 5M; proxy_read_timeout 60; @@ -18,21 +23,33 @@ server { } # Proxy websocket connections - location ~ ^/ws/(.*)$ { - proxy_pass http://10.0.0.2:8008/ws/$1; + location /ws/ { + proxy_pass http://platypush; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_redirect off; proxy_http_version 1.1; - client_max_body_size 200M; + client_max_body_size 5M; proxy_set_header Host $http_host; } # Optional SSL configuration - using Let's Encrypt certificates in this case # listen 443 ssl; - # ssl_certificate /etc/letsencrypt/live/my-platypush-host.domain.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/my-platypush-host.domain.com/privkey.pem; + # ssl_certificate /etc/letsencrypt/live/platypush.example.com/fullchain.pem; + # ssl_certificate_key /etc/letsencrypt/live/platypush.example.com/privkey.pem; # include /etc/letsencrypt/options-ssl-nginx.conf; # ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; } + +# Uncomment if you are using SSL and you want to force an HTTPS upgrade to +# clients connecting over the port 80 +# server { +# if ($host = platypush.example.com) { +# return 301 https://$host$request_uri; +# } +# +# server_name platypush.example.com; +# listen 80; +# return 404; +# } diff --git a/generate_missing_docs.py b/generate_missing_docs.py index d34eec772..3d9f82851 100644 --- a/generate_missing_docs.py +++ b/generate_missing_docs.py @@ -4,7 +4,7 @@ from typing import Iterable, Optional from platypush.backend import Backend from platypush.context import get_plugin from platypush.plugins import Plugin -from platypush.utils.manifest import get_manifests +from platypush.utils.manifest import Manifests def _get_inspect_plugin(): @@ -14,11 +14,11 @@ def _get_inspect_plugin(): def get_all_plugins(): - return sorted([mf.component_name for mf in get_manifests(Plugin)]) + return sorted([mf.component_name for mf in Manifests.by_base_class(Plugin)]) def get_all_backends(): - return sorted([mf.component_name for mf in get_manifests(Backend)]) + return sorted([mf.component_name for mf in Manifests.by_base_class(Backend)]) def get_all_events(): diff --git a/platypush/app/__main__.py b/platypush/app/__main__.py index ed2e42208..54dea00ef 100644 --- a/platypush/app/__main__.py +++ b/platypush/app/__main__.py @@ -2,6 +2,4 @@ import sys from ._app import main - -if __name__ == '__main__': - sys.exit(main(*sys.argv[1:])) +sys.exit(main(*sys.argv[1:])) diff --git a/platypush/app/_app.py b/platypush/app/_app.py index e95d0db3f..c11e1af0e 100644 --- a/platypush/app/_app.py +++ b/platypush/app/_app.py @@ -42,6 +42,7 @@ class Application: config_file: Optional[str] = None, workdir: Optional[str] = None, logsdir: Optional[str] = None, + device_id: Optional[str] = None, pidfile: Optional[str] = None, requests_to_process: Optional[int] = None, no_capture_stdout: bool = False, @@ -61,6 +62,10 @@ class Application: ``filename`` setting under the ``logging`` section of the configuration file is used. If not set, logging will be sent to stdout and stderr. + :param device_id: Override the device ID used to identify this + instance. If not passed here, it is inferred from the configuration + (device_id field). If not present there either, it is inferred from + the hostname. :param pidfile: File where platypush will store its PID upon launch, useful if you're planning to integrate the application within a service or a launcher script (default: None). @@ -97,11 +102,14 @@ class Application: os.path.abspath(os.path.expanduser(logsdir)) if logsdir else None ) - Config.init(self.config_file) - Config.set('ctrl_sock', ctrl_sock) - - if workdir: - Config.set('workdir', os.path.abspath(os.path.expanduser(workdir))) + Config.init( + self.config_file, + device_id=device_id, + workdir=os.path.abspath(os.path.expanduser(workdir)) if workdir else None, + ctrl_sock=os.path.abspath(os.path.expanduser(ctrl_sock)) + if ctrl_sock + else None, + ) self.no_capture_stdout = no_capture_stdout self.no_capture_stderr = no_capture_stderr @@ -199,6 +207,7 @@ class Application: config_file=opts.config, workdir=opts.workdir, logsdir=opts.logsdir, + device_id=opts.device_id, pidfile=opts.pidfile, no_capture_stdout=opts.no_capture_stdout, no_capture_stderr=opts.no_capture_stderr, diff --git a/platypush/backend/assistant/google/manifest.yaml b/platypush/backend/assistant/google/manifest.yaml index c97ca42ef..6de6807f1 100644 --- a/platypush/backend/assistant/google/manifest.yaml +++ b/platypush/backend/assistant/google/manifest.yaml @@ -10,7 +10,7 @@ manifest: times out platypush.message.event.assistant.MicMutedEvent: when the microphone is muted. platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted. - platypush.message.event.assistant.NoResponse: when a conversation returned no + platypush.message.event.assistant.NoResponseEvent: when a conversation returned no response platypush.message.event.assistant.ResponseEvent: when the assistant is speaking a response diff --git a/platypush/backend/http/__init__.py b/platypush/backend/http/__init__.py index 6bda90657..a68226275 100644 --- a/platypush/backend/http/__init__.py +++ b/platypush/backend/http/__init__.py @@ -192,7 +192,7 @@ class HttpBackend(Backend): """ - _DEFAULT_HTTP_PORT = 8008 + DEFAULT_HTTP_PORT = 8008 """The default listen port for the webserver.""" _STOP_TIMEOUT = 5 @@ -200,7 +200,7 @@ class HttpBackend(Backend): def __init__( self, - port: int = _DEFAULT_HTTP_PORT, + port: int = DEFAULT_HTTP_PORT, bind_address: str = '0.0.0.0', resource_dirs: Optional[Mapping[str, str]] = None, secret_key_file: Optional[str] = None, diff --git a/platypush/backend/http/app/utils/routes.py b/platypush/backend/http/app/utils/routes.py index f7de249da..68a3fd9e1 100644 --- a/platypush/backend/http/app/utils/routes.py +++ b/platypush/backend/http/app/utils/routes.py @@ -14,7 +14,7 @@ def get_http_port(): from platypush.backend.http import HttpBackend http_conf = Config.get('backend.http') or {} - return http_conf.get('port', HttpBackend._DEFAULT_HTTP_PORT) + return http_conf.get('port', HttpBackend.DEFAULT_HTTP_PORT) def get_routes(): diff --git a/platypush/backend/music/spotify/manifest.yaml b/platypush/backend/music/spotify/manifest.yaml index 8c165cec2..83e16714f 100644 --- a/platypush/backend/music/spotify/manifest.yaml +++ b/platypush/backend/music/spotify/manifest.yaml @@ -18,7 +18,7 @@ manifest: pacman: - sudo - cargo - exec: + after: - sudo cargo install librespot package: platypush.backend.music.spotify type: backend diff --git a/platypush/backend/pushbullet/manifest.yaml b/platypush/backend/pushbullet/manifest.yaml index 27f5a0f25..e49e855aa 100644 --- a/platypush/backend/pushbullet/manifest.yaml +++ b/platypush/backend/pushbullet/manifest.yaml @@ -1,6 +1,12 @@ manifest: events: platypush.message.event.pushbullet.PushbulletEvent: if a new push is received + apk: + - git + apt: + - git + pacman: + - git install: pip: - git+https://github.com/rbrcsk/pushbullet.py diff --git a/platypush/builder/__init__.py b/platypush/builder/__init__.py new file mode 100644 index 000000000..6be2232a0 --- /dev/null +++ b/platypush/builder/__init__.py @@ -0,0 +1,3 @@ +from ._base import BaseBuilder + +__all__ = ["BaseBuilder"] diff --git a/platypush/builder/_base.py b/platypush/builder/_base.py new file mode 100644 index 000000000..73cf69c8a --- /dev/null +++ b/platypush/builder/_base.py @@ -0,0 +1,198 @@ +from abc import ABC, abstractmethod +import argparse +import inspect +import logging +import os +import pathlib +import sys +from typing import Final, Optional, Sequence + +from platypush.config import Config +from platypush.utils.manifest import ( + Dependencies, + InstallContext, +) + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger() + + +class BaseBuilder(ABC): + """ + Base interface and utility methods for Platypush builders. + + A Platypush builder is a script/piece of logic that can build a Platypush + installation, with all its base and required extra dependencies, given a + configuration file. + + This class is currently implemented by the :module:`platypush.platyvenv` + and :module:`platypush.platydock` modules/scripts. + """ + + REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git' + """ + We use the Github URL here rather than the self-hosted Gitea URL to prevent + too many requests to the Gitea server. + """ + + def __init__( + self, + cfgfile: str, + gitref: str, + output: str, + install_context: InstallContext, + *_, + verbose: bool = False, + device_id: Optional[str] = None, + **__, + ) -> None: + """ + :param cfgfile: The path to the configuration file. + :param gitref: The git reference to use. It can be a branch name, a tag + name or a commit hash. + :param output: The path to the output file or directory. + :param install_context: The installation context for this builder. + :param verbose: Whether to log debug traces. + :param device_id: A device name that will be used to uniquely identify + this installation. + """ + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.output = os.path.abspath(os.path.expanduser(output)) + self.gitref = gitref + self.install_context = install_context + self.device_id = device_id + logger.setLevel(logging.DEBUG if verbose else logging.INFO) + + @classmethod + @abstractmethod + def get_name(cls) -> str: + """ + :return: The name of the builder. + """ + + @classmethod + @abstractmethod + def get_description(cls) -> str: + """ + :return: The description of the builder. + """ + + @property + def deps(self) -> Dependencies: + """ + :return: The dependencies for this builder, given the configuration + file and the installation context. + """ + return Dependencies.from_config( + self.cfgfile, + install_context=self.install_context, + ) + + def _print_instructions(self, s: str) -> None: + GREEN = '\033[92m' + NORM = '\033[0m' + + helper_lines = s.split('\n') + wrapper_line = '=' * max(len(t) for t in helper_lines) + helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n' + print(GREEN + helper + NORM) + + @abstractmethod + def build(self): + """ + Builds the application. To be implemented by the subclasses. + """ + + @classmethod + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=cls.get_name(), + add_help=False, + description=cls.get_description(), + ) + + parser.add_argument( + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + + parser.add_argument( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='Enable debug traces', + ) + + parser.add_argument( + '-c', + '--config', + type=str, + dest='cfgfile', + required=False, + default=None, + help='The path to the configuration file. If not specified, a minimal ' + 'installation including only the base dependencies will be generated.', + ) + + parser.add_argument( + '-o', + '--output', + dest='output', + type=str, + required=False, + default='.', + help='Target directory (default: current directory). For Platydock, ' + 'this is the directory where the Dockerfile will be generated. For ' + 'Platyvenv, this is the base directory of the new virtual ' + 'environment.', + ) + + parser.add_argument( + '-d', + '--device-id', + dest='device_id', + type=str, + required=False, + default=None, + help='A name that will be used to uniquely identify this device. ' + 'Default: a random name for Docker containers, and the ' + 'hostname of the machine for virtual environments.', + ) + + parser.add_argument( + '-r', + '--ref', + dest='gitref', + required=False, + type=str, + default='master', + help='If the script is not run from a Platypush installation directory, ' + 'it will clone the sources via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + + return parser + + @classmethod + def from_cmdline(cls, args: Sequence[str]): + """ + Create a builder instance from command line arguments. + + :param args: Command line arguments. + :return: A builder instance. + """ + parser = cls._get_arg_parser() + opts, _ = parser.parse_known_args(args) + if opts.show_usage: + parser.print_help() + sys.exit(0) + + if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.yaml', + ) + + logger.info('No configuration file specified. Using %s.', opts.cfgfile) + + return cls(**vars(opts)) diff --git a/platypush/cli.py b/platypush/cli.py index 9864c16be..b9bec5564 100644 --- a/platypush/cli.py +++ b/platypush/cli.py @@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace: """ Parse command-line arguments from a list of strings. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + 'platypush', + description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.', + ) parser.add_argument( '--config', @@ -29,6 +32,17 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace: help='Custom working directory to be used for the application', ) + parser.add_argument( + '--device-id', + '-d', + dest='device_id', + required=False, + default=None, + help='Override the device ID used to identify this instance. If not ' + 'passed here, it is inferred from the configuration (device_id field).' + 'If not present there either, it is inferred from the hostname.', + ) + parser.add_argument( '--logsdir', '-l', diff --git a/platypush/commands/_stream.py b/platypush/commands/_stream.py index 3ae185d3e..b4b120142 100644 --- a/platypush/commands/_stream.py +++ b/platypush/commands/_stream.py @@ -1,4 +1,4 @@ -from multiprocessing import Queue +from multiprocessing import RLock, Queue import os from queue import Empty import socket @@ -35,6 +35,7 @@ class CommandStream(ControllableProcess): self.path = os.path.abspath(os.path.expanduser(path or self._default_sock_path)) self._sock: Optional[socket.socket] = None self._cmd_queue: Queue["Command"] = Queue() + self._close_lock = RLock() def reset(self): if self._sock is not None: @@ -68,9 +69,18 @@ class CommandStream(ControllableProcess): return self def __exit__(self, *_, **__): - self.terminate() - self.join() - self.close() + with self._close_lock: + self.terminate() + + try: + self.close() + except Exception as e: + self.logger.warning(str(e)) + + try: + self.kill() + except Exception as e: + self.logger.warning(str(e)) def _serve(self, sock: socket.socket): """ diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 4e6e1ee53..fce5fe92c 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -20,6 +20,7 @@ from platypush.utils import ( is_functional_procedure, is_functional_hook, is_functional_cron, + is_root, ) @@ -65,13 +66,14 @@ class Config: _included_files: Set[str] = set() - def __init__(self, cfgfile: Optional[str] = None): + def __init__(self, cfgfile: Optional[str] = None, workdir: Optional[str] = None): """ Constructor. Always use the class as a singleton (i.e. through Config.init), you won't probably need to call the constructor directly :param cfgfile: Config file path (default: retrieve the first available location in _cfgfile_locations). + :param workdir: Overrides the default working directory. """ self.backends = {} @@ -83,13 +85,13 @@ class Config: self.dashboards = {} self._plugin_manifests = {} self._backend_manifests = {} - self._cfgfile = '' + self.config_file = '' self._init_cfgfile(cfgfile) - self._config = self._read_config_file(self._cfgfile) + self._config = self._read_config_file(self.config_file) self._init_secrets() - self._init_dirs() + self._init_dirs(workdir=workdir) self._init_db() self._init_logging() self._init_device_id() @@ -104,10 +106,10 @@ class Config: if cfgfile is None: cfgfile = self._get_default_cfgfile() - if cfgfile is None: + if cfgfile is None or not os.path.exists(cfgfile): cfgfile = self._create_default_config() - self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.config_file = os.path.abspath(os.path.expanduser(cfgfile)) def _init_logging(self): logging_config = { @@ -163,21 +165,26 @@ class Config: for k, v in self._config['environment'].items(): os.environ[k] = str(v) - def _init_dirs(self): - if 'workdir' not in self._config: + def _init_dirs(self, workdir: Optional[str] = None): + if workdir: + self._config['workdir'] = workdir + if not self._config.get('workdir'): self._config['workdir'] = self._workdir_location - self._config['workdir'] = os.path.expanduser(self._config['workdir']) - os.makedirs(self._config['workdir'], exist_ok=True) + + self._config['workdir'] = os.path.expanduser( + os.path.expanduser(self._config['workdir']) + ) + pathlib.Path(self._config['workdir']).mkdir(parents=True, exist_ok=True) if 'scripts_dir' not in self._config: self._config['scripts_dir'] = os.path.join( - os.path.dirname(self._cfgfile), 'scripts' + os.path.dirname(self.config_file), 'scripts' ) os.makedirs(self._config['scripts_dir'], mode=0o755, exist_ok=True) if 'dashboards_dir' not in self._config: self._config['dashboards_dir'] = os.path.join( - os.path.dirname(self._cfgfile), 'dashboards' + os.path.dirname(self.config_file), 'dashboards' ) os.makedirs(self._config['dashboards_dir'], mode=0o755, exist_ok=True) @@ -206,17 +213,29 @@ class Config: def _create_default_config(self): cfg_mod_dir = os.path.dirname(os.path.abspath(__file__)) - cfgfile = self._cfgfile_locations[0] + # Use /etc/platypush/config.yaml if the user is running as root, + # otherwise ~/.config/platypush/config.yaml + cfgfile = ( + ( + os.path.join(os.environ['XDG_CONFIG_HOME'], 'config.yaml') + if os.environ.get('XDG_CONFIG_HOME') + else os.path.join( + os.path.expanduser('~'), '.config', 'platypush', 'config.yaml' + ) + ) + if not is_root() + else os.path.join(os.sep, 'etc', 'platypush', 'config.yaml') + ) + cfgdir = pathlib.Path(cfgfile).parent cfgdir.mkdir(parents=True, exist_ok=True) - for cfgfile in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')): - shutil.copy(cfgfile, str(cfgdir)) + for cf in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')): + shutil.copy(cf, str(cfgdir)) return cfgfile def _read_config_file(self, cfgfile): cfgfile_dir = os.path.dirname(os.path.abspath(os.path.expanduser(cfgfile))) - config = {} try: @@ -397,14 +416,17 @@ class Config: @classmethod def _get_instance( - cls, cfgfile: Optional[str] = None, force_reload: bool = False + cls, + cfgfile: Optional[str] = None, + workdir: Optional[str] = None, + force_reload: bool = False, ) -> "Config": """ Lazy getter/setter for the default configuration instance. """ if force_reload or cls._instance is None: cfg_args = [cfgfile] if cfgfile else [] - cls._instance = Config(*cfg_args) + cls._instance = Config(*cfg_args, workdir=workdir) return cls._instance @classmethod @@ -463,17 +485,34 @@ class Config: return None @classmethod - def init(cls, cfgfile: Optional[str] = None): + def init( + cls, + cfgfile: Optional[str] = None, + device_id: Optional[str] = None, + workdir: Optional[str] = None, + ctrl_sock: Optional[str] = None, + **_, + ): """ Initializes the config object singleton - Params: - cfgfile -- path to the config file - default: _cfgfile_locations + + :param cfgfile: Path to the config file (default: _cfgfile_locations) + :param device_id: Override the configured device_id. + :param workdir: Override the configured working directory. + :param ctrl_sock: Override the configured control socket. """ - return cls._get_instance(cfgfile, force_reload=True) + cfg = cls._get_instance(cfgfile, workdir=workdir, force_reload=True) + if device_id: + cfg.set('device_id', device_id) + if workdir: + cfg.set('workdir', workdir) + if ctrl_sock: + cfg.set('ctrl_sock', ctrl_sock) + + return cfg @classmethod - @property - def workdir(cls) -> str: + def get_workdir(cls) -> str: """ :return: The path of the configured working directory. """ @@ -505,5 +544,12 @@ class Config: # pylint: disable=protected-access cls._get_instance()._config[key] = value + @classmethod + def get_file(cls) -> str: + """ + :return: The main configuration file path. + """ + return cls._get_instance().config_file + # vim:sw=4:ts=4:et: diff --git a/platypush/config/config.auto.yaml b/platypush/config/config.auto.yaml deleted file mode 100644 index e9bbcd0ce..000000000 --- a/platypush/config/config.auto.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-generated configuration file. -# Do not edit manually - use the config.yaml file for manual modifications -# instead - -backend.http: - enabled: True diff --git a/platypush/config/config.yaml b/platypush/config/config.yaml index 1e36005fe..7f31c3cd3 100644 --- a/platypush/config/config.yaml +++ b/platypush/config/config.yaml @@ -1,2 +1,920 @@ -include: - - config.auto.yaml +################################################################################ +# Sample Platypush configuration file. +# +# Edit it and: +# - Copy it to /etc/platypush/config.yaml for system installation. +# - Copy it to ~/.config/platypush/config.yaml for user installation. +# - Start the application with `-c `. +# +# Since the configuration file also includes the custom integrations, you can +# create a Platypush custom installation, with all the extra dependencies +# required by the configured integrations, using the `platydock` or `platyvenv` +# commands and passing this file as an argument. These commands will build a +# Docker image or a Python virtual environment respectively, with all the +# required extra dependencies inferred from your configuration file. +# +# A `scripts` directory with an empty `__init__.py` script will also be created +# under the same directory as the configuration file. This directory can be +# used to add custom scripts containing procedures, hooks and crons if you want +# a full Python interface to define your logic rather than a YAML file. +# +# Please refer to the `scripts` directory provided under this file's directory +# for some examples that use the Python API. +################################################################################ + +### ------------------ +### Include directives +### ------------------ + +# # You can split your configuration over multiple files and use the include +# # directive to import other files into your configuration. +# +# # Files referenced via relative paths will be searched in the directory of +# # the configuration file that references them. Symbolic links are also +# # supported. +# +# include: +# - logging.yaml +# - media.yaml +# - sensors.yaml + +### ----------------- +### Working directory +### ----------------- + +# # Working directory of the application. This is where the main database will be +# # stored by default (if the default SQLite configuration is used), and it's +# # where the integrations will store their state. +# +# # Note that the working directory can also be specified at runtime using the +# # -w/--workdir option. +# # +# # If not specified, then one of the following will be used: +# # +# # - $XDG_DATA_HOME/platypush if the XDG_DATA_HOME environment variable is set. +# # - $HOME/.local/share/platypush otherwise. +# +# workdir: ~/.local/share/platypush + +### ---------------------- +### Database configuration +### ---------------------- + +# # By default Platypush will use a SQLite database named `main.db` under the +# # `workdir`. You can specify any other engine string here - the application has +# # been tested against SQLite, Postgres and MariaDB/MySQL >= 8. +# # +# # NOTE: If you want to use a DBMS other than SQLite, then you will also need to +# # ensure that a compatible Python driver is installed on the system where +# # Platypush is running. For example, Postgres will require the Python pg8000, +# # psycopg or another compatible driver. +# +# main.db: +# engine: sqlite:///home/user/.local/share/platypush/main.db +# # OR, if you want to use e.g. Postgres with the pg8000 driver: +# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname + +### --------------------- +### Logging configuration +### --------------------- + +# # Platypush logs on stdout by default. You can use the logging section to +# # specify an alternative file or change the logging level. +# +# # Note that a custom logging directory can also be specified at runtime using +# # the -l/--logsdir option. +# +# logging: +# filename: ~/.local/log/platypush/platypush.log +# level: INFO + +### ----------------------- +### device_id configuration +### ----------------------- + +# # The device_id is used by many components of Platypush and it should uniquely +# # identify a device in your network. If nothing is specified then the hostname +# # will be used. +# +# # Note that a custom device ID can also be specified at runtime using the +# # -d/--device-id option. +# +# device_id: my_device + +### ------------------- +### Redis configuration +### ------------------- + +# # Platypush needs a Redis instance for inter-process communication. +# # +# # By default, the application will try and connect to a Redis server listening +# # on localhost:6379. +# # +# # Platypush can also start the service on the fly if instructed to do so +# # through the `--start-redis` option. You can also specify a custom port +# # through the `--redis-port` option. +# # +# # If you are running Platypush in a Docker image built through Platydock, then +# # `--start-redis` is the default behaviour and you won't need any extra +# # documentation here. +# +# redis: +# host: localhost +# port: 6379 +# username: user +# password: secret + +### ------------------------ +### Web server configuration +### ------------------------ + +# Platypush comes with a versatile Web server that is used to: +# +# - Serve the main UI and the UIs for the plugins that provide one. +# - Serve custom user-configured dashboards. +# - Expose the `/execute` RPC endpoint to send synchronous requests. +# - Expose the `/ws/events` and `/ws/requests` Websocket paths, which can be +# respectively by other clients to subscribe to the application's events or +# send asynchronous requests. +# - Stream media files provided by other plugins, as well as camera and audio +# feeds. +# - Serve custom directories of static files that can be accessed by other +# clients. +# - Provide a versatile API for hooks - the user can easily create custom HTTP +# hooks by creating a hook with their custom logic that reacts when a +# `platypush.message.event.http.hook.WebhookEvent` is received. The `hook` +# parameter of the event specifies which URL will be served by the hook. +# +# The Web server is enabled by default, but you can disable it simply by +# commenting/removing the `backend.http` section. The default listen port is +# 8008. +# +# After starting the application, you can access the UI at +# http://localhost:8008, set up your username and password, and also create an +# access or session token from the configuration panel. +# +# This token can be used to authenticate calls to the available APIs. +# For example, to turn on the lights using the /execute endpoint: +# +# curl -XPOST -H "Authorization: Bearer $TOKEN" \ +# -H "Content-Type: application/json" \ +# -d ' +# { +# "type": "request", +# "action": "light.hue.on", +# "args": { +# "lights": ["Bedroom"] +# } +# }' http://localhost:8008/execute +# +# If you want to serve the Web server behind a reverse proxy, you can copy the +# reference configuration available at +# https://git.platypush.tech/platypush/platypush/src/branch/master/examples/nginx/nginx.sample.conf + +backend.http: + # # Bind address (default: 0.0.0.0) + # bind_address: 0.0.0.0 + # # Listen port (default: 8008) + port: 8008 + + # # resource_dirs can be used to specify directories on the host system that + # # you want to expose through the Web server. For example, you may want to + # # expose directories that contain photos or images if you want to make a + # # carousel dashboard, or a directory containing some files that you want to + # # share with someone (or other systems) using a simple Web server. + # # + # # In the following example, we're exposing a directory with photos on an + # # external hard drive other the `/photos` URL. An image like e.g. + # # `/mnt/hd/photos/IMG_1234.jpg` will be served over e.g. + # # `http://localhost:8008/photos/IMG_1234.jpg` in this case. + # resource_dirs: + # photos: /mnt/hd/photos + + # # Number of WSGI workers. Default: (#cpus * 2) + 1 + # num_workers: 4 + +### ----------------------------- +### Plugin configuration examples +### ----------------------------- + +### +# # The configuration of a plugin matches one-to-one the parameters required by +# # its constructor. +# # +# # Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html +# # +# # For example, there is a `light.hue` plugin +# # (https://docs.platypush.tech/platypush/plugins/light.hue.html) whose +# # constructor takes the following parameters: `bridge`, `lights` (default +# # target lights for the commands), `groups` (default target groups for the +# # commands) and `poll_interval` (how often the plugin should poll for updates). +# # +# # This means that the `light.hue` plugin can be configured here as: +# +# light.hue: +# # IP address or hostname of the Hue bridge +# # NOTE: The first run will require you to register the application with +# # your bridge - that's usually done by pressing a physical button on your +# # bridge while the application is pairing. +# bridge: 192.168.1.3 +# # Groups that will be handled by default if nothing is specified on the request +# groups: +# - Living Room +# +# # How often we should poll for updates (default: 20 seconds) +# poll_interval: 20 +### + +### +# # Example configuration of music.mpd plugin, a plugin to interact with MPD and +# # Mopidy music server instances. See +# # https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html +# # You can easily install the dependencies through pip install 'platypush[mpd]' +# +# music.mpd: +# host: localhost +# port: 6600 +### + +### +# # Plugins with empty configuration can also be explicitly enabled by specifying +# # `enabled: true` or `disabled: false`. An integration with no items will be +# # enabled with no configuration. +# +# clipboard: +### + +### +# # Example configuration of the MQTT plugin. This specifies a server that the +# # application will use by default (if not specified on the request body). +# +# mqtt: +# host: 192.168.1.2 +# port: 1883 +# username: user +# password: secret +### + +### +# # Enable the system plugin if you want your device to periodically report +# # system statistics (CPU load, disk usage, memory usage etc.) +# # +# # When new data is gathered, an `EntityUpdateEvent` with `plugin='system'` will +# # be triggered with the new data, and you can subscribe a hook to these events +# # to run your custom logic. +# +# system: +# # How often we should poll for new data +# poll_interval: 60 +### + +### +# # Example configuration for the calendar plugin. In this case, we have +# # registered a Google calendar that uses the `google.calendar` integration, and +# # a Facebook plugin and a NextCloud (WebDAV) plugin exposed over iCal format. +# # Installing the dependencies: pip install 'platypush[ical,google]' +# calendar: +# calendars: +# - type: platypush.plugins.google.calendar.GoogleCalendarPlugin +# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin +# url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key +# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin +# url: http://riemann/nextcloud/remote.php/dav/public-calendars/9JBWHR7iioM88Y4D?export +### + +### +# # Torrent plugin configuration, with the default directory that should be used +# # to store downloaded files. +# +# torrent: +# download_dir: ~/Downloads +### + +### +# # List of RSS/Atom subscriptions. These feeds will be monitored for changes and +# # a `platypush.message.event.rss.NewFeedEntryEvent` +# # (https://docs.platypush.tech/platypush/events/rss.html#platypush.message.event.rss.NewFeedEntryEvent) +# # will be triggered when one of these feeds has new entries - you can subscribe +# # the event to run your custom logic. +# +# rss: +# # How often we should check for updates (default: 5 minutes) +# poll_seconds: 300 +# # List of feeds to monitor +# subscriptions: +# - https://www.theguardian.com/rss/world +# - https://phys.org/rss-feed/ +# - https://news.ycombinator.com/rss +# - https://www.technologyreview.com/stories.rss +# - https://api.quantamagazine.org/feed/ +### + +### +# # Example configuration of a weather plugin +# +# weather.openweathermap: +# token: secret +# lat: lat +# long: long +### + +### +# # You can add IFTTT integrations to your routines quite easily. +# # +# # Register an API key for IFTTT, paste it here, and you can run an +# # `ifttt.trigger_event` action to fire an event on IFTTT. +# # +# # You can also create IFTTT routines that call your Platypush instance, by +# # using Web hooks (i.e. event hooks that subscribe to +# # `platypush.message.event.http.hook.WebhookEvent` events), provided that the +# # Web server is listening on a publicly accessible address. +# +# ifttt: +# ifttt_key: SECRET +### + +### +# # The `http.webpage` integration comes with the mercury-parser JavaScript library. +# # It allows you to "distill" the content of a Web page and export it in readable format (in simplified HTML, Markdown or PDF) through the +# +# http.webpage: +### + +### +# # Example configuration of the zigbee.mqtt integration. +# # This integration listens for the events pushed by zigbee2mqtt service to an +# # MQTT broker. It can forward those events to native Platypush events (see +# # https://docs.platypush.tech/platypush/events/zigbee.mqtt.html) that you can +# # build automation routines on. You can also use Platypush to control your +# # Zigbee devices, either through the Web interface or programmatically through +# # the available plugin actions. +# +# zigbee.mqtt: +# # Host of the MQTT broker +# host: riemann +# # Listen port of the MQTT broker +# port: 1883 +# # Base topic, as specified in `/data/configuration.yaml` +# base_topic: zigbee2mqtt +### + +### +# # Example configuration of the zwave.mqtt integration. +# # This integration listens for the events pushed by ZWaveJS service to an MQTT +# # broker. It can forward those events to native Platypush events (see +# # https://docs.platypush.tech/platypush/events/zwave.html) that you can build +# # automation routines on. +# # You can also use Platypush to control your Z-Wave devices, either through the +# # Web interface or programmatically through the available plugin actions. +# +# zwave.mqtt: +# # Host of the MQTT broker +# host: riemann +# # Listen port of the MQTT broker +# port: 1883 +# # Gateway name, usually configured in the ZWaveJS-UI through `Settings -> +# # MQTT -> Name` +# name: zwavejs2mqtt +# # The prefix of the published topics, usually configured in the ZWaveJS-UI +# # through `Settings -> MQTT -> Prefix`. +# topic_prefix: zwave +### + +### -------------------- +### Camera configuration +### -------------------- + +### +# # There are several providers for the camera integration - you can choose +# # between ffmpeg, gstreamer, PiCamera etc., and they all expose the same +# # interface/configuration options. +# # +# # It is advised to use the ffmpeg integration, as it's the one that provides +# # the highest degree of features and supported hardware. +# # +# # If the plugin is correctly configured, you can access your camera feed from +# # the Platypush Web panel, programmatically start/stop recording sessions, take +# # photos, get a feed stream URL etc. +# +# # The camera feed will be available at `/camera//video[.extension]`, +# # for example `/camera/ffmpeg/video.mjpeg` for MJPEG (usually faster), or +# # `camera/ffmpeg/video.mp4` for MP4. +# +# # You can also capture images by connecting to the +# # `/camera//photo[.extension]`, for example `/camera/ffmpeg/photo.jpg`. +# +# camera.ffmpeg: +# # Default video device to use +# device: /dev/video0 +# # Default resolution +# resolution: +# - 640 +# - 480 +# # The directory that will be used to store captured frames/images +# frames_dir: ~/Camera/Photos +# # Default image scaling factors (default: 1, no scaling) +# scale_x: 1.5 +# scale_y: 1.5 +# # Default rotation of the image, in degrees (default: 0, no rotation) +# rotate: 90 +# # Grayscale mode (default: False): +# grayscale: false +# # Default frames per second (default: 16) +# fps: 16 +# # Whether to flip the image along the horizontal axis (default: False) +# horizontal_flip: false +# # Whether to flip the image along the horizontal axis (default: False) +# vertical_flip: false +### + +### ----------------- +### Sound integration +### ----------------- + +### +# # The sound plugin allows you to stream from an audio source connected to the +# # machine, play audio files or synthetic audio waves or MIDI sounds. +# +# # After enabling the plugin, you can access the audio stream at +# # `/sound/stream[.extension]` (e.g. `/sound/stream.mp3`) if you want to get a +# # live recording of the captured sound from the configured audio +# # `input_device`. +# +# sound: +# enabled: true +### + +### ----------------------------------- +### Some examples of media integrations +### ----------------------------------- + +### +# # Example configuration for the media.vlc plugin. You can replace `vlc` with +# # `mpv`, `mplayer`, `omxplayer` or `gstreamer` if you want to use another +# # player - the supported configuration option are the same across all these +# # players. +# +# media.vlc: +# # Volume level, between 0 and 100 +# volume: 50 +# # Where to store downloaded files +# download_dir: ~/Downloads +# # Play videos in fullscreen by default +# fullscreen: True +# # If youtube-dl or any compatible application is installed, play requested +# # videos in this format by default. Default: `best`. +# youtube_format: 'mp4[height<=?480]' +# # Extra arguments to pass to the executable. --play-and-exit may be a good +# # idea with VLC, so the player terminates upon stop instead of lingering in +# # the background. +# args: +# - --play-and-exit +# # List of directories to search for media files. The media files in these +# # folders can be searched through the `media..search` command, or +# # through the Web interface. +# media_dirs: +# - /mnt/hd/media/movies +# - /mnt/hd/media/series +# - /mnt/hd/media/videos +# - ~/Downloads +### + +### +# # Example configuration for the media.chromecast plugin, see +# # https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html +# # You can easily install the dependencies through pip install 'platypush[chromecast]' +# +# media.chromecast: +# chromecast: Living Room TV +### + +### +# # Example Kodi configuration. This makes it possible to control and query a +# # Kodi instance, from your automation hooks, from the Platypush APIs or from +# # the Platypush Web interface. It requires you to enable the JSON API service +# # from Kodi's settings. +# +# media.kodi: +# host: localhost +# http_port: 8080 +# username: kodi +# password: secret +### + +### +# # Example configuration for a Plex media server. This integration makes it +# # possible to navigate and search media items from your Plex library in the +# # media UI. +# +# media.plex: +# server: localhost +# username: plex +# password: secret +### + +### +# # Jellyfin media server configuration. +# +# media.jellyfin: +# server: https://media.example.com +# api_key: secret +### + +### --------------------- +### Sensors configuration +### --------------------- + +### +# # The serial plugin can be used to read sensor data from a device connected +# # over serial/USB interface. +# # +# # It can be used, for example, to connect to an Arduino or ESP device over +# # serial port, where the remote microcontroller periodically sends sensor data +# # over the serial interface. +# # +# # The data can be sent on the wire either as raw string-encoded numbers (one +# # per line), or (better) in JSON format. For example, you can program your +# # microcontroller to periodically send JSON strings like these when you get new +# # readings from your sensors: +# # +# # {"temperature": 25.0, "humidity": 20.0, "smoke": 0.01, "luminosity": 45} +# # +# # The JSON will be automatically unpacked by the application, and the relevant +# # `platypush.message.event.sensor.SensorDataChangeEvent` events will be +# # triggered when the data changes - you can subscribe to them in your custom +# # hooks. +# +# serial: +# # The path to the USB interface with e.g. an Arduino or ESP microcontroller +# # connected. +# # A way to get a deterministic path name on Linux, instead of +# # `/dev/ttyUSB`, can be the following: +# # +# # - Get the vendor and product ID of your device via e.g. `lsusb`. For +# # example, for an Arduino-compatible microcontroller: +# # +# # Bus 001 Device 008: ID 1a86:7523 QinHeng Electronics CH340 serial converter +# # +# # - In the case above, `1a86` is the vendor ID and `7523` is the product +# # ID. Create a new udev rule for it, so every time the device is +# # connected it will also be symlinked to e.g. `/dev/arduino`: +# # +# # echo 'SUBSYSTEM=="tty", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", SYMLINK+="arduino"' | \ +# # sudo tee /etc/udev/rules.d/98-usb-serial.rules +# device: /dev/ttyUSB0 +# # How often the interface should be polled for updates, in seconds +# poll_interval: 1 +# # The tolerance argument can be used to tune when you want to be notified +# # of data changes through `SensorDataChangeEvent` events. In the case +# # below, if the microcontroller sends two consecutive temperature reads, +# # one for 22.0 and one for 22.2, then only one `SensorDataChangeEvent` will +# # be triggered (for the first read), since the absolute value of the +# # difference between the two events is less than the configured tolerance. +# # However, if the device sends two temperature reads, one for 22.0 and one +# # for 22.7, then two `SensorDataChangeEvent` events will be triggered. +# # The tolerance for all the metrics is set to a value close to zero by +# # default - i.e. any read, unless it's exactly the same as the previous +# # one, will trigger a new event. +# tolerance: +# temperature: 0.5 +# humidity: 0.75 +# luminosity: 5 +# +# # If a threshold is defined for a sensor, and the value of that sensor goes +# # below/above that temperature between two reads, then a +# # `SensorDataBelowThresholdEvent` or a `SensorDataAboveThresholdEvent` will +# # be triggered respectively. +# thresholds: +# temperature: 25.0 +### + +### +# # Alternatively to the serial plugin, you can also use the arduino plugin if +# # you want to specifically interface with Arduino. +# # +# # This plugin won't require you to write any logic for your microcontroller. +# # However, it requires your microcontroller to be flash with the Firmata +# # firmware, which allows programmatic external control. +# # +# # Note that the interface of this plugin is basically the same as the serial +# # plugin, and any other plugin that extends `SensorPlugin` in general. +# # Therefore, poll_interval, tolerance and thresholds are supported here too. +# +# arduino: +# board: /dev/ttyUSB0 +# # name -> PIN number mapping (similar for digital_pins). +# # It allows you to pick a common name for your PINs that will be used in +# # the forwarded events. +# analog_pins: +# temperature: 7 +# +# tolerance: +# temperature: 0.5 +# +# thresholds: +# temperature: 25.0 +### + +### +# # Another example: the LTR559 is a common sensor for proximity and luminosity +# # that can be wired to a Raspberry Pi or similar devices over SPI or I2C +# # interface. It exposes the same base interface as all other sensor plugins. +# +# sensor.ltr559: +# poll_interval: 1.0 +# tolerance: +# light: 7.0 +# proximity: 5.0 +# +# thresholds: +# proximity: 10.0 +### + +### -------------------------------- +### Some text-to-speech integrations +### -------------------------------- + +### +# # `tts` is the simplest TTS integration. It leverages the Google Translate open +# # "say" endpoint to render text as audio speech. +# +# tts: +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +# # The default language of the voice +# language: en-gb +### + +### +# # `tts.google` leverages Google's official text-to-speech API to render audio +# # speech from text. +# # +# # Install its dependencies via 'pip install "platypush[google-tts]"'. +# # +# # Like all other Google integrations, it requires you to register an app on the +# # Google developers console, create an API key, and follow the instruction +# # logged on the next restart to give your app the required permissions to your +# # account. +# +# tts.google: +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +# # The default language of the voice +# language: en-US +# # The gender of the voice (MALE or FEMALE) +# gender: FEMALE +# # The path to the JSON file containing your Google API credentials +# credentials_file: '~/.credentials/platypush/google/platypush-tts.json' +### + +### +# # This TTS integration leverages mimic3, an open-source TTS Web server +# # developed by Mycroft (RIP). +# # +# # Follow the instructions at +# # https://docs.platypush.tech/platypush/plugins/tts.mimic3.html to quickly +# # bootstrap a mimic3 server. +# +# tts.mimic3: +# # The base URL of the mimic3 server +# server_url: http://riemann:59125 +# # Path of the default voice that should be used +# voice: 'en_UK/apope_low' +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +### + +## ---------- +## Procedures +## ---------- + +# Procedures are lists of actions that are executed sequentially. +# +# This section shows how to define procedures directly in your YAML +# configuration file(s). However, you can also put your procedures into Python +# scripts inside of the `/scripts` directory if you want access to +# a full-blown Python syntax. They will be automatically discovered at startup +# and available to the application. +# +# You can also access Python variables and evaluate Python expressions by using +# `${}` context expressions. +# +# The `context` special variable is a name->value dictionary containing the +# items returned from previous actions. For example, if an action returned +# `{"status": "ok", "temperature": 21.5}`, then the following actions can access +# those variables through `${context["status"]}` or +# `${context["temperature"]}`, or simply `${status}` and `${temperature}`, +# respectively. +# +# You can also add statements like `- if ${temperature > 20.0}` or +# `- for ${temp in temperature_values}` in your procedures. +# +# Besides the `context` variables, the following special variables are also +# available to the `${}` constructs when running a procedure: +# +# - `output`: It contains the parsed output of the previous action. +# - `errors`: It contains the errors of the previous action +# - `event`: If the procedure is an event hook (or it is executed within an +# event hook), it contains the event that triggered the hook + +### +# # An example procedure that can be called when you arrive home. +# # +# # You can run this procedure from the Platypush `execute` Web panel, or +# # programmatically by sending a JSON request to your Web server (or to the +# # `/ws/requests` Websocket route, or to the TCP backend) +# # +# # curl -XPOST \ +# # -H "Authorization: Bearer $YOUR_TOKEN" \ +# # -d '{"type": "request", "action": "procedure.at_home"}' +# # +# # A use-case can be the one where you have a Tasker automation running on your +# # Android device that detects when your phone enters or exits a certain area, +# # and sends the appropriate request to your Platypush server. +# +# procedure.at_home: +# # Set the db variable AT_HOME to 1. +# # Variables are flexible entities with a name and a value that will be +# # stored on the database and persisted across sessions. +# # You can access them in other procedures, scripts or hooks and run +# # custom logic on the basis of their value. +# - action: variable.set +# args: +# AT_HOME: 1 +# +# # Check the luminosity level from e.g. a connected LTR559 sensor. +# # It could also be a Bluetooth, Zigbee, Z-Wave, serial etc. sensor. +# - action: sensor.ltr559.get_measurement +# +# # If it's below a certain threshold, turn on the lights. +# # In this case, `light` is a parameter returned by the previous response, +# # so we can directly access it here through the `${}` context operator. +# # ${light} in this case is equivalent to ${context["light"]} or +# # ${output["light"]}. +# - if ${int(light or 0) < 110}: +# - action: light.hue.on +# +# # Say a welcome home message +# - action: tts.mimic3.say +# args: +# text: Welcome home +# +# # Start the music +# - action: music.mpd.play +### + +### +# # Procedure that will be execute when you walk outside your home. +# +# procedure.outside_home: +# # Unset the db variable AT_HOME +# - action: variable.unset +# args: +# name: AT_HOME +# +# # Stop the music +# - action: music.mpd.stop +# +# # Turn off the lights +# - action: light.hue.off +### + +### +# # Procedures can also take optional arguments. The example below shows a +# # generic procedure that broadcasts measurements from a sensor through an +# MQTT broker. +# +# # A listener on this topic can react to an `MQTTMessageEvent` and, for +# # example, store the event on a centralized storage. +# # +# # See the event hook section below for a sample hook that listens for messages +# # sent by other clients using this procedure. +# +# procedure.send_sensor_data(name, value): +# - action: mqtt.send_message +# args: +# topic: platypush/sensors +# host: mqtt-server +# port: 1883 +# msg: +# name: ${name} +# value: ${value} +# source: ${Config.get("device_id")} +### + +## ------------------- +## Event hook examples +## ------------------- + +# Event hooks are procedures that are run when a certain condition is met. +# +# Check the documentation of your configured backends and plugins to see which +# events they can trigger, and check https://docs.platypush.tech/events.html +# for the full list of available events with their schemas. +# +# Just like procedures, event hooks can be defined either using the YAML +# syntax, or in Python snippets in your `scripts` folder. +# +# A YAML event hook consists of two parts: an `if` field that specifies on +# which event the hook will be triggered (type and attribute values), and a +# `then` field that uses the same syntax as procedures to specify a list of +# actions to execute when the event is matched. + +### +# # This example is a hook that reacts when an `MQTTMessageEvent` is received on +# # a topic named `platypush/sensor` (see `send_sensor_data` example from the +# # procedures section). +# # +# # It will store the event on a centralized Postgres database. +# # +# # Note that, for this event to be triggered, the application must first +# # subscribe to the `platypush/sensor` topic - e.g. by adding `platypush/sensor` +# # to the active subscriptions in the `mqtt` configurations. +# +# event.hook.OnSensorDataReceived: +# if: +# type: platypush.message.event.mqtt.MQTTMessageEvent +# topic: platypush/sensor +# then: +# - action: db.insert +# args: +# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname +# table: sensor_data +# records: +# - name: ${msg["name"]} +# value: ${msg["value"]} +# source: ${msg["source"]} +### + +### +# # The example below plays the music on mpd/mopidy when your voice assistant +# # triggers a speech recognized event with "play the music" content. +# +# event.hook.PlayMusicAssistantCommand: +# if: +# type: platypush.message.event.assistant.SpeechRecognizedEvent +# # Note that basic regexes are supported for `SpeechRecognizedEvent`, +# # so the hook will be triggered both if you say "play the music" and +# # "play music" +# phrase: "play (the)? music" +# then: +# - action: music.mpd.play +### + +### +# # This will turn on the lights when you say "turn on the lights" +# +# event.hook.TurnOnLightsCommand: +# if: +# type: platypush.message.event.assistant.SpeechRecognizedEvent +# phrase: "turn on (the)? lights?" +# then: +# - action: light.hue.on +### + +### +# # The WebhookEvent is a special type of event. It allows you to dynamically +# # register a Web hook that can be invoked by other clients, if the HTTP backend +# # is active. +# # +# # In this case, we are registering a hook under `/hook/test-hook` that accepts +# # POST requests, gets the body of the requests and logs it. +# # +# # NOTE: Since Web hooks are supposed to be called by external (and potentially +# # untrusted) parties, they aren't designed to use the standard authentication +# # mechanism used by all other routes. +# # +# # By default they don't have an authentication layer at all. You are however +# # advised to create your custom passphrase and checks the request's headers or +# # query string for it - preferably one passphrase per endpoint. +# +# event.hook.WebhookExample: +# if: +# type: platypush.message.event.http.hook.WebhookEvent +# hook: test-hook +# method: POST +# then: +# # Check the token/passphrase +# - if ${args.get('headers', {}).get('X-Token') == 'SECRET': +# - action: logger.info +# args: +# msg: ${data} +### + +### ------------- +### Cron examples +### ------------- + +### +# # Cronjobs allow you to execute procedures at periodic intervals. +# # Standard UNIX cron syntax is supported, plus an optional 6th indicator +# # at the end of the expression to run jobs with second granularity. +# # The example below executes a script at intervals of 1 minute. +# +# cron.TestCron: +# cron_expression: '* * * * *' +# actions: +# - action: shell.exec +# args: +# cmd: ~/bin/myscript.sh +### diff --git a/platypush/entities/_base.py b/platypush/entities/_base.py index c6d5c18b3..2a739124a 100644 --- a/platypush/entities/_base.py +++ b/platypush/entities/_base.py @@ -24,10 +24,12 @@ from sqlalchemy import ( UniqueConstraint, inspect as schema_inspect, ) +from sqlalchemy.engine import Engine from sqlalchemy.orm import ColumnProperty, backref, relationship from sqlalchemy.orm.exc import ObjectDeletedError import platypush +from platypush.config import Config from platypush.common.db import Base from platypush.message import JSONAble, Message @@ -303,6 +305,24 @@ def _discover_entity_types(): entities_registry[obj] = {} # type: ignore +def _get_db(): + """ + Utility method to get the db plugin. + """ + from platypush.context import get_plugin + + db = get_plugin('db') + assert db + return db + + +def _get_db_engine() -> Engine: + """ + Utility method to get the db engine. + """ + return _get_db().get_engine() + + def get_entities_registry() -> EntityRegistryType: """ :returns: A copy of the entities registry. @@ -314,13 +334,9 @@ def init_entities_db(): """ Initializes the entities database. """ - from platypush.context import get_plugin - run_db_migrations() _discover_entity_types() - db = get_plugin('db') - assert db - db.create_all(db.get_engine(), Base) + _get_db().create_all(_get_db_engine(), Base) def run_db_migrations(): @@ -339,6 +355,10 @@ def run_db_migrations(): 'alembic', '-c', alembic_ini, + '-x', + f'CFGFILE={Config.get_file()}', + '-x', + f'DBNAME={_get_db_engine().url}', 'upgrade', 'head', ], diff --git a/platypush/install/docker/alpine.Dockerfile b/platypush/install/docker/alpine.Dockerfile new file mode 100644 index 000000000..5092f782a --- /dev/null +++ b/platypush/install/docker/alpine.Dockerfile @@ -0,0 +1,23 @@ +FROM alpine + +ADD . /install +WORKDIR /var/lib/platypush + +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + +RUN apk update +RUN /install/platypush/install/scripts/alpine/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install +RUN rm -rf /var/cache/apk + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/docker/debian.Dockerfile b/platypush/install/docker/debian.Dockerfile new file mode 100644 index 000000000..49e59035c --- /dev/null +++ b/platypush/install/docker/debian.Dockerfile @@ -0,0 +1,27 @@ +FROM debian + +ADD . /install +WORKDIR /var/lib/platypush + +ARG DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + +RUN apt update +RUN /install/platypush/install/scripts/debian/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . --break-system-packages +RUN rm -rf /install +RUN apt autoclean -y +RUN apt autoremove -y +RUN apt clean + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/docker/ubuntu.Dockerfile b/platypush/install/docker/ubuntu.Dockerfile new file mode 100644 index 000000000..14b400802 --- /dev/null +++ b/platypush/install/docker/ubuntu.Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu + +ADD . /install +WORKDIR /var/lib/platypush + +ARG DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + +RUN apt update +RUN /install/platypush/install/scripts/debian/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install +RUN apt autoclean -y +RUN apt autoremove -y +RUN apt clean + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/requirements/alpine.txt b/platypush/install/requirements/alpine.txt new file mode 100644 index 000000000..e9f2b4773 --- /dev/null +++ b/platypush/install/requirements/alpine.txt @@ -0,0 +1,26 @@ +python3 +py3-pip +py3-alembic +py3-bcrypt +py3-dateutil +py3-docutils +py3-flask +py3-frozendict +py3-greenlet +py3-magic +py3-mypy-extensions +py3-psutil +py3-redis +py3-requests +py3-rsa +py3-sqlalchemy +py3-tornado +py3-typing-extensions +py3-tz +py3-websocket-client +py3-websockets +py3-wheel +py3-yaml +py3-zeroconf +redis +sudo diff --git a/platypush/install/requirements/arch.txt b/platypush/install/requirements/arch.txt new file mode 100644 index 000000000..cc1a45ea6 --- /dev/null +++ b/platypush/install/requirements/arch.txt @@ -0,0 +1,24 @@ +python +python-alembic +python-bcrypt +python-dateutil +python-docutils +python-flask +python-frozendict +python-magic +python-marshmallow +python-pip +python-psutil +python-pytz +python-redis +python-requests +python-rsa +python-sqlalchemy +python-tornado +python-websocket-client +python-websockets +python-wheel +python-yaml +python-zeroconf +redis +sudo diff --git a/platypush/install/requirements/debian.txt b/platypush/install/requirements/debian.txt new file mode 100644 index 000000000..d7fdd1aa7 --- /dev/null +++ b/platypush/install/requirements/debian.txt @@ -0,0 +1,28 @@ +python3 +python3-pip +python3-alembic +python3-bcrypt +python3-dateutil +python3-docutils +python3-flask +python3-frozendict +python3-greenlet +python3-magic +python3-marshmallow +python3-mypy-extensions +python3-psutil +python3-redis +python3-requests +python3-rsa +python3-sqlalchemy +python3-tornado +python3-typing-extensions +python3-typing-inspect +python3-tz +python3-websocket +python3-websockets +python3-wheel +python3-yaml +python3-zeroconf +redis +sudo diff --git a/platypush/install/requirements/ubuntu.txt b/platypush/install/requirements/ubuntu.txt new file mode 120000 index 000000000..22a908cb6 --- /dev/null +++ b/platypush/install/requirements/ubuntu.txt @@ -0,0 +1 @@ +debian.txt \ No newline at end of file diff --git a/platypush/install/scripts/alpine/PKGCMD b/platypush/install/scripts/alpine/PKGCMD new file mode 100644 index 000000000..eba0c3d56 --- /dev/null +++ b/platypush/install/scripts/alpine/PKGCMD @@ -0,0 +1 @@ +apk add --update --no-interactive --no-cache diff --git a/platypush/install/scripts/alpine/install.sh b/platypush/install/scripts/alpine/install.sh new file mode 120000 index 000000000..3f44f994d --- /dev/null +++ b/platypush/install/scripts/alpine/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/arch/PKGCMD b/platypush/install/scripts/arch/PKGCMD new file mode 100644 index 000000000..eb711b620 --- /dev/null +++ b/platypush/install/scripts/arch/PKGCMD @@ -0,0 +1 @@ +pacman -S --noconfirm --needed diff --git a/platypush/install/scripts/arch/install.sh b/platypush/install/scripts/arch/install.sh new file mode 120000 index 000000000..3f44f994d --- /dev/null +++ b/platypush/install/scripts/arch/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/debian/PKGCMD b/platypush/install/scripts/debian/PKGCMD new file mode 100644 index 000000000..333026745 --- /dev/null +++ b/platypush/install/scripts/debian/PKGCMD @@ -0,0 +1 @@ +apt install -y diff --git a/platypush/install/scripts/debian/install.sh b/platypush/install/scripts/debian/install.sh new file mode 120000 index 000000000..3f44f994d --- /dev/null +++ b/platypush/install/scripts/debian/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/install.sh b/platypush/install/scripts/install.sh new file mode 100755 index 000000000..5eb688269 --- /dev/null +++ b/platypush/install/scripts/install.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# This script parses the system requirements for a specific OS and it runs the +# appropriate package manager command to install them. + +# This script is usually symlinked in the folders of the individual operating +# systems, and it's not supposed to be invoked directly. +# Instead, it will be called either by the root install.sh script or by a +# Dockerfile. + +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +OS="$(basename "$SCRIPT_PATH")" +CMD="$(cat "${SCRIPT_PATH}/PKGCMD")" +REQUIREMENTS="$(cat "${SCRIPT_PATH}/../../requirements/${OS}.txt" | tr '\n' ' ')" +SUDO= + +# If we aren't running in a Docker context, or the user is not root, we should +# use sudo to install system packages. +if [ $(id -u) -ne 0 ] || [ -z "$DOCKER_CTX" ]; then + if ! type sudo >/dev/null; then + echo "sudo executable not found, I can't install system packages" >&2 + exit 1 + fi + + SUDO="sudo" +fi + +${SUDO_ARGS} ${CMD} ${REQUIREMENTS} diff --git a/platypush/install/scripts/ubuntu b/platypush/install/scripts/ubuntu new file mode 120000 index 000000000..b2f7fd3e9 --- /dev/null +++ b/platypush/install/scripts/ubuntu @@ -0,0 +1 @@ +debian \ No newline at end of file diff --git a/platypush/migrations/alembic/env.py b/platypush/migrations/alembic/env.py index a9de8c4f2..d4aba7078 100644 --- a/platypush/migrations/alembic/env.py +++ b/platypush/migrations/alembic/env.py @@ -74,14 +74,19 @@ def run_migrations_online() -> None: def set_db_engine(): - db_conf = Config.get('db') - assert db_conf, 'Could not retrieve the database configuration' - engine = db_conf['engine'] - assert engine, 'No database engine configured' + app_conf_file = context.get_x_argument(as_dictionary=True).get('CFGFILE') + if app_conf_file: + Config.init(app_conf_file) + + engine_url = context.get_x_argument(as_dictionary=True).get('DBNAME') + if not engine_url: + db_conf = Config.get('db') + assert db_conf, 'Could not retrieve the database configuration' + engine_url = db_conf['engine'] + assert engine_url, 'No database engine configured' - config = context.config section = config.config_ini_section - config.set_section_option(section, 'DB_ENGINE', engine) + config.set_section_option(section, 'DB_ENGINE', engine_url) set_db_engine() diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 77e9778c9..7ad3eb74c 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -1,462 +1,360 @@ """ -Platydock - -Platydock is a helper that allows you to easily manage (create, destroy, start, -stop and list) Platypush instances as Docker images. +Platydock is a helper script that allows you to automatically create a +Dockerfile for Platypush starting from a configuration file. """ import argparse -import enum +from contextlib import contextmanager +import inspect +import logging import os import pathlib import re -import shutil import subprocess import sys import textwrap -import traceback as tb -import yaml +from typing import IO, Generator, Iterable +from typing_extensions import override +from platypush.builder import BaseBuilder from platypush.config import Config -from platypush.utils import manifest - -workdir = os.path.join( - os.path.expanduser('~'), '.local', 'share', 'platypush', 'platydock' +from platypush.utils.manifest import ( + BaseImage, + Dependencies, + InstallContext, + PackageManagers, ) - -class Action(enum.Enum): - build = 'build' - start = 'start' - stop = 'stop' - rm = 'rm' - ls = 'ls' - - def __str__(self): - return self.value +logger = logging.getLogger() -def _parse_deps(cls): - deps = [] +class DockerBuilder(BaseBuilder): + """ + Creates a Platypush Docker image from a configuration file. + """ - for line in cls.__doc__.split('\n'): - m = re.search(r'\(``pip install (.+)``\)', line) - if m: - deps.append(m.group(1)) + _pkg_manager_by_base_image = { + BaseImage.ALPINE: PackageManagers.APK, + BaseImage.DEBIAN: PackageManagers.APT, + BaseImage.UBUNTU: PackageManagers.APT, + } - return deps + _header = textwrap.dedent( + """ + # This Dockerfile was automatically generated by Platydock. + # + # You can build a Platypush image from it by running + # `docker build -t platypush .` in the same folder as this file, + # or copy it to the root a Platypush source folder to install the + # checked out version instead of downloading it first. + # + # You can then run your new image through: + # docker run --rm --name platypush \\ + # -v /path/to/your/config/dir:/etc/platypush \\ + # -v /path/to/your/workdir:/var/lib/platypush \\ + # -p 8008:8008 \\ + # platypush\n + """ + ) + _footer = textwrap.dedent( + """ + # You can customize the name of your installation by passing + # --device-id=... to the launched command. + """ + ) -def generate_dockerfile(deps, ports, cfgfile, device_dir, python_version): - device_id = Config.get('device_id') - if not device_id: - raise RuntimeError( - ( - 'You need to specify a device_id in {} - Docker ' - + 'containers cannot rely on hostname' - ).format(cfgfile) + def __init__( + self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs + ): + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) + self.image = image + self.tag = tag + self.print_only = print_only # TODO + + @override + @classmethod + def get_name(cls): + return "platydock" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush Docker image from a configuration file." + + @property + def dockerfile_dir(self) -> str: + """ + Proxy property for the output Dockerfile directory. + """ + output = self.output + parent = os.path.dirname(output) + + if os.path.isfile(output): + return parent + + if os.path.isdir(output): + return output + + logger.info('%s directory does not exist, creating it', output) + pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True) + return output + + @property + def dockerfile(self) -> str: + """ + Proxy property for the output Dockerfile. + """ + return os.path.join(self.dockerfile_dir, 'Dockerfile') + + @property + def pkg_manager(self) -> PackageManagers: + """ + Proxy property for the package manager to use. + """ + return self._pkg_manager_by_base_image[self.image] + + def _read_base_dockerfile_lines(self) -> Generator[str, None, None]: + """ + :return: The lines of the base Dockerfile. + """ + import platypush + + base_file = os.path.join( + str(pathlib.Path(inspect.getfile(platypush)).parent), + 'install', + 'docker', + f'{self.image}.Dockerfile', ) - os.makedirs(device_dir, exist_ok=True) - content = textwrap.dedent( - ''' - FROM python:{python_version}-slim-bookworm + with open(base_file, 'r') as f: + for line in f: + yield line.rstrip() - RUN mkdir -p /app - RUN mkdir -p /etc/platypush - RUN mkdir -p /usr/local/share/platypush\n - '''.format( - python_version=python_version + @property + @override + def deps(self) -> Dependencies: + return Dependencies.from_config( + self.cfgfile, + pkg_manager=self.pkg_manager, + install_context=InstallContext.DOCKER, + base_image=self.image, ) - ).lstrip() - srcdir = os.path.dirname(cfgfile) - cfgfile_copy = os.path.join(device_dir, 'config.yaml') - shutil.copy(cfgfile, cfgfile_copy, follow_symlinks=True) - content += 'COPY config.yaml /etc/platypush/\n' - backend_config = Config.get_backends() + def _create_dockerfile_parser(self): + """ + Closure for a context-aware parser for the default Dockerfile. + """ + is_after_expose_cmd = False + deps = self.deps + ports = self._get_exposed_ports() - # Redis configuration for Docker - if 'redis' not in backend_config: - backend_config['redis'] = { - 'redis_args': { - 'host': 'redis', - 'port': 6379, - } - } + def parser(): + nonlocal is_after_expose_cmd - with open(cfgfile_copy, 'a') as f: - f.write( - '\n# Automatically added by platydock, do not remove\n' - + yaml.dump( - { - 'backend.redis': backend_config['redis'], - } - ) - + '\n' + for line in self._read_base_dockerfile_lines(): + if re.match( + r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', + line.strip(), + ): + yield self._generate_git_clone_command() + elif line.startswith('RUN cd /install '): + for new_line in deps.before: + yield 'RUN ' + new_line + + for new_line in deps.to_pkg_install_commands(): + yield 'RUN ' + new_line + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + yield 'RUN ' + new_line + + for new_line in deps.after: + yield 'RUN' + new_line + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + yield from [f'EXPOSE {port}' for port in ports] + is_after_expose_cmd = True + + continue + elif line.startswith('CMD'): + yield from self._footer.split('\n') + + yield line + + if line.startswith('CMD') and self.device_id: + yield f'\t--device-id {self.device_id} \\' + + return parser + + @override + def build(self): + """ + Build a Dockerfile based on a configuration file. + + :return: The content of the generated Dockerfile. + """ + + # Set the DOCKER_CTX environment variable so any downstream logic knows + # that we're running in a Docker build context. + os.environ['DOCKER_CTX'] = '1' + + self._generate_dockerfile() + if self.print_only: + return + + self._build_image() + self._print_instructions( + textwrap.dedent( + f""" + A Docker image has been built from the configuration file {self.cfgfile}. + The Dockerfile is available under {self.dockerfile}. + You can run the Docker image with the following command: + + docker run \\ + --rm --name platypush \\ + -v {os.path.dirname(self.cfgfile)}:/etc/platypush \\ + -v /path/to/your/workdir:/var/lib/platypush \\ + -p 8008:8008 \\ + platypush + """ ) - - # Main database configuration - has_main_db = False - with open(cfgfile_copy, 'r') as f: - for line in f.readlines(): - if re.match(r'^(main.)?db.*', line): - has_main_db = True - break - - if not has_main_db: - with open(cfgfile_copy, 'a') as f: - f.write( - '\n# Automatically added by platydock, do not remove\n' - + yaml.dump( - { - 'main.db': { - 'engine': 'sqlite:////platypush.db', - } - } - ) - + '\n' - ) - - # Copy included files - # noinspection PyProtectedMember - for include in Config._included_files: - incdir = os.path.relpath(os.path.dirname(include), srcdir) - destdir = os.path.join(device_dir, incdir) - pathlib.Path(destdir).mkdir(parents=True, exist_ok=True) - shutil.copy(include, destdir, follow_symlinks=True) - content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n' - content += ( - 'COPY ' - + os.path.relpath(include, srcdir) - + ' /etc/platypush/' - + incdir - + '\n' ) - # Copy script files - scripts_dir = os.path.join(os.path.dirname(cfgfile), 'scripts') - if os.path.isdir(scripts_dir): - local_scripts_dir = os.path.join(device_dir, 'scripts') - remote_scripts_dir = '/etc/platypush/scripts' - shutil.copytree( - scripts_dir, local_scripts_dir, symlinks=True, dirs_exist_ok=True - ) - content += f'RUN mkdir -p {remote_scripts_dir}\n' - content += f'COPY scripts/ {remote_scripts_dir}\n' - - packages = deps.pop('packages', None) - pip = deps.pop('pip', None) - exec_cmds = deps.pop('exec', None) - pkg_cmd = ( - f'\n\t&& apt-get install --no-install-recommends -y {" ".join(packages)} \\' - if packages - else '' - ) - pip_cmd = f'\n\t&& pip install {" ".join(pip)} \\' if pip else '' - content += f''' -RUN dpkg --configure -a \\ - && apt-get -f install \\ - && apt-get --fix-missing install \\ - && apt-get clean \\ - && apt-get update \\ - && apt-get -y upgrade \\ - && apt-get -y dist-upgrade \\ - && apt-get install --no-install-recommends -y apt-utils \\ - && apt-get install --no-install-recommends -y build-essential \\ - && apt-get install --no-install-recommends -y git \\ - && apt-get install --no-install-recommends -y sudo \\ - && apt-get install --no-install-recommends -y libffi-dev \\ - && apt-get install --no-install-recommends -y libcap-dev \\ - && apt-get install --no-install-recommends -y libjpeg-dev \\{pkg_cmd}{pip_cmd}''' - - for exec_cmd in exec_cmds: - content += f'\n\t&& {exec_cmd} \\' - content += ''' - && apt-get install --no-install-recommends -y zlib1g-dev - -RUN git clone --recursive https://git.platypush.tech/platypush/platypush.git /app \\ - && cd /app \\ - && pip install -r requirements.txt - -RUN apt-get remove -y git \\ - && apt-get remove -y build-essential \\ - && apt-get remove -y libffi-dev \\ - && apt-get remove -y libjpeg-dev \\ - && apt-get remove -y libcap-dev \\ - && apt-get remove -y zlib1g-dev \\ - && apt-get remove -y apt-utils \\ - && apt-get clean \\ - && apt-get autoremove -y \\ - && rm -rf /var/lib/apt/lists/* -''' - - for port in ports: - content += 'EXPOSE {}\n'.format(port) - - content += textwrap.dedent( - ''' - - ENV PYTHONPATH /app:$PYTHONPATH - CMD ["python", "-m", "platypush"] - ''' - ) - - dockerfile = os.path.join(device_dir, 'Dockerfile') - print('Generating Dockerfile {}'.format(dockerfile)) - - with open(dockerfile, 'w') as f: - f.write(content) - - -def build(args): - global workdir - - ports = set() - parser = argparse.ArgumentParser( - prog='platydock build', description='Build a Platypush image from a config.yaml' - ) - - parser.add_argument( - '-c', - '--config', - type=str, - required=True, - help='Path to the platypush configuration file', - ) - parser.add_argument( - '-p', - '--python-version', - type=str, - default='3.9', - help='Python version to be used', - ) - - opts, args = parser.parse_known_args(args) - - cfgfile = os.path.abspath(os.path.expanduser(opts.config)) - manifest._available_package_manager = ( - 'apt' # Force apt for Debian-based Docker images - ) - install_cmds = manifest.get_dependencies_from_conf(cfgfile) - python_version = opts.python_version - backend_config = Config.get_backends() - - # Container exposed ports - if backend_config.get('http'): - from platypush.backend.http import HttpBackend - - # noinspection PyProtectedMember - ports.add(backend_config['http'].get('port', HttpBackend._DEFAULT_HTTP_PORT)) - - if backend_config.get('tcp'): - ports.add(backend_config['tcp']['port']) - - dev_dir = os.path.join(workdir, Config.get('device_id')) - generate_dockerfile( - deps=dict(install_cmds), - ports=ports, - cfgfile=cfgfile, - device_dir=dev_dir, - python_version=python_version, - ) - - subprocess.call( - [ + def _build_image(self): + """ + Build a Platypush Docker image from the generated Dockerfile. + """ + logger.info('Building Docker image...') + cmd = [ 'docker', 'build', + '-f', + self.dockerfile, '-t', - 'platypush-{}'.format(Config.get('device_id')), - dev_dir, + self.tag, + '.', ] - ) + subprocess.run(cmd, check=True) -def start(args): - global workdir + def _generate_dockerfile(self): + """ + Parses the configuration file and generates a Dockerfile based on it. + """ - parser = argparse.ArgumentParser( - prog='platydock start', - description='Start a Platypush container', - epilog=textwrap.dedent( - ''' - You can append additional options that - will be passed to the docker container. - Example: + @contextmanager + def open_writer() -> Generator[IO, None, None]: + # flake8: noqa + f = sys.stdout if self.print_only else open(self.dockerfile, 'w') - --add-host='myhost:192.168.1.1' - ''' - ), - ) + try: + yield f + finally: + if f is not sys.stdout: + f.close() - parser.add_argument('image', type=str, help='Platypush image to start') - parser.add_argument( - '-p', - '--publish', - action='append', - nargs='*', - default=[], - help=textwrap.dedent( - ''' - Container's ports to expose to the host. - Note that the default exposed ports from - the container service will be exposed unless - these mappings override them (e.g. port 8008 - on the container will be mapped to 8008 on - the host). + if not self.print_only: + logger.info('Parsing configuration file %s...', self.cfgfile) - Example: + Config.init(self.cfgfile) - -p 18008:8008 - ''' - ), - ) + if not self.print_only: + logger.info('Generating Dockerfile %s...', self.dockerfile) - parser.add_argument( - '-a', - '--attach', - action='store_true', - default=False, - help=textwrap.dedent( - ''' - If set, then attach to the container after starting it up (default: false). - ''' - ), - ) + parser = self._create_dockerfile_parser() - opts, args = parser.parse_known_args(args) - ports = {} - dockerfile = os.path.join(workdir, opts.image, 'Dockerfile') + with open_writer() as f: + f.write(self._header + '\n') + for line in parser(): + f.write(line + '\n') - with open(dockerfile) as f: - for line in f: - m = re.match(r'expose (\d+)', line.strip().lower()) - if m: - ports[m.group(1)] = m.group(1) + def _generate_git_clone_command(self) -> str: + """ + Generates a git clone command in Dockerfile that checks out the repo + and the right git reference, if the application sources aren't already + available under /install. + """ + install_cmd = ' '.join(self.pkg_manager.value.install) + uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall) + return textwrap.dedent( + f""" + RUN if [ ! -f "/install/setup.py" ]; then \\ + echo "Platypush source not found under the current directory, downloading it" && \\ + {install_cmd} git && \\ + rm -rf /install && \\ + git clone --recursive https://github.com/BlackLight/platypush.git /install && \\ + cd /install && \\ + git checkout {self.gitref} && \\ + {uninstall_cmd} git; \\ + fi + """ + ) - for mapping in opts.publish: - host_port, container_port = mapping[0].split(':') - ports[container_port] = host_port + @classmethod + @override + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = super()._get_arg_parser() - print('Preparing Redis support container') - subprocess.call(['docker', 'pull', 'redis']) - subprocess.call( - ['docker', 'run', '--rm', '--name', 'redis-' + opts.image, '-d', 'redis'] - ) + parser.add_argument( + '-i', + '--image', + dest='image', + required=False, + type=BaseImage, + choices=list(BaseImage), + default=BaseImage.ALPINE, + help='Base image to use for the Dockerfile (default: alpine).', + ) - docker_cmd = [ - 'docker', - 'run', - '--rm', - '--name', - opts.image, - '-it', - '--link', - 'redis-' + opts.image + ':redis', - ] + parser.add_argument( + '-t', + '--tag', + dest='tag', + required=False, + type=str, + default='platypush:latest', + help='Tag name to be used for the built image ' + '(default: "platypush:latest").', + ) - for container_port, host_port in ports.items(): - docker_cmd += ['-p', host_port + ':' + container_port] + parser.add_argument( + '--print', + dest='print_only', + action='store_true', + help='Use this flag if you only want to print the Dockerfile to ' + 'stdout instead of generating an image.', + ) - docker_cmd += args - docker_cmd += ['-d', 'platypush-' + opts.image] + return parser - print('Starting Platypush container {}'.format(opts.image)) - subprocess.call(docker_cmd) - - if opts.attach: - subprocess.call(['docker', 'attach', opts.image]) - - -def stop(args): - parser = argparse.ArgumentParser( - prog='platydock stop', description='Stop a Platypush container' - ) - - parser.add_argument('container', type=str, help='Platypush container to stop') - opts, args = parser.parse_known_args(args) - - print('Stopping Platypush container {}'.format(opts.container)) - subprocess.call(['docker', 'kill', '{}'.format(opts.container)]) - - print('Stopping Redis support container') - subprocess.call(['docker', 'stop', 'redis-{}'.format(opts.container)]) - - -def rm(args): - global workdir - - parser = argparse.ArgumentParser( - prog='platydock rm', - description='Remove a Platypush image. ' - + 'NOTE: make sure that no container is ' - + 'running nor linked to the image before ' - + 'removing it', - ) - - parser.add_argument('image', type=str, help='Platypush image to remove') - opts, args = parser.parse_known_args(args) - - subprocess.call(['docker', 'rmi', 'platypush-{}'.format(opts.image)]) - shutil.rmtree(os.path.join(workdir, opts.image), ignore_errors=True) - - -def ls(args): - parser = argparse.ArgumentParser( - prog='platydock ls', description='List available Platypush containers' - ) - parser.add_argument('filter', type=str, nargs='?', help='Image name filter') - - opts, args = parser.parse_known_args(args) - - p = subprocess.Popen(['docker', 'images'], stdout=subprocess.PIPE) - output = p.communicate()[0].decode().split('\n') - header = output.pop(0) - images = [] - - for line in output: - if re.match(r'^platypush-(.+?)\s.*', line) and ( - not opts.filter or (opts.filter and opts.filter in line) - ): - images.append(line) - - if images: - print(header) - - for image in images: - print(image) + @staticmethod + def _get_exposed_ports() -> Iterable[int]: + """ + :return: The listen ports used by the backends enabled in the configuration + file. + """ + backends_config = Config.get_backends() + return { + int(port) + for port in ( + backends_config.get('http', {}).get('port'), + backends_config.get('tcp', {}).get('port'), + ) + if port + } def main(): - parser = argparse.ArgumentParser( - prog='platydock', - add_help=False, - description='Manage Platypush docker containers', - epilog='Use platydock --help to ' + 'get additional help', - ) - - # noinspection PyTypeChecker - parser.add_argument( - 'action', nargs='?', type=Action, choices=list(Action), help='Action to execute' - ) - parser.add_argument('-h', '--help', action='store_true', help='Show usage') - opts, args = parser.parse_known_args(sys.argv[1:]) - - if (opts.help and not opts.action) or (not opts.help and not opts.action): - parser.print_help() - return 1 - - globals()[str(opts.action)](sys.argv[2:]) + """ + Generates a Dockerfile based on the configuration file. + """ + DockerBuilder.from_cmdline(sys.argv[1:]).build() + return 0 if __name__ == '__main__': - ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m ' - ERR_SUFFIX = '\033[0m' + sys.exit(main()) - try: - main() - except Exception as e: - tb.print_exc(file=sys.stdout) - print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr) # vim:sw=4:ts=4:et: diff --git a/platypush/platydock/__main__.py b/platypush/platydock/__main__.py index f216837d4..bc0461ab7 100644 --- a/platypush/platydock/__main__.py +++ b/platypush/platydock/__main__.py @@ -3,4 +3,3 @@ from platypush.platydock import main main() # vim:sw=4:ts=4:et: - diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py new file mode 100755 index 000000000..0564b719f --- /dev/null +++ b/platypush/platyvenv/__init__.py @@ -0,0 +1,193 @@ +""" +Platyvenv is a helper script that allows you to automatically create a +virtual environment for Platypush starting from a configuration file. +""" + +from contextlib import contextmanager +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +from typing import Generator, Sequence +import venv + +from typing_extensions import override + +from platypush.builder import BaseBuilder +from platypush.config import Config +from platypush.utils.manifest import ( + Dependencies, + InstallContext, +) + +logger = logging.getLogger() + + +class VenvBuilder(BaseBuilder): + """ + Build a virtual environment from on a configuration file. + """ + + def __init__(self, *args, **kwargs) -> None: + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) + + @override + @classmethod + def get_name(cls): + return "platyvenv" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush virtual environment from a configuration file." + + @property + def _pip_cmd(self) -> Sequence[str]: + """ + :return: The pip install command to use for the selected environment. + """ + return ( + os.path.join(self.output, 'bin', 'python'), + '-m', + 'pip', + 'install', + '-U', + '--no-cache-dir', + '--no-input', + ) + + def _install_system_packages(self, deps: Dependencies): + """ + Install the required system packages. + """ + for cmd in deps.to_pkg_install_commands(): + logger.info('Installing system packages: %s', cmd) + subprocess.call(re.split(r'\s+', cmd.strip())) + + @contextmanager + def _prepare_src_dir(self) -> Generator[str, None, None]: + """ + Prepare the source directory used to install the virtual enviornment. + + If platyvenv is launched from a local checkout of the Platypush source + code, then that checkout will be used. + + Otherwise, the source directory will be cloned from git into a + temporary folder. + """ + setup_py_path = os.path.join(os.getcwd(), 'setup.py') + if os.path.isfile(setup_py_path): + logger.info('Using local checkout of the Platypush source code') + yield os.getcwd() + else: + checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git') + logger.info('Cloning Platypush source code from git into %s', checkout_dir) + subprocess.call( + [ + 'git', + 'clone', + '--recursive', + 'https://github.com/BlackLight/platypush.git', + checkout_dir, + ] + ) + + pwd = os.getcwd() + os.chdir(checkout_dir) + subprocess.call(['git', 'checkout', self.gitref]) + yield checkout_dir + + os.chdir(pwd) + logger.info('Cleaning up %s', checkout_dir) + shutil.rmtree(checkout_dir, ignore_errors=True) + + def _prepare_venv(self) -> None: + """ + Install the virtual environment under the configured output. + """ + logger.info('Creating virtual environment under %s...', self.output) + + venv.create( + self.output, + symlinks=True, + with_pip=True, + upgrade_deps=True, + ) + + logger.info('Installing base Python dependencies under %s...', self.output) + subprocess.call([*self._pip_cmd, 'pip', '.']) + + def _install_extra_pip_packages(self, deps: Dependencies): + """ + Install the extra pip dependencies inferred from the configured + extensions. + """ + pip_deps = list(deps.to_pip_install_commands(full_command=False)) + if not pip_deps: + return + + logger.info( + 'Installing extra pip dependencies under %s: %s', + self.output, + ' '.join(pip_deps), + ) + + subprocess.call([*self._pip_cmd, *pip_deps]) + + def build(self): + """ + Build a Dockerfile based on a configuration file. + """ + Config.init(self.cfgfile) + + deps = Dependencies.from_config( + self.cfgfile, + install_context=InstallContext.VENV, + ) + + self._install_system_packages(deps) + + with self._prepare_src_dir(): + self._prepare_venv() + + self._install_extra_pip_packages(deps) + self._print_instructions( + textwrap.dedent( + f""" + Virtual environment created under {self.output}. + To run the application: + + source {self.output}/bin/activate + platypush -c {self.cfgfile} { + "--device_id " + self.device_id if self.device_id else "" + } + + Platypush requires a Redis instance. If you don't want to use a + stand-alone server, you can pass the --start-redis option to + the executable (optionally with --redis-port). + + Platypush will then start its own local instance and it will + terminate it once the application is stopped. + """ + ) + ) + + +def main(): + """ + Generates a virtual environment based on the configuration file. + """ + VenvBuilder.from_cmdline(sys.argv[1:]).build() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + + +# vim:sw=4:ts=4:et: diff --git a/platypush/platyvenv/__main__.py b/platypush/platyvenv/__main__.py new file mode 100644 index 000000000..b6726678b --- /dev/null +++ b/platypush/platyvenv/__main__.py @@ -0,0 +1,5 @@ +from platypush.platyvenv import main + +main() + +# vim:sw=4:ts=4:et: diff --git a/platypush/plugins/__init__.py b/platypush/plugins/__init__.py index a66a7e34e..b0090f60b 100644 --- a/platypush/plugins/__init__.py +++ b/platypush/plugins/__init__.py @@ -15,7 +15,7 @@ from platypush.message.response import Response from platypush.utils import get_decorators, get_plugin_name_by_class PLUGIN_STOP_TIMEOUT = 5 # Plugin stop timeout in seconds -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def action(f: Callable[..., Any]) -> Callable[..., Response]: @@ -33,7 +33,7 @@ def action(f: Callable[..., Any]) -> Callable[..., Response]: try: result = f(*args, **kwargs) except TypeError as e: - logger.exception(e) + _logger.exception(e) result = Response(errors=[str(e)]) if result and isinstance(result, Response): diff --git a/platypush/plugins/bluetooth/manifest.yaml b/platypush/plugins/bluetooth/manifest.yaml index c2e98a348..893d4e074 100644 --- a/platypush/plugins/bluetooth/manifest.yaml +++ b/platypush/plugins/bluetooth/manifest.yaml @@ -15,12 +15,15 @@ manifest: install: apk: - py3-pydbus + - git apt: - libbluetooth-dev - python3-pydbus + - git pacman: - python-pydbus - python-bleak + - git pip: - bleak - bluetooth-numbers diff --git a/platypush/plugins/gpio/manifest.yaml b/platypush/plugins/gpio/manifest.yaml index dfe358cc9..dc52185a2 100644 --- a/platypush/plugins/gpio/manifest.yaml +++ b/platypush/plugins/gpio/manifest.yaml @@ -1,7 +1,6 @@ manifest: events: - - platypush.message.event.gpio.GPIOEvent: - When the value of a monitored PIN changes. + - platypush.message.event.gpio.GPIOEvent install: pip: - RPi.GPIO diff --git a/platypush/plugins/http/request/rss/__init__.py b/platypush/plugins/http/request/rss/__init__.py index 498579a2e..f1ec501f0 100644 --- a/platypush/plugins/http/request/rss/__init__.py +++ b/platypush/plugins/http/request/rss/__init__.py @@ -1,6 +1,7 @@ from platypush.plugins import action from platypush.plugins.http.request import HttpRequestPlugin + class HttpRequestRssPlugin(HttpRequestPlugin): """ Plugin to programmatically retrieve and parse an RSS feed URL. @@ -11,12 +12,12 @@ class HttpRequestRssPlugin(HttpRequestPlugin): """ @action - def get(self, url): + def get(self, url, **_): import feedparser + response = super().get(url, output='text').output feed = feedparser.parse(response) return feed.entries # vim:sw=4:ts=4:et: - diff --git a/platypush/plugins/http/webpage/__init__.py b/platypush/plugins/http/webpage/__init__.py index 33589b504..425fbade4 100644 --- a/platypush/plugins/http/webpage/__init__.py +++ b/platypush/plugins/http/webpage/__init__.py @@ -1,13 +1,66 @@ +from dataclasses import dataclass import datetime +from enum import Enum import json import os import re import subprocess import tempfile +import textwrap +from typing import Iterable, Optional, Union from urllib.parse import urlparse -from platypush.plugins import action -from platypush.plugins.http.request import Plugin +from platypush.plugins import Plugin, action + + +@dataclass +class OutputFormat: + """ + Definition of a supported output format. + """ + + name: str + cmd_fmt: str + extensions: Iterable[str] = () + + +class OutputFormats(Enum): + """ + Supported output formats. + """ + + HTML = OutputFormat('html', extensions=('html', 'htm'), cmd_fmt='html') + # PDF will first be exported to HTML and then converted to PDF + PDF = OutputFormat('pdf', extensions=('pdf',), cmd_fmt='html') + TEXT = OutputFormat('text', extensions=('txt',), cmd_fmt='text') + MARKDOWN = OutputFormat('markdown', extensions=('md',), cmd_fmt='markdown') + + @classmethod + def parse( + cls, + type: Union[str, "OutputFormats"], # pylint: disable=redefined-builtin + outfile: Optional[str] = None, + ) -> "OutputFormats": + """ + Parse the format given a type argument and and output file name. + """ + try: + fmt = ( + getattr(OutputFormats, type.upper()) if isinstance(type, str) else type + ) + except AttributeError as e: + raise AssertionError( + f'Unsupported output format: {type}. Supported formats: ' + + f'{[f.name for f in OutputFormats]}' + ) from e + + by_extension = {ext.lower(): f for f in cls for ext in f.value.extensions} + if outfile: + fmt_by_ext = by_extension.get(os.path.splitext(outfile)[1].lower()[1:]) + if fmt_by_ext: + return fmt_by_ext + + return fmt class HttpWebpagePlugin(Plugin): @@ -24,34 +77,71 @@ class HttpWebpagePlugin(Plugin): """ - _mercury_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js') + _mercury_script = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js' + ) @staticmethod def _parse(proc): + """ + Runs the mercury-parser script and returns the result as a string. + """ with subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=None) as parser: return parser.communicate()[0].decode() @staticmethod def _fix_relative_links(markdown: str, url: str) -> str: - url = urlparse(url) - base_url = f'{url.scheme}://{url.netloc}' + """ + Fix relative links to match the base URL of the page (Markdown only). + """ + parsed_url = urlparse(url) + base_url = f'{parsed_url.scheme}://{parsed_url.netloc}' return re.sub(r'(\[.+?])\((/.+?)\)', fr'\1({base_url}\2)', markdown) - # noinspection PyShadowingBuiltins @action - def simplify(self, url, type='html', html=None, outfile=None): + def simplify( + self, + url: str, + type: Union[ # pylint: disable=redefined-builtin + str, OutputFormats + ] = OutputFormats.HTML, + html: Optional[str] = None, + outfile: Optional[str] = None, + font_size: str = '19px', + font_family: Union[str, Iterable[str]] = ( + '-apple-system', + 'Segoe UI', + 'Roboto', + 'Oxygen', + 'Ubuntu', + 'Cantarell', + "Fira Sans", + 'Open Sans', + 'Droid Sans', + 'Helvetica Neue', + 'Helvetica', + 'Arial', + 'sans-serif', + ), + ): """ Parse the readable content of a web page removing any extra HTML elements using Mercury. :param url: URL to parse. - :param type: Output format. Supported types: ``html``, ``markdown``, ``text`` (default: ``html``). - :param html: Set this parameter if you want to parse some HTML content already fetched. Note - that URL is still required by Mercury to properly style the output, but it won't be used - to actually fetch the content. - - :param outfile: If set then the output will be written to the specified file. If the file extension - is ``.pdf`` then the content will be exported in PDF format. If the output ``type`` is not - specified then it can also be inferred from the extension of the output file. + :param type: Output format. Supported types: ``html``, ``markdown``, + ``text``, ``pdf`` (default: ``html``). + :param html: Set this parameter if you want to parse some HTML content + already fetched. Note that URL is still required by Mercury to + properly style the output, but it won't be used to actually fetch + the content. + :param outfile: If set then the output will be written to the specified + file. If the file extension is ``.pdf`` then the content will be + exported in PDF format. If the output ``type`` is not specified + then it can also be inferred from the extension of the output file. + :param font_size: Font size to use for the output (default: 19px). + :param font_family: Custom font family (or list of font families, in + decreasing order) to use for the output. It only applies to HTML + and PDF. :return: dict Example if outfile is not specified:: @@ -74,48 +164,46 @@ class HttpWebpagePlugin(Plugin): """ - self.logger.info('Parsing URL {}'.format(url)) - wants_pdf = False - - if outfile: - wants_pdf = outfile.lower().endswith('.pdf') - if ( - wants_pdf # HTML will be exported to PDF - or outfile.lower().split('.')[-1].startswith('htm') - ): - type = 'html' - elif outfile.lower().endswith('.md'): - type = 'markdown' - elif outfile.lower().endswith('.txt'): - type = 'text' - - proc = ['node', self._mercury_script, url, type] - f = None + self.logger.info('Parsing URL %s', url) + fmt = OutputFormats.parse(type=type, outfile=outfile) + proc = ['node', self._mercury_script, url, fmt.value.cmd_fmt] + tmp_file = None if html: - f = tempfile.NamedTemporaryFile('w+', delete=False) - f.write(html) - f.flush() - proc.append(f.name) + with tempfile.NamedTemporaryFile('w+', delete=False) as f: + tmp_file = f.name + f.write(html) + f.flush() + proc.append(f.name) try: response = self._parse(proc) finally: - if f: - os.unlink(f.name) + if tmp_file: + os.unlink(tmp_file) try: response = json.loads(response.strip()) except Exception as e: - raise RuntimeError('Could not parse JSON: {}. Response: {}'.format(str(e), response)) + raise RuntimeError( + f'Could not parse JSON: {e}. Response: {response}' + ) from e - if type == 'markdown': + if fmt == OutputFormats.MARKDOWN: response['content'] = self._fix_relative_links(response['content'], url) - self.logger.debug('Got response from Mercury API: {}'.format(response)) - title = response.get('title', '{} on {}'.format( - 'Published' if response.get('date_published') else 'Generated', - response.get('date_published', datetime.datetime.now().isoformat()))) + self.logger.debug('Got response from Mercury API: %s', response) + title = response.get( + 'title', + ( + ('Published' if response.get('date_published') else 'Generated') + + ' on ' + + ( + response.get('date_published') + or datetime.datetime.now().isoformat() + ) + ), + ) content = response.get('content', '') @@ -126,46 +214,134 @@ class HttpWebpagePlugin(Plugin): 'content': content, } - outfile = os.path.abspath(os.path.expanduser(outfile)) - style = ''' - body { - font-size: 22px; - font-family: 'Merriweather', Georgia, 'Times New Roman', Times, serif; - } - ''' + return self._process_outfile( + url=url, + fmt=fmt, + title=title, + content=content, + outfile=outfile, + font_size=font_size, + font_family=tuple( + font_family, + ) + if isinstance(font_family, str) + else tuple(font_family), + ) - if type == 'html': - content = ( + @staticmethod + def _style_by_format( + fmt: OutputFormats, + font_size: str, + font_family: Iterable[str], + ) -> str: + """ + :return: The CSS style to be used for the given output format. + """ + style = textwrap.dedent( + f''' + ._parsed-content-container {{ + font-size: {font_size}; + font-family: {', '.join(f'"{f}"' for f in font_family)}; + }} + + ._parsed-content {{ + text-align: justify; + }} + + pre {{ + white-space: pre-wrap; + }} + ''' + ) + + if fmt == OutputFormats.HTML: + style += textwrap.dedent( ''' + ._parsed-content-container { + margin: 1em; + display: flex; + flex-direction: column; + align-items: center; + } + + ._parsed-content { + max-width: 800px; + } + + h1 { + max-width: 800px; + } + ''' + ) + + return style + + @classmethod + def _process_outfile( + cls, + url: str, + fmt: OutputFormats, + title: str, + content: str, + outfile: str, + font_size: str, + font_family: Iterable[str], + ): + """ + Process the output file. + + :param url: URL to parse. + :param fmt: Output format. Supported types: ``html``, ``markdown``, + ``text``, ``pdf`` (default: ``html``). + :param title: Page title. + :param content: Page content. + :param outfile: Output file path. + :param font_size: Font size to use for the output (default: 19px). + :param font_family: Custom font family (or list of font families, in + decreasing order) to use for the output. It only applies to HTML + and PDF. + :return: dict + """ + outfile = os.path.abspath(os.path.expanduser(outfile)) + style = cls._style_by_format(fmt, font_size, font_family) + + if fmt in {OutputFormats.HTML, OutputFormats.PDF}: + content = textwrap.dedent( + f''' +

{title}

{content}
- '''.format(title=title, url=url, content=content) +
+ ''' ) - if not wants_pdf: - content = ''' - - {title} - - '''.format(title=title, style=style) + \ - '{{' + content + '}}' - elif type == 'markdown': - content = '# [{title}]({url})\n\n{content}'.format( - title=title, url=url, content=content - ) + if fmt == OutputFormats.PDF: + content = textwrap.dedent( + f''' + + + {title} + + + {content} + + + ''' + ) + else: + content = textwrap.dedent( + f''' + + {content} + ''' + ) + elif fmt == OutputFormats.MARKDOWN: + content = f'# [{title}]({url})\n\n{content}' - if wants_pdf: - import weasyprint - try: - from weasyprint.fonts import FontConfiguration - except ImportError: - from weasyprint.document import FontConfiguration - - font_config = FontConfiguration() - css = [weasyprint.CSS('https://fonts.googleapis.com/css?family=Merriweather'), - weasyprint.CSS(string=style, font_config=font_config)] - - weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css) + if fmt == OutputFormats.PDF: + cls._process_pdf(content, outfile, style) else: with open(outfile, 'w', encoding='utf-8') as f: f.write(content) @@ -176,5 +352,28 @@ class HttpWebpagePlugin(Plugin): 'outfile': outfile, } + @staticmethod + def _process_pdf(content: str, outfile: str, style: str): + """ + Convert the given HTML content to a PDF document. + + :param content: Page content. + :param outfile: Output file path. + :param style: CSS style to use for the output. + """ + import weasyprint + + try: + from weasyprint.fonts import FontConfiguration # pylint: disable + except ImportError: + from weasyprint.document import FontConfiguration + + font_config = FontConfiguration() + css = [ + weasyprint.CSS(string=style, font_config=font_config), + ] + + weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css) + # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/http/webpage/manifest.yaml b/platypush/plugins/http/webpage/manifest.yaml index 04560d806..c266b4ace 100644 --- a/platypush/plugins/http/webpage/manifest.yaml +++ b/platypush/plugins/http/webpage/manifest.yaml @@ -15,7 +15,7 @@ manifest: - npm pip: - weasyprint - exec: + after: - sudo npm install -g @postlight/mercury-parser package: platypush.plugins.http.webpage type: plugin diff --git a/platypush/plugins/inspect/__init__.py b/platypush/plugins/inspect/__init__.py index 01bb6cfdb..ad0f1492a 100644 --- a/platypush/plugins/inspect/__init__.py +++ b/platypush/plugins/inspect/__init__.py @@ -20,7 +20,7 @@ from platypush.utils import ( get_plugin_class_by_name, get_plugin_name_by_class, ) -from platypush.utils.manifest import Manifest, scan_manifests +from platypush.utils.manifest import Manifests from ._context import ComponentContext from ._model import ( @@ -116,8 +116,7 @@ class InspectPlugin(Plugin): A generator that scans the manifest files given a ``base_type`` (``Plugin`` or ``Backend``) and yields the parsed submodules. """ - for mf_file in scan_manifests(base_type): - manifest = Manifest.from_file(mf_file) + for manifest in Manifests.by_base_class(base_type): try: yield importlib.import_module(manifest.package) except Exception as e: diff --git a/platypush/plugins/music/mpd/manifest.yaml b/platypush/plugins/music/mpd/manifest.yaml index f96099e6c..b9d837d80 100644 --- a/platypush/plugins/music/mpd/manifest.yaml +++ b/platypush/plugins/music/mpd/manifest.yaml @@ -2,7 +2,7 @@ manifest: events: {} install: apt: - - python-mpd + - python3-mpd pacman: - python-mpd2 pip: diff --git a/platypush/plugins/music/tidal/manifest.yaml b/platypush/plugins/music/tidal/manifest.yaml index e047ad5dd..ba06b2575 100644 --- a/platypush/plugins/music/tidal/manifest.yaml +++ b/platypush/plugins/music/tidal/manifest.yaml @@ -1,7 +1,6 @@ manifest: events: - - platypush.message.event.music.TidalPlaylistUpdatedEvent: when a user playlist - is updated. + - platypush.message.event.music.tidal.TidalPlaylistUpdatedEvent install: pip: - tidalapi >= 0.7.0 diff --git a/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml b/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml index f041778e2..4aa00b1ab 100644 --- a/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml +++ b/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml @@ -1,6 +1,6 @@ manifest: events: - platypush.message.event.sensor import SensorDataChangeEvent: + - platypush.message.event.sensor.SensorDataChangeEvent install: pip: diff --git a/platypush/plugins/serial/manifest.yaml b/platypush/plugins/serial/manifest.yaml index 1af0e50f7..2756fc834 100644 --- a/platypush/plugins/serial/manifest.yaml +++ b/platypush/plugins/serial/manifest.yaml @@ -1,6 +1,6 @@ manifest: events: - - platypush.message.event.sensor.SensorDataChangeEvent: + - platypush.message.event.sensor.SensorDataChangeEvent install: apk: - py3-pyserial diff --git a/platypush/plugins/tts/mimic3/__init__.py b/platypush/plugins/tts/mimic3/__init__.py index d5fdea2b0..0b0fdd049 100644 --- a/platypush/plugins/tts/mimic3/__init__.py +++ b/platypush/plugins/tts/mimic3/__init__.py @@ -1,8 +1,9 @@ -import requests from typing import Optional from urllib.parse import urljoin, urlencode -from platypush.backend.http.app.utils import get_local_base_url +import requests + +from platypush.backend.http.app.utils import get_local_base_url from platypush.context import get_backend from platypush.plugins import action from platypush.plugins.tts import TtsPlugin @@ -10,7 +11,7 @@ from platypush.schemas.tts.mimic3 import Mimic3VoiceSchema class TtsMimic3Plugin(TtsPlugin): - """ + r""" TTS plugin that uses the `Mimic3 webserver `_ provided by `Mycroft `_ as a text-to-speech engine. @@ -42,7 +43,7 @@ class TtsMimic3Plugin(TtsPlugin): voice: str = 'en_UK/apope_low', media_plugin: Optional[str] = None, player_args: Optional[dict] = None, - **kwargs + **kwargs, ): """ :param server_url: Base URL of the web server that runs the Mimic3 engine. @@ -69,6 +70,7 @@ class TtsMimic3Plugin(TtsPlugin): def say( self, text: str, + *_, server_url: Optional[str] = None, voice: Optional[str] = None, player_args: Optional[dict] = None, diff --git a/platypush/plugins/xmpp/__init__.py b/platypush/plugins/xmpp/__init__.py index 2d8ecc9de..a7de00766 100644 --- a/platypush/plugins/xmpp/__init__.py +++ b/platypush/plugins/xmpp/__init__.py @@ -117,7 +117,7 @@ class XmppPlugin(AsyncRunnablePlugin, XmppBasePlugin): auto_accept_invites=auto_accept_invites, restore_state=restore_state, state_file=os.path.expanduser( - state_file or os.path.join(Config.workdir, 'xmpp', 'state.json') + state_file or os.path.join(Config.get_workdir(), 'xmpp', 'state.json') ), ) self._loaded_state = SerializedState() diff --git a/platypush/runner/_app.py b/platypush/runner/_app.py index 6ff9207be..718780e76 100644 --- a/platypush/runner/_app.py +++ b/platypush/runner/_app.py @@ -32,7 +32,7 @@ class ApplicationProcess(ControllableProcess): self.logger.info('Starting application...') with subprocess.Popen( - ['python', '-m', 'platypush.app', *self.args], + [sys.executable, '-m', 'platypush.app', *self.args], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, diff --git a/platypush/runner/_runner.py b/platypush/runner/_runner.py index ce2a443ab..cf89b6a4b 100644 --- a/platypush/runner/_runner.py +++ b/platypush/runner/_runner.py @@ -1,4 +1,6 @@ import logging +import os +import signal import sys from threading import Thread from typing import Optional @@ -23,6 +25,7 @@ class ApplicationRunner: logging.basicConfig(level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger('platypush:runner') self._proc: Optional[ApplicationProcess] = None + self._stream: Optional[CommandStream] = None def _listen(self, stream: CommandStream): """ @@ -48,12 +51,16 @@ class ApplicationRunner: if parsed_args.version: self._print_version() + signal.signal(signal.SIGTERM, lambda *_: self.stop()) + while True: - with CommandStream(parsed_args.ctrl_sock) as stream, ApplicationProcess( + with CommandStream( + parsed_args.ctrl_sock + ) as self._stream, ApplicationProcess( *args, pidfile=parsed_args.pidfile, timeout=self._default_timeout ) as self._proc: try: - self._listen(stream) + self._listen(self._stream) except KeyboardInterrupt: pass @@ -63,6 +70,8 @@ class ApplicationRunner: break + self._stream = None + def run(self, *args: str) -> None: try: self._run(*args) @@ -73,6 +82,10 @@ class ApplicationRunner: if self._proc is not None: self._proc.stop() + if self._stream and self._stream.pid: + os.kill(self._stream.pid, signal.SIGKILL) + self._stream = None + def restart(self): if self._proc is not None: self._proc.mark_for_restart() diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py index 20763fc90..45ffbedb9 100644 --- a/platypush/utils/__init__.py +++ b/platypush/utils/__init__.py @@ -523,7 +523,7 @@ def get_or_generate_jwt_rsa_key_pair(): """ from platypush.config import Config - key_dir = os.path.join(Config.workdir, 'jwt') + key_dir = os.path.join(Config.get_workdir(), 'jwt') priv_key_file = os.path.join(key_dir, 'id_rsa') pub_key_file = priv_key_file + '.pub' @@ -646,4 +646,20 @@ def get_remaining_timeout( return cls(max(0, timeout - (time.time() - start))) +def get_src_root() -> str: + """ + :return: The root source folder of the application. + """ + import platypush + + return os.path.dirname(inspect.getfile(platypush)) + + +def is_root() -> bool: + """ + :return: True if the current user is root/administrator. + """ + return os.getuid() == 0 + + # vim:sw=4:ts=4:et: diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index b26573e7e..b5f6c39b2 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -1,200 +1,617 @@ -import enum +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +import importlib import inspect import json import logging import os import pathlib +import re import shutil +import subprocess +import sys + +from typing import ( + Callable, + Dict, + Generator, + List, + Optional, + Iterable, + Mapping, + Sequence, + Set, + Type, + Union, +) +from typing_extensions import override + import yaml -from abc import ABC, abstractmethod -from typing import Optional, Iterable, Mapping, Callable, Type - -supported_package_managers = { - 'pacman': 'pacman -S', - 'apt': 'apt-get install', -} +from platypush.message.event import Event +from platypush.utils import get_src_root, is_root _available_package_manager = None +logger = logging.getLogger(__name__) -class ManifestType(enum.Enum): +class BaseImage(Enum): + """ + Supported base images for Dockerfiles. + """ + + ALPINE = 'alpine' + DEBIAN = 'debian' + UBUNTU = 'ubuntu' + + def __str__(self) -> str: + """ + Explicit __str__ override for argparse purposes. + """ + return self.value + + +@dataclass +class PackageManager: + """ + Representation of a package manager. + """ + + executable: str + """ The executable name. """ + default_os: str + """ + The default distro whose configuration we should use if this package + manager is detected. + """ + install: Sequence[str] = field(default_factory=tuple) + """ The install command, as a sequence of strings. """ + uninstall: Sequence[str] = field(default_factory=tuple) + """ The uninstall command, as a sequence of strings. """ + list: Sequence[str] = field(default_factory=tuple) + """ The command to list the installed packages. """ + parse_list_line: Callable[[str], str] = field(default_factory=lambda: lambda s: s) + """ + Internal package-manager dependent function that parses the base package + name from a line returned by the list command. + """ + + def _get_installed(self) -> Sequence[str]: + """ + :return: The install context-aware list of installed packages. + It should only used within the context of :meth:`.get_installed`. + """ + + if os.environ.get('DOCKER_CTX'): + # If we're running in a Docker build context, don't run the package + # manager to retrieve the list of installed packages, as the host + # and guest systems have different environments. + return () + + return tuple( + line.strip() + for line in subprocess.Popen( # pylint: disable=consider-using-with + self.list, stdout=subprocess.PIPE + ) + .communicate()[0] + .decode() + .split('\n') + if line.strip() + ) + + def get_installed(self) -> Sequence[str]: + """ + :return: The list of installed packages. + """ + return tuple(self.parse_list_line(line) for line in self._get_installed()) + + +class PackageManagers(Enum): + """ + Supported package managers. + """ + + APK = PackageManager( + executable='apk', + install=('apk', 'add', '--update', '--no-interactive', '--no-cache'), + uninstall=('apk', 'del', '--no-interactive'), + list=('apk', 'list', '--installed'), + default_os='alpine', + parse_list_line=lambda line: re.sub(r'.*\s*\{(.+?)\}\s*.*', r'\1', line), + ) + + APT = PackageManager( + executable='apt', + install=('apt', 'install', '-y'), + uninstall=('apt', 'remove', '-y'), + list=('apt', 'list', '--installed'), + default_os='debian', + parse_list_line=lambda line: line.split('/')[0], + ) + + PACMAN = PackageManager( + executable='pacman', + install=('pacman', '-S', '--noconfirm', '--needed'), + uninstall=('pacman', '-R', '--noconfirm'), + list=('pacman', '-Q'), + default_os='arch', + parse_list_line=lambda line: line.split(' ')[0], + ) + + @classmethod + def get_command(cls, name: str) -> Iterable[str]: + """ + :param name: The name of the package manager executable to get the + command for. + :return: The base command to execute, as a sequence of strings. + """ + pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None) + if not pkg_manager: + raise ValueError(f'Unknown package manager: {name}') + + return pkg_manager.value.install + + @classmethod + def scan(cls) -> Optional["PackageManagers"]: + """ + Get the name of the available package manager on the system, if supported. + """ + # pylint: disable=global-statement + global _available_package_manager + if _available_package_manager: + return _available_package_manager + + available_package_managers = [ + pkg_manager + for pkg_manager in cls + if shutil.which(pkg_manager.value.executable) + ] + + if not available_package_managers: + logger.warning( + '\nYour OS does not provide any of the supported package managers.\n' + 'You may have to install some optional dependencies manually.\n' + 'Supported package managers: %s.\n', + ', '.join([pm.value.executable for pm in cls]), + ) + + return None + + _available_package_manager = available_package_managers[0] + return _available_package_manager + + +class InstallContext(Enum): + """ + Supported installation contexts. + """ + + NONE = None + DOCKER = 'docker' + VENV = 'venv' + + +class ManifestType(Enum): + """ + Manifest types. + """ + PLUGIN = 'plugin' BACKEND = 'backend' +@dataclass +class Dependencies: + """ + Dependencies for a plugin/backend. + """ + + before: List[str] = field(default_factory=list) + """ Commands to execute before the component is installed. """ + packages: Set[str] = field(default_factory=set) + """ System packages required by the component. """ + pip: Set[str] = field(default_factory=set) + """ pip dependencies. """ + after: List[str] = field(default_factory=list) + """ Commands to execute after the component is installed. """ + pkg_manager: Optional[PackageManagers] = None + """ Override the default package manager detected on the system. """ + install_context: InstallContext = InstallContext.NONE + """ The installation context - Docker, virtual environment or bare metal. """ + base_image: Optional[BaseImage] = None + """ Base image used in case of Docker installations. """ + + @property + def _is_venv(self) -> bool: + """ + :return: True if the dependencies scanning logic is running either in a + virtual environment or in a virtual environment preparation + context. + """ + return ( + self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix + ) + + @property + def _is_docker(self) -> bool: + """ + :return: True if the dependencies scanning logic is running either in a + Docker environment. + """ + return ( + self.install_context == InstallContext.DOCKER + or bool(os.environ.get('DOCKER_CTX')) + or os.path.isfile('/.dockerenv') + ) + + @property + def _wants_sudo(self) -> bool: + """ + :return: True if the system dependencies should be installed with sudo. + """ + return not (self._is_docker or is_root()) + + @staticmethod + def _get_requirements_dir() -> str: + """ + :return: The root folder for the base installation requirements. + """ + return os.path.join(get_src_root(), 'install', 'requirements') + + @classmethod + def _parse_requirements_file( + cls, + requirements_file: str, + install_context: InstallContext = InstallContext.NONE, + ) -> Iterable[str]: + """ + :param requirements_file: The path to the requirements file. + :return: The list of requirements to install. + """ + with open(requirements_file, 'r') as f: + return { + line.strip() + for line in f + if not ( + not line.strip() + or line.strip().startswith('#') + # Virtual environments will install all the Python + # dependencies via pip, so we should skip them here + or ( + install_context == InstallContext.VENV + and cls._is_python_pkg(line.strip()) + ) + ) + } + + @classmethod + def _get_base_system_dependencies( + cls, + pkg_manager: Optional[PackageManagers] = None, + install_context: InstallContext = InstallContext.NONE, + ) -> Iterable[str]: + """ + :return: The base system dependencies that should be installed on the + system. + """ + + # Docker images will install the base packages through their own + # dedicated shell script, so don't report their base system + # requirements here. + if not (pkg_manager and install_context != InstallContext.DOCKER): + return set() + + return cls._parse_requirements_file( + os.path.join( + cls._get_requirements_dir(), pkg_manager.value.default_os + '.txt' + ), + install_context, + ) + + @staticmethod + def _is_python_pkg(pkg: str) -> bool: + """ + Utility function that returns True if a given package is a Python + system package. These should be skipped during a virtual + environment installation, as the virtual environment will be + installed via pip. + """ + tokens = pkg.split('-') + return len(tokens) > 1 and tokens[0] in {'py3', 'python3', 'python'} + + @classmethod + def from_config( + cls, + conf_file: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, + install_context: InstallContext = InstallContext.NONE, + base_image: Optional[BaseImage] = None, + ) -> "Dependencies": + """ + Parse the required dependencies from a configuration file. + """ + if not pkg_manager: + pkg_manager = PackageManagers.scan() + + base_system_deps = cls._get_base_system_dependencies( + pkg_manager=pkg_manager, install_context=install_context + ) + + deps = cls( + packages=set(base_system_deps), + pkg_manager=pkg_manager, + install_context=install_context, + base_image=base_image, + ) + + for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager): + deps.before += manifest.install.before + deps.pip.update(manifest.install.pip) + deps.packages.update(manifest.install.packages) + deps.after += manifest.install.after + + return deps + + def to_pkg_install_commands(self) -> Generator[str, None, None]: + """ + Generates the package manager commands required to install the given + dependencies on the system. + """ + + wants_sudo = not (self._is_docker or is_root()) + pkg_manager = self.pkg_manager or PackageManagers.scan() + + if self.packages and pkg_manager: + installed_packages = pkg_manager.value.get_installed() + to_install = sorted( + pkg + for pkg in self.packages # type: ignore + if pkg not in installed_packages + and not ( + self.install_context == InstallContext.VENV + and self._is_python_pkg(pkg) + ) + ) + + if to_install: + yield ' '.join( + [ + *(['sudo'] if wants_sudo else []), + *pkg_manager.value.install, + *to_install, + ] + ) + + def to_pip_install_commands(self, full_command=True) -> Generator[str, None, None]: + """ + Generates the pip commands required to install the given dependencies on + the system. + + :param full_command: Whether to return the full pip command to execute + (as a single string) or the list of packages that will be installed + through another script. + """ + wants_break_system_packages = not ( + # Docker installations shouldn't require --break-system-packages in + # pip, except for Debian + (self._is_docker and self.base_image != BaseImage.DEBIAN) + # --break-system-packages has been introduced in Python 3.10 + or sys.version_info < (3, 11) + # If we're in a virtual environment then we don't need + # --break-system-packages + or self._is_venv + ) + + if self.pip: + deps = sorted(self.pip) + if full_command: + yield ( + 'pip install -U --no-input --no-cache-dir ' + + ( + '--break-system-packages ' + if wants_break_system_packages + else '' + ) + + ' '.join(deps) + ) + else: + for dep in deps: + yield dep + + def to_install_commands(self) -> Generator[str, None, None]: + """ + Generates the commands required to install the given dependencies on + this system. + """ + for cmd in self.before: + yield cmd + + for cmd in self.to_pkg_install_commands(): + yield cmd + + for cmd in self.to_pip_install_commands(): + yield cmd + + for cmd in self.after: + yield cmd + + class Manifest(ABC): """ Base class for plugin/backend manifests. """ - def __init__(self, package: str, description: Optional[str] = None, - install: Optional[Iterable[str]] = None, events: Optional[Mapping] = None, **_): + + def __init__( + self, + package: str, + description: Optional[str] = None, + install: Optional[Dict[str, Iterable[str]]] = None, + events: Optional[Mapping] = None, + pkg_manager: Optional[PackageManagers] = None, + **_, + ): + self._pkg_manager = pkg_manager or PackageManagers.scan() self.description = description - self.install = install or {} - self.events = events or {} - self.logger = logging.getLogger(__name__) + self.install = self._init_deps(install or {}) + self.events = self._init_events(events or {}) self.package = package self.component_name = '.'.join(package.split('.')[2:]) self.component = None - @classmethod @property @abstractmethod - def component_getter(self) -> Callable[[str], object]: - raise NotImplementedError + def manifest_type(self) -> ManifestType: + """ + :return: The type of the manifest. + """ + + def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies: + deps = Dependencies() + for key, items in install.items(): + if key == 'pip': + deps.pip.update(items) + elif key == 'before': + deps.before += items + elif key == 'after': + deps.after += items + elif self._pkg_manager and key == self._pkg_manager.value.executable: + deps.packages.update(items) + + return deps + + @staticmethod + def _init_events( + events: Union[Iterable[str], Mapping[str, Optional[str]]] + ) -> Dict[Type[Event], str]: + evt_dict = events if isinstance(events, Mapping) else {e: None for e in events} + ret = {} + + for evt_name, doc in evt_dict.items(): + evt_module_name, evt_class_name = evt_name.rsplit('.', 1) + try: + evt_module = importlib.import_module(evt_module_name) + evt_class = getattr(evt_module, evt_class_name) + except Exception as e: + raise AssertionError(f'Could not load event {evt_name}: {e}') from e + + ret[evt_class] = doc or evt_class.__doc__ + + return ret @classmethod - def from_file(cls, filename: str) -> "Manifest": + def from_file( + cls, filename: str, pkg_manager: Optional[PackageManagers] = None + ) -> "Manifest": + """ + Parse a manifest filename into a ``Manifest`` class. + """ with open(str(filename), 'r') as f: manifest = yaml.safe_load(f).get('manifest', {}) assert 'type' in manifest, f'Manifest file {filename} has no type field' comp_type = ManifestType(manifest.pop('type')) - manifest_class = _manifest_class_by_type[comp_type] - return manifest_class(**manifest) + manifest_class = cls.by_type(comp_type) + return manifest_class(**manifest, pkg_manager=pkg_manager) @classmethod - def from_class(cls, clazz) -> "Manifest": - return cls.from_file(os.path.dirname(inspect.getfile(clazz))) + def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]: + """ + :return: The manifest class corresponding to the given manifest type. + """ + if manifest_type == ManifestType.PLUGIN: + return PluginManifest + if manifest_type == ManifestType.BACKEND: + return BackendManifest - @classmethod - def from_component(cls, comp) -> "Manifest": - return cls.from_class(comp.__class__) - - def get_component(self): - try: - self.component = self.component_getter(self.component_name) - except Exception as e: - self.logger.warning(f'Could not load {self.component_name}: {e}') - - return self.component + raise ValueError(f'Unknown manifest type: {manifest_type}') def __repr__(self): - return json.dumps({ - 'description': self.description, - 'install': self.install, - 'events': self.events, - 'type': _manifest_type_by_class[self.__class__].value, - 'package': self.package, - 'component_name': self.component_name, - }) + """ + :return: A JSON serialized representation of the manifest. + """ + return json.dumps( + { + 'description': self.description, + 'install': self.install, + 'events': { + '.'.join([evt_type.__module__, evt_type.__name__]): doc + for evt_type, doc in self.events.items() + }, + 'type': self.manifest_type.value, + 'package': self.package, + 'component_name': self.component_name, + } + ) class PluginManifest(Manifest): - @classmethod + """ + Plugin manifest. + """ + @property - def component_getter(self): - from platypush.context import get_plugin - return get_plugin + @override + def manifest_type(self) -> ManifestType: + return ManifestType.PLUGIN +# pylint: disable=too-few-public-methods class BackendManifest(Manifest): - @classmethod + """ + Backend manifest. + """ + @property - def component_getter(self): - from platypush.context import get_backend - return get_backend + @override + def manifest_type(self) -> ManifestType: + return ManifestType.BACKEND -_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = { - ManifestType.PLUGIN: PluginManifest, - ManifestType.BACKEND: BackendManifest, -} +class Manifests: + """ + General-purpose manifests utilities. + """ -_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = { - cls: t for t, cls in _manifest_class_by_type.items() -} + @staticmethod + def by_base_class( + base_class: Type, pkg_manager: Optional[PackageManagers] = None + ) -> Generator[Manifest, None, None]: + """ + Get all the manifest files declared under the base path of a given class + and parse them into :class:`Manifest` objects. + """ + for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob( + 'manifest.yaml' + ): + yield Manifest.from_file(str(mf), pkg_manager=pkg_manager) + @staticmethod + def by_config( + conf_file: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, + ) -> Generator[Manifest, None, None]: + """ + Get all the manifest objects associated to the extensions declared in a + given configuration file. + """ + from platypush.config import Config -def scan_manifests(base_class: Type) -> Iterable[str]: - for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob('manifest.yaml'): - yield str(mf) + conf_args = [] + if conf_file: + conf_args.append(conf_file) + Config.init(*conf_args) + app_dir = get_src_root() -def get_manifests(base_class: Type) -> Iterable[Manifest]: - return [ - Manifest.from_file(mf) - for mf in scan_manifests(base_class) - ] + for name in Config.get_backends().keys(): + yield Manifest.from_file( + os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'), + pkg_manager=pkg_manager, + ) - -def get_components(base_class: Type) -> Iterable: - manifests = get_manifests(base_class) - components = {mf.get_component() for mf in manifests} - return {comp for comp in components if comp is not None} - - -def get_manifests_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Manifest]: - import platypush - from platypush.config import Config - - conf_args = [] - if conf_file: - conf_args.append(conf_file) - - Config.init(*conf_args) - app_dir = os.path.dirname(inspect.getfile(platypush)) - manifest_files = set() - - for name in Config.get_backends().keys(): - manifest_files.add(os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml')) - - for name in Config.get_plugins().keys(): - manifest_files.add(os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml')) - - return { - manifest_file: Manifest.from_file(manifest_file) - for manifest_file in manifest_files - } - - -def get_dependencies_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Iterable[str]]: - manifests = get_manifests_from_conf(conf_file) - deps = { - 'pip': set(), - 'packages': set(), - 'exec': set(), - } - - for manifest in manifests.values(): - deps['pip'].update(manifest.install.get('pip', set())) - deps['exec'].update(manifest.install.get('exec', set())) - has_requires_packages = len([ - section for section in manifest.install.keys() - if section in supported_package_managers - ]) > 0 - - if has_requires_packages: - pkg_manager = get_available_package_manager() - deps['packages'].update(manifest.install.get(pkg_manager, set())) - - return deps - - -def get_install_commands_from_conf(conf_file: Optional[str] = None) -> Mapping[str, str]: - deps = get_dependencies_from_conf(conf_file) - return { - 'pip': f'pip install {" ".join(deps["pip"])}', - 'exec': deps["exec"], - 'packages': f'{supported_package_managers[_available_package_manager]} {" ".join(deps["packages"])}' - if deps['packages'] else None, - } - - -def get_available_package_manager() -> str: - global _available_package_manager - if _available_package_manager: - return _available_package_manager - - available_package_managers = [ - pkg_manager for pkg_manager in supported_package_managers.keys() - if shutil.which(pkg_manager) - ] - - assert available_package_managers, ( - 'Your OS does not provide any of the supported package managers. ' - f'Supported package managers: {supported_package_managers.keys()}' - ) - - _available_package_manager = available_package_managers[0] - return _available_package_manager + for name in Config.get_plugins().keys(): + yield Manifest.from_file( + os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'), + pkg_manager=pkg_manager, + ) diff --git a/setup.cfg b/setup.cfg index e80481961..a6014566e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,5 +11,6 @@ max-line-length = 120 extend-ignore = E203 W503 + SIM104 SIM105 diff --git a/setup.py b/setup.py index 1f2bf8d99..b9ea662eb 100755 --- a/setup.py +++ b/setup.py @@ -49,9 +49,9 @@ setup( 'console_scripts': [ 'platypush=platypush:main', 'platydock=platypush.platydock:main', + 'platyvenv=platypush.platyvenv:main', ], }, - scripts=['bin/platyvenv'], long_description=readfile('README.md'), long_description_content_type='text/markdown', classifiers=[