revil-O/platypush
1
0
Fork 0
forked from platypush/platypush
Code Pull requests Activity

Merge pull request 'Better Docker support' (#277) from 276/better-docker into master

Browse source 
Reviewed-on: platypush/platypush#277
This commit is contained in:
Fabio Manganiello 2023-09-04 02:49:14 +02:00
commit 82ef928d5b
66 changed files with 2919 additions and 1378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.dockerignore Normal file
View file

@ -0,0 +1,6 @@
**/.git
**/node_modules
**/__pycache__
**/venv
**/.mypy_cache
**/build

2
.drone.yml
View file

@ -29,7 +29,7 @@ steps:
- ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
- git config --global --add safe.directory $PWD
- git remote add github git@github.com:/BlackLight/platypush.git
- git pull github master
- git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github"
- git push --all -v github
- name: docs

3
.gitignore vendored
View file

@ -10,7 +10,7 @@ package.sh
platypush/backend/http/static/resources/*
docs/build
.idea/
config
/config
platypush/backend/http/static/css/*/.sass-cache/
.vscode
platypush/backend/http/static/js/lib/vue.js
@ -24,3 +24,4 @@ coverage.xml
Session.vim
/jsconfig.json
/package.json
/Dockerfile

1
MANIFEST.in
View file

@ -1,4 +1,5 @@
recursive-include platypush/backend/http/webapp/dist *
recursive-include platypush/install *
include platypush/plugins/http/webpage/mercury-parser.js
include platypush/config/*.yaml
global-include manifest.yaml

249
bin/platyvenv
View file

@ -1,249 +0,0 @@
#!/bin/bash
##############################################################################
# This script allows you to easily manage Platypush instances through Python #
# virtual environment. You can build environments from a config.yaml file #
# and automatically managed the required dependencies, as well as start, #
# stop and remove them #
# #
# @author: Fabio Manganiello <fabio@platypush.tech> #
# @licence: MIT #
##############################################################################
workdir="$HOME/.local/share/platypush/venv"
function build {
cfgfile=
while getopts ':c:' opt; do
case ${opt} in
c)
cfgfile=$OPTARG;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1;;
:)
echo "Option -$OPTARG requires the path to a Platypush configuration file" >&2
exit 1;;
esac
done
if [[ -z "$cfgfile" ]]; then
echo "Usage: $0 build -c <path-to-platypush-config-file>" >&2
exit 1
fi
echo "Parsing configuration file"
pip_cmd=
pkg_cmd=
includes=()
cmd_exec=()
while read -r line; do
if echo "$line" | grep -E "^pip:\s*"; then
pip_cmd="$(echo "$line" | sed -r -e 's/^pip:\s*(.*)'/\\1/)"
elif echo "$line" | grep -E "^packages:\s*"; then
pkg_cmd="$(echo "$line" | sed -r -e 's/^packages:\s*(.*)'/\\1/)"
elif echo "$line" | grep -E "^exec:\s*"; then
cmd_exec+=("$(echo "$line" | sed -r -e 's/^exec:\s*(.*)'/\\1/)")
elif echo "$line" | grep -E "^include:\s*"; then
includes+=("$(echo "$line" | sed -r -e 's/^include:\s*(.*)'/\\1/)")
elif echo "$line" | grep -E "^device_id:\s*"; then
device_id="$(echo "$line" | sed -r -e 's/^device_id:\s*(.*)'/\\1/)"
fi
done <<< "$(python <<EOF
from platypush.config import Config
from platypush.utils.manifest import get_install_commands_from_conf
deps = get_install_commands_from_conf('$(realpath "${cfgfile}")')
print(f'device_id: {Config.get("device_id")}')
if deps.get('pip'):
print(f'pip: {deps["pip"]}')
if deps.get('packages'):
print(f'packages: {deps["packages"]}')
for cmd in deps.get('exec', []):
print(f'exec: {cmd}')
for include in Config._included_files:
print(f'include: {include}')
EOF
)"
envdir="${workdir}/${device_id}"
etcdir="${envdir}/etc/platypush"
echo "Preparing virtual environment for device $device_id"
mkdir -p "$envdir"
mkdir -p "$etcdir"
srcdir=$(dirname "$cfgfile")
for ((i=0; i < ${#includes[@]}; i++)); do
incdir=$(dirname "${includes[$i]}")
incdir=$(realpath --relative-to="$srcdir" "$incdir")
destdir="$etcdir/$incdir"
mkdir -p "$destdir"
cp "${includes[$i]}" "$destdir"
done
cp "$cfgfile" "$etcdir/config.yaml"
cfgfile="${etcdir}/config.yaml"
python3 -m venv "${envdir}"
cd "${envdir}" || exit 1
source "${envdir}/bin/activate"
echo "Installing required dependencies"
# shellcheck disable=SC2086
[ -n "${pkg_cmd}" ] && sudo ${pkg_cmd}
[ -n "${pip_cmd}" ] && ${pip_cmd}
for ((i=0; i < ${#cmd_exec[@]}; i++)); do
${cmd_exec[$i]}
done
pip install --upgrade git+https://git.platypush.tech/platypush/platypush.git
echo "Platypush virtual environment prepared under $envdir"
}
function start {
if [[ -z "$1" ]]; then
echo "Usage: $0 start <env-name>" >&2
exit 1
fi
env=$1
envdir="${workdir}/${env}"
rundir="${envdir}/var/run"
pidfile="${rundir}/platypush.pid"
cfgfile="${envdir}/etc/platypush/config.yaml"
if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2
exit 1
fi
mkdir -p "${rundir}"
if [[ -f "$pidfile" ]]; then
if pgrep -F "${pidfile}"; then
echo "Another instance (PID $(cat "${pidfile}")) is running, please stop that instance first"
exit 1
fi
echo "A PID file was found but the process does not seem to be running, starting anyway"
rm -f "$pidfile"
fi
python3 -m venv "${envdir}"
cd "${envdir}" || exit 1
source bin/activate
bin/platypush -c "$cfgfile" -P "$pidfile" &
start_time=$(date +'%s')
timeout=30
while :; do
[[ -f "$pidfile" ]] && break
now=$(date +'%s')
elapsed=$(( now-start_time ))
if (( elapsed >= timeout )); then
echo "Platypush instance '$env' did not start within $timeout seconds" >&2
exit 1
fi
echo -n '.'
sleep 1
done
pid=$(cat "$pidfile")
echo
echo "Platypush environment $env started with PID $pid"
wait "${pid}"
echo "Platypush environment $env terminated"
}
function stop {
if [[ -z "$1" ]]; then
echo "Usage: $0 stop <env-name>" >&2
exit 1
fi
env=$1
envdir="${workdir}/${env}"
rundir="${envdir}/var/run"
pidfile="${rundir}/platypush.pid"
if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2
exit 1
fi
if [[ ! -f "$pidfile" ]]; then
echo "No pidfile found for instance \"${env}\""
exit 1
fi
pid=$(cat "$pidfile")
pids="$pid $(ps --no-headers -o pid= --ppid "$pid")"
# shellcheck disable=SC2086
kill -9 ${pids}
rm -f "$pidfile"
echo "Instance '$env' with PID $pid stopped"
}
function rme {
if [[ -z "$1" ]]; then
echo "Usage: $0 rm <env-name>" >&2
exit 1
fi
envdir="${workdir}/$1"
rundir="${envdir}/var/run"
pidfile="${rundir}/platypush.pid"
if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2
exit 1
fi
if [[ -f "$pidfile" ]]; then
if pgrep -F "${pidfile}"; then
echo "Another instance (PID $(cat "$pidfile")) is running, please stop that instance first"
exit 1
fi
echo "A PID file was found but the process does not seem to be running, removing anyway"
fi
echo "WARNING: This operation will permanently remove the Platypush environment $1"
echo -n "Are you sure you want to continue? (y/N) "
IFS= read -r answer
echo "$answer" | grep -E '^[yY]' >/dev/null || exit 0
rm -rf "$envdir"
echo "$envdir removed"
}
function usage {
echo "Usage: $0 <build|start|stop|rm> [options]" >&2
exit 1
}
if (( $# < 1 )); then
usage
fi
action=$1
shift
mkdir -p "${workdir}"
# shellcheck disable=SC2048,SC2086
case ${action} in
'build') build $*;;
'start') start $*;;
'stop') stop $*;;
'rm') rme $*;;
*) usage;;
esac

379
examples/conf/config.yaml
View file

@ -1,379 +0,0 @@
#################################################################################
# Sample platypush configuration file.
# Edit it and copy it to /etc/platypush/config.yaml for system installation or to
# ~/.config/platypush/config.yaml for user installation (recommended).
#################################################################################
# --
# include directive example
# --
#
# You can split your configuration over multiple files
# and use the include directive to import them in your configuration.
# Relative paths are also supported, and computed using the config.yaml
# installation directory as base folder. Symlinks are also supported.
#
# Using multiple files is encouraged in the case of large configurations
# that can easily end up in a messy config.yaml file, as they help you
# keep your configuration more organized.
#include:
# - include/logging.yaml
# - include/media.yaml
# - include/sensors.yaml
# platypush logs on stdout by default. You can use the logging section to specify
# an alternative file or change the logging level.
#logging:
# filename: ~/.local/log/platypush/platypush.log
# level: INFO
# The device_id is used by many components of platypush and it should uniquely
# identify a device in your network. If nothing is specified then the hostname
# will be used.
#device_id: my_device
## --
## Plugin configuration examples
## --
#
# Plugins configuration is very straightforward. Each plugin is mapped to
# a plugin class. The methods of the class with @action annotation will
# be exported as runnable actions, while the __init__ parameters are
# configuration attributes that you can initialize in your config.yaml.
# Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html
#
# In this example we'll configure the light.hue plugin, see
# https://docs.platypush.tech/en/latest/platypush/plugins/light.hue.html
# for reference. You can easily install the required dependencies for the plugin through
# pip install 'platypush[hue]'
light.hue:
# IP address or hostname of the Hue bridge
bridge: 192.168.1.10
# Groups that will be handled by default if nothing is specified on the request
groups:
- Living Room
# Example configuration of music.mpd plugin, see
# https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html
# You can easily install the dependencies through pip install 'platypush[mpd]'
music.mpd:
host: localhost
port: 6600
# Example configuration of media.chromecast plugin, see
# https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html
# You can easily install the dependencies through pip install 'platypush[chromecast]'
media.chromecast:
chromecast: Living Room TV
# Plugins with empty configuration can also be explicitly enabled by specifying
# enabled=True or disabled=False (it's a good practice if you want the
# corresponding web panel to be enabled, if available)
camera.pi:
enabled: True
# Support for calendars - in this case Google and Facebook calendars
# Installing the dependencies: pip install 'platypush[ical,google]'
calendar:
calendars:
- type: platypush.plugins.google.calendar.GoogleCalendarPlugin
- type: platypush.plugins.calendar.ical.CalendarIcalPlugin
url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key
## --
## Backends configuration examples
## --
#
# Backends are basically threads that run in the background and listen for something
# to happen and either trigger events or provide additional services on top of platypush.
# Just like plugins, backends are classes whose configuration matches one-to-one the
# supported parameters on the __init__ methods. You can check the documentation for the
# available backends here: https://docs.platypush.tech/en/latest/backends.html.
# Moreover, most of the backends will generate events that you can react to through custom
# event hooks. Check here for the events documentation:
# https://docs.platypush.tech/en/latest/events.html
#
# You may usually want to enable the HTTP backend, as it provides many useful features on
# top of platypush. Among those:
#
# - Expose the /execute endpoint, that allows you to send requests to platypush through a
# JSON-RPC interface.
# - Web panel, one of the key additiona features of platypush. Many plugins will expose web
# panel tabs for e.g. accessing and controlling lights, music, media and sensors.
# - Dashboard: platypush can be configured to show a custom dashboard on your screens with
# e.g. music platypush and weather info, news, upcoming calendar events and photo carousel.
# - Streaming support - the HTTP backend makes it possible to stream local media to other
# devices - e.g. Chromecasts and external browsers.
#
# To install the HTTP backend dependencies simply run 'pip install "platypush[http]"'
backend.http:
# Listening port
port: 8008
# Through resource_dirs you can specify external folders whose content can be accessed on
# the web server through a custom URL. In the case below we have a Dropbox folder containing
# our pictures and we mount it to the '/carousel' endpoint.
resource_dirs:
carousel: /mnt/hd/photos/carousel
# The HTTP poll backend is a versatile backend that can monitor for HTTP-based resources and
# trigger events whenever new entries are available. In the example below we show how to use
# the backend to listen for changes on a set of RSS feeds. New content will be stored by default
# on a SQLite database under ~/.local/share/platypush/feeds/rss.db.
# Install the required dependencies through 'pip install "platypush[rss,db]"'
backend.http.poll:
requests:
- type: platypush.backend.http.request.rss.RssUpdates # HTTP poll type (RSS)
# Remote URL
url: http://www.theguardian.com/rss/world
# Custom title
title: The Guardian - World News
# How often we should check for changes
poll_seconds: 600
# Maximum number of new entries to be processed
max_entries: 10
- type: platypush.backend.http.request.rss.RssUpdates
url: http://www.physorg.com/rss-feed
title: Phys.org
poll_seconds: 600
max_entries: 10
- type: platypush.backend.http.request.rss.RssUpdates
url: http://feeds.feedburner.com/Techcrunch
title: Tech Crunch
poll_seconds: 600
max_entries: 10
- type: platypush.backend.http.request.rss.RssUpdates
url: http://www.nytimes.com/services/xml/rss/nyt/HomePage.xml
title: The New York Times
poll_seconds: 300
max_entries: 10
# MQTT backend. Installed required dependencies through 'pip install "platypush[mqtt]"'
backend.mqtt:
# Remote MQTT server IP or hostname
host: mqtt-server
# By default the backend will listen for messages on the platypush_bus_mq/device_id
# topic, but you can change the prefix using the topic attribute
# topic: MyBus
# Raw TCP socket backend. It can run commands sent as JSON over telnet or netcat
#backend.tcp:
# port: 3333
## --
## Assistant configuration examples
## --
#
# Both Google Assistant and Alexa voice assistant interfaces are supported by platypush.
# You can easily make your custom voice assistant with a RaspberryPi and a USB microphone,
# or on your laptop. Note however that the Alexa integration is still experimental
# (mostly because of glitches and bugs on the avs package provided by Amazon), while the
# Google Assistant support should be more robust. The recommended way of triggering a
# hotword ('OK Google', 'Alexa' or any custom hotword you like) is through the snowboy
# backend (install it through 'pip install "platypush[hotword]"'). You can download custom
# voice model files (.umdl) from https://snowboy.kitt.ai.
backend.assistant.snowboy:
# Microphone audio gain
audio_gain: 1.1
models:
# "Computer" hotword model
computer:
# UMDL file path
voice_model_file: ~/.local/share/snowboy/models/computer.umdl
# Plugin to use (Google Assistant)
assistant_plugin: assistant.google.pushtotalk
# Language assistant (Italian)
assistant_language: it-IT
# Sound to play when the hotword is detected
detect_sound: ~/.local/share/sounds/hotword.wav
# Model sensitivity
sensitivity: 0.4
# "OK Google" hotword model
ok_google:
voice_model_file: ~/.local/share/snowboy/models/OK Google.pmdl
assistant_plugin: assistant.google.pushtotalk
assistant_language: en-US
detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav
sensitivity: 0.4
# "Alexa" voice model
alexa:
voice_model_file: ~/.local/share/snowboy/models/Alexa.pmdl
assistant_plugin: assistant.echo
assistant_language: en-US
detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav
sensitivity: 0.5
# Install Alexa dependencies with 'pip install "platypush[alexa]"'
assistant.echo:
audio_player: mplayer
# Install Google Assistant dependencies with 'pip install "platypush[google-assistant-legacy]"'
assistant.google:
enabled: True
backend.assistant.google:
enabled: True
## --
## Procedure examples
## --
#
# Procedures are lists of actions that can be executed synchronously (default) or in parallel
# (procedure.async. prefix). Basic flow control operators (if/else/for) are also available.
# You can also access Python variables and evaluate Python expressions by using the ${} expressions.
# The 'context' special variable is a name->value dictionary containing the items returned from
# previous actions - for example if an action returned '{"status": "ok", "temperature":21.5}' then
# the following actions can access those variables through ${status} and ${temperature} respectively,
# and you can also add things like '- if ${temperature > 20.0}' or '- for ${temp in temperature_values}'.
# Alternatively, you can access those variable also through ${context.get('status')} or ${context.get('temperature')}.
# Other special variables that you can use in your procedures:
#
# - output: Will contain the parsed output of the previous action
# - errors: Will contain the errors of the previous action
# - event: If the procedure is executed within an event hook, it contains the event that triggered the hook
#
# An example procedure that can be called when you arrive home. You can run this procedure by sending a JSON
# message like this on whichever backend you like (HTTP, websocket, TCP, Redis, MQTT, Node-RED, Pushbullet...)
# {"type":"request", "action":"procedure.at_home"}
# You can for instance install Tasker+AutoLocation on your mobile and send this message whenever you enter
# your home area.
procedure.at_home:
# Set the db variable HOME to 1
- action: variable.set
args:
HOME: 1
# Check the luminosity level from a connected LTR559 sensor
- action: gpio.sensor.ltr559.get_data
# If it's below a certain threshold turn on the lights
- if ${int(light or 0) < 110}:
- action: light.hue.on
# Say a welcome home message. Install dependencies through 'pip install "platypush[google-tts]"'
- action: tts.google.say
args:
text: Welcome home
# Start the music
- action: music.mpd.play
# Procedure that will be execute when you're outside of home
procedure.outside_home:
# Unset the db variable HOME
- action: variable.unset
args:
name: HOME
# Stop the music
- action: music.mpd.stop
# Turn off the lights
- action: light.hue.off
# Start the camera streaming. Install the Pi Camera dependencies through
# 'pip install "platypush[picamera]"'
- action: camera.pi.start_streaming
args:
listen_port: 2222
# Procedures can also take optional arguments. The example below show a
# generic procedure to send a request to another platypush host over MQTT
# given target, action and args
procedure.send_request(target, action, args):
- action: mqtt.send_message
args:
topic: platypush_bus_mq/${target}
host: mqtt-server
port: 1883
msg:
type: request
target: ${target}
action: ${action}
args: ${args}
## --
## Event hook examples
## --
#
# Event hooks are procedures that are run when a certain condition is met.
# Check the documentation of the backends to see which events they can trigger.
# An event hook consists of two parts: an 'if' field that specifies on which
# event the hook will be triggered (type and attributes content), and a 'then'
# field that uses the same syntax as procedures to specify a list of actions to
# execute when the event is matched.
#
# The example below plays the music on mpd/mopidy when your voice assistant
# triggers a speech recognized event with "play the music" content.
event.hook.PlayMusicAssistantCommand:
if:
type: platypush.message.event.assistant.SpeechRecognizedEvent
# Note that basic regexes are supported, so the hook will be triggered
# both if you say "play the music" and "play music"
phrase: "play (the)? music"
then:
- action: music.mpd.play
# This will turn on the lights when you say "turn on the lights"
event.hook.TurnOnLightsCommand:
if:
type: platypush.message.event.assistant.SpeechRecognizedEvent
phrase: "turn on (the)? lights?"
then:
- action: light.hue.on
# This will play a song by a specified artist
event.hook.SearchSongVoiceCommand:
if:
type: platypush.message.event.assistant.SpeechRecognizedEvent
# Note that you can use the ${} operator in event matching to
# extract part of the matched string into context variables that
# can be accessed in your event hook.
phrase: "play ${title} by ${artist}"
then:
- action: music.mpd.clear
- action: music.mpd.search
args:
filter:
artist: ${artist}
title: ${title}
# Play the first search result
- action: music.mpd.play
args:
resource: ${output[0]['file']}
# This event will scrobble newly listened tracks on mpd/mopidy to last.fm
event.hook.ScrobbleNewTrack:
if:
type: platypush.message.event.music.NewPlayingTrackEvent
then:
- action: lastfm.scrobble
args:
artist: ${track['artist']}
title: ${track['title']}
- action: lastfm.update_now_playing
args:
artist: ${track['artist']}
title: ${track['title']}
## --
## Cron examples
## --
#
# Cronjobs allow you to execute procedures at periodic intervals.
# Standard UNIX cron syntax is supported, plus an optional 6th indicator
# at the end of the expression to run jobs with second granularity.
# The example below executes a script at intervals of 1 minute.
cron.TestCron:
cron_expression: '* * * * *'
actions:
- action: shell.exec
args:
cmd: ~/bin/myscript.sh

1
examples/config/config.yaml Symbolic link
View file

@ -0,0 +1 @@
../../platypush/config/config.yaml

0
examples/conf/dashboard.xml → examples/config/dashboard.xml
View file

16
examples/conf/hook.py → examples/config/hook.py
View file

@ -12,7 +12,10 @@ from platypush.utils import run
from platypush.event.hook import hook
# Event types that you want to react to
from platypush.message.event.assistant import ConversationStartEvent, SpeechRecognizedEvent
from platypush.message.event.assistant import (
ConversationStartEvent,
SpeechRecognizedEvent,
)
@hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
@ -23,10 +26,13 @@ def on_music_play_command(event, title=None, artist=None, **context):
Note that in this specific case we can leverage the token-extraction feature of SpeechRecognizedEvent through
${} that operates on regex-like principles to extract any text that matches the pattern into context variables.
"""
results = run('music.mpd.search', filter={
'artist': artist,
'title': title,
})
results = run(
'music.mpd.search',
filter={
'artist': artist,
'title': title,
},
)
if results:
run('music.mpd.play', results[0]['file'])

19
examples/docker/Dockerfile
View file

@ -1,19 +0,0 @@
# Sample Dockerfile. Use platydock -c /path/to/custom/config.yaml
# to generate your custom Dockerfile.
FROM python:3.11-alpine
RUN mkdir -p /install /app
COPY . /install
RUN apk add --update --no-cache redis
RUN apk add --update --no-cache --virtual build-base g++ rust linux-headers
RUN pip install -U pip
RUN cd /install && pip install .
RUN apk del build-base g++ rust linux-headers
EXPOSE 8008
VOLUME /app/config
VOLUME /app/workdir
CMD python -m platypush --start-redis --config-file /app/config/config.yaml --workdir /app/workdir

35
examples/nginx/nginx.sample.conf
View file

@ -1,12 +1,17 @@
# An nginx configuration that can be used to reverse proxy connections to your
# Platypush' HTTP service.
server {
server_name my-platypush-host.domain.com;
upstream platypush {
# The address and port where the HTTP backend is listening
server 127.0.0.1:8008;
}
# Proxy standard HTTP connections to your Platypush IP
server {
server_name platypush.example.com;
# Proxy standard HTTP connections
location / {
proxy_pass http://my-platypush-host:8008/;
proxy_pass http://platypush;
client_max_body_size 5M;
proxy_read_timeout 60;
@ -18,21 +23,33 @@ server {
}
# Proxy websocket connections
location ~ ^/ws/(.*)$ {
proxy_pass http://10.0.0.2:8008/ws/$1;
location /ws/ {
proxy_pass http://platypush;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_redirect off;
proxy_http_version 1.1;
client_max_body_size 200M;
client_max_body_size 5M;
proxy_set_header Host $http_host;
}
# Optional SSL configuration - using Let's Encrypt certificates in this case
# listen 443 ssl;
# ssl_certificate /etc/letsencrypt/live/my-platypush-host.domain.com/fullchain.pem;
# ssl_certificate_key /etc/letsencrypt/live/my-platypush-host.domain.com/privkey.pem;
# ssl_certificate /etc/letsencrypt/live/platypush.example.com/fullchain.pem;
# ssl_certificate_key /etc/letsencrypt/live/platypush.example.com/privkey.pem;
# include /etc/letsencrypt/options-ssl-nginx.conf;
# ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
}
# Uncomment if you are using SSL and you want to force an HTTPS upgrade to
# clients connecting over the port 80
# server {
# if ($host = platypush.example.com) {
# return 301 https://$host$request_uri;
# }
#
# server_name platypush.example.com;
# listen 80;
# return 404;
# }

6
generate_missing_docs.py
View file

@ -4,7 +4,7 @@ from typing import Iterable, Optional
from platypush.backend import Backend
from platypush.context import get_plugin
from platypush.plugins import Plugin
from platypush.utils.manifest import get_manifests
from platypush.utils.manifest import Manifests
def _get_inspect_plugin():
@ -14,11 +14,11 @@ def _get_inspect_plugin():
def get_all_plugins():
return sorted([mf.component_name for mf in get_manifests(Plugin)])
return sorted([mf.component_name for mf in Manifests.by_base_class(Plugin)])
def get_all_backends():
return sorted([mf.component_name for mf in get_manifests(Backend)])
return sorted([mf.component_name for mf in Manifests.by_base_class(Backend)])
def get_all_events():

4
platypush/app/__main__.py
View file

@ -2,6 +2,4 @@ import sys
from ._app import main
if __name__ == '__main__':
sys.exit(main(*sys.argv[1:]))
sys.exit(main(*sys.argv[1:]))

19
platypush/app/_app.py
View file

@ -42,6 +42,7 @@ class Application:
config_file: Optional[str] = None,
workdir: Optional[str] = None,
logsdir: Optional[str] = None,
device_id: Optional[str] = None,
pidfile: Optional[str] = None,
requests_to_process: Optional[int] = None,
no_capture_stdout: bool = False,
@ -61,6 +62,10 @@ class Application:
``filename`` setting under the ``logging`` section of the
configuration file is used. If not set, logging will be sent to
stdout and stderr.
:param device_id: Override the device ID used to identify this
instance. If not passed here, it is inferred from the configuration
(device_id field). If not present there either, it is inferred from
the hostname.
:param pidfile: File where platypush will store its PID upon launch,
useful if you're planning to integrate the application within a
service or a launcher script (default: None).
@ -97,11 +102,14 @@ class Application:
os.path.abspath(os.path.expanduser(logsdir)) if logsdir else None
)
Config.init(self.config_file)
Config.set('ctrl_sock', ctrl_sock)
if workdir:
Config.set('workdir', os.path.abspath(os.path.expanduser(workdir)))
Config.init(
self.config_file,
device_id=device_id,
workdir=os.path.abspath(os.path.expanduser(workdir)) if workdir else None,
ctrl_sock=os.path.abspath(os.path.expanduser(ctrl_sock))
if ctrl_sock
else None,
)
self.no_capture_stdout = no_capture_stdout
self.no_capture_stderr = no_capture_stderr
@ -199,6 +207,7 @@ class Application:
config_file=opts.config,
workdir=opts.workdir,
logsdir=opts.logsdir,
device_id=opts.device_id,
pidfile=opts.pidfile,
no_capture_stdout=opts.no_capture_stdout,
no_capture_stderr=opts.no_capture_stderr,

2
platypush/backend/assistant/google/manifest.yaml
View file

@ -10,7 +10,7 @@ manifest:
times out
platypush.message.event.assistant.MicMutedEvent: when the microphone is muted.
platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted.
platypush.message.event.assistant.NoResponse: when a conversation returned no
platypush.message.event.assistant.NoResponseEvent: when a conversation returned no
response
platypush.message.event.assistant.ResponseEvent: when the assistant is speaking
a response

4
platypush/backend/http/__init__.py
View file

@ -192,7 +192,7 @@ class HttpBackend(Backend):
"""
_DEFAULT_HTTP_PORT = 8008
DEFAULT_HTTP_PORT = 8008
"""The default listen port for the webserver."""
_STOP_TIMEOUT = 5
@ -200,7 +200,7 @@ class HttpBackend(Backend):
def __init__(
self,
port: int = _DEFAULT_HTTP_PORT,
port: int = DEFAULT_HTTP_PORT,
bind_address: str = '0.0.0.0',
resource_dirs: Optional[Mapping[str, str]] = None,
secret_key_file: Optional[str] = None,

2
platypush/backend/http/app/utils/routes.py
View file

@ -14,7 +14,7 @@ def get_http_port():
from platypush.backend.http import HttpBackend
http_conf = Config.get('backend.http') or {}
return http_conf.get('port', HttpBackend._DEFAULT_HTTP_PORT)
return http_conf.get('port', HttpBackend.DEFAULT_HTTP_PORT)
def get_routes():

2
platypush/backend/music/spotify/manifest.yaml
View file

@ -18,7 +18,7 @@ manifest:
pacman:
- sudo
- cargo
exec:
after:
- sudo cargo install librespot
package: platypush.backend.music.spotify
type: backend

6
platypush/backend/pushbullet/manifest.yaml
View file

@ -1,6 +1,12 @@
manifest:
events:
platypush.message.event.pushbullet.PushbulletEvent: if a new push is received
apk:
- git
apt:
- git
pacman:
- git
install:
pip:
- git+https://github.com/rbrcsk/pushbullet.py

3
platypush/builder/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from ._base import BaseBuilder
__all__ = ["BaseBuilder"]

198
platypush/builder/_base.py Normal file
View file

@ -0,0 +1,198 @@
from abc import ABC, abstractmethod
import argparse
import inspect
import logging
import os
import pathlib
import sys
from typing import Final, Optional, Sequence
from platypush.config import Config
from platypush.utils.manifest import (
Dependencies,
InstallContext,
)
logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger()
class BaseBuilder(ABC):
"""
Base interface and utility methods for Platypush builders.
A Platypush builder is a script/piece of logic that can build a Platypush
installation, with all its base and required extra dependencies, given a
configuration file.
This class is currently implemented by the :module:`platypush.platyvenv`
and :module:`platypush.platydock` modules/scripts.
"""
REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git'
"""
We use the Github URL here rather than the self-hosted Gitea URL to prevent
too many requests to the Gitea server.
"""
def __init__(
self,
cfgfile: str,
gitref: str,
output: str,
install_context: InstallContext,
*_,
verbose: bool = False,
device_id: Optional[str] = None,
**__,
) -> None:
"""
:param cfgfile: The path to the configuration file.
:param gitref: The git reference to use. It can be a branch name, a tag
name or a commit hash.
:param output: The path to the output file or directory.
:param install_context: The installation context for this builder.
:param verbose: Whether to log debug traces.
:param device_id: A device name that will be used to uniquely identify
this installation.
"""
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
self.output = os.path.abspath(os.path.expanduser(output))
self.gitref = gitref
self.install_context = install_context
self.device_id = device_id
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
@classmethod
@abstractmethod
def get_name(cls) -> str:
"""
:return: The name of the builder.
"""
@classmethod
@abstractmethod
def get_description(cls) -> str:
"""
:return: The description of the builder.
"""
@property
def deps(self) -> Dependencies:
"""
:return: The dependencies for this builder, given the configuration
file and the installation context.
"""
return Dependencies.from_config(
self.cfgfile,
install_context=self.install_context,
)
def _print_instructions(self, s: str) -> None:
GREEN = '\033[92m'
NORM = '\033[0m'
helper_lines = s.split('\n')
wrapper_line = '=' * max(len(t) for t in helper_lines)
helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n'
print(GREEN + helper + NORM)
@abstractmethod
def build(self):
"""
Builds the application. To be implemented by the subclasses.
"""
@classmethod
def _get_arg_parser(cls) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog=cls.get_name(),
add_help=False,
description=cls.get_description(),
)
parser.add_argument(
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
)
parser.add_argument(
'-v',
'--verbose',
dest='verbose',
action='store_true',
help='Enable debug traces',
)
parser.add_argument(
'-c',
'--config',
type=str,
dest='cfgfile',
required=False,
default=None,
help='The path to the configuration file. If not specified, a minimal '
'installation including only the base dependencies will be generated.',
)
parser.add_argument(
'-o',
'--output',
dest='output',
type=str,
required=False,
default='.',
help='Target directory (default: current directory). For Platydock, '
'this is the directory where the Dockerfile will be generated. For '
'Platyvenv, this is the base directory of the new virtual '
'environment.',
)
parser.add_argument(
'-d',
'--device-id',
dest='device_id',
type=str,
required=False,
default=None,
help='A name that will be used to uniquely identify this device. '
'Default: a random name for Docker containers, and the '
'hostname of the machine for virtual environments.',
)
parser.add_argument(
'-r',
'--ref',
dest='gitref',
required=False,
type=str,
default='master',
help='If the script is not run from a Platypush installation directory, '
'it will clone the sources via git. You can specify through this '
'option which branch, tag or commit hash to use. Defaults to master.',
)
return parser
@classmethod
def from_cmdline(cls, args: Sequence[str]):
"""
Create a builder instance from command line arguments.
:param args: Command line arguments.
:return: A builder instance.
"""
parser = cls._get_arg_parser()
opts, _ = parser.parse_known_args(args)
if opts.show_usage:
parser.print_help()
sys.exit(0)
if not opts.cfgfile:
opts.cfgfile = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent),
'config.yaml',
)
logger.info('No configuration file specified. Using %s.', opts.cfgfile)
return cls(**vars(opts))

16
platypush/cli.py
View file

@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace:
"""
Parse command-line arguments from a list of strings.
"""
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(
'platypush',
description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.',
)
parser.add_argument(
'--config',
@ -29,6 +32,17 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace:
help='Custom working directory to be used for the application',
)
parser.add_argument(
'--device-id',
'-d',
dest='device_id',
required=False,
default=None,
help='Override the device ID used to identify this instance. If not '
'passed here, it is inferred from the configuration (device_id field).'
'If not present there either, it is inferred from the hostname.',
)
parser.add_argument(
'--logsdir',
'-l',

18
platypush/commands/_stream.py
View file

@ -1,4 +1,4 @@
from multiprocessing import Queue
from multiprocessing import RLock, Queue
import os
from queue import Empty
import socket
@ -35,6 +35,7 @@ class CommandStream(ControllableProcess):
self.path = os.path.abspath(os.path.expanduser(path or self._default_sock_path))
self._sock: Optional[socket.socket] = None
self._cmd_queue: Queue["Command"] = Queue()
self._close_lock = RLock()
def reset(self):
if self._sock is not None:
@ -68,9 +69,18 @@ class CommandStream(ControllableProcess):
return self
def __exit__(self, *_, **__):
self.terminate()
self.join()
self.close()
with self._close_lock:
self.terminate()
try:
self.close()
except Exception as e:
self.logger.warning(str(e))
try:
self.kill()
except Exception as e:
self.logger.warning(str(e))
def _serve(self, sock: socket.socket):
"""

94
platypush/config/__init__.py
View file

@ -20,6 +20,7 @@ from platypush.utils import (
is_functional_procedure,
is_functional_hook,
is_functional_cron,
is_root,
)
@ -65,13 +66,14 @@ class Config:
_included_files: Set[str] = set()
def __init__(self, cfgfile: Optional[str] = None):
def __init__(self, cfgfile: Optional[str] = None, workdir: Optional[str] = None):
"""
Constructor. Always use the class as a singleton (i.e. through
Config.init), you won't probably need to call the constructor directly
:param cfgfile: Config file path (default: retrieve the first available
location in _cfgfile_locations).
:param workdir: Overrides the default working directory.
"""
self.backends = {}
@ -83,13 +85,13 @@ class Config:
self.dashboards = {}
self._plugin_manifests = {}
self._backend_manifests = {}
self._cfgfile = ''
self.config_file = ''
self._init_cfgfile(cfgfile)
self._config = self._read_config_file(self._cfgfile)
self._config = self._read_config_file(self.config_file)
self._init_secrets()
self._init_dirs()
self._init_dirs(workdir=workdir)
self._init_db()
self._init_logging()
self._init_device_id()
@ -104,10 +106,10 @@ class Config:
if cfgfile is None:
cfgfile = self._get_default_cfgfile()
if cfgfile is None:
if cfgfile is None or not os.path.exists(cfgfile):
cfgfile = self._create_default_config()
self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
self.config_file = os.path.abspath(os.path.expanduser(cfgfile))
def _init_logging(self):
logging_config = {
@ -163,21 +165,26 @@ class Config:
for k, v in self._config['environment'].items():
os.environ[k] = str(v)
def _init_dirs(self):
if 'workdir' not in self._config:
def _init_dirs(self, workdir: Optional[str] = None):
if workdir:
self._config['workdir'] = workdir
if not self._config.get('workdir'):
self._config['workdir'] = self._workdir_location
self._config['workdir'] = os.path.expanduser(self._config['workdir'])
os.makedirs(self._config['workdir'], exist_ok=True)
self._config['workdir'] = os.path.expanduser(
os.path.expanduser(self._config['workdir'])
)
pathlib.Path(self._config['workdir']).mkdir(parents=True, exist_ok=True)
if 'scripts_dir' not in self._config:
self._config['scripts_dir'] = os.path.join(
os.path.dirname(self._cfgfile), 'scripts'
os.path.dirname(self.config_file), 'scripts'
)
os.makedirs(self._config['scripts_dir'], mode=0o755, exist_ok=True)
if 'dashboards_dir' not in self._config:
self._config['dashboards_dir'] = os.path.join(
os.path.dirname(self._cfgfile), 'dashboards'
os.path.dirname(self.config_file), 'dashboards'
)
os.makedirs(self._config['dashboards_dir'], mode=0o755, exist_ok=True)
@ -206,17 +213,29 @@ class Config:
def _create_default_config(self):
cfg_mod_dir = os.path.dirname(os.path.abspath(__file__))
cfgfile = self._cfgfile_locations[0]
# Use /etc/platypush/config.yaml if the user is running as root,
# otherwise ~/.config/platypush/config.yaml
cfgfile = (
(
os.path.join(os.environ['XDG_CONFIG_HOME'], 'config.yaml')
if os.environ.get('XDG_CONFIG_HOME')
else os.path.join(
os.path.expanduser('~'), '.config', 'platypush', 'config.yaml'
)
)
if not is_root()
else os.path.join(os.sep, 'etc', 'platypush', 'config.yaml')
)
cfgdir = pathlib.Path(cfgfile).parent
cfgdir.mkdir(parents=True, exist_ok=True)
for cfgfile in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')):
shutil.copy(cfgfile, str(cfgdir))
for cf in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')):
shutil.copy(cf, str(cfgdir))
return cfgfile
def _read_config_file(self, cfgfile):
cfgfile_dir = os.path.dirname(os.path.abspath(os.path.expanduser(cfgfile)))
config = {}
try:
@ -397,14 +416,17 @@ class Config:
@classmethod
def _get_instance(
cls, cfgfile: Optional[str] = None, force_reload: bool = False
cls,
cfgfile: Optional[str] = None,
workdir: Optional[str] = None,
force_reload: bool = False,
) -> "Config":
"""
Lazy getter/setter for the default configuration instance.
"""
if force_reload or cls._instance is None:
cfg_args = [cfgfile] if cfgfile else []
cls._instance = Config(*cfg_args)
cls._instance = Config(*cfg_args, workdir=workdir)
return cls._instance
@classmethod
@ -463,17 +485,34 @@ class Config:
return None
@classmethod
def init(cls, cfgfile: Optional[str] = None):
def init(
cls,
cfgfile: Optional[str] = None,
device_id: Optional[str] = None,
workdir: Optional[str] = None,
ctrl_sock: Optional[str] = None,
**_,
):
"""
Initializes the config object singleton
Params:
cfgfile -- path to the config file - default: _cfgfile_locations
:param cfgfile: Path to the config file (default: _cfgfile_locations)
:param device_id: Override the configured device_id.
:param workdir: Override the configured working directory.
:param ctrl_sock: Override the configured control socket.
"""
return cls._get_instance(cfgfile, force_reload=True)
cfg = cls._get_instance(cfgfile, workdir=workdir, force_reload=True)
if device_id:
cfg.set('device_id', device_id)
if workdir:
cfg.set('workdir', workdir)
if ctrl_sock:
cfg.set('ctrl_sock', ctrl_sock)
return cfg
@classmethod
@property
def workdir(cls) -> str:
def get_workdir(cls) -> str:
"""
:return: The path of the configured working directory.
"""
@ -505,5 +544,12 @@ class Config:
# pylint: disable=protected-access
cls._get_instance()._config[key] = value
@classmethod
def get_file(cls) -> str:
"""
:return: The main configuration file path.
"""
return cls._get_instance().config_file
# vim:sw=4:ts=4:et:

6
platypush/config/config.auto.yaml
View file

@ -1,6 +0,0 @@
# Auto-generated configuration file.
# Do not edit manually - use the config.yaml file for manual modifications
# instead
backend.http:
enabled: True

922
platypush/config/config.yaml
View file

@ -1,2 +1,920 @@
include:
- config.auto.yaml
################################################################################
# Sample Platypush configuration file.
#
# Edit it and:
# - Copy it to /etc/platypush/config.yaml for system installation.
# - Copy it to ~/.config/platypush/config.yaml for user installation.
# - Start the application with `-c <path-to-this-file>`.
#
# Since the configuration file also includes the custom integrations, you can
# create a Platypush custom installation, with all the extra dependencies
# required by the configured integrations, using the `platydock` or `platyvenv`
# commands and passing this file as an argument. These commands will build a
# Docker image or a Python virtual environment respectively, with all the
# required extra dependencies inferred from your configuration file.
#
# A `scripts` directory with an empty `__init__.py` script will also be created
# under the same directory as the configuration file. This directory can be
# used to add custom scripts containing procedures, hooks and crons if you want
# a full Python interface to define your logic rather than a YAML file.
#
# Please refer to the `scripts` directory provided under this file's directory
# for some examples that use the Python API.
################################################################################
### ------------------
### Include directives
### ------------------
# # You can split your configuration over multiple files and use the include
# # directive to import other files into your configuration.
#
# # Files referenced via relative paths will be searched in the directory of
# # the configuration file that references them. Symbolic links are also
# # supported.
#
# include:
# - logging.yaml
# - media.yaml
# - sensors.yaml
### -----------------
### Working directory
### -----------------
# # Working directory of the application. This is where the main database will be
# # stored by default (if the default SQLite configuration is used), and it's
# # where the integrations will store their state.
#
# # Note that the working directory can also be specified at runtime using the
# # -w/--workdir option.
# #
# # If not specified, then one of the following will be used:
# #
# # - $XDG_DATA_HOME/platypush if the XDG_DATA_HOME environment variable is set.
# # - $HOME/.local/share/platypush otherwise.
#
# workdir: ~/.local/share/platypush
### ----------------------
### Database configuration
### ----------------------
# # By default Platypush will use a SQLite database named `main.db` under the
# # `workdir`. You can specify any other engine string here - the application has
# # been tested against SQLite, Postgres and MariaDB/MySQL >= 8.
# #
# # NOTE: If you want to use a DBMS other than SQLite, then you will also need to
# # ensure that a compatible Python driver is installed on the system where
# # Platypush is running. For example, Postgres will require the Python pg8000,
# # psycopg or another compatible driver.
#
# main.db:
# engine: sqlite:///home/user/.local/share/platypush/main.db
# # OR, if you want to use e.g. Postgres with the pg8000 driver:
# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname
### ---------------------
### Logging configuration
### ---------------------
# # Platypush logs on stdout by default. You can use the logging section to
# # specify an alternative file or change the logging level.
#
# # Note that a custom logging directory can also be specified at runtime using
# # the -l/--logsdir option.
#
# logging:
# filename: ~/.local/log/platypush/platypush.log
# level: INFO
### -----------------------
### device_id configuration
### -----------------------
# # The device_id is used by many components of Platypush and it should uniquely
# # identify a device in your network. If nothing is specified then the hostname
# # will be used.
#
# # Note that a custom device ID can also be specified at runtime using the
# # -d/--device-id option.
#
# device_id: my_device
### -------------------
### Redis configuration
### -------------------
# # Platypush needs a Redis instance for inter-process communication.
# #
# # By default, the application will try and connect to a Redis server listening
# # on localhost:6379.
# #
# # Platypush can also start the service on the fly if instructed to do so
# # through the `--start-redis` option. You can also specify a custom port
# # through the `--redis-port` option.
# #
# # If you are running Platypush in a Docker image built through Platydock, then
# # `--start-redis` is the default behaviour and you won't need any extra
# # documentation here.
#
# redis:
# host: localhost
# port: 6379
# username: user
# password: secret
### ------------------------
### Web server configuration
### ------------------------
# Platypush comes with a versatile Web server that is used to:
#
# - Serve the main UI and the UIs for the plugins that provide one.
# - Serve custom user-configured dashboards.
# - Expose the `/execute` RPC endpoint to send synchronous requests.
# - Expose the `/ws/events` and `/ws/requests` Websocket paths, which can be
# respectively by other clients to subscribe to the application's events or
# send asynchronous requests.
# - Stream media files provided by other plugins, as well as camera and audio
# feeds.
# - Serve custom directories of static files that can be accessed by other
# clients.
# - Provide a versatile API for hooks - the user can easily create custom HTTP
# hooks by creating a hook with their custom logic that reacts when a
# `platypush.message.event.http.hook.WebhookEvent` is received. The `hook`
# parameter of the event specifies which URL will be served by the hook.
#
# The Web server is enabled by default, but you can disable it simply by
# commenting/removing the `backend.http` section. The default listen port is
# 8008.
#
# After starting the application, you can access the UI at
# http://localhost:8008, set up your username and password, and also create an
# access or session token from the configuration panel.
#
# This token can be used to authenticate calls to the available APIs.
# For example, to turn on the lights using the /execute endpoint:
#
# curl -XPOST -H "Authorization: Bearer $TOKEN" \
# -H "Content-Type: application/json" \
# -d '
# {
# "type": "request",
# "action": "light.hue.on",
# "args": {
# "lights": ["Bedroom"]
# }
# }' http://localhost:8008/execute
#
# If you want to serve the Web server behind a reverse proxy, you can copy the
# reference configuration available at
# https://git.platypush.tech/platypush/platypush/src/branch/master/examples/nginx/nginx.sample.conf
backend.http:
# # Bind address (default: 0.0.0.0)
# bind_address: 0.0.0.0
# # Listen port (default: 8008)
port: 8008
# # resource_dirs can be used to specify directories on the host system that
# # you want to expose through the Web server. For example, you may want to
# # expose directories that contain photos or images if you want to make a
# # carousel dashboard, or a directory containing some files that you want to
# # share with someone (or other systems) using a simple Web server.
# #
# # In the following example, we're exposing a directory with photos on an
# # external hard drive other the `/photos` URL. An image like e.g.
# # `/mnt/hd/photos/IMG_1234.jpg` will be served over e.g.
# # `http://localhost:8008/photos/IMG_1234.jpg` in this case.
# resource_dirs:
# photos: /mnt/hd/photos
# # Number of WSGI workers. Default: (#cpus * 2) + 1
# num_workers: 4
### -----------------------------
### Plugin configuration examples
### -----------------------------
###
# # The configuration of a plugin matches one-to-one the parameters required by
# # its constructor.
# #
# # Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html
# #
# # For example, there is a `light.hue` plugin
# # (https://docs.platypush.tech/platypush/plugins/light.hue.html) whose
# # constructor takes the following parameters: `bridge`, `lights` (default
# # target lights for the commands), `groups` (default target groups for the
# # commands) and `poll_interval` (how often the plugin should poll for updates).
# #
# # This means that the `light.hue` plugin can be configured here as:
#
# light.hue:
# # IP address or hostname of the Hue bridge
# # NOTE: The first run will require you to register the application with
# # your bridge - that's usually done by pressing a physical button on your
# # bridge while the application is pairing.
# bridge: 192.168.1.3
# # Groups that will be handled by default if nothing is specified on the request
# groups:
# - Living Room
#
# # How often we should poll for updates (default: 20 seconds)
# poll_interval: 20
###
###
# # Example configuration of music.mpd plugin, a plugin to interact with MPD and
# # Mopidy music server instances. See
# # https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html
# # You can easily install the dependencies through pip install 'platypush[mpd]'
#
# music.mpd:
# host: localhost
# port: 6600
###
###
# # Plugins with empty configuration can also be explicitly enabled by specifying
# # `enabled: true` or `disabled: false`. An integration with no items will be
# # enabled with no configuration.
#
# clipboard:
###
###
# # Example configuration of the MQTT plugin. This specifies a server that the
# # application will use by default (if not specified on the request body).
#
# mqtt:
# host: 192.168.1.2
# port: 1883
# username: user
# password: secret
###
###
# # Enable the system plugin if you want your device to periodically report
# # system statistics (CPU load, disk usage, memory usage etc.)
# #
# # When new data is gathered, an `EntityUpdateEvent` with `plugin='system'` will
# # be triggered with the new data, and you can subscribe a hook to these events
# # to run your custom logic.
#
# system:
# # How often we should poll for new data
# poll_interval: 60
###
###
# # Example configuration for the calendar plugin. In this case, we have
# # registered a Google calendar that uses the `google.calendar` integration, and
# # a Facebook plugin and a NextCloud (WebDAV) plugin exposed over iCal format.
# # Installing the dependencies: pip install 'platypush[ical,google]'
# calendar:
# calendars:
# - type: platypush.plugins.google.calendar.GoogleCalendarPlugin
# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin
# url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key
# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin
# url: http://riemann/nextcloud/remote.php/dav/public-calendars/9JBWHR7iioM88Y4D?export
###
###
# # Torrent plugin configuration, with the default directory that should be used
# # to store downloaded files.
#
# torrent:
# download_dir: ~/Downloads
###
###
# # List of RSS/Atom subscriptions. These feeds will be monitored for changes and
# # a `platypush.message.event.rss.NewFeedEntryEvent`
# # (https://docs.platypush.tech/platypush/events/rss.html#platypush.message.event.rss.NewFeedEntryEvent)
# # will be triggered when one of these feeds has new entries - you can subscribe
# # the event to run your custom logic.
#
# rss:
# # How often we should check for updates (default: 5 minutes)
# poll_seconds: 300
# # List of feeds to monitor
# subscriptions:
# - https://www.theguardian.com/rss/world
# - https://phys.org/rss-feed/
# - https://news.ycombinator.com/rss
# - https://www.technologyreview.com/stories.rss
# - https://api.quantamagazine.org/feed/
###
###
# # Example configuration of a weather plugin
#
# weather.openweathermap:
# token: secret
# lat: lat
# long: long
###
###
# # You can add IFTTT integrations to your routines quite easily.
# #
# # Register an API key for IFTTT, paste it here, and you can run an
# # `ifttt.trigger_event` action to fire an event on IFTTT.
# #
# # You can also create IFTTT routines that call your Platypush instance, by
# # using Web hooks (i.e. event hooks that subscribe to
# # `platypush.message.event.http.hook.WebhookEvent` events), provided that the
# # Web server is listening on a publicly accessible address.
#
# ifttt:
# ifttt_key: SECRET
###
###
# # The `http.webpage` integration comes with the mercury-parser JavaScript library.
# # It allows you to "distill" the content of a Web page and export it in readable format (in simplified HTML, Markdown or PDF) through the
#
# http.webpage:
###
###
# # Example configuration of the zigbee.mqtt integration.
# # This integration listens for the events pushed by zigbee2mqtt service to an
# # MQTT broker. It can forward those events to native Platypush events (see
# # https://docs.platypush.tech/platypush/events/zigbee.mqtt.html) that you can
# # build automation routines on. You can also use Platypush to control your
# # Zigbee devices, either through the Web interface or programmatically through
# # the available plugin actions.
#
# zigbee.mqtt:
# # Host of the MQTT broker
# host: riemann
# # Listen port of the MQTT broker
# port: 1883
# # Base topic, as specified in `<zigbee2mqtt_dir>/data/configuration.yaml`
# base_topic: zigbee2mqtt
###
###
# # Example configuration of the zwave.mqtt integration.
# # This integration listens for the events pushed by ZWaveJS service to an MQTT
# # broker. It can forward those events to native Platypush events (see
# # https://docs.platypush.tech/platypush/events/zwave.html) that you can build
# # automation routines on.
# # You can also use Platypush to control your Z-Wave devices, either through the
# # Web interface or programmatically through the available plugin actions.
#
# zwave.mqtt:
# # Host of the MQTT broker
# host: riemann
# # Listen port of the MQTT broker
# port: 1883
# # Gateway name, usually configured in the ZWaveJS-UI through `Settings ->
# # MQTT -> Name`
# name: zwavejs2mqtt
# # The prefix of the published topics, usually configured in the ZWaveJS-UI
# # through `Settings -> MQTT -> Prefix`.
# topic_prefix: zwave
###
### --------------------
### Camera configuration
### --------------------
###
# # There are several providers for the camera integration - you can choose
# # between ffmpeg, gstreamer, PiCamera etc., and they all expose the same
# # interface/configuration options.
# #
# # It is advised to use the ffmpeg integration, as it's the one that provides
# # the highest degree of features and supported hardware.
# #
# # If the plugin is correctly configured, you can access your camera feed from
# # the Platypush Web panel, programmatically start/stop recording sessions, take
# # photos, get a feed stream URL etc.
#
# # The camera feed will be available at `/camera/<plugin>/video[.extension]`,
# # for example `/camera/ffmpeg/video.mjpeg` for MJPEG (usually faster), or
# # `camera/ffmpeg/video.mp4` for MP4.
#
# # You can also capture images by connecting to the
# # `/camera/<plugin>/photo[.extension]`, for example `/camera/ffmpeg/photo.jpg`.
#
# camera.ffmpeg:
# # Default video device to use
# device: /dev/video0
# # Default resolution
# resolution:
# - 640
# - 480
# # The directory that will be used to store captured frames/images
# frames_dir: ~/Camera/Photos
# # Default image scaling factors (default: 1, no scaling)
# scale_x: 1.5
# scale_y: 1.5
# # Default rotation of the image, in degrees (default: 0, no rotation)
# rotate: 90
# # Grayscale mode (default: False):
# grayscale: false
# # Default frames per second (default: 16)
# fps: 16
# # Whether to flip the image along the horizontal axis (default: False)
# horizontal_flip: false
# # Whether to flip the image along the horizontal axis (default: False)
# vertical_flip: false
###
### -----------------
### Sound integration
### -----------------
###
# # The sound plugin allows you to stream from an audio source connected to the
# # machine, play audio files or synthetic audio waves or MIDI sounds.
#
# # After enabling the plugin, you can access the audio stream at
# # `/sound/stream[.extension]` (e.g. `/sound/stream.mp3`) if you want to get a
# # live recording of the captured sound from the configured audio
# # `input_device`.
#
# sound:
# enabled: true
###
### -----------------------------------
### Some examples of media integrations
### -----------------------------------
###
# # Example configuration for the media.vlc plugin. You can replace `vlc` with
# # `mpv`, `mplayer`, `omxplayer` or `gstreamer` if you want to use another
# # player - the supported configuration option are the same across all these
# # players.
#
# media.vlc:
# # Volume level, between 0 and 100
# volume: 50
# # Where to store downloaded files
# download_dir: ~/Downloads
# # Play videos in fullscreen by default
# fullscreen: True
# # If youtube-dl or any compatible application is installed, play requested
# # videos in this format by default. Default: `best`.
# youtube_format: 'mp4[height<=?480]'
# # Extra arguments to pass to the executable. --play-and-exit may be a good
# # idea with VLC, so the player terminates upon stop instead of lingering in
# # the background.
# args:
# - --play-and-exit
# # List of directories to search for media files. The media files in these
# # folders can be searched through the `media.<player>.search` command, or
# # through the Web interface.
# media_dirs:
# - /mnt/hd/media/movies
# - /mnt/hd/media/series
# - /mnt/hd/media/videos
# - ~/Downloads
###
###
# # Example configuration for the media.chromecast plugin, see
# # https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html
# # You can easily install the dependencies through pip install 'platypush[chromecast]'
#
# media.chromecast:
# chromecast: Living Room TV
###
###
# # Example Kodi configuration. This makes it possible to control and query a
# # Kodi instance, from your automation hooks, from the Platypush APIs or from
# # the Platypush Web interface. It requires you to enable the JSON API service
# # from Kodi's settings.
#
# media.kodi:
# host: localhost
# http_port: 8080
# username: kodi
# password: secret
###
###
# # Example configuration for a Plex media server. This integration makes it
# # possible to navigate and search media items from your Plex library in the
# # media UI.
#
# media.plex:
# server: localhost
# username: plex
# password: secret
###
###
# # Jellyfin media server configuration.
#
# media.jellyfin:
# server: https://media.example.com
# api_key: secret
###
### ---------------------
### Sensors configuration
### ---------------------
###
# # The serial plugin can be used to read sensor data from a device connected
# # over serial/USB interface.
# #
# # It can be used, for example, to connect to an Arduino or ESP device over
# # serial port, where the remote microcontroller periodically sends sensor data
# # over the serial interface.
# #
# # The data can be sent on the wire either as raw string-encoded numbers (one
# # per line), or (better) in JSON format. For example, you can program your
# # microcontroller to periodically send JSON strings like these when you get new
# # readings from your sensors:
# #
# # {"temperature": 25.0, "humidity": 20.0, "smoke": 0.01, "luminosity": 45}
# #
# # The JSON will be automatically unpacked by the application, and the relevant
# # `platypush.message.event.sensor.SensorDataChangeEvent` events will be
# # triggered when the data changes - you can subscribe to them in your custom
# # hooks.
#
# serial:
# # The path to the USB interface with e.g. an Arduino or ESP microcontroller
# # connected.
# # A way to get a deterministic path name on Linux, instead of
# # `/dev/ttyUSB<n>`, can be the following:
# #
# # - Get the vendor and product ID of your device via e.g. `lsusb`. For
# # example, for an Arduino-compatible microcontroller:
# #
# # Bus 001 Device 008: ID 1a86:7523 QinHeng Electronics CH340 serial converter
# #
# # - In the case above, `1a86` is the vendor ID and `7523` is the product
# # ID. Create a new udev rule for it, so every time the device is
# # connected it will also be symlinked to e.g. `/dev/arduino`:
# #
# # echo 'SUBSYSTEM=="tty", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", SYMLINK+="arduino"' | \
# # sudo tee /etc/udev/rules.d/98-usb-serial.rules
# device: /dev/ttyUSB0
# # How often the interface should be polled for updates, in seconds
# poll_interval: 1
# # The tolerance argument can be used to tune when you want to be notified
# # of data changes through `SensorDataChangeEvent` events. In the case
# # below, if the microcontroller sends two consecutive temperature reads,
# # one for 22.0 and one for 22.2, then only one `SensorDataChangeEvent` will
# # be triggered (for the first read), since the absolute value of the
# # difference between the two events is less than the configured tolerance.
# # However, if the device sends two temperature reads, one for 22.0 and one
# # for 22.7, then two `SensorDataChangeEvent` events will be triggered.
# # The tolerance for all the metrics is set to a value close to zero by
# # default - i.e. any read, unless it's exactly the same as the previous
# # one, will trigger a new event.
# tolerance:
# temperature: 0.5
# humidity: 0.75
# luminosity: 5
#
# # If a threshold is defined for a sensor, and the value of that sensor goes
# # below/above that temperature between two reads, then a
# # `SensorDataBelowThresholdEvent` or a `SensorDataAboveThresholdEvent` will
# # be triggered respectively.
# thresholds:
# temperature: 25.0
###
###
# # Alternatively to the serial plugin, you can also use the arduino plugin if
# # you want to specifically interface with Arduino.
# #
# # This plugin won't require you to write any logic for your microcontroller.
# # However, it requires your microcontroller to be flash with the Firmata
# # firmware, which allows programmatic external control.
# #
# # Note that the interface of this plugin is basically the same as the serial
# # plugin, and any other plugin that extends `SensorPlugin` in general.
# # Therefore, poll_interval, tolerance and thresholds are supported here too.
#
# arduino:
# board: /dev/ttyUSB0
# # name -> PIN number mapping (similar for digital_pins).
# # It allows you to pick a common name for your PINs that will be used in
# # the forwarded events.
# analog_pins:
# temperature: 7
#
# tolerance:
# temperature: 0.5
#
# thresholds:
# temperature: 25.0
###
###
# # Another example: the LTR559 is a common sensor for proximity and luminosity
# # that can be wired to a Raspberry Pi or similar devices over SPI or I2C
# # interface. It exposes the same base interface as all other sensor plugins.
#
# sensor.ltr559:
# poll_interval: 1.0
# tolerance:
# light: 7.0
# proximity: 5.0
#
# thresholds:
# proximity: 10.0
###
### --------------------------------
### Some text-to-speech integrations
### --------------------------------
###
# # `tts` is the simplest TTS integration. It leverages the Google Translate open
# # "say" endpoint to render text as audio speech.
#
# tts:
# # The media plugin that should be used to play the audio response
# media_plugin: media.vlc
# # The default language of the voice
# language: en-gb
###
###
# # `tts.google` leverages Google's official text-to-speech API to render audio
# # speech from text.
# #
# # Install its dependencies via 'pip install "platypush[google-tts]"'.
# #
# # Like all other Google integrations, it requires you to register an app on the
# # Google developers console, create an API key, and follow the instruction
# # logged on the next restart to give your app the required permissions to your
# # account.
#
# tts.google:
# # The media plugin that should be used to play the audio response
# media_plugin: media.vlc
# # The default language of the voice
# language: en-US
# # The gender of the voice (MALE or FEMALE)
# gender: FEMALE
# # The path to the JSON file containing your Google API credentials
# credentials_file: '~/.credentials/platypush/google/platypush-tts.json'
###
###
# # This TTS integration leverages mimic3, an open-source TTS Web server
# # developed by Mycroft (RIP).
# #
# # Follow the instructions at
# # https://docs.platypush.tech/platypush/plugins/tts.mimic3.html to quickly
# # bootstrap a mimic3 server.
#
# tts.mimic3:
# # The base URL of the mimic3 server
# server_url: http://riemann:59125
# # Path of the default voice that should be used
# voice: 'en_UK/apope_low'
# # The media plugin that should be used to play the audio response
# media_plugin: media.vlc
###
## ----------
## Procedures
## ----------
# Procedures are lists of actions that are executed sequentially.
#
# This section shows how to define procedures directly in your YAML
# configuration file(s). However, you can also put your procedures into Python
# scripts inside of the `<config-dir>/scripts` directory if you want access to
# a full-blown Python syntax. They will be automatically discovered at startup
# and available to the application.
#
# You can also access Python variables and evaluate Python expressions by using
# `${}` context expressions.
#
# The `context` special variable is a name->value dictionary containing the
# items returned from previous actions. For example, if an action returned
# `{"status": "ok", "temperature": 21.5}`, then the following actions can access
# those variables through `${context["status"]}` or
# `${context["temperature"]}`, or simply `${status}` and `${temperature}`,
# respectively.
#
# You can also add statements like `- if ${temperature > 20.0}` or
# `- for ${temp in temperature_values}` in your procedures.
#
# Besides the `context` variables, the following special variables are also
# available to the `${}` constructs when running a procedure:
#
# - `output`: It contains the parsed output of the previous action.
# - `errors`: It contains the errors of the previous action
# - `event`: If the procedure is an event hook (or it is executed within an
# event hook), it contains the event that triggered the hook
###
# # An example procedure that can be called when you arrive home.
# #
# # You can run this procedure from the Platypush `execute` Web panel, or
# # programmatically by sending a JSON request to your Web server (or to the
# # `/ws/requests` Websocket route, or to the TCP backend)
# #
# # curl -XPOST \
# # -H "Authorization: Bearer $YOUR_TOKEN" \
# # -d '{"type": "request", "action": "procedure.at_home"}'
# #
# # A use-case can be the one where you have a Tasker automation running on your
# # Android device that detects when your phone enters or exits a certain area,
# # and sends the appropriate request to your Platypush server.
#
# procedure.at_home:
# # Set the db variable AT_HOME to 1.
# # Variables are flexible entities with a name and a value that will be
# # stored on the database and persisted across sessions.
# # You can access them in other procedures, scripts or hooks and run
# # custom logic on the basis of their value.
# - action: variable.set
# args:
# AT_HOME: 1
#
# # Check the luminosity level from e.g. a connected LTR559 sensor.
# # It could also be a Bluetooth, Zigbee, Z-Wave, serial etc. sensor.
# - action: sensor.ltr559.get_measurement
#
# # If it's below a certain threshold, turn on the lights.
# # In this case, `light` is a parameter returned by the previous response,
# # so we can directly access it here through the `${}` context operator.
# # ${light} in this case is equivalent to ${context["light"]} or
# # ${output["light"]}.
# - if ${int(light or 0) < 110}:
# - action: light.hue.on
#
# # Say a welcome home message
# - action: tts.mimic3.say
# args:
# text: Welcome home
#
# # Start the music
# - action: music.mpd.play
###
###
# # Procedure that will be execute when you walk outside your home.
#
# procedure.outside_home:
# # Unset the db variable AT_HOME
# - action: variable.unset
# args:
# name: AT_HOME
#
# # Stop the music
# - action: music.mpd.stop
#
# # Turn off the lights
# - action: light.hue.off
###
###
# # Procedures can also take optional arguments. The example below shows a
# # generic procedure that broadcasts measurements from a sensor through an
# MQTT broker.
#
# # A listener on this topic can react to an `MQTTMessageEvent` and, for
# # example, store the event on a centralized storage.
# #
# # See the event hook section below for a sample hook that listens for messages
# # sent by other clients using this procedure.
#
# procedure.send_sensor_data(name, value):
# - action: mqtt.send_message
# args:
# topic: platypush/sensors
# host: mqtt-server
# port: 1883
# msg:
# name: ${name}
# value: ${value}
# source: ${Config.get("device_id")}
###
## -------------------
## Event hook examples
## -------------------
# Event hooks are procedures that are run when a certain condition is met.
#
# Check the documentation of your configured backends and plugins to see which
# events they can trigger, and check https://docs.platypush.tech/events.html
# for the full list of available events with their schemas.
#
# Just like procedures, event hooks can be defined either using the YAML
# syntax, or in Python snippets in your `scripts` folder.
#
# A YAML event hook consists of two parts: an `if` field that specifies on
# which event the hook will be triggered (type and attribute values), and a
# `then` field that uses the same syntax as procedures to specify a list of
# actions to execute when the event is matched.
###
# # This example is a hook that reacts when an `MQTTMessageEvent` is received on
# # a topic named `platypush/sensor` (see `send_sensor_data` example from the
# # procedures section).
# #
# # It will store the event on a centralized Postgres database.
# #
# # Note that, for this event to be triggered, the application must first
# # subscribe to the `platypush/sensor` topic - e.g. by adding `platypush/sensor`
# # to the active subscriptions in the `mqtt` configurations.
#
# event.hook.OnSensorDataReceived:
# if:
# type: platypush.message.event.mqtt.MQTTMessageEvent
# topic: platypush/sensor
# then:
# - action: db.insert
# args:
# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname
# table: sensor_data
# records:
# - name: ${msg["name"]}
# value: ${msg["value"]}
# source: ${msg["source"]}
###
###
# # The example below plays the music on mpd/mopidy when your voice assistant
# # triggers a speech recognized event with "play the music" content.
#
# event.hook.PlayMusicAssistantCommand:
# if:
# type: platypush.message.event.assistant.SpeechRecognizedEvent
# # Note that basic regexes are supported for `SpeechRecognizedEvent`,
# # so the hook will be triggered both if you say "play the music" and
# # "play music"
# phrase: "play (the)? music"
# then:
# - action: music.mpd.play
###
###
# # This will turn on the lights when you say "turn on the lights"
#
# event.hook.TurnOnLightsCommand:
# if:
# type: platypush.message.event.assistant.SpeechRecognizedEvent
# phrase: "turn on (the)? lights?"
# then:
# - action: light.hue.on
###
###
# # The WebhookEvent is a special type of event. It allows you to dynamically
# # register a Web hook that can be invoked by other clients, if the HTTP backend
# # is active.
# #
# # In this case, we are registering a hook under `/hook/test-hook` that accepts
# # POST requests, gets the body of the requests and logs it.
# #
# # NOTE: Since Web hooks are supposed to be called by external (and potentially
# # untrusted) parties, they aren't designed to use the standard authentication
# # mechanism used by all other routes.
# #
# # By default they don't have an authentication layer at all. You are however
# # advised to create your custom passphrase and checks the request's headers or
# # query string for it - preferably one passphrase per endpoint.
#
# event.hook.WebhookExample:
# if:
# type: platypush.message.event.http.hook.WebhookEvent
# hook: test-hook
# method: POST
# then:
# # Check the token/passphrase
# - if ${args.get('headers', {}).get('X-Token') == 'SECRET':
# - action: logger.info
# args:
# msg: ${data}
###
### -------------
### Cron examples
### -------------
###
# # Cronjobs allow you to execute procedures at periodic intervals.
# # Standard UNIX cron syntax is supported, plus an optional 6th indicator
# # at the end of the expression to run jobs with second granularity.
# # The example below executes a script at intervals of 1 minute.
#
# cron.TestCron:
# cron_expression: '* * * * *'
# actions:
# - action: shell.exec
# args:
# cmd: ~/bin/myscript.sh
###

30
platypush/entities/_base.py
View file

@ -24,10 +24,12 @@ from sqlalchemy import (
UniqueConstraint,
inspect as schema_inspect,
)
from sqlalchemy.engine import Engine
from sqlalchemy.orm import ColumnProperty, backref, relationship
from sqlalchemy.orm.exc import ObjectDeletedError
import platypush
from platypush.config import Config
from platypush.common.db import Base
from platypush.message import JSONAble, Message
@ -303,6 +305,24 @@ def _discover_entity_types():
entities_registry[obj] = {} # type: ignore
def _get_db():
"""
Utility method to get the db plugin.
"""
from platypush.context import get_plugin
db = get_plugin('db')
assert db
return db
def _get_db_engine() -> Engine:
"""
Utility method to get the db engine.
"""
return _get_db().get_engine()
def get_entities_registry() -> EntityRegistryType:
"""
:returns: A copy of the entities registry.
@ -314,13 +334,9 @@ def init_entities_db():
"""
Initializes the entities database.
"""
from platypush.context import get_plugin
run_db_migrations()
_discover_entity_types()
db = get_plugin('db')
assert db
db.create_all(db.get_engine(), Base)
_get_db().create_all(_get_db_engine(), Base)
def run_db_migrations():
@ -339,6 +355,10 @@ def run_db_migrations():
'alembic',
'-c',
alembic_ini,
'-x',
f'CFGFILE={Config.get_file()}',
'-x',
f'DBNAME={_get_db_engine().url}',
'upgrade',
'head',
],

23
platypush/install/docker/alpine.Dockerfile Normal file
View file

@ -0,0 +1,23 @@
FROM alpine
ADD . /install
WORKDIR /var/lib/platypush
ARG DOCKER_CTX=1
ENV DOCKER_CTX=1
RUN apk update
RUN /install/platypush/install/scripts/alpine/install.sh
RUN cd /install && pip install -U --no-input --no-cache-dir .
RUN rm -rf /install
RUN rm -rf /var/cache/apk
EXPOSE 8008
VOLUME /etc/platypush
VOLUME /var/lib/platypush
CMD platypush \
--start-redis \
--config /etc/platypush/config.yaml \
--workdir /var/lib/platypush

27
platypush/install/docker/debian.Dockerfile Normal file
View file

@ -0,0 +1,27 @@
FROM debian
ADD . /install
WORKDIR /var/lib/platypush
ARG DEBIAN_FRONTEND=noninteractive
ENV DEBIAN_FRONTEND=noninteractive
ARG DOCKER_CTX=1
ENV DOCKER_CTX=1
RUN apt update
RUN /install/platypush/install/scripts/debian/install.sh
RUN cd /install && pip install -U --no-input --no-cache-dir . --break-system-packages
RUN rm -rf /install
RUN apt autoclean -y
RUN apt autoremove -y
RUN apt clean
EXPOSE 8008
VOLUME /etc/platypush
VOLUME /var/lib/platypush
CMD platypush \
--start-redis \
--config /etc/platypush/config.yaml \
--workdir /var/lib/platypush

27
platypush/install/docker/ubuntu.Dockerfile Normal file
View file

@ -0,0 +1,27 @@
FROM ubuntu
ADD . /install
WORKDIR /var/lib/platypush
ARG DEBIAN_FRONTEND=noninteractive
ENV DEBIAN_FRONTEND=noninteractive
ARG DOCKER_CTX=1
ENV DOCKER_CTX=1
RUN apt update
RUN /install/platypush/install/scripts/debian/install.sh
RUN cd /install && pip install -U --no-input --no-cache-dir .
RUN rm -rf /install
RUN apt autoclean -y
RUN apt autoremove -y
RUN apt clean
EXPOSE 8008
VOLUME /etc/platypush
VOLUME /var/lib/platypush
CMD platypush \
--start-redis \
--config /etc/platypush/config.yaml \
--workdir /var/lib/platypush

26
platypush/install/requirements/alpine.txt Normal file
View file

@ -0,0 +1,26 @@
python3
py3-pip
py3-alembic
py3-bcrypt
py3-dateutil
py3-docutils
py3-flask
py3-frozendict
py3-greenlet
py3-magic
py3-mypy-extensions
py3-psutil
py3-redis
py3-requests
py3-rsa
py3-sqlalchemy
py3-tornado
py3-typing-extensions
py3-tz
py3-websocket-client
py3-websockets
py3-wheel
py3-yaml
py3-zeroconf
redis
sudo

24
platypush/install/requirements/arch.txt Normal file
View file

@ -0,0 +1,24 @@
python
python-alembic
python-bcrypt
python-dateutil
python-docutils
python-flask
python-frozendict
python-magic
python-marshmallow
python-pip
python-psutil
python-pytz
python-redis
python-requests
python-rsa
python-sqlalchemy
python-tornado
python-websocket-client
python-websockets
python-wheel
python-yaml
python-zeroconf
redis
sudo

28
platypush/install/requirements/debian.txt Normal file
View file

@ -0,0 +1,28 @@
python3
python3-pip
python3-alembic
python3-bcrypt
python3-dateutil
python3-docutils
python3-flask
python3-frozendict
python3-greenlet
python3-magic
python3-marshmallow
python3-mypy-extensions
python3-psutil
python3-redis
python3-requests
python3-rsa
python3-sqlalchemy
python3-tornado
python3-typing-extensions
python3-typing-inspect
python3-tz
python3-websocket
python3-websockets
python3-wheel
python3-yaml
python3-zeroconf
redis
sudo

1
platypush/install/requirements/ubuntu.txt Symbolic link
View file

@ -0,0 +1 @@
debian.txt

1
platypush/install/scripts/alpine/PKGCMD Normal file
View file

@ -0,0 +1 @@
apk add --update --no-interactive --no-cache

1
platypush/install/scripts/alpine/install.sh Symbolic link
View file

@ -0,0 +1 @@
../install.sh

1
platypush/install/scripts/arch/PKGCMD Normal file
View file

@ -0,0 +1 @@
pacman -S --noconfirm --needed

1
platypush/install/scripts/arch/install.sh Symbolic link
View file

@ -0,0 +1 @@
../install.sh

1
platypush/install/scripts/debian/PKGCMD Normal file
View file

@ -0,0 +1 @@
apt install -y

1
platypush/install/scripts/debian/install.sh Symbolic link
View file

@ -0,0 +1 @@
../install.sh

28
platypush/install/scripts/install.sh Executable file
View file

@ -0,0 +1,28 @@
#!/bin/sh
# This script parses the system requirements for a specific OS and it runs the
# appropriate package manager command to install them.
# This script is usually symlinked in the folders of the individual operating
# systems, and it's not supposed to be invoked directly.
# Instead, it will be called either by the root install.sh script or by a
# Dockerfile.
SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
OS="$(basename "$SCRIPT_PATH")"
CMD="$(cat "${SCRIPT_PATH}/PKGCMD")"
REQUIREMENTS="$(cat "${SCRIPT_PATH}/../../requirements/${OS}.txt" | tr '\n' ' ')"
SUDO=
# If we aren't running in a Docker context, or the user is not root, we should
# use sudo to install system packages.
if [ $(id -u) -ne 0 ] || [ -z "$DOCKER_CTX" ]; then
if ! type sudo >/dev/null; then
echo "sudo executable not found, I can't install system packages" >&2
exit 1
fi
SUDO="sudo"
fi
${SUDO_ARGS} ${CMD} ${REQUIREMENTS}

1
platypush/install/scripts/ubuntu Symbolic link
View file

@ -0,0 +1 @@
debian

17
platypush/migrations/alembic/env.py
View file

@ -74,14 +74,19 @@ def run_migrations_online() -> None:
def set_db_engine():
db_conf = Config.get('db')
assert db_conf, 'Could not retrieve the database configuration'
engine = db_conf['engine']
assert engine, 'No database engine configured'
app_conf_file = context.get_x_argument(as_dictionary=True).get('CFGFILE')
if app_conf_file:
Config.init(app_conf_file)
engine_url = context.get_x_argument(as_dictionary=True).get('DBNAME')
if not engine_url:
db_conf = Config.get('db')
assert db_conf, 'Could not retrieve the database configuration'
engine_url = db_conf['engine']
assert engine_url, 'No database engine configured'
config = context.config
section = config.config_ini_section
config.set_section_option(section, 'DB_ENGINE', engine)
config.set_section_option(section, 'DB_ENGINE', engine_url)
set_db_engine()

704
platypush/platydock/__init__.py
View file

@ -1,462 +1,360 @@
"""
Platydock
Platydock is a helper that allows you to easily manage (create, destroy, start,
stop and list) Platypush instances as Docker images.
Platydock is a helper script that allows you to automatically create a
Dockerfile for Platypush starting from a configuration file.
"""
import argparse
import enum
from contextlib import contextmanager
import inspect
import logging
import os
import pathlib
import re
import shutil
import subprocess
import sys
import textwrap
import traceback as tb
import yaml
from typing import IO, Generator, Iterable
from typing_extensions import override
from platypush.builder import BaseBuilder
from platypush.config import Config
from platypush.utils import manifest
workdir = os.path.join(
os.path.expanduser('~'), '.local', 'share', 'platypush', 'platydock'
from platypush.utils.manifest import (
BaseImage,
Dependencies,
InstallContext,
PackageManagers,
)
class Action(enum.Enum):
build = 'build'
start = 'start'
stop = 'stop'
rm = 'rm'
ls = 'ls'
def __str__(self):
return self.value
logger = logging.getLogger()
def _parse_deps(cls):
deps = []
class DockerBuilder(BaseBuilder):
"""
Creates a Platypush Docker image from a configuration file.
"""
for line in cls.__doc__.split('\n'):
m = re.search(r'\(``pip install (.+)``\)', line)
if m:
deps.append(m.group(1))
_pkg_manager_by_base_image = {
BaseImage.ALPINE: PackageManagers.APK,
BaseImage.DEBIAN: PackageManagers.APT,
BaseImage.UBUNTU: PackageManagers.APT,
}
return deps
_header = textwrap.dedent(
"""
# This Dockerfile was automatically generated by Platydock.
#
# You can build a Platypush image from it by running
# `docker build -t platypush .` in the same folder as this file,
# or copy it to the root a Platypush source folder to install the
# checked out version instead of downloading it first.
#
# You can then run your new image through:
# docker run --rm --name platypush \\
# -v /path/to/your/config/dir:/etc/platypush \\
# -v /path/to/your/workdir:/var/lib/platypush \\
# -p 8008:8008 \\
# platypush\n
"""
)
_footer = textwrap.dedent(
"""
# You can customize the name of your installation by passing
# --device-id=... to the launched command.
"""
)
def generate_dockerfile(deps, ports, cfgfile, device_dir, python_version):
device_id = Config.get('device_id')
if not device_id:
raise RuntimeError(
(
'You need to specify a device_id in {} - Docker '
+ 'containers cannot rely on hostname'
).format(cfgfile)
def __init__(
self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs
):
kwargs['install_context'] = InstallContext.DOCKER
super().__init__(*args, **kwargs)
self.image = image
self.tag = tag
self.print_only = print_only # TODO
@override
@classmethod
def get_name(cls):
return "platydock"
@override
@classmethod
def get_description(cls):
return "Build a Platypush Docker image from a configuration file."
@property
def dockerfile_dir(self) -> str:
"""
Proxy property for the output Dockerfile directory.
"""
output = self.output
parent = os.path.dirname(output)
if os.path.isfile(output):
return parent
if os.path.isdir(output):
return output
logger.info('%s directory does not exist, creating it', output)
pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True)
return output
@property
def dockerfile(self) -> str:
"""
Proxy property for the output Dockerfile.
"""
return os.path.join(self.dockerfile_dir, 'Dockerfile')
@property
def pkg_manager(self) -> PackageManagers:
"""
Proxy property for the package manager to use.
"""
return self._pkg_manager_by_base_image[self.image]
def _read_base_dockerfile_lines(self) -> Generator[str, None, None]:
"""
:return: The lines of the base Dockerfile.
"""
import platypush
base_file = os.path.join(
str(pathlib.Path(inspect.getfile(platypush)).parent),
'install',
'docker',
f'{self.image}.Dockerfile',
)
os.makedirs(device_dir, exist_ok=True)
content = textwrap.dedent(
'''
FROM python:{python_version}-slim-bookworm
with open(base_file, 'r') as f:
for line in f:
yield line.rstrip()
RUN mkdir -p /app
RUN mkdir -p /etc/platypush
RUN mkdir -p /usr/local/share/platypush\n
'''.format(
python_version=python_version
@property
@override
def deps(self) -> Dependencies:
return Dependencies.from_config(
self.cfgfile,
pkg_manager=self.pkg_manager,
install_context=InstallContext.DOCKER,
base_image=self.image,
)
).lstrip()
srcdir = os.path.dirname(cfgfile)
cfgfile_copy = os.path.join(device_dir, 'config.yaml')
shutil.copy(cfgfile, cfgfile_copy, follow_symlinks=True)
content += 'COPY config.yaml /etc/platypush/\n'
backend_config = Config.get_backends()
def _create_dockerfile_parser(self):
"""
Closure for a context-aware parser for the default Dockerfile.
"""
is_after_expose_cmd = False
deps = self.deps
ports = self._get_exposed_ports()
# Redis configuration for Docker
if 'redis' not in backend_config:
backend_config['redis'] = {
'redis_args': {
'host': 'redis',
'port': 6379,
}
}
def parser():
nonlocal is_after_expose_cmd
with open(cfgfile_copy, 'a') as f:
f.write(
'\n# Automatically added by platydock, do not remove\n'
+ yaml.dump(
{
'backend.redis': backend_config['redis'],
}
)
+ '\n'
for line in self._read_base_dockerfile_lines():
if re.match(
r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh',
line.strip(),
):
yield self._generate_git_clone_command()
elif line.startswith('RUN cd /install '):
for new_line in deps.before:
yield 'RUN ' + new_line
for new_line in deps.to_pkg_install_commands():
yield 'RUN ' + new_line
elif line == 'RUN rm -rf /install':
for new_line in deps.to_pip_install_commands():
yield 'RUN ' + new_line
for new_line in deps.after:
yield 'RUN' + new_line
elif line.startswith('EXPOSE ') and ports:
if not is_after_expose_cmd:
yield from [f'EXPOSE {port}' for port in ports]
is_after_expose_cmd = True
continue
elif line.startswith('CMD'):
yield from self._footer.split('\n')
yield line
if line.startswith('CMD') and self.device_id:
yield f'\t--device-id {self.device_id} \\'
return parser
@override
def build(self):
"""
Build a Dockerfile based on a configuration file.
:return: The content of the generated Dockerfile.
"""
# Set the DOCKER_CTX environment variable so any downstream logic knows
# that we're running in a Docker build context.
os.environ['DOCKER_CTX'] = '1'
self._generate_dockerfile()
if self.print_only:
return
self._build_image()
self._print_instructions(
textwrap.dedent(
f"""
A Docker image has been built from the configuration file {self.cfgfile}.
The Dockerfile is available under {self.dockerfile}.
You can run the Docker image with the following command:
docker run \\
--rm --name platypush \\
-v {os.path.dirname(self.cfgfile)}:/etc/platypush \\
-v /path/to/your/workdir:/var/lib/platypush \\
-p 8008:8008 \\
platypush
"""
)
# Main database configuration
has_main_db = False
with open(cfgfile_copy, 'r') as f:
for line in f.readlines():
if re.match(r'^(main.)?db.*', line):
has_main_db = True
break
if not has_main_db:
with open(cfgfile_copy, 'a') as f:
f.write(
'\n# Automatically added by platydock, do not remove\n'
+ yaml.dump(
{
'main.db': {
'engine': 'sqlite:////platypush.db',
}
}
)
+ '\n'
)
# Copy included files
# noinspection PyProtectedMember
for include in Config._included_files:
incdir = os.path.relpath(os.path.dirname(include), srcdir)
destdir = os.path.join(device_dir, incdir)
pathlib.Path(destdir).mkdir(parents=True, exist_ok=True)
shutil.copy(include, destdir, follow_symlinks=True)
content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n'
content += (
'COPY '
+ os.path.relpath(include, srcdir)
+ ' /etc/platypush/'
+ incdir
+ '\n'
)
# Copy script files
scripts_dir = os.path.join(os.path.dirname(cfgfile), 'scripts')
if os.path.isdir(scripts_dir):
local_scripts_dir = os.path.join(device_dir, 'scripts')
remote_scripts_dir = '/etc/platypush/scripts'
shutil.copytree(
scripts_dir, local_scripts_dir, symlinks=True, dirs_exist_ok=True
)
content += f'RUN mkdir -p {remote_scripts_dir}\n'
content += f'COPY scripts/ {remote_scripts_dir}\n'
packages = deps.pop('packages', None)
pip = deps.pop('pip', None)
exec_cmds = deps.pop('exec', None)
pkg_cmd = (
f'\n\t&& apt-get install --no-install-recommends -y {" ".join(packages)} \\'
if packages
else ''
)
pip_cmd = f'\n\t&& pip install {" ".join(pip)} \\' if pip else ''
content += f'''
RUN dpkg --configure -a \\
&& apt-get -f install \\
&& apt-get --fix-missing install \\
&& apt-get clean \\
&& apt-get update \\
&& apt-get -y upgrade \\
&& apt-get -y dist-upgrade \\
&& apt-get install --no-install-recommends -y apt-utils \\
&& apt-get install --no-install-recommends -y build-essential \\
&& apt-get install --no-install-recommends -y git \\
&& apt-get install --no-install-recommends -y sudo \\
&& apt-get install --no-install-recommends -y libffi-dev \\
&& apt-get install --no-install-recommends -y libcap-dev \\
&& apt-get install --no-install-recommends -y libjpeg-dev \\{pkg_cmd}{pip_cmd}'''
for exec_cmd in exec_cmds:
content += f'\n\t&& {exec_cmd} \\'
content += '''
&& apt-get install --no-install-recommends -y zlib1g-dev
RUN git clone --recursive https://git.platypush.tech/platypush/platypush.git /app \\
&& cd /app \\
&& pip install -r requirements.txt
RUN apt-get remove -y git \\
&& apt-get remove -y build-essential \\
&& apt-get remove -y libffi-dev \\
&& apt-get remove -y libjpeg-dev \\
&& apt-get remove -y libcap-dev \\
&& apt-get remove -y zlib1g-dev \\
&& apt-get remove -y apt-utils \\
&& apt-get clean \\
&& apt-get autoremove -y \\
&& rm -rf /var/lib/apt/lists/*
'''
for port in ports:
content += 'EXPOSE {}\n'.format(port)
content += textwrap.dedent(
'''
ENV PYTHONPATH /app:$PYTHONPATH
CMD ["python", "-m", "platypush"]
'''
)
dockerfile = os.path.join(device_dir, 'Dockerfile')
print('Generating Dockerfile {}'.format(dockerfile))
with open(dockerfile, 'w') as f:
f.write(content)
def build(args):
global workdir
ports = set()
parser = argparse.ArgumentParser(
prog='platydock build', description='Build a Platypush image from a config.yaml'
)
parser.add_argument(
'-c',
'--config',
type=str,
required=True,
help='Path to the platypush configuration file',
)
parser.add_argument(
'-p',
'--python-version',
type=str,
default='3.9',
help='Python version to be used',
)
opts, args = parser.parse_known_args(args)
cfgfile = os.path.abspath(os.path.expanduser(opts.config))
manifest._available_package_manager = (
'apt' # Force apt for Debian-based Docker images
)
install_cmds = manifest.get_dependencies_from_conf(cfgfile)
python_version = opts.python_version
backend_config = Config.get_backends()
# Container exposed ports
if backend_config.get('http'):
from platypush.backend.http import HttpBackend
# noinspection PyProtectedMember
ports.add(backend_config['http'].get('port', HttpBackend._DEFAULT_HTTP_PORT))
if backend_config.get('tcp'):
ports.add(backend_config['tcp']['port'])
dev_dir = os.path.join(workdir, Config.get('device_id'))
generate_dockerfile(
deps=dict(install_cmds),
ports=ports,
cfgfile=cfgfile,
device_dir=dev_dir,
python_version=python_version,
)
subprocess.call(
[
def _build_image(self):
"""
Build a Platypush Docker image from the generated Dockerfile.
"""
logger.info('Building Docker image...')
cmd = [
'docker',
'build',
'-f',
self.dockerfile,
'-t',
'platypush-{}'.format(Config.get('device_id')),
dev_dir,
self.tag,
'.',
]
)
subprocess.run(cmd, check=True)
def start(args):
global workdir
def _generate_dockerfile(self):
"""
Parses the configuration file and generates a Dockerfile based on it.
"""
parser = argparse.ArgumentParser(
prog='platydock start',
description='Start a Platypush container',
epilog=textwrap.dedent(
'''
You can append additional options that
will be passed to the docker container.
Example:
@contextmanager
def open_writer() -> Generator[IO, None, None]:
# flake8: noqa
f = sys.stdout if self.print_only else open(self.dockerfile, 'w')
--add-host='myhost:192.168.1.1'
'''
),
)
try:
yield f
finally:
if f is not sys.stdout:
f.close()
parser.add_argument('image', type=str, help='Platypush image to start')
parser.add_argument(
'-p',
'--publish',
action='append',
nargs='*',
default=[],
help=textwrap.dedent(
'''
Container's ports to expose to the host.
Note that the default exposed ports from
the container service will be exposed unless
these mappings override them (e.g. port 8008
on the container will be mapped to 8008 on
the host).
if not self.print_only:
logger.info('Parsing configuration file %s...', self.cfgfile)
Example:
Config.init(self.cfgfile)
-p 18008:8008
'''
),
)
if not self.print_only:
logger.info('Generating Dockerfile %s...', self.dockerfile)
parser.add_argument(
'-a',
'--attach',
action='store_true',
default=False,
help=textwrap.dedent(
'''
If set, then attach to the container after starting it up (default: false).
'''
),
)
parser = self._create_dockerfile_parser()
opts, args = parser.parse_known_args(args)
ports = {}
dockerfile = os.path.join(workdir, opts.image, 'Dockerfile')
with open_writer() as f:
f.write(self._header + '\n')
for line in parser():
f.write(line + '\n')
with open(dockerfile) as f:
for line in f:
m = re.match(r'expose (\d+)', line.strip().lower())
if m:
ports[m.group(1)] = m.group(1)
def _generate_git_clone_command(self) -> str:
"""
Generates a git clone command in Dockerfile that checks out the repo
and the right git reference, if the application sources aren't already
available under /install.
"""
install_cmd = ' '.join(self.pkg_manager.value.install)
uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall)
return textwrap.dedent(
f"""
RUN if [ ! -f "/install/setup.py" ]; then \\
echo "Platypush source not found under the current directory, downloading it" && \\
{install_cmd} git && \\
rm -rf /install && \\
git clone --recursive https://github.com/BlackLight/platypush.git /install && \\
cd /install && \\
git checkout {self.gitref} && \\
{uninstall_cmd} git; \\
fi
"""
)
for mapping in opts.publish:
host_port, container_port = mapping[0].split(':')
ports[container_port] = host_port
@classmethod
@override
def _get_arg_parser(cls) -> argparse.ArgumentParser:
parser = super()._get_arg_parser()
print('Preparing Redis support container')
subprocess.call(['docker', 'pull', 'redis'])
subprocess.call(
['docker', 'run', '--rm', '--name', 'redis-' + opts.image, '-d', 'redis']
)
parser.add_argument(
'-i',
'--image',
dest='image',
required=False,
type=BaseImage,
choices=list(BaseImage),
default=BaseImage.ALPINE,
help='Base image to use for the Dockerfile (default: alpine).',
)
docker_cmd = [
'docker',
'run',
'--rm',
'--name',
opts.image,
'-it',
'--link',
'redis-' + opts.image + ':redis',
]
parser.add_argument(
'-t',
'--tag',
dest='tag',
required=False,
type=str,
default='platypush:latest',
help='Tag name to be used for the built image '
'(default: "platypush:latest").',
)
for container_port, host_port in ports.items():
docker_cmd += ['-p', host_port + ':' + container_port]
parser.add_argument(
'--print',
dest='print_only',
action='store_true',
help='Use this flag if you only want to print the Dockerfile to '
'stdout instead of generating an image.',
)
docker_cmd += args
docker_cmd += ['-d', 'platypush-' + opts.image]
return parser
print('Starting Platypush container {}'.format(opts.image))
subprocess.call(docker_cmd)
if opts.attach:
subprocess.call(['docker', 'attach', opts.image])
def stop(args):
parser = argparse.ArgumentParser(
prog='platydock stop', description='Stop a Platypush container'
)
parser.add_argument('container', type=str, help='Platypush container to stop')
opts, args = parser.parse_known_args(args)
print('Stopping Platypush container {}'.format(opts.container))
subprocess.call(['docker', 'kill', '{}'.format(opts.container)])
print('Stopping Redis support container')
subprocess.call(['docker', 'stop', 'redis-{}'.format(opts.container)])
def rm(args):
global workdir
parser = argparse.ArgumentParser(
prog='platydock rm',
description='Remove a Platypush image. '
+ 'NOTE: make sure that no container is '
+ 'running nor linked to the image before '
+ 'removing it',
)
parser.add_argument('image', type=str, help='Platypush image to remove')
opts, args = parser.parse_known_args(args)
subprocess.call(['docker', 'rmi', 'platypush-{}'.format(opts.image)])
shutil.rmtree(os.path.join(workdir, opts.image), ignore_errors=True)
def ls(args):
parser = argparse.ArgumentParser(
prog='platydock ls', description='List available Platypush containers'
)
parser.add_argument('filter', type=str, nargs='?', help='Image name filter')
opts, args = parser.parse_known_args(args)
p = subprocess.Popen(['docker', 'images'], stdout=subprocess.PIPE)
output = p.communicate()[0].decode().split('\n')
header = output.pop(0)
images = []
for line in output:
if re.match(r'^platypush-(.+?)\s.*', line) and (
not opts.filter or (opts.filter and opts.filter in line)
):
images.append(line)
if images:
print(header)
for image in images:
print(image)
@staticmethod
def _get_exposed_ports() -> Iterable[int]:
"""
:return: The listen ports used by the backends enabled in the configuration
file.
"""
backends_config = Config.get_backends()
return {
int(port)
for port in (
backends_config.get('http', {}).get('port'),
backends_config.get('tcp', {}).get('port'),
)
if port
}
def main():
parser = argparse.ArgumentParser(
prog='platydock',
add_help=False,
description='Manage Platypush docker containers',
epilog='Use platydock <action> --help to ' + 'get additional help',
)
# noinspection PyTypeChecker
parser.add_argument(
'action', nargs='?', type=Action, choices=list(Action), help='Action to execute'
)
parser.add_argument('-h', '--help', action='store_true', help='Show usage')
opts, args = parser.parse_known_args(sys.argv[1:])
if (opts.help and not opts.action) or (not opts.help and not opts.action):
parser.print_help()
return 1
globals()[str(opts.action)](sys.argv[2:])
"""
Generates a Dockerfile based on the configuration file.
"""
DockerBuilder.from_cmdline(sys.argv[1:]).build()
return 0
if __name__ == '__main__':
ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m '
ERR_SUFFIX = '\033[0m'
sys.exit(main())
try:
main()
except Exception as e:
tb.print_exc(file=sys.stdout)
print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr)
# vim:sw=4:ts=4:et:

1
platypush/platydock/__main__.py
View file

@ -3,4 +3,3 @@ from platypush.platydock import main
main()
# vim:sw=4:ts=4:et:

193
platypush/platyvenv/__init__.py Executable file
View file

@ -0,0 +1,193 @@
"""
Platyvenv is a helper script that allows you to automatically create a
virtual environment for Platypush starting from a configuration file.
"""
from contextlib import contextmanager
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
from typing import Generator, Sequence
import venv
from typing_extensions import override
from platypush.builder import BaseBuilder
from platypush.config import Config
from platypush.utils.manifest import (
Dependencies,
InstallContext,
)
logger = logging.getLogger()
class VenvBuilder(BaseBuilder):
"""
Build a virtual environment from on a configuration file.
"""
def __init__(self, *args, **kwargs) -> None:
kwargs['install_context'] = InstallContext.DOCKER
super().__init__(*args, **kwargs)
@override
@classmethod
def get_name(cls):
return "platyvenv"
@override
@classmethod
def get_description(cls):
return "Build a Platypush virtual environment from a configuration file."
@property
def _pip_cmd(self) -> Sequence[str]:
"""
:return: The pip install command to use for the selected environment.
"""
return (
os.path.join(self.output, 'bin', 'python'),
'-m',
'pip',
'install',
'-U',
'--no-cache-dir',
'--no-input',
)
def _install_system_packages(self, deps: Dependencies):
"""
Install the required system packages.
"""
for cmd in deps.to_pkg_install_commands():
logger.info('Installing system packages: %s', cmd)
subprocess.call(re.split(r'\s+', cmd.strip()))
@contextmanager
def _prepare_src_dir(self) -> Generator[str, None, None]:
"""
Prepare the source directory used to install the virtual enviornment.
If platyvenv is launched from a local checkout of the Platypush source
code, then that checkout will be used.
Otherwise, the source directory will be cloned from git into a
temporary folder.
"""
setup_py_path = os.path.join(os.getcwd(), 'setup.py')
if os.path.isfile(setup_py_path):
logger.info('Using local checkout of the Platypush source code')
yield os.getcwd()
else:
checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git')
logger.info('Cloning Platypush source code from git into %s', checkout_dir)
subprocess.call(
[
'git',
'clone',
'--recursive',
'https://github.com/BlackLight/platypush.git',
checkout_dir,
]
)
pwd = os.getcwd()
os.chdir(checkout_dir)
subprocess.call(['git', 'checkout', self.gitref])
yield checkout_dir
os.chdir(pwd)
logger.info('Cleaning up %s', checkout_dir)
shutil.rmtree(checkout_dir, ignore_errors=True)
def _prepare_venv(self) -> None:
"""
Install the virtual environment under the configured output.
"""
logger.info('Creating virtual environment under %s...', self.output)
venv.create(
self.output,
symlinks=True,
with_pip=True,
upgrade_deps=True,
)
logger.info('Installing base Python dependencies under %s...', self.output)
subprocess.call([*self._pip_cmd, 'pip', '.'])
def _install_extra_pip_packages(self, deps: Dependencies):
"""
Install the extra pip dependencies inferred from the configured
extensions.
"""
pip_deps = list(deps.to_pip_install_commands(full_command=False))
if not pip_deps:
return
logger.info(
'Installing extra pip dependencies under %s: %s',
self.output,
' '.join(pip_deps),
)
subprocess.call([*self._pip_cmd, *pip_deps])
def build(self):
"""
Build a Dockerfile based on a configuration file.
"""
Config.init(self.cfgfile)
deps = Dependencies.from_config(
self.cfgfile,
install_context=InstallContext.VENV,
)
self._install_system_packages(deps)
with self._prepare_src_dir():
self._prepare_venv()
self._install_extra_pip_packages(deps)
self._print_instructions(
textwrap.dedent(
f"""
Virtual environment created under {self.output}.
To run the application:
source {self.output}/bin/activate
platypush -c {self.cfgfile} {
"--device_id " + self.device_id if self.device_id else ""
}
Platypush requires a Redis instance. If you don't want to use a
stand-alone server, you can pass the --start-redis option to
the executable (optionally with --redis-port).
Platypush will then start its own local instance and it will
terminate it once the application is stopped.
"""
)
)
def main():
"""
Generates a virtual environment based on the configuration file.
"""
VenvBuilder.from_cmdline(sys.argv[1:]).build()
return 0
if __name__ == '__main__':
sys.exit(main())
# vim:sw=4:ts=4:et:

5
platypush/platyvenv/__main__.py Normal file
View file

@ -0,0 +1,5 @@
from platypush.platyvenv import main
main()
# vim:sw=4:ts=4:et:

4
platypush/plugins/__init__.py
View file

@ -15,7 +15,7 @@ from platypush.message.response import Response
from platypush.utils import get_decorators, get_plugin_name_by_class
PLUGIN_STOP_TIMEOUT = 5 # Plugin stop timeout in seconds
logger = logging.getLogger(__name__)
_logger = logging.getLogger(__name__)
def action(f: Callable[..., Any]) -> Callable[..., Response]:
@ -33,7 +33,7 @@ def action(f: Callable[..., Any]) -> Callable[..., Response]:
try:
result = f(*args, **kwargs)
except TypeError as e:
logger.exception(e)
_logger.exception(e)
result = Response(errors=[str(e)])
if result and isinstance(result, Response):

3
platypush/plugins/bluetooth/manifest.yaml
View file

@ -15,12 +15,15 @@ manifest:
install:
apk:
- py3-pydbus
- git
apt:
- libbluetooth-dev
- python3-pydbus
- git
pacman:
- python-pydbus
- python-bleak
- git
pip:
- bleak
- bluetooth-numbers

3
platypush/plugins/gpio/manifest.yaml
View file

@ -1,7 +1,6 @@
manifest:
events:
- platypush.message.event.gpio.GPIOEvent:
When the value of a monitored PIN changes.
- platypush.message.event.gpio.GPIOEvent
install:
pip:
- RPi.GPIO

5
platypush/plugins/http/request/rss/__init__.py
View file

@ -1,6 +1,7 @@
from platypush.plugins import action
from platypush.plugins.http.request import HttpRequestPlugin
class HttpRequestRssPlugin(HttpRequestPlugin):
"""
Plugin to programmatically retrieve and parse an RSS feed URL.
@ -11,12 +12,12 @@ class HttpRequestRssPlugin(HttpRequestPlugin):
"""
@action
def get(self, url):
def get(self, url, **_):
import feedparser
response = super().get(url, output='text').output
feed = feedparser.parse(response)
return feed.entries
# vim:sw=4:ts=4:et:

353
platypush/plugins/http/webpage/__init__.py
View file

@ -1,13 +1,66 @@
from dataclasses import dataclass
import datetime
from enum import Enum
import json
import os
import re
import subprocess
import tempfile
import textwrap
from typing import Iterable, Optional, Union
from urllib.parse import urlparse
from platypush.plugins import action
from platypush.plugins.http.request import Plugin
from platypush.plugins import Plugin, action
@dataclass
class OutputFormat:
"""
Definition of a supported output format.
"""
name: str
cmd_fmt: str
extensions: Iterable[str] = ()
class OutputFormats(Enum):
"""
Supported output formats.
"""
HTML = OutputFormat('html', extensions=('html', 'htm'), cmd_fmt='html')
# PDF will first be exported to HTML and then converted to PDF
PDF = OutputFormat('pdf', extensions=('pdf',), cmd_fmt='html')
TEXT = OutputFormat('text', extensions=('txt',), cmd_fmt='text')
MARKDOWN = OutputFormat('markdown', extensions=('md',), cmd_fmt='markdown')
@classmethod
def parse(
cls,
type: Union[str, "OutputFormats"], # pylint: disable=redefined-builtin
outfile: Optional[str] = None,
) -> "OutputFormats":
"""
Parse the format given a type argument and and output file name.
"""
try:
fmt = (
getattr(OutputFormats, type.upper()) if isinstance(type, str) else type
)
except AttributeError as e:
raise AssertionError(
f'Unsupported output format: {type}. Supported formats: '
+ f'{[f.name for f in OutputFormats]}'
) from e
by_extension = {ext.lower(): f for f in cls for ext in f.value.extensions}
if outfile:
fmt_by_ext = by_extension.get(os.path.splitext(outfile)[1].lower()[1:])
if fmt_by_ext:
return fmt_by_ext
return fmt
class HttpWebpagePlugin(Plugin):
@ -24,34 +77,71 @@ class HttpWebpagePlugin(Plugin):
"""
_mercury_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js')
_mercury_script = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js'
)
@staticmethod
def _parse(proc):
"""
Runs the mercury-parser script and returns the result as a string.
"""
with subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=None) as parser:
return parser.communicate()[0].decode()
@staticmethod
def _fix_relative_links(markdown: str, url: str) -> str:
url = urlparse(url)
base_url = f'{url.scheme}://{url.netloc}'
"""
Fix relative links to match the base URL of the page (Markdown only).
"""
parsed_url = urlparse(url)
base_url = f'{parsed_url.scheme}://{parsed_url.netloc}'
return re.sub(r'(\[.+?])\((/.+?)\)', fr'\1({base_url}\2)', markdown)
# noinspection PyShadowingBuiltins
@action
def simplify(self, url, type='html', html=None, outfile=None):
def simplify(
self,
url: str,
type: Union[ # pylint: disable=redefined-builtin
str, OutputFormats
] = OutputFormats.HTML,
html: Optional[str] = None,
outfile: Optional[str] = None,
font_size: str = '19px',
font_family: Union[str, Iterable[str]] = (
'-apple-system',
'Segoe UI',
'Roboto',
'Oxygen',
'Ubuntu',
'Cantarell',
"Fira Sans",
'Open Sans',
'Droid Sans',
'Helvetica Neue',
'Helvetica',
'Arial',
'sans-serif',
),
):
"""
Parse the readable content of a web page removing any extra HTML elements using Mercury.
:param url: URL to parse.
:param type: Output format. Supported types: ``html``, ``markdown``, ``text`` (default: ``html``).
:param html: Set this parameter if you want to parse some HTML content already fetched. Note
that URL is still required by Mercury to properly style the output, but it won't be used
to actually fetch the content.
:param outfile: If set then the output will be written to the specified file. If the file extension
is ``.pdf`` then the content will be exported in PDF format. If the output ``type`` is not
specified then it can also be inferred from the extension of the output file.
:param type: Output format. Supported types: ``html``, ``markdown``,
``text``, ``pdf`` (default: ``html``).
:param html: Set this parameter if you want to parse some HTML content
already fetched. Note that URL is still required by Mercury to
properly style the output, but it won't be used to actually fetch
the content.
:param outfile: If set then the output will be written to the specified
file. If the file extension is ``.pdf`` then the content will be
exported in PDF format. If the output ``type`` is not specified
then it can also be inferred from the extension of the output file.
:param font_size: Font size to use for the output (default: 19px).
:param font_family: Custom font family (or list of font families, in
decreasing order) to use for the output. It only applies to HTML
and PDF.
:return: dict
Example if outfile is not specified::
@ -74,48 +164,46 @@ class HttpWebpagePlugin(Plugin):
"""
self.logger.info('Parsing URL {}'.format(url))
wants_pdf = False
if outfile:
wants_pdf = outfile.lower().endswith('.pdf')
if (
wants_pdf # HTML will be exported to PDF
or outfile.lower().split('.')[-1].startswith('htm')
):
type = 'html'
elif outfile.lower().endswith('.md'):
type = 'markdown'
elif outfile.lower().endswith('.txt'):
type = 'text'
proc = ['node', self._mercury_script, url, type]
f = None
self.logger.info('Parsing URL %s', url)
fmt = OutputFormats.parse(type=type, outfile=outfile)
proc = ['node', self._mercury_script, url, fmt.value.cmd_fmt]
tmp_file = None
if html:
f = tempfile.NamedTemporaryFile('w+', delete=False)
f.write(html)
f.flush()
proc.append(f.name)
with tempfile.NamedTemporaryFile('w+', delete=False) as f:
tmp_file = f.name
f.write(html)
f.flush()
proc.append(f.name)
try:
response = self._parse(proc)
finally:
if f:
os.unlink(f.name)
if tmp_file:
os.unlink(tmp_file)
try:
response = json.loads(response.strip())
except Exception as e:
raise RuntimeError('Could not parse JSON: {}. Response: {}'.format(str(e), response))
raise RuntimeError(
f'Could not parse JSON: {e}. Response: {response}'
) from e
if type == 'markdown':
if fmt == OutputFormats.MARKDOWN:
response['content'] = self._fix_relative_links(response['content'], url)
self.logger.debug('Got response from Mercury API: {}'.format(response))
title = response.get('title', '{} on {}'.format(
'Published' if response.get('date_published') else 'Generated',
response.get('date_published', datetime.datetime.now().isoformat())))
self.logger.debug('Got response from Mercury API: %s', response)
title = response.get(
'title',
(
('Published' if response.get('date_published') else 'Generated')
+ ' on '
+ (
response.get('date_published')
or datetime.datetime.now().isoformat()
)
),
)
content = response.get('content', '')
@ -126,46 +214,134 @@ class HttpWebpagePlugin(Plugin):
'content': content,
}
outfile = os.path.abspath(os.path.expanduser(outfile))
style = '''
body {
font-size: 22px;
font-family: 'Merriweather', Georgia, 'Times New Roman', Times, serif;
}
'''
return self._process_outfile(
url=url,
fmt=fmt,
title=title,
content=content,
outfile=outfile,
font_size=font_size,
font_family=tuple(
font_family,
)
if isinstance(font_family, str)
else tuple(font_family),
)
if type == 'html':
content = (
@staticmethod
def _style_by_format(
fmt: OutputFormats,
font_size: str,
font_family: Iterable[str],
) -> str:
"""
:return: The CSS style to be used for the given output format.
"""
style = textwrap.dedent(
f'''
._parsed-content-container {{
font-size: {font_size};
font-family: {', '.join(f'"{f}"' for f in font_family)};
}}
._parsed-content {{
text-align: justify;
}}
pre {{
white-space: pre-wrap;
}}
'''
)
if fmt == OutputFormats.HTML:
style += textwrap.dedent(
'''
._parsed-content-container {
margin: 1em;
display: flex;
flex-direction: column;
align-items: center;
}
._parsed-content {
max-width: 800px;
}
h1 {
max-width: 800px;
}
'''
)
return style
@classmethod
def _process_outfile(
cls,
url: str,
fmt: OutputFormats,
title: str,
content: str,
outfile: str,
font_size: str,
font_family: Iterable[str],
):
"""
Process the output file.
:param url: URL to parse.
:param fmt: Output format. Supported types: ``html``, ``markdown``,
``text``, ``pdf`` (default: ``html``).
:param title: Page title.
:param content: Page content.
:param outfile: Output file path.
:param font_size: Font size to use for the output (default: 19px).
:param font_family: Custom font family (or list of font families, in
decreasing order) to use for the output. It only applies to HTML
and PDF.
:return: dict
"""
outfile = os.path.abspath(os.path.expanduser(outfile))
style = cls._style_by_format(fmt, font_size, font_family)
if fmt in {OutputFormats.HTML, OutputFormats.PDF}:
content = textwrap.dedent(
f'''
<div class="_parsed-content-container">
<h1><a href="{url}" target="_blank">{title}</a></h1>
<div class="_parsed-content">{content}</div>
'''.format(title=title, url=url, content=content)
</div>
'''
)
if not wants_pdf:
content = '''<html>
<head>
<title>{title}</title>
<style>{style}</style>
</head>'''.format(title=title, style=style) + \
'<body>{{' + content + '}}</body></html>'
elif type == 'markdown':
content = '# [{title}]({url})\n\n{content}'.format(
title=title, url=url, content=content
)
if fmt == OutputFormats.PDF:
content = textwrap.dedent(
f'''<html>
<head>
<style>{style}</style>
<title>{title}</title>
</head>
<body>
{content}
</body>
</html>
'''
)
else:
content = textwrap.dedent(
f'''
<style>
{style}
</style>
{content}
'''
)
elif fmt == OutputFormats.MARKDOWN:
content = f'# [{title}]({url})\n\n{content}'
if wants_pdf:
import weasyprint
try:
from weasyprint.fonts import FontConfiguration
except ImportError:
from weasyprint.document import FontConfiguration
font_config = FontConfiguration()
css = [weasyprint.CSS('https://fonts.googleapis.com/css?family=Merriweather'),
weasyprint.CSS(string=style, font_config=font_config)]
weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css)
if fmt == OutputFormats.PDF:
cls._process_pdf(content, outfile, style)
else:
with open(outfile, 'w', encoding='utf-8') as f:
f.write(content)
@ -176,5 +352,28 @@ class HttpWebpagePlugin(Plugin):
'outfile': outfile,
}
@staticmethod
def _process_pdf(content: str, outfile: str, style: str):
"""
Convert the given HTML content to a PDF document.
:param content: Page content.
:param outfile: Output file path.
:param style: CSS style to use for the output.
"""
import weasyprint
try:
from weasyprint.fonts import FontConfiguration # pylint: disable
except ImportError:
from weasyprint.document import FontConfiguration
font_config = FontConfiguration()
css = [
weasyprint.CSS(string=style, font_config=font_config),
]
weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css)
# vim:sw=4:ts=4:et:

2
platypush/plugins/http/webpage/manifest.yaml
View file

@ -15,7 +15,7 @@ manifest:
- npm
pip:
- weasyprint
exec:
after:
- sudo npm install -g @postlight/mercury-parser
package: platypush.plugins.http.webpage
type: plugin

5
platypush/plugins/inspect/__init__.py
View file

@ -20,7 +20,7 @@ from platypush.utils import (
get_plugin_class_by_name,
get_plugin_name_by_class,
)
from platypush.utils.manifest import Manifest, scan_manifests
from platypush.utils.manifest import Manifests
from ._context import ComponentContext
from ._model import (
@ -116,8 +116,7 @@ class InspectPlugin(Plugin):
A generator that scans the manifest files given a ``base_type``
(``Plugin`` or ``Backend``) and yields the parsed submodules.
"""
for mf_file in scan_manifests(base_type):
manifest = Manifest.from_file(mf_file)
for manifest in Manifests.by_base_class(base_type):
try:
yield importlib.import_module(manifest.package)
except Exception as e:

2
platypush/plugins/music/mpd/manifest.yaml
View file

@ -2,7 +2,7 @@ manifest:
events: {}
install:
apt:
- python-mpd
- python3-mpd
pacman:
- python-mpd2
pip:

3
platypush/plugins/music/tidal/manifest.yaml
View file

@ -1,7 +1,6 @@
manifest:
events:
- platypush.message.event.music.TidalPlaylistUpdatedEvent: when a user playlist
is updated.
- platypush.message.event.music.tidal.TidalPlaylistUpdatedEvent
install:
pip:
- tidalapi >= 0.7.0

2
platypush/plugins/sensor/distance/vl53l1x/manifest.yaml
View file

@ -1,6 +1,6 @@
manifest:
events:
platypush.message.event.sensor import SensorDataChangeEvent:
- platypush.message.event.sensor.SensorDataChangeEvent
install:
pip:

2
platypush/plugins/serial/manifest.yaml
View file

@ -1,6 +1,6 @@
manifest:
events:
- platypush.message.event.sensor.SensorDataChangeEvent:
- platypush.message.event.sensor.SensorDataChangeEvent
install:
apk:
- py3-pyserial

10
platypush/plugins/tts/mimic3/__init__.py
View file

@ -1,8 +1,9 @@
import requests
from typing import Optional
from urllib.parse import urljoin, urlencode
from platypush.backend.http.app.utils import get_local_base_url
import requests
from platypush.backend.http.app.utils import get_local_base_url
from platypush.context import get_backend
from platypush.plugins import action
from platypush.plugins.tts import TtsPlugin
@ -10,7 +11,7 @@ from platypush.schemas.tts.mimic3 import Mimic3VoiceSchema
class TtsMimic3Plugin(TtsPlugin):
"""
r"""
TTS plugin that uses the `Mimic3 webserver
<https://github.com/MycroftAI/mimic3>`_ provided by `Mycroft
<https://mycroft.ai/>`_ as a text-to-speech engine.
@ -42,7 +43,7 @@ class TtsMimic3Plugin(TtsPlugin):
voice: str = 'en_UK/apope_low',
media_plugin: Optional[str] = None,
player_args: Optional[dict] = None,
**kwargs
**kwargs,
):
"""
:param server_url: Base URL of the web server that runs the Mimic3 engine.
@ -69,6 +70,7 @@ class TtsMimic3Plugin(TtsPlugin):
def say(
self,
text: str,
*_,
server_url: Optional[str] = None,
voice: Optional[str] = None,
player_args: Optional[dict] = None,

2
platypush/plugins/xmpp/__init__.py
View file

@ -117,7 +117,7 @@ class XmppPlugin(AsyncRunnablePlugin, XmppBasePlugin):
auto_accept_invites=auto_accept_invites,
restore_state=restore_state,
state_file=os.path.expanduser(
state_file or os.path.join(Config.workdir, 'xmpp', 'state.json')
state_file or os.path.join(Config.get_workdir(), 'xmpp', 'state.json')
),
)
self._loaded_state = SerializedState()

2
platypush/runner/_app.py
View file

@ -32,7 +32,7 @@ class ApplicationProcess(ControllableProcess):
self.logger.info('Starting application...')
with subprocess.Popen(
['python', '-m', 'platypush.app', *self.args],
[sys.executable, '-m', 'platypush.app', *self.args],
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,

17
platypush/runner/_runner.py
View file

@ -1,4 +1,6 @@
import logging
import os
import signal
import sys
from threading import Thread
from typing import Optional
@ -23,6 +25,7 @@ class ApplicationRunner:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
self.logger = logging.getLogger('platypush:runner')
self._proc: Optional[ApplicationProcess] = None
self._stream: Optional[CommandStream] = None
def _listen(self, stream: CommandStream):
"""
@ -48,12 +51,16 @@ class ApplicationRunner:
if parsed_args.version:
self._print_version()
signal.signal(signal.SIGTERM, lambda *_: self.stop())
while True:
with CommandStream(parsed_args.ctrl_sock) as stream, ApplicationProcess(
with CommandStream(
parsed_args.ctrl_sock
) as self._stream, ApplicationProcess(
*args, pidfile=parsed_args.pidfile, timeout=self._default_timeout
) as self._proc:
try:
self._listen(stream)
self._listen(self._stream)
except KeyboardInterrupt:
pass
@ -63,6 +70,8 @@ class ApplicationRunner:
break
self._stream = None
def run(self, *args: str) -> None:
try:
self._run(*args)
@ -73,6 +82,10 @@ class ApplicationRunner:
if self._proc is not None:
self._proc.stop()
if self._stream and self._stream.pid:
os.kill(self._stream.pid, signal.SIGKILL)
self._stream = None
def restart(self):
if self._proc is not None:
self._proc.mark_for_restart()

18
platypush/utils/__init__.py
View file

@ -523,7 +523,7 @@ def get_or_generate_jwt_rsa_key_pair():
"""
from platypush.config import Config
key_dir = os.path.join(Config.workdir, 'jwt')
key_dir = os.path.join(Config.get_workdir(), 'jwt')
priv_key_file = os.path.join(key_dir, 'id_rsa')
pub_key_file = priv_key_file + '.pub'
@ -646,4 +646,20 @@ def get_remaining_timeout(
return cls(max(0, timeout - (time.time() - start)))
def get_src_root() -> str:
"""
:return: The root source folder of the application.
"""
import platypush
return os.path.dirname(inspect.getfile(platypush))
def is_root() -> bool:
"""
:return: True if the current user is root/administrator.
"""
return os.getuid() == 0
# vim:sw=4:ts=4:et:

707
platypush/utils/manifest.py
View file

@ -1,200 +1,617 @@
import enum
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import Enum
import importlib
import inspect
import json
import logging
import os
import pathlib
import re
import shutil
import subprocess
import sys
from typing import (
Callable,
Dict,
Generator,
List,
Optional,
Iterable,
Mapping,
Sequence,
Set,
Type,
Union,
)
from typing_extensions import override
import yaml
from abc import ABC, abstractmethod
from typing import Optional, Iterable, Mapping, Callable, Type
supported_package_managers = {
'pacman': 'pacman -S',
'apt': 'apt-get install',
}
from platypush.message.event import Event
from platypush.utils import get_src_root, is_root
_available_package_manager = None
logger = logging.getLogger(__name__)
class ManifestType(enum.Enum):
class BaseImage(Enum):
"""
Supported base images for Dockerfiles.
"""
ALPINE = 'alpine'
DEBIAN = 'debian'
UBUNTU = 'ubuntu'
def __str__(self) -> str:
"""
Explicit __str__ override for argparse purposes.
"""
return self.value
@dataclass
class PackageManager:
"""
Representation of a package manager.
"""
executable: str
""" The executable name. """
default_os: str
"""
The default distro whose configuration we should use if this package
manager is detected.
"""
install: Sequence[str] = field(default_factory=tuple)
""" The install command, as a sequence of strings. """
uninstall: Sequence[str] = field(default_factory=tuple)
""" The uninstall command, as a sequence of strings. """
list: Sequence[str] = field(default_factory=tuple)
""" The command to list the installed packages. """
parse_list_line: Callable[[str], str] = field(default_factory=lambda: lambda s: s)
"""
Internal package-manager dependent function that parses the base package
name from a line returned by the list command.
"""
def _get_installed(self) -> Sequence[str]:
"""
:return: The install context-aware list of installed packages.
It should only used within the context of :meth:`.get_installed`.
"""
if os.environ.get('DOCKER_CTX'):
# If we're running in a Docker build context, don't run the package
# manager to retrieve the list of installed packages, as the host
# and guest systems have different environments.
return ()
return tuple(
line.strip()
for line in subprocess.Popen( # pylint: disable=consider-using-with
self.list, stdout=subprocess.PIPE
)
.communicate()[0]
.decode()
.split('\n')
if line.strip()
)
def get_installed(self) -> Sequence[str]:
"""
:return: The list of installed packages.
"""
return tuple(self.parse_list_line(line) for line in self._get_installed())
class PackageManagers(Enum):
"""
Supported package managers.
"""
APK = PackageManager(
executable='apk',
install=('apk', 'add', '--update', '--no-interactive', '--no-cache'),
uninstall=('apk', 'del', '--no-interactive'),
list=('apk', 'list', '--installed'),
default_os='alpine',
parse_list_line=lambda line: re.sub(r'.*\s*\{(.+?)\}\s*.*', r'\1', line),
)
APT = PackageManager(
executable='apt',
install=('apt', 'install', '-y'),
uninstall=('apt', 'remove', '-y'),
list=('apt', 'list', '--installed'),
default_os='debian',
parse_list_line=lambda line: line.split('/')[0],
)
PACMAN = PackageManager(
executable='pacman',
install=('pacman', '-S', '--noconfirm', '--needed'),
uninstall=('pacman', '-R', '--noconfirm'),
list=('pacman', '-Q'),
default_os='arch',
parse_list_line=lambda line: line.split(' ')[0],
)
@classmethod
def get_command(cls, name: str) -> Iterable[str]:
"""
:param name: The name of the package manager executable to get the
command for.
:return: The base command to execute, as a sequence of strings.
"""
pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None)
if not pkg_manager:
raise ValueError(f'Unknown package manager: {name}')
return pkg_manager.value.install
@classmethod
def scan(cls) -> Optional["PackageManagers"]:
"""
Get the name of the available package manager on the system, if supported.
"""
# pylint: disable=global-statement
global _available_package_manager
if _available_package_manager:
return _available_package_manager
available_package_managers = [
pkg_manager
for pkg_manager in cls
if shutil.which(pkg_manager.value.executable)
]
if not available_package_managers:
logger.warning(
'\nYour OS does not provide any of the supported package managers.\n'
'You may have to install some optional dependencies manually.\n'
'Supported package managers: %s.\n',
', '.join([pm.value.executable for pm in cls]),
)
return None
_available_package_manager = available_package_managers[0]
return _available_package_manager
class InstallContext(Enum):
"""
Supported installation contexts.
"""
NONE = None
DOCKER = 'docker'
VENV = 'venv'
class ManifestType(Enum):
"""
Manifest types.
"""
PLUGIN = 'plugin'
BACKEND = 'backend'
@dataclass
class Dependencies:
"""
Dependencies for a plugin/backend.
"""
before: List[str] = field(default_factory=list)
""" Commands to execute before the component is installed. """
packages: Set[str] = field(default_factory=set)
""" System packages required by the component. """
pip: Set[str] = field(default_factory=set)
""" pip dependencies. """
after: List[str] = field(default_factory=list)
""" Commands to execute after the component is installed. """
pkg_manager: Optional[PackageManagers] = None
""" Override the default package manager detected on the system. """
install_context: InstallContext = InstallContext.NONE
""" The installation context - Docker, virtual environment or bare metal. """
base_image: Optional[BaseImage] = None
""" Base image used in case of Docker installations. """
@property
def _is_venv(self) -> bool:
"""
:return: True if the dependencies scanning logic is running either in a
virtual environment or in a virtual environment preparation
context.
"""
return (
self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix
)
@property
def _is_docker(self) -> bool:
"""
:return: True if the dependencies scanning logic is running either in a
Docker environment.
"""
return (
self.install_context == InstallContext.DOCKER
or bool(os.environ.get('DOCKER_CTX'))
or os.path.isfile('/.dockerenv')
)
@property
def _wants_sudo(self) -> bool:
"""
:return: True if the system dependencies should be installed with sudo.
"""
return not (self._is_docker or is_root())
@staticmethod
def _get_requirements_dir() -> str:
"""
:return: The root folder for the base installation requirements.
"""
return os.path.join(get_src_root(), 'install', 'requirements')
@classmethod
def _parse_requirements_file(
cls,
requirements_file: str,
install_context: InstallContext = InstallContext.NONE,
) -> Iterable[str]:
"""
:param requirements_file: The path to the requirements file.
:return: The list of requirements to install.
"""
with open(requirements_file, 'r') as f:
return {
line.strip()
for line in f
if not (
not line.strip()
or line.strip().startswith('#')
# Virtual environments will install all the Python
# dependencies via pip, so we should skip them here
or (
install_context == InstallContext.VENV
and cls._is_python_pkg(line.strip())
)
)
}
@classmethod
def _get_base_system_dependencies(
cls,
pkg_manager: Optional[PackageManagers] = None,
install_context: InstallContext = InstallContext.NONE,
) -> Iterable[str]:
"""
:return: The base system dependencies that should be installed on the
system.
"""
# Docker images will install the base packages through their own
# dedicated shell script, so don't report their base system
# requirements here.
if not (pkg_manager and install_context != InstallContext.DOCKER):
return set()
return cls._parse_requirements_file(
os.path.join(
cls._get_requirements_dir(), pkg_manager.value.default_os + '.txt'
),
install_context,
)
@staticmethod
def _is_python_pkg(pkg: str) -> bool:
"""
Utility function that returns True if a given package is a Python
system package. These should be skipped during a virtual
environment installation, as the virtual environment will be
installed via pip.
"""
tokens = pkg.split('-')
return len(tokens) > 1 and tokens[0] in {'py3', 'python3', 'python'}
@classmethod
def from_config(
cls,
conf_file: Optional[str] = None,
pkg_manager: Optional[PackageManagers] = None,
install_context: InstallContext = InstallContext.NONE,
base_image: Optional[BaseImage] = None,
) -> "Dependencies":
"""
Parse the required dependencies from a configuration file.
"""
if not pkg_manager:
pkg_manager = PackageManagers.scan()
base_system_deps = cls._get_base_system_dependencies(
pkg_manager=pkg_manager, install_context=install_context
)
deps = cls(
packages=set(base_system_deps),
pkg_manager=pkg_manager,
install_context=install_context,
base_image=base_image,
)
for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager):
deps.before += manifest.install.before
deps.pip.update(manifest.install.pip)
deps.packages.update(manifest.install.packages)
deps.after += manifest.install.after
return deps
def to_pkg_install_commands(self) -> Generator[str, None, None]:
"""
Generates the package manager commands required to install the given
dependencies on the system.
"""
wants_sudo = not (self._is_docker or is_root())
pkg_manager = self.pkg_manager or PackageManagers.scan()
if self.packages and pkg_manager:
installed_packages = pkg_manager.value.get_installed()
to_install = sorted(
pkg
for pkg in self.packages # type: ignore
if pkg not in installed_packages
and not (
self.install_context == InstallContext.VENV
and self._is_python_pkg(pkg)
)
)
if to_install:
yield ' '.join(
[
*(['sudo'] if wants_sudo else []),
*pkg_manager.value.install,
*to_install,
]
)
def to_pip_install_commands(self, full_command=True) -> Generator[str, None, None]:
"""
Generates the pip commands required to install the given dependencies on
the system.
:param full_command: Whether to return the full pip command to execute
(as a single string) or the list of packages that will be installed
through another script.
"""
wants_break_system_packages = not (
# Docker installations shouldn't require --break-system-packages in
# pip, except for Debian
(self._is_docker and self.base_image != BaseImage.DEBIAN)
# --break-system-packages has been introduced in Python 3.10
or sys.version_info < (3, 11)
# If we're in a virtual environment then we don't need
# --break-system-packages
or self._is_venv
)
if self.pip:
deps = sorted(self.pip)
if full_command:
yield (
'pip install -U --no-input --no-cache-dir '
+ (
'--break-system-packages '
if wants_break_system_packages
else ''
)
+ ' '.join(deps)
)
else:
for dep in deps:
yield dep
def to_install_commands(self) -> Generator[str, None, None]:
"""
Generates the commands required to install the given dependencies on
this system.
"""
for cmd in self.before:
yield cmd
for cmd in self.to_pkg_install_commands():
yield cmd
for cmd in self.to_pip_install_commands():
yield cmd
for cmd in self.after:
yield cmd
class Manifest(ABC):
"""
Base class for plugin/backend manifests.
"""
def __init__(self, package: str, description: Optional[str] = None,
install: Optional[Iterable[str]] = None, events: Optional[Mapping] = None, **_):
def __init__(
self,
package: str,
description: Optional[str] = None,
install: Optional[Dict[str, Iterable[str]]] = None,
events: Optional[Mapping] = None,
pkg_manager: Optional[PackageManagers] = None,
**_,
):
self._pkg_manager = pkg_manager or PackageManagers.scan()
self.description = description
self.install = install or {}
self.events = events or {}
self.logger = logging.getLogger(__name__)
self.install = self._init_deps(install or {})
self.events = self._init_events(events or {})
self.package = package
self.component_name = '.'.join(package.split('.')[2:])
self.component = None
@classmethod
@property
@abstractmethod
def component_getter(self) -> Callable[[str], object]:
raise NotImplementedError
def manifest_type(self) -> ManifestType:
"""
:return: The type of the manifest.
"""
def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
deps = Dependencies()
for key, items in install.items():
if key == 'pip':
deps.pip.update(items)
elif key == 'before':
deps.before += items
elif key == 'after':
deps.after += items
elif self._pkg_manager and key == self._pkg_manager.value.executable:
deps.packages.update(items)
return deps
@staticmethod
def _init_events(
events: Union[Iterable[str], Mapping[str, Optional[str]]]
) -> Dict[Type[Event], str]:
evt_dict = events if isinstance(events, Mapping) else {e: None for e in events}
ret = {}
for evt_name, doc in evt_dict.items():
evt_module_name, evt_class_name = evt_name.rsplit('.', 1)
try:
evt_module = importlib.import_module(evt_module_name)
evt_class = getattr(evt_module, evt_class_name)
except Exception as e:
raise AssertionError(f'Could not load event {evt_name}: {e}') from e
ret[evt_class] = doc or evt_class.__doc__
return ret
@classmethod
def from_file(cls, filename: str) -> "Manifest":
def from_file(
cls, filename: str, pkg_manager: Optional[PackageManagers] = None
) -> "Manifest":
"""
Parse a manifest filename into a ``Manifest`` class.
"""
with open(str(filename), 'r') as f:
manifest = yaml.safe_load(f).get('manifest', {})
assert 'type' in manifest, f'Manifest file {filename} has no type field'
comp_type = ManifestType(manifest.pop('type'))
manifest_class = _manifest_class_by_type[comp_type]
return manifest_class(**manifest)
manifest_class = cls.by_type(comp_type)
return manifest_class(**manifest, pkg_manager=pkg_manager)
@classmethod
def from_class(cls, clazz) -> "Manifest":
return cls.from_file(os.path.dirname(inspect.getfile(clazz)))
def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]:
"""
:return: The manifest class corresponding to the given manifest type.
"""
if manifest_type == ManifestType.PLUGIN:
return PluginManifest
if manifest_type == ManifestType.BACKEND:
return BackendManifest
@classmethod
def from_component(cls, comp) -> "Manifest":
return cls.from_class(comp.__class__)
def get_component(self):
try:
self.component = self.component_getter(self.component_name)
except Exception as e:
self.logger.warning(f'Could not load {self.component_name}: {e}')
return self.component
raise ValueError(f'Unknown manifest type: {manifest_type}')
def __repr__(self):
return json.dumps({
'description': self.description,
'install': self.install,
'events': self.events,
'type': _manifest_type_by_class[self.__class__].value,
'package': self.package,
'component_name': self.component_name,
})
"""
:return: A JSON serialized representation of the manifest.
"""
return json.dumps(
{
'description': self.description,
'install': self.install,
'events': {
'.'.join([evt_type.__module__, evt_type.__name__]): doc
for evt_type, doc in self.events.items()
},
'type': self.manifest_type.value,
'package': self.package,
'component_name': self.component_name,
}
)
class PluginManifest(Manifest):
@classmethod
"""
Plugin manifest.
"""
@property
def component_getter(self):
from platypush.context import get_plugin
return get_plugin
@override
def manifest_type(self) -> ManifestType:
return ManifestType.PLUGIN
# pylint: disable=too-few-public-methods
class BackendManifest(Manifest):
@classmethod
"""
Backend manifest.
"""
@property
def component_getter(self):
from platypush.context import get_backend
return get_backend
@override
def manifest_type(self) -> ManifestType:
return ManifestType.BACKEND
_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = {
ManifestType.PLUGIN: PluginManifest,
ManifestType.BACKEND: BackendManifest,
}
class Manifests:
"""
General-purpose manifests utilities.
"""
_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = {
cls: t for t, cls in _manifest_class_by_type.items()
}
@staticmethod
def by_base_class(
base_class: Type, pkg_manager: Optional[PackageManagers] = None
) -> Generator[Manifest, None, None]:
"""
Get all the manifest files declared under the base path of a given class
and parse them into :class:`Manifest` objects.
"""
for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob(
'manifest.yaml'
):
yield Manifest.from_file(str(mf), pkg_manager=pkg_manager)
@staticmethod
def by_config(
conf_file: Optional[str] = None,
pkg_manager: Optional[PackageManagers] = None,
) -> Generator[Manifest, None, None]:
"""
Get all the manifest objects associated to the extensions declared in a
given configuration file.
"""
from platypush.config import Config
def scan_manifests(base_class: Type) -> Iterable[str]:
for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob('manifest.yaml'):
yield str(mf)
conf_args = []
if conf_file:
conf_args.append(conf_file)
Config.init(*conf_args)
app_dir = get_src_root()
def get_manifests(base_class: Type) -> Iterable[Manifest]:
return [
Manifest.from_file(mf)
for mf in scan_manifests(base_class)
]
for name in Config.get_backends().keys():
yield Manifest.from_file(
os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'),
pkg_manager=pkg_manager,
)
def get_components(base_class: Type) -> Iterable:
manifests = get_manifests(base_class)
components = {mf.get_component() for mf in manifests}
return {comp for comp in components if comp is not None}
def get_manifests_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Manifest]:
import platypush
from platypush.config import Config
conf_args = []
if conf_file:
conf_args.append(conf_file)
Config.init(*conf_args)
app_dir = os.path.dirname(inspect.getfile(platypush))
manifest_files = set()
for name in Config.get_backends().keys():
manifest_files.add(os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'))
for name in Config.get_plugins().keys():
manifest_files.add(os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'))
return {
manifest_file: Manifest.from_file(manifest_file)
for manifest_file in manifest_files
}
def get_dependencies_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Iterable[str]]:
manifests = get_manifests_from_conf(conf_file)
deps = {
'pip': set(),
'packages': set(),
'exec': set(),
}
for manifest in manifests.values():
deps['pip'].update(manifest.install.get('pip', set()))
deps['exec'].update(manifest.install.get('exec', set()))
has_requires_packages = len([
section for section in manifest.install.keys()
if section in supported_package_managers
]) > 0
if has_requires_packages:
pkg_manager = get_available_package_manager()
deps['packages'].update(manifest.install.get(pkg_manager, set()))
return deps
def get_install_commands_from_conf(conf_file: Optional[str] = None) -> Mapping[str, str]:
deps = get_dependencies_from_conf(conf_file)
return {
'pip': f'pip install {" ".join(deps["pip"])}',
'exec': deps["exec"],
'packages': f'{supported_package_managers[_available_package_manager]} {" ".join(deps["packages"])}'
if deps['packages'] else None,
}
def get_available_package_manager() -> str:
global _available_package_manager
if _available_package_manager:
return _available_package_manager
available_package_managers = [
pkg_manager for pkg_manager in supported_package_managers.keys()
if shutil.which(pkg_manager)
]
assert available_package_managers, (
'Your OS does not provide any of the supported package managers. '
f'Supported package managers: {supported_package_managers.keys()}'
)
_available_package_manager = available_package_managers[0]
return _available_package_manager
for name in Config.get_plugins().keys():
yield Manifest.from_file(
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
pkg_manager=pkg_manager,
)

1
setup.cfg
View file

@ -11,5 +11,6 @@ max-line-length = 120
extend-ignore =
E203
W503
SIM104
SIM105

2
setup.py
View file

@ -49,9 +49,9 @@ setup(
'console_scripts': [
'platypush=platypush:main',
'platydock=platypush.platydock:main',
'platyvenv=platypush.platyvenv:main',
],
},
scripts=['bin/platyvenv'],
long_description=readfile('README.md'),
long_description_content_type='text/markdown',
classifiers=[