A more robust cron start logic

If may happen (usually because of a race condition) that a cronjob has
already been started, but it hasn't yet changed its status from IDLE to
RUNNING when the scheduler checks it.

This fix guards the application against such events. If they occur, we
should just report them and move on, not terminate the whole scheduler.

.gitignore

@@ -2,11 +2,23 @@
 *.swp
 *.pyc
 __pycache__
-config.yaml
 build/
-dist/
+/dist/
 *.egg-info/
 package.sh
 .pypirc
 platypush/backend/http/static/resources/*
 docs/build
+.idea/
+config
+platypush/backend/http/static/css/*/.sass-cache/
+.vscode
+platypush/backend/http/static/js/lib/vue.js
+platypush/notebooks
+platypush/requests
+/http-client.env.json
+/platypush/backend/http/static/css/dist
+/tests/etc/dashboards
+.coverage
+coverage.xml
+Session.vim

.gitlab-ci.yml

@@ -0,0 +1,72 @@
+sync-to-github:
+  stage: build
+  script:
+    - echo "Synchronizing repo state to Github"
+    - export REPO_DIR="$(mktemp -d /tmp/platypush-XXXXX)"
+    - git clone git@git.platypush.tech:platypush/platypush.git "$REPO_DIR"
+    - cd "$REPO_DIR"
+    - git remote add github git@github.com:/BlackLight/platypush.git
+    - git checkout $CI_COMMIT_BRANCH
+    - git pull
+    - git push --mirror -v github
+
+run-tests:
+  stage: test
+  script:
+    - ./.gitlab/run_ci_tests.sh
+
+rebuild-docs:
+  stage: deploy
+  only:
+    - master
+  script:
+    - ./.gitlab/rebuild_docs.sh
+
+update-aur-packages:
+  stage: deploy
+  only:
+    - master
+    - tags
+  script:
+    - echo "Updating AUR packages"
+    - export REPO_DIR="$(mktemp -d /tmp/platypush-distutils-XXXXX)"
+    - git clone git@fabiomanganiello.com:/home/git/platypush-distutils.git "$REPO_DIR"
+    - cd "$REPO_DIR"
+    - git submodule init
+    - git submodule update
+    - cd distro/arch/git
+    - git checkout master
+    - git pull --rebase
+    - cd ../../../
+    - cd distro/arch/stable
+    - git checkout master
+    - git pull --rebase
+    - cd ../../../
+    - ./update.sh
+    - cd distro/arch/git
+    - changes="$(git status --porcelain --untracked-files=no)"
+    - "[[ -n \"$changes\" ]] && git commit -a -m '[Automatic] Package updated' && git push || echo 'No changes'"
+    - cd ../../../
+    - cd distro/arch/stable
+    - changes="$(git status --porcelain --untracked-files=no)"
+    - "[[ -n \"$changes\" ]] && git commit -a -m '[Automatic] Package updated' && git push || echo 'No changes'"
+
+upload-pip-package:
+  stage: deploy
+  only:
+    - tags
+  script:
+    # Update the CI/CD configuration
+    - cd ~/platypush-ci-cd
+    - git pull
+    - cd -
+    # Build the package
+    - rm -rf build dist *.egg-info
+    - export VERSION=$(grep -e '^\s*__version__\s*=' platypush/__init__.py | sed -r -e 's/^\s*__version__\s*=\s*.(.+?).\s*$/\1/')
+    - source ~/.credentials/pypi.env
+    - python setup.py sdist bdist_wheel
+    # Upload to PyPI
+    - twine upload --repository platypush ./dist/platypush-${VERSION}.tar.gz
+    # Upload to the local package repository
+    - TWINE_USERNAME=$LOCAL_TWINE_USERNAME TWINE_PASSWORD=$LOCAL_TWINE_PASSWORD twine upload --repository-url https://git.platypush.tech/api/v4/projects/3/packages/pypi dist/platypush-${VERSION}.tar.gz
+

.gitlab/rebuild_docs.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+LOGFILE="./docs.log"
+STATUS_IMG_PATH="./docs-status.svg"
+
+build_docs() {
+    cd ./docs || exit 1
+    make html 2>&1 | tee "../$LOGFILE"
+    ret=$?
+    cd .. || exit 1
+    return $?
+}
+
+########
+# MAIN #
+########
+
+build_docs
+ret=$?
+
+log_base_path="$(date +/opt/tests/platypush/logs/docs/%Y-%m-%dT%H:%M:%S.%m)"
+if [[ $ret == 0 ]]; then
+    wget -O "$STATUS_IMG_PATH" https://ci.platypush.tech/docs/passed.svg
+    cp "$LOGFILE" "${log_base_path}_PASSED.log"
+else
+    wget -O "$STATUS_IMG_PATH" https://ci.platypush.tech/docs/failed.svg
+    cp "$LOGFILE" "${log_base_path}_FAILED.log"
+fi
+
+mv "$STATUS_IMG_PATH" /opt/tests/platypush/logs/docs/
+mv "$LOGFILE" /opt/tests/platypush/logs/latest.log
+cp -r docs/build/html /opt/repos/platypush/docs/build/
+exit $ret

.gitlab/run_ci_tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+BASE_DIR="$(mktemp -d '/tmp/platypush-ci-tests-XXXXX')"
+VENV_DIR="$BASE_DIR/venv"
+TEST_LOG="./test.log"
+STATUS_IMG_PATH="./status.svg"
+
+cleanup() {
+    echo "Cleaning up environment"
+    rm -rf "$BASE_DIR"
+}
+
+prepare_venv() {
+    echo "Preparing virtual environment"
+    python -m venv "$VENV_DIR"
+    cd "$VENV_DIR" || exit 1
+    source ./bin/activate
+    cd - || exit 1
+}
+
+install_repo() {
+    echo "Installing latest version of the repository"
+    pip install '.[http]'
+}
+
+run_tests() {
+    echo "Running tests"
+    pytest 2>&1 | tee "$TEST_LOG"
+    deactivate
+
+    if grep -e '^FAILED ' "$TEST_LOG"; then
+      return 2
+    fi
+
+    return 0  # PASSED
+}
+
+########
+# MAIN #
+########
+
+cleanup
+prepare_venv
+install_repo
+run_tests
+ret=$?
+cleanup
+
+log_base_path="$(date +/opt/tests/platypush/logs/%Y-%m-%dT%H:%M:%S.%m)"
+if [[ $ret == 0 ]]; then
+    wget -O "$STATUS_IMG_PATH" https://ci.platypush.tech/passed.svg
+    cp "$TEST_LOG" "${log_base_path}_PASSED.log"
+else
+    wget -O "$STATUS_IMG_PATH" https://ci.platypush.tech/failed.svg
+    cp "$TEST_LOG" "${log_base_path}_FAILED.log"
+fi
+
+mv "$STATUS_IMG_PATH" /opt/tests/platypush/logs/status.svg
+mv "$TEST_LOG" /opt/tests/platypush/logs/latest.log
+exit $ret

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "platypush/backend/http/static/font-awesome"]
-	path = platypush/backend/http/static/font-awesome
-	url = https://github.com/FortAwesome/Font-Awesome
-[submodule "docs/wiki"]
-	path = docs/wiki
-	url = https://github.com/BlackLight/platypush.wiki.git
+[submodule "platypush/plugins/gpio/sensor/ir/mlx90640/lib"]
+	path = platypush/plugins/camera/ir/mlx90640/lib
+	url = https://github.com/pimoroni/mlx90640-library

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+    # -   id: trailing-whitespace
+    # -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-json
+    -   id: check-xml
+    -   id: check-symlinks
+    -   id: check-added-large-files
+
+-   repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs
+    rev: v1.1.2
+    hooks:
+    -   id: markdown-toc
+
+- repo: https://github.com/pycqa/flake8
+  rev: 4.0.1
+  hooks:
+    - id: flake8
+      additional_dependencies:
+        - flake8-bugbear
+        - flake8-comprehensions
+        - flake8-simplify
+
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black

.readthedocs.yml

@@ -1,7 +0,0 @@
-build:
-  image: latest
-
-python:
-  version: 3.6
-  setup_py_install: true
-

.travis.requirements

@@ -1,7 +0,0 @@
-pyyaml
-requires
-requests
-flask
-redis
-python-dateutil
-websockets

.travis.yml

@@ -1,21 +0,0 @@
-language: python
-python:
-  - "3.6"
-
-install: "pip install -r .travis.requirements"
-
-script: ./run_tests.sh
-
-notifications:
-    email:
-        recipients:
-            - blacklight86@gmail.com
-        on_success: change
-        on_failure: change
-
-services:
-    - redis-server
-
-git:
-    submodules: false
-

CHANGELOG.md

@@ -0,0 +1,441 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+Given the high speed of development in the first phase, changes are being
+reported only starting from v0.20.2.
+
+## [Unreleased]
+
+### Added
+
+- Added [Wallabag integration](https://git.platypush.tech/platypush/platypush/issues/224).
+- Added [Mimic3 TTS integration](https://git.platypush.tech/platypush/platypush/issues/226).
+
+## [0.23.6] - 2022-09-19
+
+### Fixed
+
+- Fixed album_id and list of tracks on `music.tidal.get_album`.
+
+## [0.23.5] - 2022-09-18
+
+### Added
+
+- Added support for web hooks returning their hook method responses back to the
+  HTTP client.
+
+- Added [Tidal integration](https://git.platypush.tech/platypush/platypush/pulls/223)
+
+- Added support for [OPML
+  subscriptions](https://git.platypush.tech/platypush/platypush/pulls/220) to
+  the `rss` plugin.
+
+- Better support for bulk database operations on the `db` plugin.
+
+### Fixed
+
+- Now supporting YAML sections with empty configurations.
+
+## [0.23.4] - 2022-08-28
+
+### Added
+
+- Added `matrix` integration
+  ([issue](https://git.platypush.tech/platypush/platypush/issues/2),
+  [PR](https://git.platypush.tech/platypush/platypush/pulls/217)).
+
+### Changed
+
+- Removed `clipboard` backend. Enabling the `clipboard` plugin will also enable
+  clipboard monitoring, with no need for an additional backend.
+
+## [0.23.3] - 2022-06-01
+
+### Added
+
+- Added `ntfy` integration (see #219).
+- Support for a default `config.yaml` if one isn't specified in the default
+  locations.
+
+### Changed
+
+- The HTTP server dependencies are now marked as required, since the default
+  `config.yaml` will have the HTTP backend enabled by default in order to allow
+  the creation of a first user.
+- Updated Vue.js frontend dependencies to the latest version.
+- Removed bulma from the frontend dependencies, making the frontend much
+  lighter and loading times much faster.
+- Other UI improvements.
+
+### Fixed
+
+- More reliable cronjobs in case of DST change or any clock changes in general
+  (see #217).
+- Fixed `--redis-queue` argument.
+
+## [0.23.2] - 2022-03-27
+
+### Added
+
+- Support for asynchronous events over GPIO PINs. It is now possible to specify
+  a list of `monitored_pins` in the [`gpio`
+  plugin](https://git.platypush.tech/platypush/platypush/-/blob/master/platypush/plugins/gpio/__init__.py)
+  configuration. A change in the value on those GPIO PINs (caused by e.g. a
+  button, a binary sensor or a probe) will trigger a
+  `platypush.message.event.gpio.GPIOEvent` that you can use in your automation
+  scripts.
+
+- Simplified script API to interact with platform variables
+  (closes [#206](https://git.platypush.tech/platypush/platypush/-/issues/206)).
+  You can now read and write stored variables in your Platypush scripts through
+  a much more intuitive interface compared to explicitly using the `variable`
+  plugin explicitly:
+
+```python
+from platypush.context import Variable
+
+# ...
+
+my_var = Variable.get('my_var')
+my_var = int(my_var) + 1
+Variable.set(my_var=my_var)
+```
+
+## [0.23.0] - 2022-03-01
+
+### Added
+
+- Added [Jellyfin integration](https://git.platypush.tech/platypush/platypush/-/issues/208).
+
+### Fixed
+
+- Merged several PRs from `dependabot`.
+
+- Fixed management of the `CN` field in the `calendar.ical` plugin.
+
+## [0.22.10] - 2022-02-07
+
+### Added
+
+- Refactored the `dbus` integration. The plugin and backend have been merged into a
+  single plugin component, and the ability to subscribe to custom signals has been
+  added.
+
+### Fixed
+
+- Proper support for empty payloads on the integrations that trigger a `SensorDataChangeEvent`.
+- Fixed possible infinite recursion on the Pushbullet integration in case of errors where the
+  error and close handlers keep calling each other in a loop.
+
+## [0.22.9] - 2022-01-06
+
+### Added
+
+- Added `rss` integration (replaces the cumbersome and deprecated `backend.http.poll`).
+
+### Fixed
+
+- Fixed timezone handling in calendar integrations.
+- Fixed handling of ignored directories in the `file.monitor` backend.
+
+## [0.22.8] - 2021-12-13
+
+### Added
+
+- Added support for audio tracks in Plex integration.
+
+### Changed
+
+- Web server uWSGI wrapper changed from `uwsgi` to `gunicorn`.
+
+### Fixed
+
+- Fixed client ID assignment logic in MQTT backends to prevent client ID clashes and reconnections
+  (closes #205).
+- Updated LTR559 integration to be compatible with the new API.
+- Updated Chromecast integration to be compatible with `pychromecast >= 10`.
+- Better handling of media errors.
+
+## [0.22.6] - 2021-11-27
+
+### Added
+
+- Added support for converting webpages to markdown in `http.webpage.simplify`
+  even if no `outfile` is specified.
+
+### Fixed
+
+- Improved robustness of the ICal calendar parser in case some fields (e.g. `*status`)
+  are not defined.
+
+## [0.22.5] - 2021-11-15
+
+### Added
+
+- Added `mastodon` plugin.
+- Added `chat.irc` plugin.
+- Added `mailgun` plugin.
+
+### Fixed
+
+- Fixed `switchbot.status` method in case of virtual devices.
+- Fixed `platypush[alexa]` optional package installation.
+
+## [0.22.4] - 2021-10-19
+
+### Added
+
+- Support for IR virtual devices in Switchbot plugin.
+- Added [`google.maps.get_travel_time`](https://docs.platypush.tech/platypush/plugins/google.maps.html#platypush.plugins.google.maps.GoogleMapsPlugin.get_travel_time)
+  method (closes #115).
+- Support for custom YouTube video/audio formats on media plugins.
+
+### Fixed
+
+- Responses for requests received over an MQTT backend are now delivered to the right topic
+  (`<device_base_topic>/responses/<msg_id>`).
+- Various fixes on media plugins.
+
+## [0.22.3] - 2021-10-01
+
+### Added
+
+- `gotify` integration (see #198).
+
+## [0.22.2] - 2021-09-25
+
+### Added
+
+- `ngrok` integration (see #196).
+
+## [0.22.1] - 2021-09-22
+
+### Fixed
+
+- `zigbee.mqtt` backend now no longer requires the MQTT backend/plugin to be enabled.
+
+- Fixed bug on empty popcorn API responses.
+
+### Changed
+
+- Created CI Gitlab pipeline to replace the Platypush event-based pre-existing pipeline.
+
+### Removed
+
+- Removed docs references to removed/abstract integrations.
+
+## [0.22.0] - 2021-09-16
+
+### Changed
+
+- Platypush now uses manifest files to describe plugins and backends. Each extension is now
+  expected to provide a `manifest.yaml` file in its folder, reporting its package name, pip
+  dependencies, required system packages and optional extra installation commands.
+
+- Refactored `platyvenv`, `platydock`, documentation generation and plugin management engine.
+  They are now both faster and more robust, since they can rely on the manifest definition to
+  operate instead of pydoc strings conventions or `config.yaml` conventions.
+
+- `platyvenv start`  now starts the environment process synchronously and it prints
+  stdout/stderr instead of redirecting it to the logs dir (previous behaviour:
+  `platyvenv start` used to start the process asynchronously and the logs were stored
+  to `~/.local/share/platypush/venv/<env>/logs/<stdout|stderr>.log`).
+
+### Removed
+
+- Removed `Homeseer` integration - it was based on a Python integration that has now been
+  pulled out of PyPI and GitHub. A new integration may come in the future if there is enough
+  demand for it.
+
+## [0.21.4] - 2021-08-24
+
+### Fixed
+
+- Fixed JWT token generation, since python-jwt >= 2.0 always returns strings (not bytes) upon `jwt.encode`.
+
+## [0.21.3] - 2021-07-28
+
+### Added
+
+- Added `sun` plugin for sunrise/sunset events.
+
+- Added `slack` integration.
+
+## [0.21.2] - 2021-07-20
+
+### Added
+
+- Added `music.spotify` backend to emulate a Spotify Connect receiver through Platypush.
+
+- Added `music.spotify` plugin.
+
+- Added `music.spotify` UI integration.
+
+## [0.21.1] - 2021-06-22
+
+### Added
+
+- Added `switchbot` plugin to interact with Switchbot devices over the cloud API instead of
+  directly accessing the device's Bluetooth interface.
+
+- Added `marshmallow` dependency - it will be used from now own to dump and document schemas
+  and responses instead of the currently mixed approach with `Response` objects and plain
+  dictionaries and lists.
+
+- Support for custom MQTT timeout on all the `zwavejs2mqtt` calls.
+
+- Added generic joystick backend `backend.joystick.jstest` which uses `jstest` from the
+  standard `joystick` system package to read the state of joysticks not compatible with
+  `python-inputs`.
+
+- Added PWM PCA9685 plugin.
+
+- Added Linux native joystick plugin, ``backend.joystick.linux``, for the cases where
+  ``python-inputs`` doesn't work and ``jstest`` is too slow.
+
+### Changed
+
+- `switch.switchbot` plugin renamed to `switchbot.bluetooth` plugin, while the new plugin
+  that uses the Switchbot API is simply named `switchbot`.
+
+### Fixed
+
+- More robust reconnection logic on the Pushbullet backend in case of websocket errors.
+
+## [0.21.0] - 2021-05-06
+
+### Added
+
+- Support for custom PopcornTime API mirror/base URL.
+
+- Full support for TV series search.
+
+### Fixed
+
+- Fixed torrent search (now using a different PopcornTime API mirror).
+
+- Migrated SASS engine from `node-sass` (currently deprecated and broken on Node 16) to `sass`.
+
+- Fixed alignment of Z-Wave UI header on Chrome/Webkit.
+
+## [0.20.10] - 2021-04-28
+
+### Fixed
+
+- Fixed zwave/zwavejs2mqtt interoperability.
+
+## [0.20.9] - 2021-04-12
+
+### Added
+
+- Added zwavejs2mqtt integration (see [#186](https://git.platypush.tech/platypush/platypush/-/issues/186).
+
+### Fixed
+
+- Major LINT fixes.
+
+### Removed
+
+- Removed unmaintained integrations: TorrentCast and Booking.com
+
+## [0.20.8] - 2021-04-04
+
+### Added
+
+- Added `<Camera>` dashboard widget.
+
+- Added support for custom dashboard widgets with customized (see https://git.platypush.tech/platypush/platypush/wiki/Backends#creating-custom-widgets).
+
+- Added support for controls on `music.mpd` dashboard widget.
+
+### Fixed
+
+- Fixed zigbee2mqtt backend error in case of messages with empty payload (see [#184](https://git.platypush.tech/platypush/platypush/-/issues/184)).
+
+- Fixed compatibility with all versions of websocket-client - versions >= 0.58.0 pass a `WebSocketApp` object as a first
+  argument to the callbacks, as well as versions < 0.54.0 do, but the versions in between don't pass this argument.
+
+## [0.20.7] - 2021-03-26
+
+### Fixed
+
+- Fixed race condition on `media.vlc.stop` when clearing the VLC instance.
+
+- Fixed dashboard widgets custom classes being propagated both to the container and to the widget content [see #179]
+
+- Fixed compatibility with SQLAlchemy >= 1.4.
+
+## [0.20.6] - 2021-03-16
+
+### Added
+
+- Added `log.http` backend to monitor changes to HTTP log files
+  (see [#167](https://git.platypush.tech/platypush/platypush/-/issues/167)).
+
+- Added `file.monitor` backend, which replaces the `inotify` backend
+  (see [#172](https://git.platypush.tech/platypush/platypush/-/issues/172)).
+
+### Removed
+
+- Removed legacy `pusher` script and `local` backend.
+
+### Fixed
+
+- Fixed support for Z-Wave switches.
+
+- Fixed possible race condition on VLC stop.
+
+## [0.20.5] - 2021-03-12
+
+### Added
+
+- Added support for a static list of devices to actively scan to the `bluetooth.scanner` backend
+  (see [#174](https://git.platypush.tech/platypush/platypush/-/issues/174)).
+
+- Added `weather.openweathermap` plugin and backend, which replaces `weather.darksky`, since the
+  Darksky API will be completely shut down by the end of 2021.
+
+### Fixed
+
+- Cron expressions should adhere to the UNIX cronjob standard and use the machine local time,
+  not UTC, as a reference (closes [#173](https://git.platypush.tech/platypush/platypush/-/issues/173)).
+
+- Better management of Z-Wave values types from the UI.
+
+- Disable logging for `ZwaveValueEvent` events - they tend to be very verbose and
+  can impact the performance on slower devices. They will still be published to the
+  websocket clients though, so you can still debug Z-Wave values issues from the browser
+  developer console (enable debug traces).
+
+- Added suffix to the `zigbee.mqtt` backend default `client_id` to prevent clashes with
+  the default `mqtt` backend `client_id`.
+
+## [0.20.4] - 2021-03-08
+
+### Added
+
+- Added SmartThings integration.
+- Support for custom Redis message queue name over the `--redis-queue` argument.
+
+### Fixed
+
+- Refactored tests to use `pytest` instead of `unittest`.
+- Some major bug fixes on procedures and hooks context evaluation.
+
+## [0.20.3] - 2021-02-28
+
+### Fixed
+
+- Several bug fixes on the VLC plugin, including proper management of stop/end-of-stream, volume set and missing integration requirements in `requirements.txt` and `setup.py`.
+
+## [0.20.2] - 2021-02-27
+
+### Fixed
+
+- More stable ZeroConf backends registration logic in case of partial or missing results.
+- Improved and refactored integration tests.
+
+### Added
+
+- Support for passing context variables (${}) from YAML procedures/hooks/crons to Python procedure/hooks/crons.
+- New integration test for testing procedures.

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+Thanks for considering contributing your work to make Platypush a better product!
+
+Contributions are more than welcome, and the follow the standard Gitlab procedure:
+
+- [Fork the repo](https://git.platypush.tech/platypush/platypush).
+- Prepare your changes.
+- [Submit a merge request](https://git.platypush.tech/platypush/platypush/-/merge_requests).
+
+Guidelines:
+
+- The code should ideally have no LINT warnings/issues.
+
+- Project conventions:
+  - 4 spaces to indent.
+  - RST format for classes and methods documentation 
+  - Run `python generate_missing_docs.py` if you are adding new plugins/backends to automatically
+    generate the doc templates. Make sure that you don't accidentally remove lines elements from
+    the docs because of missing dependencies on the machine you use to generate the docs.
+  - Naming conventions: plugin classes are named `<Module>Plugin` and backend classes are
+    named `<Module>Backend`, with `<Module>` being the (camel-case) representation of the
+    Python module of the plugin without the prefix - for example, the plugin under
+    `platypush.plugins.light.hue` must be named `LightHuePlugin`.
+
+- If possible, [add a test](https://git.platypush.tech/platypush/platypush/-/tree/master/tests)
+  for the new functionality. If you have built a new functionality that works with some specific
+  device or service then it's not required to write a test that mocks the whole service, but if
+  you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons
+  or the bus) then make sure to add tests and not to break the existing tests.
+
+- If the feature requires an optional dependency then make sure to document it:
+
+  - In the class docstring (see other plugins and backends for examples).
+  - In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as
+    an `extras_require` entry.
+  - In the plugin/backend class pydoc string.
+  - In the `manifest.yaml` - refer to the Wiki (how to write
+    [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
+    and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
+    for examples on how to write an extension manifest file.

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017, 2018 Fabio Manganiello
+Copyright (c) 2017, 2020 Fabio Manganiello
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

@@ -1,2 +1,4 @@
-recursive-include platypush/backend/http/static *
-recursive-include platypush/backend/http/templates *
+recursive-include platypush/backend/http/webapp/dist *
+include platypush/plugins/http/webpage/mercury-parser.js
+include platypush/config/*.yaml
+global-include manifest.yaml

README.md

@@ -1,63 +1,650 @@
 Platypush
 =========
 
-[![Build Status](https://travis-ci.org/BlackLight/platypush.svg?branch=master)](https://travis-ci.org/BlackLight/platypush)
-[![Documentation Status](https://readthedocs.org/projects/platypush/badge/?version=latest)](https://platypush.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://ci.platypush.tech/status.svg)](https://ci.platypush.tech/latest.log)
+[![Documentation Status](https://ci.platypush.tech/docs/status.svg)](https://ci.platypush.tech/docs/latest.log)
+[![pip version](https://img.shields.io/pypi/v/platypush.svg?style=flat)](https://pypi.python.org/pypi/platypush/)
+[![License](https://img.shields.io/github/license/BlackLight/platypush.svg)](https://git.platypush.tech/platypush/platypush/-/blob/master/LICENSE.txt)
+[![Last Commit](https://img.shields.io/github/last-commit/BlackLight/platypush.svg)](https://git.platypush.tech/platypush/platypush/-/commits/master/)
+[![Join chat on Matrix](https://img.shields.io/matrix/:platypush?server_fqdn=matrix.platypush.tech)](https://matrix.to/#/#platypush:matrix.platypush.tech)
+[![Contributions](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://git.platypush.tech/platypush/platypush/-/blob/master/CONTRIBUTING.md)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/BlackLight/platypush.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/BlackLight/platypush/context:python)
+[![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/BlackLight/platypush.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/BlackLight/platypush/context:javascript)
 
+<!-- toc -->
 
-Imagine Platypush as some kind of [IFTTT](https://ifttt.com) on steroids - or [Tasker](https://tasker.joaoapps.com/), or [Microsoft Flow](https://flow.microsoft.com), or [PushBullet](https://pushbullet.com) on steroids.
-Platypush aims to turn any device in a smart hub that can control things, interact with cloud services and send messages to other devices. It's a general-purpose lightweight platform to process any request and run any logic triggered by custom events.
+- [Architecture](#architecture)
+  * [Plugins](#plugins)
+  * [Actions](#actions)
+  * [Backends](#backends)
+  * [Events](#events)
+  * [Hooks](#hooks)
+  * [Procedures](#procedures)
+  * [Cronjobs](#cronjobs)
+  * [The web interface](#the-web-interface)
+- [Installation](#installation)
+  * [System installation](#system-installation)
+    + [Install through `pip`](#install-through-pip)
+    + [Install through a system package manager](#install-through-a-system-package-manager)
+    + [Install from sources](#install-from-sources)
+  * [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions)
+    + [Install via `extras` name](#install-via-extras-name)
+    + [Install via `manifest.yaml`](#install-via-manifestyaml)
+    + [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation)
+  * [Virtual environment installation](#virtual-environment-installation)
+  * [Docker installation](#docker-installation)
+- [Mobile app](#mobile-app)
+- [Tests](#tests)
+- [Funding](#funding)
 
-Imagine the ability of running any task you like, or automate any routine you like, on any of your devices. And the flexibility of executing actions through a cloud service, with the power of running them from your laptop, Raspberry Pi, smart home device or smartphone.
+<!-- tocstop -->
+
+- Recommended read: [**Getting started with Platypush**](https://blog.platypush.tech/article/Ultimate-self-hosted-automation-with-Platypush).
+
+- The [blog](https://blog.platypush.tech) is in general a good place to get
+  more insights on what you can build with it and inspiration about possible
+  usages.
+
+- The [wiki](https://git.platypush.tech/platypush/platypush/wiki) also
+  contains many resources on getting started.
+
+- Extensive documentation for all the available integrations and messages [is
+  available](https://docs.platypush.tech/).
+
+- If you have issues/feature requests/enhancement ideas please [create an
+  issue](https://git.platypush.tech/platypush/platypush/-/issues).
+
+- A [Reddit channel](https://www.reddit.com/r/platypush) is also available for
+  more general questions.
+
+- A [Matrix instance](https://matrix.to/#/#platypush:matrix.platypush.tech) is
+  also available if you are looking for more interactive support.
+
+---
+
+Platypush is a general-purpose extensible platform for automation and
+integration across multiple services and devices.
+
+It enables users to create their own self-hosted pieces of automation based on
+events (*if this happens then do that*)
+and it provides a comprehensive and customizable user interface that collects
+everything you need to visualize and control under one roof.
+
+It takes some concepts from [IFTTT](https://ifttt.com),
+[Tasker](https://tasker.joaoapps.com/), [Microsoft
+Flow](https://flow.microsoft.com), [PushBullet](https://pushbullet.com) and
+[Home Assistant](https://www.home-assistant.io/) to provide an environment
+where the user can easily connect things together.
+
+Its ideal home is a single-board computer like a RaspberryPi that you can
+configure to orchestrate any home automation and cloud automation in your own
+living room or garage, but it can easily run on any device that can run a
+Python interpreter, and the bar for the hardware requirements is very low as
+well - I use it to run pieces of automation on devices as powerful as a
+RaspberryPi Zero or an old Nokia N900 with Linux.
 
 You can use Platypush to do things like:
 
-- Control your smart home lights
-- Control your favourite music player
-- Interact with your voice assistant
-- Get events from your Google or Facebook calendars
-- Read data from your sensors and trigger custom events whenever they go above or below some custom thresholds
-- Control the motors of your robot
-- Send automated emails
-- Synchronize the clipboards on your devices
-- Control your smart switches
-- Implement custom text-to-speech commands
-- Build any kind of interaction with your Android device using Tasker
-- Play local videos, YouTube videos and torrent links
-- Get weather forecast for your location
-- Build your own web dashboard with calendar, weather, news and music controls (basically, anything that has a Platypush web widget)
-- ...and much more (basically, anything that comes with a [Platypush plugin](https://platypush.readthedocs.io/en/latest/plugins.html))
+- [Control your smart home lights](https://blog.platypush.tech/article/Ultimate-self-hosted-automation-with-Platypush)
+- [Control your music and synchronize it to multiple devices](https://blog.platypush.tech/article/Build-your-open-source-multi-room-and-multi-provider-sound-server-with-Platypush-Mopidy-and-Snapcast)
+- [Create custom and privacy-secure voice assistants that run custom hooks on your phrases](https://blog.platypush.tech/article/Build-custom-voice-assistants)
+- Build integrations between [sensors](https://docs.platypush.tech/en/latest/platypush/backend/sensor.html),
+  [cameras](https://docs.platypush.tech/en/latest/platypush/plugins/camera.pi.html),
+  [microphones](https://docs.platypush.tech/en/latest/platypush/plugins/sound.html)
+  and [machine learning
+  models](https://docs.platypush.tech/en/latest/platypush/plugins/tensorflow.html)
+  to create smart
+  pieces of automation for e.g.
+  [people detection](https://blog.platypush.tech/article/Detect-people-with-a-RaspberryPi-a-thermal-camera-Platypush-and-a-pinch-of-machine-learning)
+  or [sound detection](https://blog.platypush.tech/article/Create-your-smart-baby-monitor-with-Platypush-and-Tensorflow)
+- [Get events from your Google or Facebook calendars](https://docs.platypush.tech/en/latest/platypush/plugins/calendar.html)
+- [Read data from your sensors and trigger custom events whenever they go above or below some custom thresholds](https://blog.platypush.tech/article/How-to-build-your-personal-infrastructure-for-data-collection-and-visualization)
+- [Control and automate a self-built robot](https://docs.platypush.tech/en/latest/platypush/plugins/gpio.zeroborg.html)
+- [Deliver automated newsletters from custom RSS digests](https://blog.platypush.tech/article/Deliver-customized-newsletters-from-RSS-feeds-with-Platypush)
+- [Synchronize the clipboards on your devices](https://docs.platypush.tech/en/latest/platypush/plugins/clipboard.html)
+- [Control your smart switches](https://docs.platypush.tech/en/latest/platypush/plugins/switch.html)
+- [Implement automated custom text-to-speech routines](https://docs.platypush.tech/en/latest/platypush/plugins/tts.html)
+- [Build any kind of interactions and automation routines with your Android device using Tasker](https://blog.platypush.tech/article/How-to-build-your-personal-infrastructure-for-data-collection-and-visualization)
+- Play [local
+  videos](https://docs.platypush.tech/en/latest/platypush/plugins/media.mpv.html),
+  YouTube videos and torrent media from any device and service, to any device
+- [Get weather forecast events for your location and build automation routines on them](https://docs.platypush.tech/en/latest/platypush/plugins/weather.darksky.html)
+- [Create a custom single hub for Zigbee and Z-Wave smart devices](https://blog.platypush.tech/article/Transform-a-RaspberryPi-into-a-universal-Zigbee-and-Z-Wave-bridge)
+- Build your own web dashboard with calendar, weather, news and music controls
+  (basically, anything that has a Platypush web widget)
+- ...and much more (basically, anything that comes with a [Platypush plugin](https://docs.platypush.tech/en/latest/plugins.html))
 
-Imagine the ability of executing all the actions above through messages delivered through:
+## Architecture
 
-- A web interface
-- Raw HTTP requests
-- Web sockets
-- [PushBullet](https://pushbullet.com)
-- [Kafka](https://kafka.apache.org)
-- [Redis](https://redis.io)
-- [MQTT](https://mqtt.org)
-- ...amd much more (basically, anything that comes with a [Platypush backend](https://platypush.readthedocs.io/en/latest/backends.html))
+The architecture of Platypush consists of a few simple pieces, orchestrated by
+a configuration file stored by default under
+[`~/.config/platypush/config.yaml`](https://git.platypush.tech/platypush/platypush/-/blob/master/examples/conf/config.yaml):
 
-Imagine the ability of building custom event hooks to automatically trigger any actions:
+### Plugins
 
-- When your voice assistant recognizes some text
-- When you start playing a new song
-- When a new event is added to your calendar
-- When a new article is published on your favourite feed
-- When the weather conditions change
-- When your press a [Flic button](https://flic.io) with a certain pattern
-- When you receive a new push on your Pushbullet account
-- When your GPS signal enters a certain area
-- Whenever a new MIDI event is received (yes, you heard well :) )
-- Whenever a sensor sends new data
-- At a specific date or time
-- ...and so on (basically, anything can send events that can be used to build hooks)
+[Full list](https://docs.platypush.tech/en/latest/plugins.html)
 
-Imagine the ability of running the application, with lots of those bundled features, on any device that can comes with Python (_only compatible with version 3 and higher_). Platypush has been designed with performance in mind, it's been heavily tested on slower devices like Raspberry Pis, and it can run the web server features, multiple backends and plugins quite well even on a Raspberry Pi Zero - it's even been tested with some quite impressive performance on an older [Nokia N900](https://en.wikipedia.org/wiki/Nokia_N900), and of course you can run it on any laptop, desktop, server environment. It's been developed mainly with IoT in mind (and some of its features overlap with IoT frameworks like [Mozilla IoT](https://iot.mozilla.com) and [Android Things](https://developer.android.com/things/)), but nothing prevents you from automating any task on any device and environment.
+Plugins are integrations that do things - like [modify
+files](https://docs.platypush.tech/en/latest/platypush/plugins/file.html),
+[train and evaluate machine learning
+models](https://docs.platypush.tech/en/latest/platypush/plugins/tensorflow.html),
+[control
+cameras](https://docs.platypush.tech/en/latest/platypush/plugins/camera.pi.html),
+[read
+sensors](https://docs.platypush.tech/en/latest/platypush/plugins/gpio.sensor.dht.html),
+[parse a web
+page](https://docs.platypush.tech/en/latest/platypush/plugins/http.webpage.html),
+[control
+lights](https://docs.platypush.tech/en/latest/platypush/plugins/light.hue.html),
+[send
+emails](https://docs.platypush.tech/en/latest/platypush/plugins/mail.smtp.html),
+[control
+Chromecasts](https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html),
+[run voice
+queries](https://docs.platypush.tech/en/latest/platypush/plugins/assistant.google.html),
+[handle torrent
+transfers](https://docs.platypush.tech/en/latest/platypush/plugins/torrent.html)
+or control
+[Zigbee](https://docs.platypush.tech/en/latest/platypush/plugins/zigbee.mqtt.html)
+or [Z-Wave](https://docs.platypush.tech/en/latest/platypush/plugins/zwave.html)
+devices.
 
-To get started:
+The configuration of a plugin matches one-on-one that of its documented class
+constructor, so it's very straightforward to write a configuration for a plugin
+by reading its documentation:
 
-- [Wiki](https://github.com/BlackLight/platypush/wiki) for installation notes, quick start, examples and architecture reference
-- [Read the docs](https://platypush.readthedocs.io/en/latest/) for a complete reference on the available plugins and backends
+```yaml
+light.hue:
+  # Groups that will be controlled by default
+  groups:
+    - Living Room
+    - Hall
+```
 
+### Actions
+
+Plugins expose *actions*, that match one-on-one the plugin class methods
+denoted by `@action`, so it's very straightforward to invoke plugin actions by
+just reading the plugin documentation. They can be invoked directly from your
+own scripts or they can be sent to the platform through any supported channel
+as simple JSON messages:
+
+```json
+{
+  "type": "request",
+  "action": "light.hue.on",
+  "args": {
+    "lights": ["Entrance Bulb"]
+  }
+}
+```
+
+### Backends
+
+[Full list](https://docs.platypush.tech/en/latest/backends.html)
+
+They are background services that either listen for messages on channels (like
+an [HTTP
+backend](https://docs.platypush.tech/en/latest/platypush/backend/http.html), an
+[MQTT
+instance](https://docs.platypush.tech/en/latest/platypush/backend/mqtt.html), a
+[Kafka
+instance](https://docs.platypush.tech/en/latest/platypush/backend/kafka.html),
+a [Websocket
+service](https://docs.platypush.tech/en/latest/platypush/backend/websocket.html),
+[Pushbullet](https://docs.platypush.tech/en/latest/platypush/backend/pushbullet.html)
+etc.) or monitor a device or a service for events (like a
+[sensor](https://docs.platypush.tech/en/latest/platypush/backend/sensor.html),
+a custom [voice
+assistant](https://docs.platypush.tech/en/latest/platypush/backend/assistant.google.html),
+a bridge running on a
+[Zigbee](https://docs.platypush.tech/en/latest/platypush/backend/zigbee.mqtt.html)
+or
+[Z-Wave](https://docs.platypush.tech/en/latest/platypush/backend/zwave.html),
+an [NFC card
+reader](https://docs.platypush.tech/en/latest/platypush/backend/nfc.html), a
+[MIDI
+device](https://docs.platypush.tech/en/latest/platypush/backend/midi.html), a
+[Telegram
+channel](https://docs.platypush.tech/en/latest/platypush/backend/chat.telegram.html),
+a [Bluetooth
+scanner](https://docs.platypush.tech/en/latest/platypush/backend/bluetooth.scanner.ble.html)
+etc.).
+
+If a backend supports the execution of requests (e.g. HTTP, MQTT, Kafka,
+Websocket and TCP) then you can send requests to these services in JSON format.
+For example, in the case of the HTTP backend:
+
+```shell
+    # Get a token
+curl -XPOST -H 'Content-Type: application/json' -d '
+  {
+    "username": "$YOUR_USER",
+    "password": "$YOUR_PASSWORD"
+  }' http://host:8008/auth
+
+    # Execute a request
+
+curl -XPOST -H 'Content-Type: application/json' \
+    -H "Authorization: Bearer $YOUR_TOKEN" -d '
+  {
+    "type": "request",
+    "action": "tts.say",
+    "args": {
+      "text": "This is a test"
+    }
+  }' http://host:8008/execute
+```
+
+### Events
+
+[Full list](https://docs.platypush.tech/en/latest/events.html)
+
+When a certain event occurs (e.g. a JSON request is received, or a [Bluetooth
+device is
+connected](https://docs.platypush.tech/en/latest/platypush/events/bluetooth.html#platypush.message.event.bluetooth.BluetoothDeviceConnectedEvent),
+or a [Flic button is
+pressed](https://docs.platypush.tech/en/latest/platypush/events/button.flic.html#platypush.message.event.button.flic.FlicButtonEvent),
+or some [speech is detected on the voice assistant
+service](https://docs.platypush.tech/en/latest/platypush/events/assistant.html#platypush.message.event.assistant.SpeechRecognizedEvent),
+or an [RSS feed has new
+items](https://docs.platypush.tech/en/latest/platypush/events/http.rss.html#platypush.message.event.http.rss.NewFeedEvent),
+or a [new email is
+received](https://docs.platypush.tech/en/latest/platypush/events/mail.html#platypush.message.event.mail.MailReceivedEvent),
+or a [new track is
+played](https://docs.platypush.tech/en/latest/platypush/events/music.html#platypush.message.event.music.NewPlayingTrackEvent),
+or an [NFC tag is
+detected](https://docs.platypush.tech/en/latest/platypush/events/nfc.html#platypush.message.event.nfc.NFCTagDetectedEvent),
+or [new sensor data is
+available](https://docs.platypush.tech/en/latest/platypush/events/sensor.html#platypush.message.event.sensor.SensorDataChangeEvent),
+or [a value of a Zigbee device
+changes](https://docs.platypush.tech/en/latest/platypush/events/zigbee.mqtt.html#platypush.message.event.zigbee.mqtt.ZigbeeMqttDevicePropertySetEvent),
+etc.), the associated backend will trigger an
+[event](https://docs.platypush.tech/en/latest/events.html).
+
+### Hooks
+
+Event hooks are custom pieces of logic that will be run when a certain event is
+triggered. Hooks are the glue that connects events to actions, exposing a
+paradigm similar to IFTTT (_if a certain event happens then run these
+actions_). They can declared as:
+
+- Sections of the [`config.yaml`](https://git.platypush.tech/platypush/platypush/-/blob/master/examples/conf/config.yaml).
+  Example:
+
+```yaml
+event.hook.SearchSongVoiceCommand:
+  if:
+    type: platypush.message.event.assistant.SpeechRecognizedEvent
+    phrase: "play ${title} by ${artist}"
+  then:
+    - action: music.mpd.clear
+    - action: music.mpd.search
+      args:
+        filter:
+          artist: ${artist}
+          title: ${title}
+
+    - if ${len(output)}:
+      - action: music.mpd.play
+        args:
+          resource: ${output[0]['file']}
+```
+
+- Stand-alone Python scripts stored under `~/.config/platypush/scripts` and
+  will be dynamically imported at start time.
+  [Example](https://git.platypush.tech/platypush/platypush/-/blob/master/examples/conf/hook.py):
+
+```python
+from platypush.event.hook import hook
+from platypush.utils import run
+from platypush.message.event.assistant import SpeechRecognizedEvent
+
+@hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
+def on_music_play_command(event, title=None, artist=None, **context):
+  results = run('music.mpd.search', filter={
+    'artist': artist,
+    'title': title,
+  })
+
+  if results:
+    run('music.mpd.play', results[0]['file'])
+```
+
+### Procedures
+
+Procedures are pieces of custom logic that can be executed as atomic actions
+using `procedure.<name>` as an action name.
+
+They can be defined either in the `config.yaml` or as Python scripts stored
+under `~/.config/platypush/scripts` - provided that the procedure is also
+imported in `~/.config/platypush/scripts/__init__.py` so it can be discovered
+by the service.
+
+YAML example for a procedure that can be executed when we arrive home and turns
+on the lights if the luminosity is lower that a certain thresholds, says a
+welcome home message using the TTS engine and starts playing the music:
+
+```yaml
+procedure.at_home:
+    # Get luminosity data from a sensor - e.g. LTR559
+    - action: gpio.sensor.ltr559.get_data
+
+    # If it's lower than a certain threshold, turn on the lights
+    - if ${int(light or 0) < 110}:
+        - action: light.hue.on
+
+    # Say a welcome home message
+    - action: tts.google.say
+      args:
+        text: Welcome home
+
+    # Play the music
+    - action: music.mpd.play
+```
+
+Python example:
+
+```python
+    # Content of ~/.config/platypush/scripts/home.py
+from platypush.procedure import procedure
+from platypush.utils import run
+
+@procedure
+def at_home(**context):
+  sensor_data = run('gpio.sensor.ltr559.get_data')
+  if sensor_data['light'] < 110:
+    run('light.hue.on')
+
+  run('tts.google.say', text='Welcome home')
+  run('music.mpd.play')
+```
+
+In either case, you can easily trigger the at-home procedure by sending an
+action request message to a backend - for example, over the HTTP backend:
+
+```shell
+curl -XPOST -H 'Content-Type: application/json' \
+    -H "Authorization: Bearer $YOUR_TOKEN" -d '
+  {
+    "type": "request",
+    "action": "procedure.at_home"
+  }' http://host:8008/execute
+```
+
+### Cronjobs
+
+Cronjobs are pieces of logic that will be run at regular intervals, expressed
+in crontab-compatible syntax. They can be defined either in the `config.yaml`
+or as Python scripts stored under `~/.config/platypush/scripts` as functions
+labelled by the `@cron` decorator.
+
+Note that seconds are also supported (unlike the standard crontab definition),
+but, for back-compatibility with the standard crontab format, they are at the
+end of the cron expression, so the expression is actually in the format
+`<minute> <hour> <day_of_month> <month> <day_of_week> <second>`.
+
+YAML example for a cronjob that is executed every 30 seconds and checks if a
+Bluetooth device is nearby:
+
+```yaml
+cron.check_bt_device:
+  cron_expression: '* * * * * */30'
+  actions:
+    - action: bluetooth.lookup_name
+      args:
+        addr: XX:XX:XX:XX:XX:XX
+
+    - if ${name}:
+        - action: procedure.on_device_on
+    - else:
+        - action: procedure.on_device_off
+```
+
+Python example:
+
+```python
+    # Content of ~/.config/platypush/scripts/bt_cron.py
+from platypush.cron import cron
+from platypush.utils import run
+
+@cron('* * * * * */30')
+def check_bt_device(**context):
+  name = run('bluetooth.lookup_name').get('name')
+  if name:
+    # on_device_on logic here
+  else:
+    # on_device_off logic here
+```
+
+### The web interface
+
+If
+[`backend.http`](https://docs.platypush.tech/en/latest/platypush/backend/http.html)
+is enabled then a web interface will be provided by default on
+`http://host:8008/`. Besides using the `/execute` endpoint for running
+requests, the built-in web server also provides a full-featured interface that
+groups together the controls for most of the plugins - e.g. sensors, switches,
+music controls and search, media library and torrent management, lights,
+Zigbee/Z-Wave devices and so on. The UI is responsive and mobile-friendly.
+
+The web service also provides means for the user to create [custom
+dashboards](https://git.platypush.tech/platypush/platypush/-/blob/master/examples/conf/dashboard.xml)
+that can be used to show information from multiple sources on a large screen.
+
+## Installation
+
+### System installation
+
+Platypush uses Redis to deliver and store requests and temporary messages:
+
+```yaml
+    # Example for Debian-based distributions
+[sudo] apt-get install redis-server
+
+    # Enable and start the service
+[sudo] systemctl enable redis
+[sudo] systemctl start redis
+```
+
+#### Install through `pip`
+
+```shell
+[sudo] pip3 install platypush
+```
+
+#### Install through a system package manager
+
+Note: currently only Arch Linux and derived distributions are supported.
+
+You can either install the
+[`platypush`](https://aur.archlinux.org/packages/platypush) package (for the
+latest stable version) or the
+[`platypush-git`](https://aur.archlinux.org/packages/platypush-git) package
+(for the latest git version) through your favourite AUR package manager. For
+example, using `yay`:
+
+```shell
+yay platypush
+# Or
+yay platypush-git
+```
+
+The Arch Linux packages on AUR are automatically updated upon new git commits
+or tags.
+
+#### Install from sources
+
+```shell
+git clone https://git.platypush.tech/platypush/platypush.git
+cd platypush
+[sudo] pip install .
+    # Or
+[sudo] python3 setup.py install
+```
+
+### Installing the dependencies for your extensions
+
+After installing the base platform, you may want to check the dependencies and
+configuration required by the extensions that you wish to use. There are a few
+ways to check the dependencies required by an extension:
+
+#### Install via `extras` name
+
+All the extensions that require extra dependencies are listed in the
+[`extras_require` section under
+`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72).
+
+#### Install via `manifest.yaml`
+
+All the plugins and backends have a `manifest.yaml` file in their source folder.
+Any extra dependencies are listed there
+
+If you followed the `extras` or `manifest.yaml` way to discover the
+dependencies, then you can install them in two ways:
+
+1. `pip` installation:
+
+```shell
+[sudo] pip3 install 'platypush[extra1,extra2,extra3]'
+```
+
+2. Sources installation:
+
+```shell
+cd $DIR_TO_PLATYPUSH
+[sudo] pip3 install '.[extra1,extra2,extra3]'
+```
+
+#### Check the instructions reported in the documentation
+
+If you follow this route then simply run the commands listed in the
+[plugin/backend documentation](https://docs.platypush.tech) to get the
+dependencies installed.
+
+After installing the dependencies, create a configuration file under
+`~/.config/platypush/config.yaml` (the application can load the configuration
+from another location through the `-c` option) containing the configuration of
+the backend and plugins that you want to use, and add any hooks and procedures
+for your use case.
+
+You can then start the service by simply running:
+
+```shell
+platypush
+```
+
+It's advised to run it as a systemd service though - simply copy the provided
+[`.service`
+file](https://git.platypush.tech/platypush/platypush/-/blob/master/examples/systemd/platypush.service)
+to `~/.config/systemd/user`, check if the path of `platypush` matches the path
+where it's installed on your system, and start the service via `systemctl`:
+
+```shell
+systemctl --user start platypush
+```
+
+### Virtual environment installation
+
+Platypush provides a script named `platyvenv` that can parse a `config.yaml`
+and automatically create a virtual environment (under
+`~/.local/share/platypush/venv/<device_id>`) with all the dependencies required
+by the configured integrations.
+
+1. Create the environment from a configuration file:
+
+    ```shell
+    platyvenv build -c /path/to/config.yaml
+    ```
+
+2. Start the service from the virtual environment:
+
+    ```shell
+        # device_id matches either the hostname or the device_id in config.yaml
+    platyvenv start device_id
+    ```
+
+3. Stop the instance:
+
+    ```shell
+    platyvenv stop device_id
+    ```
+
+4. Remove the instance:
+
+    ```shell
+    platyvenv rm device_id
+    ```
+
+[Wiki instructions](https://git.platypush.tech/platypush/platypush/wiki/Run-platypush-in-a-virtual-environment)
+
+### Docker installation
+
+You can also install Platypush in a container - the application provides a
+script named `platydock` that automatically creates a container instance from a
+`config.yaml`:
+
+1. Create the container from a configuration file:
+
+    ```shell
+    platydock build -c /path/to/config.yaml
+    ```
+
+2. Start the container:
+
+    ```shell
+        # device_id matches either the hostname or the device_id in config.yaml
+    platydock start device_id
+    ```
+
+3. Stop the instance:
+
+    ```shell
+    platydock stop device_id
+    ```
+
+4. Remove the instance:
+
+    ```shell
+    platydock rm device_id
+    ```
+
+Note that both the virtual environment and Docker container option offer the
+possibility to include extra YAML configuration files in the main `config.yaml`
+through the `include` directive (as long as they are in the same folder as the
+main `config.yaml`), as well as external Python scripts in a `scripts`
+directory in the same folder as the `config.yaml`.
+
+[Wiki instructions](https://git.platypush.tech/platypush/platypush/wiki/Run-platypush-in-a-container)
+
+## Mobile app
+
+An [official Android
+app](https://f-droid.org/en/packages/tech.platypush.platypush/) is provided on
+the F-Droid store. It allows to easily discover and manage multiple Platypush
+services on a network through the web interface, and it easily brings the power
+of Platypush to your fingertips.
+
+## Tests
+
+To run the tests simply run `pytest` either from the project root folder or the
+`tests/` folder. Or run the following command from the project root folder:
+
+```shell
+python -m tests
+```
+
+---
+
+## Funding
+
+If you use and love Platypush, please consider [buying me a coffee/beer](https://paypal.me/fabiomanganiello).
+
+I've been working on Platypush all by myself in my spare time for the past few
+years, and I've made sure that it remains open and free.
+
+If you like this product, please consider supporting - I'm definitely not
+planning to get rich with this project, but I'd love to have at least the costs
+for the server covered by users.
+
+Issues and requests opened by donors will also be given priority over others.

bin/platypush

@@ -0,0 +1,9 @@
+#!python3
+
+from platypush import main
+
+if __name__ == '__main__':
+    main()
+
+
+# vim:sw=4:ts=4:et:

bin/platyvenv

@@ -0,0 +1,249 @@
+#!/bin/bash
+
+##############################################################################
+# This script allows you to easily manage Platypush instances through Python #
+# virtual environment. You can build environments from a config.yaml file    #
+# and automatically managed the required dependencies, as well as start,     #
+# stop and remove them                                                       #
+#                                                                            #
+# @author: Fabio Manganiello <fabio@platypush.tech>                          #
+# @licence: MIT                                                              #
+##############################################################################
+
+
+workdir="$HOME/.local/share/platypush/venv"
+
+function build {
+    cfgfile=
+
+    while getopts ':c:' opt; do
+        case ${opt} in
+            c)
+                cfgfile=$OPTARG;;
+            \?)
+                echo "Invalid option: -$OPTARG" >&2
+                exit 1;;
+            :)
+                echo "Option -$OPTARG requires the path to a Platypush configuration file" >&2
+                exit 1;;
+        esac
+    done
+
+    if [[ -z "$cfgfile" ]]; then
+        echo "Usage: $0 build -c <path-to-platypush-config-file>" >&2
+        exit 1
+    fi
+
+    echo "Parsing configuration file"
+    pip_cmd=
+    pkg_cmd=
+    includes=()
+    cmd_exec=()
+
+    while read -r line; do
+      if echo "$line" | grep -E "^pip:\s*"; then
+        pip_cmd="$(echo "$line" | sed -r -e 's/^pip:\s*(.*)'/\\1/)"
+      elif echo "$line" | grep -E "^packages:\s*"; then
+        pkg_cmd="$(echo "$line" | sed -r -e 's/^packages:\s*(.*)'/\\1/)"
+      elif echo "$line" | grep -E "^exec:\s*"; then
+        cmd_exec+=("$(echo "$line" | sed -r -e 's/^exec:\s*(.*)'/\\1/)")
+      elif echo "$line" | grep -E "^include:\s*"; then
+        includes+=("$(echo "$line" | sed -r -e 's/^include:\s*(.*)'/\\1/)")
+      elif echo "$line" | grep -E "^device_id:\s*"; then
+        device_id="$(echo "$line" | sed -r -e 's/^device_id:\s*(.*)'/\\1/)"
+      fi
+    done <<< "$(python <<EOF
+from platypush.config import Config
+from platypush.utils.manifest import get_install_commands_from_conf
+
+deps = get_install_commands_from_conf('$(realpath "${cfgfile}")')
+print(f'device_id: {Config.get("device_id")}')
+
+if deps.get('pip'):
+    print(f'pip: {deps["pip"]}')
+
+if deps.get('packages'):
+    print(f'packages: {deps["packages"]}')
+
+for cmd in deps.get('exec', []):
+    print(f'exec: {cmd}')
+
+for include in Config._included_files:
+    print(f'include: {include}')
+EOF
+)"
+
+    envdir="${workdir}/${device_id}"
+    etcdir="${envdir}/etc/platypush"
+
+    echo "Preparing virtual environment for device $device_id"
+    mkdir -p "$envdir"
+    mkdir -p "$etcdir"
+    srcdir=$(dirname "$cfgfile")
+
+    for ((i=0; i < ${#includes[@]}; i++)); do
+        incdir=$(dirname "${includes[$i]}")
+        incdir=$(realpath --relative-to="$srcdir" "$incdir")
+        destdir="$etcdir/$incdir"
+        mkdir -p "$destdir"
+        cp "${includes[$i]}" "$destdir"
+    done
+
+    cp "$cfgfile" "$etcdir/config.yaml"
+    cfgfile="${etcdir}/config.yaml"
+
+    python3 -m venv "${envdir}"
+    cd "${envdir}" || exit 1
+    source "${envdir}/bin/activate"
+
+    echo "Installing required dependencies"
+    # shellcheck disable=SC2086
+    [ -n "${pkg_cmd}" ] && sudo ${pkg_cmd}
+    [ -n "${pip_cmd}" ] && ${pip_cmd}
+
+    for ((i=0; i < ${#cmd_exec[@]}; i++)); do
+        ${cmd_exec[$i]}
+    done
+
+    pip install --upgrade git+https://git.platypush.tech/platypush/platypush.git
+    echo "Platypush virtual environment prepared under $envdir"
+}
+
+function start {
+    if [[ -z "$1" ]]; then
+        echo "Usage: $0 start <env-name>" >&2
+        exit 1
+    fi
+
+    env=$1
+    envdir="${workdir}/${env}"
+    rundir="${envdir}/var/run"
+    pidfile="${rundir}/platypush.pid"
+    cfgfile="${envdir}/etc/platypush/config.yaml"
+
+    if [[ ! -d "$envdir" ]]; then
+        echo "No such directory: $envdir" >&2
+        exit 1
+    fi
+
+    mkdir -p "${rundir}"
+
+    if [[ -f "$pidfile" ]]; then
+        if pgrep -F "${pidfile}"; then
+            echo "Another instance (PID $(cat "${pidfile}")) is running, please stop that instance first"
+            exit 1
+        fi
+
+        echo "A PID file was found but the process does not seem to be running, starting anyway"
+        rm -f "$pidfile"
+    fi
+
+    python3 -m venv "${envdir}"
+    cd "${envdir}" || exit 1
+    source bin/activate
+    bin/platypush -c "$cfgfile" -P "$pidfile" &
+    start_time=$(date +'%s')
+    timeout=30
+
+    while :; do
+        [[ -f "$pidfile" ]] && break
+        now=$(date +'%s')
+        elapsed=$(( now-start_time ))
+        if (( elapsed >= timeout )); then
+            echo "Platypush instance '$env' did not start within $timeout seconds" >&2
+            exit 1
+        fi
+
+        echo -n '.'
+        sleep 1
+    done
+
+    pid=$(cat "$pidfile")
+    echo
+    echo "Platypush environment $env started with PID $pid"
+    wait "${pid}"
+    echo "Platypush environment $env terminated"
+}
+
+function stop {
+    if [[ -z "$1" ]]; then
+        echo "Usage: $0 stop <env-name>" >&2
+        exit 1
+    fi
+
+    env=$1
+    envdir="${workdir}/${env}"
+    rundir="${envdir}/var/run"
+    pidfile="${rundir}/platypush.pid"
+
+    if [[ ! -d "$envdir" ]]; then
+        echo "No such directory: $envdir" >&2
+        exit 1
+    fi
+
+    if [[ ! -f "$pidfile" ]]; then
+        echo "No pidfile found for instance \"${env}\""
+        exit 1
+    fi
+
+    pid=$(cat "$pidfile")
+    pids="$pid $(ps --no-headers -o pid= --ppid "$pid")"
+    # shellcheck disable=SC2086
+    kill -9 ${pids}
+    rm -f "$pidfile"
+    echo "Instance '$env' with PID $pid stopped"
+}
+
+function rme {
+    if [[ -z "$1" ]]; then
+        echo "Usage: $0 rm <env-name>" >&2
+        exit 1
+    fi
+
+    envdir="${workdir}/$1"
+    rundir="${envdir}/var/run"
+    pidfile="${rundir}/platypush.pid"
+
+    if [[ ! -d "$envdir" ]]; then
+        echo "No such directory: $envdir" >&2
+        exit 1
+    fi
+
+    if [[ -f "$pidfile" ]]; then
+        if pgrep -F "${pidfile}"; then
+            echo "Another instance (PID $(cat "$pidfile")) is running, please stop that instance first"
+            exit 1
+        fi
+
+        echo "A PID file was found but the process does not seem to be running, removing anyway"
+    fi
+
+    echo "WARNING: This operation will permanently remove the Platypush environment $1"
+    echo -n "Are you sure you want to continue? (y/N) "
+    IFS= read -r answer
+    echo "$answer" | grep -E '^[yY]' >/dev/null || exit 0
+    rm -rf "$envdir"
+    echo "$envdir removed"
+}
+
+function usage {
+    echo "Usage: $0 <build|start|stop|rm> [options]" >&2
+    exit 1
+}
+
+if (( $# < 1 )); then
+    usage
+fi
+
+action=$1
+shift
+mkdir -p "${workdir}"
+
+# shellcheck disable=SC2048,SC2086
+case ${action} in
+    'build') build $*;;
+    'start') start $*;;
+    'stop') stop $*;;
+    'rm') rme $*;;
+    *) usage;;
+esac

docs/README.md

@@ -0,0 +1,21 @@
+# Platypush self-generated reference
+====================================
+
+This directory contains the Sphinx self-generated documentation for Platypush.
+
+Dependencies required to generate the documentation:
+
+```shell
+$ [sudo] pip install sphinx 'git+https://github.com/bashtage/sphinx-material.git'
+```
+
+To generate the HTML documentation:
+
+```shell
+$ make html
+```
+
+The output will be generated under `build/html`.
+
+Type `make` with no additional arguments to get a full list of the supported output formats.
+

docs/source/_ext/sphinx_marshmallow.py

@@ -0,0 +1,87 @@
+import importlib
+import json
+import os
+import re
+import sys
+from random import randint
+from typing import Union, List
+
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from marshmallow import fields
+
+
+class SchemaDirective(Directive):
+    """
+    Support for response/message schemas in the docs. Format: ``.. schema:: rel_path.SchemaClass(arg1=value1, ...)``,
+    where ``rel_path`` is the path of the schema relative to ``platypush/schemas``.
+    """
+    has_content = True
+    _schema_regex = re.compile(r'^\s*(.+?)\s*(\((.+?)\))?\s*$')
+    _schemas_path = os.path.abspath(
+        os.path.join(
+            os.path.dirname(os.path.relpath(__file__)), '..', '..', '..', 'platypush', 'schemas'))
+
+    sys.path.insert(0, _schemas_path)
+
+    @classmethod
+    def _get_field_value(cls, field):
+        metadata = getattr(field, 'metadata', {})
+        if metadata.get('example'):
+            return metadata['example']
+        if metadata.get('description'):
+            return metadata['description']
+
+        if isinstance(field, fields.Number):
+            return randint(1, 99)
+        if isinstance(field, fields.Boolean):
+            return bool(randint(0, 1))
+        if isinstance(field, fields.URL):
+            return 'https://example.org'
+        if isinstance(field, fields.List):
+            return [cls._get_field_value(field.inner)]
+        if isinstance(field, fields.Dict):
+            return {
+                cls._get_field_value(field.key_field) if field.key_field else 'key':
+                    cls._get_field_value(field.value_field) if field.value_field else 'value'
+            }
+        if isinstance(field, fields.Nested):
+            ret = {
+                name: cls._get_field_value(f)
+                for name, f in field.nested().fields.items()
+            }
+
+            return [ret] if field.many else ret
+
+        return str(field.__class__.__name__).lower()
+
+    def _parse_schema(self) -> Union[dict, List[dict]]:
+        m = self._schema_regex.match('\n'.join(self.content))
+        schema_module_name = '.'.join(['platypush.schemas', *(m.group(1).split('.')[:-1])])
+        schema_module = importlib.import_module(schema_module_name)
+        schema_class = getattr(schema_module, m.group(1).split('.')[-1])
+        schema_args = eval(f'dict({m.group(3)})') if m.group(3) else {}
+        schema = schema_class(**schema_args)
+        output = {
+            name: self._get_field_value(field)
+            for name, field in schema.fields.items()
+            if not field.load_only
+        }
+
+        return [output] if schema.many else output
+
+    def run(self):
+        content = json.dumps(self._parse_schema(), sort_keys=True, indent=2)
+        block = nodes.literal_block(content, content)
+        block['language'] = 'json'
+        return [block]
+
+
+def setup(app):
+    app.add_directive('schema', SchemaDirective)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

docs/source/backends.rst

@@ -1,33 +1,79 @@
+
 Backends
 ========
 
 .. toctree::
-    :maxdepth: 2
+    :maxdepth: 1
     :caption: Backends:
 
-    platypush/backend.rst
-    platypush/backend/assistant.google.pushtotalk.rst
+    platypush/backend/adafruit.io.rst
+    platypush/backend/alarm.rst
     platypush/backend/assistant.google.rst
     platypush/backend/assistant.snowboy.rst
+    platypush/backend/bluetooth.fileserver.rst
+    platypush/backend/bluetooth.pushserver.rst
+    platypush/backend/bluetooth.scanner.rst
+    platypush/backend/bluetooth.scanner.ble.rst
     platypush/backend/button.flic.rst
     platypush/backend/camera.pi.rst
-    platypush/backend/http.poll.rst
+    platypush/backend/chat.telegram.rst
+    platypush/backend/covid19.rst
+    platypush/backend/file.monitor.rst
+    platypush/backend/foursquare.rst
+    platypush/backend/github.rst
+    platypush/backend/google.fit.rst
+    platypush/backend/google.pubsub.rst
+    platypush/backend/gps.rst
     platypush/backend/http.rst
+    platypush/backend/http.poll.rst
     platypush/backend/inotify.rst
+    platypush/backend/joystick.rst
+    platypush/backend/joystick.jstest.rst
+    platypush/backend/joystick.linux.rst
     platypush/backend/kafka.rst
+    platypush/backend/light.hue.rst
+    platypush/backend/linode.rst
+    platypush/backend/log.http.rst
+    platypush/backend/mail.rst
     platypush/backend/midi.rst
     platypush/backend/mqtt.rst
+    platypush/backend/music.mopidy.rst
     platypush/backend/music.mpd.rst
+    platypush/backend/music.snapcast.rst
+    platypush/backend/music.spotify.rst
+    platypush/backend/nextcloud.rst
+    platypush/backend/nfc.rst
+    platypush/backend/nodered.rst
+    platypush/backend/ping.rst
     platypush/backend/pushbullet.rst
     platypush/backend/redis.rst
     platypush/backend/scard.rst
     platypush/backend/sensor.accelerometer.rst
+    platypush/backend/sensor.arduino.rst
+    platypush/backend/sensor.battery.rst
+    platypush/backend/sensor.bme280.rst
+    platypush/backend/sensor.dht.rst
+    platypush/backend/sensor.distance.rst
+    platypush/backend/sensor.distance.vl53l1x.rst
+    platypush/backend/sensor.envirophat.rst
     platypush/backend/sensor.ir.zeroborg.rst
     platypush/backend/sensor.leap.rst
+    platypush/backend/sensor.ltr559.rst
     platypush/backend/sensor.mcp3008.rst
-    platypush/backend/sensor.rst
+    platypush/backend/sensor.motion.pmw3901.rst
     platypush/backend/sensor.serial.rst
+    platypush/backend/stt.deepspeech.rst
+    platypush/backend/stt.picovoice.hotword.rst
+    platypush/backend/stt.picovoice.speech.rst
     platypush/backend/tcp.rst
-    platypush/backend/weather.forecast.rst
+    platypush/backend/todoist.rst
+    platypush/backend/travisci.rst
+    platypush/backend/trello.rst
+    platypush/backend/weather.buienradar.rst
+    platypush/backend/weather.darksky.rst
+    platypush/backend/weather.openweathermap.rst
     platypush/backend/websocket.rst
     platypush/backend/wiimote.rst
+    platypush/backend/zigbee.mqtt.rst
+    platypush/backend/zwave.rst
+    platypush/backend/zwave.mqtt.rst

docs/source/conf.py

@@ -18,13 +18,14 @@ import sys
 # import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath("./_ext"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'platypush'
-copyright = '2018, BlackLight'
-author = 'BlackLight'
+project = 'Platypush'
+copyright = '2017-2021, Fabio Manganiello'
+author = 'Fabio Manganiello'
 
 # The short X.Y version
 version = ''
@@ -49,6 +50,8 @@ extensions = [
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'sphinx.ext.githubpages',
+    'sphinx_rtd_theme',
+    'sphinx_marshmallow',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -68,7 +71,7 @@ master_doc = 'index'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -84,21 +87,32 @@ pygments_style = 'sphinx'
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-# html_theme = 'alabaster'
-html_theme = 'nature'
+# html_theme = 'haiku'
+# html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinx_book_theme'
 
+html_title = 'Platypush Documentation'
+html_logo = '../../assets/logo-64.png'
+html_favicon = '../../platypush/backend/http/webapp/public/favicon.ico'
 html_domain_indices = True
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-# html_theme_options = {}
+html_theme_options = {
+    'toc_title': 'Platypush documentation',
+    'repository_url': 'https://git.platypush.tech/platypush/platypush',
+    'use_repository_button': True,
+    'use_issues_button': True,
+    'use_fullscreen_button': True,
+    'home_page_in_toc': True,
+}
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -108,8 +122,9 @@ html_static_path = ['_static']
 # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
 # 'searchbox.html']``.
 #
-# html_sidebars = {}
-
+# html_sidebars = {
+#     '**': ['logo-text.html', 'globaltoc.html', 'localtoc.html', 'searchbox.html']
+# }
 
 # -- Options for HTMLHelp output ---------------------------------------------
 
@@ -123,15 +138,12 @@ latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -141,8 +153,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'platypush.tex', 'platypush Documentation',
-     'BlackLight', 'manual'),
+    (master_doc, 'platypush.tex', 'platypush Documentation', 'BlackLight', 'manual'),
 ]
 
 
@@ -150,10 +161,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'platypush', 'platypush Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, 'platypush', 'platypush Documentation', [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------------
@@ -162,9 +170,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'platypush', 'platypush Documentation',
-     author, 'platypush', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        'platypush',
+        'platypush Documentation',
+        author,
+        'platypush',
+        'One line description of project.',
+        'Miscellaneous',
+    ),
 ]
 
 
@@ -180,38 +194,121 @@ intersphinx_mapping = {'https://docs.python.org/': None}
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
-autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers',
-                        'google.assistant.embedded',
-                        'google.assistant.library',
-                        'google.assistant.library.event',
-                        'google.assistant.library.file_helpers',
-                        'google.oauth2.credentials',
-                        'oauth2client',
-                        'apiclient',
-                        'tenacity',
-                        'smartcard',
-                        'Leap',
-                        'oauth2client',
-                        'rtmidi',
-                        'bluetooth',
-                        'gevent.wsgi',
-                        'Adafruit_IO',
-                        'pyperclip',
-                        'dbus',
-                        'inputs',
-                        'inotify',
-                        'omxplayer',
-                        'plexapi',
-                        'cwiid',
-                        ]
+autodoc_default_options = {
+    'members': True,
+    'show-inheritance': True,
+}
+
+autodoc_mock_imports = [
+    'googlesamples.assistant.grpc.audio_helpers',
+    'google.assistant.embedded',
+    'google.assistant.library',
+    'google.assistant.library.event',
+    'google.assistant.library.file_helpers',
+    'google.oauth2.credentials',
+    'oauth2client',
+    'apiclient',
+    'tenacity',
+    'smartcard',
+    'Leap',
+    'oauth2client',
+    'rtmidi',
+    'bluetooth',
+    'gevent.wsgi',
+    'Adafruit_IO',
+    'pyperclip',
+    'pydbus',
+    'inputs',
+    'inotify',
+    'omxplayer',
+    'plexapi',
+    'cwiid',
+    'sounddevice',
+    'soundfile',
+    'numpy',
+    'cv2',
+    'nfc',
+    'ndef',
+    'bcrypt',
+    'google',
+    'feedparser',
+    'kafka',
+    'googlesamples',
+    'icalendar',
+    'httplib2',
+    'mpd',
+    'serial',
+    'pyHS100',
+    'grpc',
+    'envirophat',
+    'gps',
+    'picamera',
+    'pmw3901',
+    'PIL',
+    'croniter',
+    'pyaudio',
+    'avs',
+    'PyOBEX',
+    'todoist',
+    'trello',
+    'telegram',
+    'telegram.ext',
+    'pyfirmata2',
+    'cups',
+    'graphyte',
+    'cpuinfo',
+    'psutil',
+    'openzwave',
+    'deepspeech',
+    'wave',
+    'pvporcupine ',
+    'pvcheetah',
+    'pyotp',
+    'linode_api4',
+    'pyzbar',
+    'tensorflow',
+    'keras',
+    'pandas',
+    'samsungtvws',
+    'paramiko',
+    'luma',
+    'zeroconf',
+    'dbus',
+    'gi',
+    'gi.repository',
+    'twilio',
+    'Adafruit_Python_DHT',
+    'RPi.GPIO',
+    'RPLCD',
+    'imapclient',
+    'pysmartthings',
+    'aiohttp',
+    'watchdog',
+    'pyngrok',
+    'irc',
+    'irc.bot',
+    'irc.strings',
+    'irc.client',
+    'irc.connection',
+    'irc.events',
+    'defusedxml',
+    'nio',
+    'aiofiles',
+    'aiofiles.os',
+    'async_lru',
+]
 
 sys.path.insert(0, os.path.abspath('../..'))
 
+
 def skip(app, what, name, obj, skip, options):
     if name == "__init__":
         return False
     return skip
 
+
 def setup(app):
     app.connect("autodoc-skip-member", skip)
 
+
+# vim:sw=4:ts=4:et:

docs/source/events.rst

@@ -1,31 +1,82 @@
+
 Events
 ======
 
 .. toctree::
-    :maxdepth: 2
+    :maxdepth: 1
     :caption: Events:
 
+    platypush/events/adafruit.rst
+    platypush/events/alarm.rst
     platypush/events/application.rst
     platypush/events/assistant.rst
+    platypush/events/bluetooth.rst
     platypush/events/button.flic.rst
+    platypush/events/camera.rst
+    platypush/events/chat.slack.rst
+    platypush/events/chat.telegram.rst
+    platypush/events/clipboard.rst
+    platypush/events/covid19.rst
+    platypush/events/custom.rst
+    platypush/events/dbus.rst
+    platypush/events/distance.rst
+    platypush/events/file.rst
+    platypush/events/foursquare.rst
     platypush/events/geo.rst
-    platypush/events/http.rss.rst
+    platypush/events/github.rst
+    platypush/events/google.rst
+    platypush/events/google.fit.rst
+    platypush/events/google.pubsub.rst
+    platypush/events/gotify.rst
+    platypush/events/gpio.rst
+    platypush/events/gps.rst
     platypush/events/http.rst
+    platypush/events/http.hook.rst
+    platypush/events/http.rss.rst
+    platypush/events/inotify.rst
+    platypush/events/irc.rst
     platypush/events/joystick.rst
     platypush/events/kafka.rst
+    platypush/events/light.rst
+    platypush/events/linode.rst
+    platypush/events/log.http.rst
+    platypush/events/mail.rst
+    platypush/events/matrix.rst
+    platypush/events/media.rst
     platypush/events/midi.rst
+    platypush/events/mqtt.rst
     platypush/events/music.rst
-    platypush/events/path.rst
+    platypush/events/music.snapcast.rst
+    platypush/events/music.tidal.rst
+    platypush/events/nextcloud.rst
+    platypush/events/nfc.rst
+    platypush/events/ngrok.rst
+    platypush/events/ntfy.rst
     platypush/events/ping.rst
     platypush/events/pushbullet.rst
+    platypush/events/qrcode.rst
+    platypush/events/rss.rst
     platypush/events/scard.rst
+    platypush/events/sensor.rst
     platypush/events/sensor.ir.rst
     platypush/events/sensor.leap.rst
     platypush/events/sensor.light.rst
-    platypush/events/sensor.rst
     platypush/events/serial.rst
+    platypush/events/sound.rst
+    platypush/events/stt.rst
+    platypush/events/sun.rst
+    platypush/events/tensorflow.rst
+    platypush/events/todoist.rst
     platypush/events/torrent.rst
+    platypush/events/travisci.rst
+    platypush/events/trello.rst
     platypush/events/video.rst
     platypush/events/weather.rst
     platypush/events/web.rst
     platypush/events/web.widget.rst
+    platypush/events/websocket.rst
+    platypush/events/wiimote.rst
+    platypush/events/zeroborg.rst
+    platypush/events/zeroconf.rst
+    platypush/events/zigbee.mqtt.rst
+    platypush/events/zwave.rst

docs/source/index.rst

@@ -3,21 +3,26 @@ Platypush
 
 Welcome to the Platypush reference of available plugins, backends and event types.
 
-For more information on Platypush please check out:
+For more information on Platypush check out:
 
-* The `GitHub page`_ of the project
+* The `main page`_ of the project
+* The `Gitea page`_ of the project
 * The `online wiki`_ for quickstart and examples
+* The `Blog articles`_ for inspiration on use-cases possible projects
 
-.. _GitHub page: https://github.com/BlackLight/platypush
-.. _online wiki: https://github.com/BlackLight/platypush/wiki
+.. _main page: https://platypush.tech
+.. _Gitea page: https://git.platypush.tech/platypush/platypush
+.. _online wiki: https://git.platypush.tech/platypush/platypush/wiki
+.. _Blog articles: https://blog.platypush.tech
 
 .. toctree::
-    :maxdepth: 3
+    :maxdepth: 2
     :caption: Contents:
 
     backends
     plugins
     events
+    responses
 
 Indices and tables
 ==================
@@ -25,4 +30,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/source/platypush/backend.rst

@@ -1,6 +0,0 @@
-``platypush.backend``
-=====================
-
-.. automodule:: platypush.backend
-    :members:
-

docs/source/platypush/backend/adafruit.io.rst

@@ -0,0 +1,6 @@
+``adafruit.io``
+=================================
+
+.. automodule:: platypush.backend.adafruit.io
+	:members:
+

docs/source/platypush/backend/alarm.rst

@@ -0,0 +1,5 @@
+``alarm``
+===========================
+
+.. automodule:: platypush.backend.alarm
+    :members:

docs/source/platypush/backend/assistant.google.pushtotalk.rst

@@ -1,6 +0,0 @@
-``platypush.backend.assistant.google.pushtotalk``
-=================================================
-
-.. automodule:: platypush.backend.assistant.google.pushtotalk
-    :members:
-

docs/source/platypush/backend/assistant.google.rst

@@ -1,4 +1,4 @@
-``platypush.backend.assistant.google``
+``assistant.google``
 ======================================
 
 .. automodule:: platypush.backend.assistant.google

docs/source/platypush/backend/assistant.snowboy.rst

@@ -1,4 +1,4 @@
-``platypush.backend.assistant.snowboy``
+``assistant.snowboy``
 =======================================
 
 .. automodule:: platypush.backend.assistant.snowboy

docs/source/platypush/backend/bluetooth.fileserver.rst

@@ -0,0 +1,6 @@
+``bluetooth.fileserver``
+==========================================
+
+.. automodule:: platypush.backend.bluetooth.fileserver
+	:members:
+

docs/source/platypush/backend/bluetooth.pushserver.rst

@@ -0,0 +1,6 @@
+``bluetooth.pushserver``
+==========================================
+
+.. automodule:: platypush.backend.bluetooth.pushserver
+	:members:
+

docs/source/platypush/backend/bluetooth.scanner.ble.rst

@@ -0,0 +1,5 @@
+``bluetooth.scanner.ble``
+===========================================
+
+.. automodule:: platypush.backend.bluetooth.scanner.ble
+    :members:

docs/source/platypush/backend/bluetooth.scanner.rst

@@ -0,0 +1,5 @@
+``bluetooth.scanner``
+=======================================
+
+.. automodule:: platypush.backend.bluetooth.scanner
+    :members:

docs/source/platypush/backend/button.flic.rst

@@ -1,4 +1,4 @@
-``platypush.backend.button.flic``
+``button.flic``
 =================================
 
 .. automodule:: platypush.backend.button.flic

docs/source/platypush/backend/camera.pi.rst

@@ -1,4 +1,4 @@
-``platypush.backend.camera.pi``
+``camera.pi``
 ===============================
 
 .. automodule:: platypush.backend.camera.pi

docs/source/platypush/backend/chat.telegram.rst

@@ -0,0 +1,5 @@
+``chat.telegram``
+===================================
+
+.. automodule:: platypush.backend.chat.telegram
+    :members:

docs/source/platypush/backend/covid19.rst

@@ -0,0 +1,5 @@
+``covid19``
+=============================
+
+.. automodule:: platypush.backend.covid19
+    :members:

docs/source/platypush/backend/file.monitor.rst

@@ -0,0 +1,5 @@
+``file.monitor``
+==================================
+
+.. automodule:: platypush.backend.file.monitor
+    :members:

docs/source/platypush/backend/foursquare.rst

@@ -0,0 +1,5 @@
+``foursquare``
+================================
+
+.. automodule:: platypush.backend.foursquare
+    :members:

docs/source/platypush/backend/github.rst

@@ -0,0 +1,5 @@
+``github``
+============================
+
+.. automodule:: platypush.backend.github
+    :members:

docs/source/platypush/backend/google.fit.rst

@@ -0,0 +1,6 @@
+``google.fit``
+================================
+
+.. automodule:: platypush.backend.google.fit
+    :members:
+

docs/source/platypush/backend/google.pubsub.rst

@@ -0,0 +1,5 @@
+``google.pubsub``
+===================================
+
+.. automodule:: platypush.backend.google.pubsub
+    :members:

docs/source/platypush/backend/gps.rst

@@ -0,0 +1,6 @@
+``gps``
+=========================
+
+.. automodule:: platypush.backend.gps
+    :members:
+

docs/source/platypush/backend/http.poll.rst

@@ -1,4 +1,4 @@
-``platypush.backend.http.poll``
+``http.poll``
 ===============================
 
 .. automodule:: platypush.backend.http.poll

docs/source/platypush/backend/http.rst

@@ -1,4 +1,4 @@
-``platypush.backend.http``
+``http``
 ==========================
 
 .. automodule:: platypush.backend.http

docs/source/platypush/backend/inotify.rst

@@ -1,4 +1,4 @@
-``platypush.backend.inotify``
+``inotify``
 =============================
 
 .. automodule:: platypush.backend.inotify

docs/source/platypush/backend/joystick.jstest.rst

@@ -0,0 +1,5 @@
+``joystick.jstest``
+=====================================
+
+.. automodule:: platypush.backend.joystick.jstest
+    :members:

docs/source/platypush/backend/joystick.linux.rst

@@ -0,0 +1,5 @@
+``joystick.linux``
+====================================
+
+.. automodule:: platypush.backend.joystick.linux
+    :members:

docs/source/platypush/backend/joystick.rst

@@ -0,0 +1,6 @@
+``joystick``
+==============================
+
+.. automodule:: platypush.backend.joystick
+    :members:
+

docs/source/platypush/backend/kafka.rst

@@ -1,4 +1,4 @@
-``platypush.backend.kafka``
+``kafka``
 ===========================
 
 .. automodule:: platypush.backend.kafka

docs/source/platypush/backend/light.hue.rst

@@ -0,0 +1,6 @@
+``light.hue``
+===============================
+
+.. automodule:: platypush.backend.light.hue
+	:members:
+

docs/source/platypush/backend/linode.rst

@@ -0,0 +1,5 @@
+``linode``
+============================
+
+.. automodule:: platypush.backend.linode
+    :members:

docs/source/platypush/backend/log.http.rst

@@ -0,0 +1,5 @@
+``log.http``
+==============================
+
+.. automodule:: platypush.backend.log.http
+    :members:

docs/source/platypush/backend/mail.rst

@@ -0,0 +1,5 @@
+``mail``
+==========================
+
+.. automodule:: platypush.backend.mail
+    :members:

docs/source/platypush/backend/midi.rst

@@ -1,4 +1,4 @@
-``platypush.backend.midi``
+``midi``
 ==========================
 
 .. automodule:: platypush.backend.midi

docs/source/platypush/backend/mqtt.rst

@@ -1,4 +1,4 @@
-``platypush.backend.mqtt``
+``mqtt``
 ==========================
 
 .. automodule:: platypush.backend.mqtt

docs/source/platypush/backend/music.mopidy.rst

@@ -0,0 +1,6 @@
+``music.mopidy``
+==================================
+
+.. automodule:: platypush.backend.music.mopidy
+	:members:
+

docs/source/platypush/backend/music.mpd.rst

@@ -1,4 +1,4 @@
-``platypush.backend.music.mpd``
+``music.mpd``
 ===============================
 
 .. automodule:: platypush.backend.music.mpd

docs/source/platypush/backend/music.snapcast.rst

@@ -0,0 +1,6 @@
+``music.snapcast``
+====================================
+
+.. automodule:: platypush.backend.music.snapcast
+	:members:
+

docs/source/platypush/backend/music.spotify.rst

@@ -0,0 +1,5 @@
+``music.spotify``
+===================================
+
+.. automodule:: platypush.backend.music.spotify
+    :members:

docs/source/platypush/backend/nextcloud.rst

@@ -0,0 +1,5 @@
+``nextcloud``
+===============================
+
+.. automodule:: platypush.backend.nextcloud
+    :members:

docs/source/platypush/backend/nfc.rst

@@ -0,0 +1,6 @@
+``nfc``
+=========================
+
+.. automodule:: platypush.backend.nfc
+    :members:
+

docs/source/platypush/backend/nodered.rst

@@ -0,0 +1,6 @@
+``nodered``
+=============================
+
+.. automodule:: platypush.backend.nodered
+	:members:
+

docs/source/platypush/backend/ping.rst

@@ -0,0 +1,5 @@
+``ping``
+==========================
+
+.. automodule:: platypush.backend.ping
+    :members:

docs/source/platypush/backend/pushbullet.rst

@@ -1,4 +1,4 @@
-``platypush.backend.pushbullet``
+``pushbullet``
 ================================
 
 .. automodule:: platypush.backend.pushbullet

docs/source/platypush/backend/redis.rst

@@ -1,4 +1,4 @@
-``platypush.backend.redis``
+``redis``
 ===========================
 
 .. automodule:: platypush.backend.redis

docs/source/platypush/backend/scard.rst

@@ -1,4 +1,4 @@
-``platypush.backend.scard``
+``scard``
 ===========================
 
 .. automodule:: platypush.backend.scard

docs/source/platypush/backend/sensor.accelerometer.rst

@@ -1,4 +1,4 @@
-``platypush.backend.sensor.accelerometer``
+``sensor.accelerometer``
 ==========================================
 
 .. automodule:: platypush.backend.sensor.accelerometer

docs/source/platypush/backend/sensor.arduino.rst

@@ -0,0 +1,5 @@
+``sensor.arduino``
+====================================
+
+.. automodule:: platypush.backend.sensor.arduino
+    :members:

docs/source/platypush/backend/sensor.battery.rst

@@ -0,0 +1,5 @@
+``sensor.battery``
+====================================
+
+.. automodule:: platypush.backend.sensor.battery
+    :members:

docs/source/platypush/backend/sensor.bme280.rst

@@ -0,0 +1,6 @@
+``sensor.bme280``
+===================================
+
+.. automodule:: platypush.backend.sensor.bme280
+	:members:
+

docs/source/platypush/backend/sensor.dht.rst

@@ -0,0 +1,5 @@
+``sensor.dht``
+================================
+
+.. automodule:: platypush.backend.sensor.dht
+    :members:

docs/source/platypush/backend/sensor.distance.rst

@@ -0,0 +1,5 @@
+``sensor.distance``
+=====================================
+
+.. automodule:: platypush.backend.sensor.distance
+    :members:

docs/source/platypush/backend/sensor.distance.vl53l1x.rst

@@ -0,0 +1,6 @@
+``sensor.distance.vl53l1x``
+=============================================
+
+.. automodule:: platypush.backend.sensor.distance.vl53l1x
+	:members:
+

docs/source/platypush/backend/sensor.envirophat.rst

@@ -0,0 +1,6 @@
+``sensor.envirophat``
+=======================================
+
+.. automodule:: platypush.backend.sensor.envirophat
+    :members:
+

docs/source/platypush/backend/sensor.ir.zeroborg.rst

@@ -1,4 +1,4 @@
-``platypush.backend.sensor.ir.zeroborg``
+``sensor.ir.zeroborg``
 ========================================
 
 .. automodule:: platypush.backend.sensor.ir.zeroborg

docs/source/platypush/backend/sensor.leap.rst

@@ -1,4 +1,4 @@
-``platypush.backend.sensor.leap``
+``sensor.leap``
 =================================
 
 .. automodule:: platypush.backend.sensor.leap

docs/source/platypush/backend/sensor.ltr559.rst

@@ -0,0 +1,6 @@
+``sensor.ltr559``
+===================================
+
+.. automodule:: platypush.backend.sensor.ltr559
+	:members:
+

docs/source/platypush/backend/sensor.mcp3008.rst

@@ -1,4 +1,4 @@
-``platypush.backend.sensor.mcp3008``
+``sensor.mcp3008``
 ====================================
 
 .. automodule:: platypush.backend.sensor.mcp3008

docs/source/platypush/backend/sensor.motion.pmw3901.rst

@@ -0,0 +1,5 @@
+``sensor.motion.pmw3901``
+=========================
+
+.. automodule:: platypush.backend.sensor.motion.pmw3901
+    :members:

docs/source/platypush/backend/sensor.rst

@@ -1,6 +0,0 @@
-``platypush.backend.sensor``
-============================
-
-.. automodule:: platypush.backend.sensor
-    :members:
-

docs/source/platypush/backend/sensor.serial.rst

@@ -1,4 +1,4 @@
-``platypush.backend.sensor.serial``
+``sensor.serial``
 ===================================
 
 .. automodule:: platypush.backend.sensor.serial

docs/source/platypush/backend/stt.deepspeech.rst

@@ -0,0 +1,5 @@
+``stt.deepspeech``
+====================================
+
+.. automodule:: platypush.backend.stt.deepspeech
+    :members:

docs/source/platypush/backend/stt.picovoice.hotword.rst

@@ -0,0 +1,5 @@
+``stt.picovoice.hotword``
+===========================================
+
+.. automodule:: platypush.backend.stt.picovoice.hotword
+    :members:

docs/source/platypush/backend/stt.picovoice.speech.rst

@@ -0,0 +1,5 @@
+``stt.picovoice.speech``
+==========================================
+
+.. automodule:: platypush.backend.stt.picovoice.speech
+    :members:

docs/source/platypush/backend/tcp.rst

@@ -1,4 +1,4 @@
-``platypush.backend.tcp``
+``tcp``
 =========================
 
 .. automodule:: platypush.backend.tcp

docs/source/platypush/backend/todoist.rst

@@ -0,0 +1,5 @@
+``todoist``
+=============================
+
+.. automodule:: platypush.backend.todoist
+    :members:

docs/source/platypush/backend/travisci.rst

@@ -0,0 +1,5 @@
+``travisci``
+==============================
+
+.. automodule:: platypush.backend.travisci
+    :members:

docs/source/platypush/backend/trello.rst

@@ -0,0 +1,5 @@
+``trello``
+============================
+
+.. automodule:: platypush.backend.trello
+    :members:

docs/source/platypush/backend/weather.buienradar.rst

@@ -0,0 +1,5 @@
+``weather.buienradar``
+========================================
+
+.. automodule:: platypush.backend.weather.buienradar
+    :members:

docs/source/platypush/backend/weather.darksky.rst

@@ -0,0 +1,5 @@
+``weather.darksky``
+=====================================
+
+.. automodule:: platypush.backend.weather.darksky
+    :members:

docs/source/platypush/backend/weather.forecast.rst

@@ -1,6 +0,0 @@
-``platypush.backend.weather.forecast``
-======================================
-
-.. automodule:: platypush.backend.weather.forecast
-    :members:
-

docs/source/platypush/backend/weather.openweathermap.rst

@@ -0,0 +1,5 @@
+``weather.openweathermap``
+============================================
+
+.. automodule:: platypush.backend.weather.openweathermap
+    :members:

docs/source/platypush/backend/websocket.rst

@@ -1,4 +1,4 @@
-``platypush.backend.websocket``
+``websocket``
 ===============================
 
 .. automodule:: platypush.backend.websocket

docs/source/platypush/backend/wiimote.rst

@@ -1,4 +1,4 @@
-``platypush.backend.wiimote``
+``wiimote``
 =============================
 
 .. automodule:: platypush.backend.wiimote

docs/source/platypush/backend/zigbee.mqtt.rst

@@ -0,0 +1,5 @@
+``zigbee.mqtt``
+=================================
+
+.. automodule:: platypush.backend.zigbee.mqtt
+    :members:

docs/source/platypush/backend/zwave.mqtt.rst

@@ -0,0 +1,5 @@
+``zwave.mqtt``
+================================
+
+.. automodule:: platypush.backend.zwave.mqtt
+    :members:

docs/source/platypush/backend/zwave.rst

@@ -0,0 +1,5 @@
+``zwave``
+===========================
+
+.. automodule:: platypush.backend.zwave
+    :members: