Fix: /var/lib/platypush is still empty at the time of package creation.

A fully self-contained 1.5k LoC Drone file isn't very maintainable, and
it makes it hard to reuse parts that are shared across multiple steps
(like SSH and git configuration).

.drone.yml

@@ -27,19 +27,7 @@ steps:
       from_secret: ssh_privkey
 
   commands:
-    - apk add --update --no-cache git openssh
-    - mkdir -p ~/.ssh
-    - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
-    - |
-      cat <<EOF > ~/.ssh/id_rsa
-      $SSH_PRIVKEY
-      EOF
-    - chmod 0600 ~/.ssh/id_rsa
-    - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
-    - git config --global --add safe.directory $PWD
-    - git remote add github git@github.com:/BlackLight/platypush.git
-    - git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github"
-    - git push --all -v github
+    - . .drone/github-mirror.sh
 
 ###
 ### Synchronize the `stable` branch to the new release
@@ -58,44 +46,7 @@ steps:
       from_secret: pgp_key_id
 
   commands:
-    - apk add --update --no-cache git bash gnupg openssh
-
-    # Backup the original git configuration before changing attributes
-    - export GIT_CONF=$PWD/.git/config
-    - cp $GIT_CONF /tmp/git.config.orig
-    - git config --global --add safe.directory $PWD
-
-    # Install the SSH and PGP keys
-    - mkdir -p ~/.ssh
-    - |
-      cat <<EOF | gpg --import --armor
-      $PGP_KEY
-      EOF
-
-    - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
-    - |
-      cat <<EOF > ~/.ssh/id_rsa
-      $SSH_PRIVKEY
-      EOF
-    - chmod 0600 ~/.ssh/id_rsa
-    - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
-
-    # Git configuration
-    - git config user.name "Platypush CI/CD Automation"
-    - git config user.email "admin@platypush.tech"
-    - git config commit.gpgsign true
-    - git config user.signingkey $PGP_KEY_ID
-    - git remote rm origin
-    - git remote add origin git@git.platypush.tech:platypush/platypush.git
-
-    # Merge and push to the `stable` branch
-    - git checkout stable
-    - git rebase master
-    - git push -u origin stable
-    - git checkout master
-
-    # Restore the original git configuration
-    - mv /tmp/git.config.orig $GIT_CONF
+    - . .drone/sync-stable-branch.sh
 
   when:
     event:
@@ -112,28 +63,7 @@ steps:
       path: /docs
 
   commands:
-    - echo "Installing required build dependencies"
-    - apk add --update --no-cache git make py3-sphinx py3-myst-parser py3-pip $(cat platypush/install/requirements/alpine.txt)
-    - pip install -U sphinx-rtd-theme sphinx-book-theme --break-system-packages
-    - pip install . --break-system-packages
-    - mkdir -p /docs/current
-    - export APPDIR=$PWD
-    - rm -rf "$APPDIR/docs/build"
-    - echo "Building the updated documentation"
-    - cd "$APPDIR/docs/source"
-    - git clone 'https://git.platypush.tech/platypush/platypush.wiki.git' wiki
-    - cd wiki
-    - ln -s Home.md index.md
-    - cd "$APPDIR/docs"
-    - make html
-    - rm -f config*.yaml
-    - cd "$APPDIR"
-    - echo "Copying the new documentation files to the target folder"
-    - mv -v "$APPDIR/docs/build" /docs/new
-    - cd /docs
-    - mv current old
-    - mv new current
-    - rm -rf old
+    - . .drone/rebuild-docs.sh
 
   when:
     event:
@@ -149,10 +79,7 @@ steps:
 - name: tests
   image: alpine
   commands:
-    - apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
-    - pip install . --break-system-packages
-    - pip install -r requirements-tests.txt --break-system-packages
-    - pytest tests
+    - . .drone/run-tests.sh
 
 ###
 ### Rebuild the UI files
@@ -181,68 +108,36 @@ steps:
     - tests
 
   commands:
-    - export SKIPCI="$PWD/.skipci"
-    - rm -rf "$SKIPCI"
-    - apk add --update --no-cache git
+    - . .drone/build-ui.sh
 
-    # Backup the original git configuration before changing attributes
-    - export GIT_CONF=$PWD/.git/config
-    - cp $GIT_CONF /tmp/git.config.orig
+###
+### Regenerate the components.json cache
+###
 
-    - git config --global --add safe.directory $PWD
-    - cd platypush/backend/http/webapp
-    - |
-      if [ $(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE . | wc -l) -eq 0 ]; then
-        echo "No UI changes detected, skipping build"
-        exit 0
-      fi
+- name: update-components-cache
+  image: alpine
 
-    - |
-      if [ "$(git log --pretty=format:%s HEAD...HEAD~1 | head -1)" == "[Automatic] Updated UI files" ]; then
-        echo "UI changes have already been committed, skipping build"
-        exit 0
-      fi
+  environment:
+    SSH_PUBKEY:
+      from_secret: ssh_pubkey
+    SSH_PRIVKEY:
+      from_secret: ssh_privkey
+    PGP_KEY:
+      from_secret: pgp_key
+    PGP_KEY_ID:
+      from_secret: pgp_key_id
 
-    - rm -rf node_modules
-    - npm install
-    - npm run build
-    - |
-      if [ $(git status --porcelain dist | wc -l) -eq 0 ]; then
-        echo "No build files have been changed"
-        exit 0
-      fi
+  when:
+    branch:
+      - master
+    event:
+      - push
 
-    # Create a .skipci file to mark the fact that the next steps should be skipped
-    # (we're going to do another push anyway, so another pipeline will be triggered)
-    - touch "$SKIPCI"
+  depends_on:
+    - build-ui
 
-    - apk add --update --no-cache bash gnupg openssh
-    - mkdir -p ~/.ssh
-    - |
-      cat <<EOF | gpg --import --armor
-      $PGP_KEY
-      EOF
-
-    - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
-    - |
-      cat <<EOF > ~/.ssh/id_rsa
-      $SSH_PRIVKEY
-      EOF
-    - chmod 0600 ~/.ssh/id_rsa
-    - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
-
-    - git config user.name "Platypush CI/CD Automation"
-    - git config user.email "admin@platypush.tech"
-    - git config commit.gpgsign true
-    - git config user.signingkey $PGP_KEY_ID
-    - git add dist
-    - git commit dist -S -m "[Automatic] Updated UI files" --no-verify
-    - git remote rm origin
-    - git remote add origin git@git.platypush.tech:platypush/platypush.git
-    - git push -f origin master
-
-    # Restore the original git configuration
-    - mv /tmp/git.config.orig $GIT_CONF
+  commands:
+    - . .drone/update-components-cache.sh
 
 ###
 ### Update the Arch packages
@@ -264,82 +159,10 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
-    - |
-      [ -f .skipci ] && exit 0
-
-    - echo "-- Installing dependencies"
-    - apk add --update --no-cache curl git openssh pacman sudo
-
-    - echo "--- Parsing metadata"
-    - git config --global --add safe.directory $PWD
-    - git pull --rebase origin master --tags
-    - export VERSION=$(python setup.py --version)
-    - export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1)
-    - export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).$HEAD"
-    - export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz"
-
-    - echo "--- Preparing environment"
-    - mkdir -p ~/.ssh
-    - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
-    - |
-      cat <<EOF > ~/.ssh/id_rsa
-      $SSH_PRIVKEY
-      EOF
-    - chmod 0600 ~/.ssh/id_rsa
-    - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
-    - ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null
-    - adduser -u 1000 -D build
-    - mkdir -p $WORKDIR
-
-    - echo "--- Updating Arch git version"
-    - export PKGDIR=$WORKDIR/git
-    - git clone ssh://aur@aur.archlinux.org/platypush-git.git $PKGDIR
-    - git config --global --add safe.directory $PKGDIR
-    - chown -R build $PKGDIR
-    - cd $PKGDIR
-    - |
-      sed -i 'PKGBUILD' -r \
-        -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \
-        -e "s/^pkgrel=.*/pkgrel=1/" \
-    - sudo -u build makepkg --printsrcinfo > .SRCINFO
-    - export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
-    - |
-      if [ $FILES_CHANGED -gt 0 ]; then
-        echo "--- Pushing git package version $GIT_VERSION"
-        git commit -a -m '[Automatic] Package update'
-        git push origin master
-      fi
-
-    - echo "--- Updating Arch stable version"
-    - export PKGDIR=$WORKDIR/stable
-    - git clone ssh://aur@aur.archlinux.org/platypush.git $PKGDIR
-    - git config --global --add safe.directory $PKGDIR
-    - chown -R build $PKGDIR
-    - cd $PKGDIR
-    - export RELEASED_VERSION=$(grep -e '^pkgver=' PKGBUILD | sed -r -e 's/^pkgver=(.*)\s*/\1/')
-    - |
-      if [ "$RELEASED_VERSION" == "$VERSION" ]; then
-        exit 0
-      fi
-
-    - export TAG_CHECKSUM=$(curl --silent "$TAG_URL" | sha512sum | awk '{print $1}')
-    - |
-      sed -i 'PKGBUILD' -r \
-        -e "s/^pkgver=.*/pkgver=$VERSION/" \
-        -e "s/^pkgrel=.*/pkgrel=1/" \
-        -e "s/^sha512sums=.*/sha512sums=('$TAG_CHECKSUM')/"
-
-    - sudo -u build makepkg --printsrcinfo > .SRCINFO
-    - export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
-    - |
-      if [ $FILES_CHANGED -gt 0 ]; then
-        echo "--- Pushing stable package version $VERSION"
-        git commit -a -m '[Automatic] Package update'
-        git push origin master
-      fi
+    - . .drone/update-arch-packages.sh
 
 ###
 ### Update the Debian (stable) packages
@@ -364,81 +187,10 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
-    - |
-      [ -f .skipci ] && exit 0
-
-    - echo "-- Installing dependencies"
-    - apt update
-    - apt install -y curl dpkg-dev gpg git python3 python3-pip
-
-    - echo "-- Copying source directory"
-    - mkdir -p "$WORKDIR"
-    - export SRCDIR="$WORKDIR/src"
-    - cp -r "$PWD" "$SRCDIR"
-    - cd "$SRCDIR"
-
-    - echo "--- Parsing metadata"
-    - git config --global --add safe.directory "$PWD"
-    - git pull --rebase origin master --tags
-    - export VERSION=$(python3 setup.py --version)
-    - export GIT_VERSION="$VERSION-$(git log --pretty=oneline HEAD...v$VERSION | wc -l)"
-    - export GIT_BUILD_DIR="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all"
-    - export GIT_DEB="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all.deb"
-    - export POOL_PATH="$APT_ROOT/pool/$DEB_VERSION/dev"
-
-    - echo "--- Building git package"
-    - pip install --prefix="$GIT_BUILD_DIR/usr" --no-cache --no-deps .
-    - |
-      find "$GIT_BUILD_DIR" -name "site-packages" | while read dir; do
-        base="$(dirname "$dir")"
-        mv "$dir" "$base/dist-packages"
-      done
-
-    - mkdir -p "$GIT_BUILD_DIR/DEBIAN"
-    - |
-      cat <<EOF > "$GIT_BUILD_DIR/DEBIAN/control"
-      Package: $PKG_NAME
-      Version: $GIT_VERSION
-      Maintainer: Fabio Manganiello <fabio@platypush.tech>
-      Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g')
-      Architecture: all
-      Homepage: https://platypush.tech
-      Description: Universal command executor and automation hub.
-      EOF
-    - mkdir -p "$POOL_PATH"
-    - rm -f "$POOL_PATH/"*.deb
-    - dpkg --build "$GIT_BUILD_DIR"
-
-    - echo "--- Copying $GIT_DEB to $POOL_PATH"
-    - cp "$GIT_DEB" "$POOL_PATH"
-
-    # If main/all/Packages doesn't exist, then we should create the first main release
-    - |
-      [ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1
-
-    - export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages"
-    - |
-      [ -z "$UPDATE_STABLE_PKG" ] &&
-        curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" |
-          grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1
-
-    # If the published release version differs from the current one, then we should publish a new main release
-    - |
-      if [ -z "$UPDATE_STABLE_PKG" ]; then
-        RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1)
-        [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1
-      fi
-
-    # Proceed and update the main release if the version number has changed
-    - |
-      if [ -n "$UPDATE_STABLE_PKG" ]; then
-        echo "--- Updating main package"
-        mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main"
-        cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-1_all.deb"
-      fi
+    - . .drone/update-deb-packages.sh
 
 ###
 ### Update the Debian (oldstable) packages
@@ -463,81 +215,10 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
-    - |
-      [ -f .skipci ] && exit 0
-
-    - echo "-- Copying source directory"
-    - mkdir -p "$WORKDIR"
-    - export SRCDIR="$WORKDIR/src"
-    - cp -r "$PWD" "$SRCDIR"
-    - cd "$SRCDIR"
-
-    - echo "-- Installing dependencies"
-    - apt update
-    - apt install -y curl dpkg-dev gpg git python3 python3-pip
-
-    - echo "--- Parsing metadata"
-    - git config --global --add safe.directory "$PWD"
-    - git pull --rebase origin master --tags
-    - export VERSION=$(python3 setup.py --version)
-    - export GIT_VERSION="$VERSION-$(git log --pretty=oneline HEAD...v$VERSION | wc -l)"
-    - export GIT_BUILD_DIR="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all"
-    - export GIT_DEB="$WORKDIR/$${PKG_NAME}_$${GIT_VERSION}_all.deb"
-    - export POOL_PATH="$APT_ROOT/pool/$DEB_VERSION/dev"
-
-    - echo "--- Building git package"
-    - pip install --prefix="$GIT_BUILD_DIR/usr" --no-cache --no-deps .
-    - |
-      find "$GIT_BUILD_DIR" -name "site-packages" | while read dir; do
-        base="$(dirname "$dir")"
-        mv "$dir" "$base/dist-packages"
-      done
-
-    - mkdir -p "$GIT_BUILD_DIR/DEBIAN"
-    - |
-      cat <<EOF > "$GIT_BUILD_DIR/DEBIAN/control"
-      Package: $PKG_NAME
-      Version: $GIT_VERSION
-      Maintainer: Fabio Manganiello <fabio@platypush.tech>
-      Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g')
-      Architecture: all
-      Homepage: https://platypush.tech
-      Description: Universal command executor and automation hub.
-      EOF
-    - mkdir -p "$POOL_PATH"
-    - rm -f "$POOL_PATH/"*.deb
-    - dpkg --build "$GIT_BUILD_DIR"
-
-    - echo "--- Copying $GIT_DEB to $POOL_PATH"
-    - cp "$GIT_DEB" "$POOL_PATH"
-
-    # If main/all/Packages doesn't exist, then we should create the first main release
-    - |
-      [ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1
-
-    - export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages"
-    - |
-      [ -z "$UPDATE_STABLE_PKG" ] &&
-        curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" |
-          grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1
-
-    # If the published release version differs from the current one, then we should publish a new main release
-    - |
-      if [ -z "$UPDATE_STABLE_PKG" ]; then
-        RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1)
-        [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1
-      fi
-
-    # Proceed and update the main release if the version number has changed
-    - |
-      if [ -n "$UPDATE_STABLE_PKG" ]; then
-        echo "--- Updating main package"
-        mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main"
-        cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/$${PKG_NAME}_$${VERSION}-1_all.deb"
-      fi
+    - . .drone/update-deb-packages.sh
 
 ###
 ### Updates the APT repository after new packages have been pushed
@@ -568,183 +249,7 @@ steps:
     - update-debian-oldstable-packages
 
   commands:
-    - |
-      [ -f .skipci ] && exit 0
-
-    - echo "-- Installing dependencies"
-    - apt update
-    - apt install -y dpkg-dev gpg
-
-    - echo "-- Creating a new apt root folder"
-    - export TMP_APT_ROOT="/tmp/apt"
-    - cp -r "$APT_ROOT" "$TMP_APT_ROOT"
-
-    - echo "-- Cleaning up older apt releases"
-    - |
-      find "$TMP_APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do
-        pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 ))
-        [ $pkg_to_remove -le 0 ] && continue
-        ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f
-      done
-
-    - echo "-- Updating Packages files"
-    - |
-      echo "stable\noldstable" | while read distro; do
-        echo "main\ndev" | while read branch; do
-          branch_dir="$TMP_APT_ROOT/pool/$distro/$branch"
-          echo "Checking pool folder: $branch_dir"
-          [ -d "$branch_dir" ] || mkdir -p "$branch_dir"
-          dist_dir="$TMP_APT_ROOT/dists/$distro/$branch/all"
-          mkdir -p "$dist_dir"
-          pkg_file="$dist_dir/Packages"
-          dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file"
-          sed -i "$pkg_file" -re "s|^Filename: $TMP_APT_ROOT/|Filename: |"
-          cat "$pkg_file" | gzip -9 > "$pkg_file.gz"
-          echo "Generated Packages file: $pkg_file"
-          cat "$pkg_file"
-        done
-      done
-
-    - echo "-- Updating Release files"
-    - |
-      add_hashes() {
-        dist_dir=$1
-        hash_cmd=$2
-        hash_label=$3
-
-        echo "$hash_label:"
-        find "$dist_dir" -name 'Packages*' | while read file; do
-          basename="$(echo "$file" | sed -r -e "s|^$dist_dir/||")"
-          hash="$($hash_cmd "$file" | cut -d" " -f1)"
-          size="$(wc -c < $file)"
-          echo " $hash $size $basename"
-          echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-i386/|')"
-          echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-amd64/|')"
-          echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armel/|')"
-          echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armhf/|')"
-          echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-arm64/|')"
-        done
-      }
-
-      echo "stable\noldstable" | while read distro; do
-        dist_dir="$TMP_APT_ROOT/dists/$distro"
-        components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ')
-        release_file="$dist_dir/Release"
-
-        cat <<EOF > "$release_file"
-      Origin: Platypush repository
-      Label: Platypush
-      Suite: $distro
-      Codename: $distro
-      Architectures: i386 amd64 armel armhf arm64
-      Components: $components
-      Description: The official APT repository for Platypush
-      Date: $(date -Ru)
-      EOF
-
-        add_hashes "$dist_dir" "md5sum" "MD5Sum" >> "$release_file"
-        add_hashes "$dist_dir" "sha1sum" "SHA1" >> "$release_file"
-        add_hashes "$dist_dir" "sha256sum" "SHA256" >> "$release_file"
-      done
-
-    - echo "-- Generating list files"
-    - mkdir -p "$TMP_APT_ROOT/lists"
-    - |
-      cat <<EOF > "$TMP_APT_ROOT/lists/platypush-stable-main.list"
-      deb https://apt.platypush.tech/ stable main
-      EOF
-
-    - |
-      cat <<EOF > "$TMP_APT_ROOT/lists/platypush-stable-dev.list"
-      deb https://apt.platypush.tech/ stable dev
-      EOF
-
-    - |
-      cat <<EOF > "$TMP_APT_ROOT/lists/platypush-oldstable-main.list"
-      deb https://apt.platypush.tech/ oldstable main
-      EOF
-
-    - |
-      cat <<EOF > "$TMP_APT_ROOT/lists/platypush-oldstable-dev.list"
-      deb https://apt.platypush.tech/ oldstable dev
-      EOF
-
-    - echo "-- Updating index file"
-    - |
-      cat <<EOF > "$TMP_APT_ROOT/index.txt"
-      Welcome to the Platypush APT repository!
-
-      Project homepage: https://platypush.tech
-      Source code: https://git.platypush.tech/platypush/platypush
-      Documentation / API reference: https://docs.platypush.tech
-
-      You can use this APT repository to install Platypush on Debian, Ubuntu or any
-      Debian-based distro.
-
-      Steps:
-
-      1. Add this repository's PGP key to your apt keyring
-      ====================================================
-
-      $ sudo wget -q -O \\\
-          /etc/apt/trusted.gpg.d/platypush.asc \\\
-          https://apt.platypush.tech/pubkey.txt
-
-      2. Add the repository to your sources
-      =====================================
-
-      $ sudo wget -q -O \\\
-          /etc/apt/sources.list.d/platypush.list \\\
-          https://apt.platypush.tech/lists/platypush-<deb_version>-<branch>.list
-
-      Where:
-
-      - deb_version can be either *stable* (for the current Debian stable version) or
-        *oldstable* (for the previous Debian stable version)
-      - branch can be either *main* (for the latest releases) or *dev* (for a package
-        that is always in sync with the git version)
-
-      For example, to install the latest stable tags on Debian stable:
-
-      $ sudo wget -q -O \\\
-          /etc/apt/sources.list.d/platypush.list \\\
-          https://apt.platypush.tech/lists/platypush-stable-main.list
-
-      3. Update your repos
-      ====================
-
-      $ sudo apt update
-
-      4. Install Platypush
-      ====================
-
-      $ sudo apt install platypush
-      EOF
-
-    - echo "-- Importing and refreshing PGP key"
-    - echo "$PGP_PUBKEY" > "$TMP_APT_ROOT/pubkey.txt"
-    - export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}')
-    - |
-      cat <<EOF | gpg --import --armor
-      $PGP_PRIVKEY
-      EOF
-
-    - echo "-- Signing Release files"
-    - |
-      find "$TMP_APT_ROOT/dists" -type f -name Release | while read file; do
-        dirname="$(dirname "$file")"
-        cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs > "$file.gpg"
-        cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease"
-      done
-
-    - echo "-- Updating the apt repo root"
-    - export OLD_APT_ROOT="$REPOS_ROOT/oldapt"
-    - rm -rf "$OLD_APT_ROOT"
-    - mv "$APT_ROOT" "$OLD_APT_ROOT"
-    - mv "$TMP_APT_ROOT" "$APT_ROOT"
-
-    - chmod -R a+r "$APT_ROOT"
-    - chmod a+x "$APT_ROOT"
+    - . .drone/update-apt-repo.sh
 
 ###
 ### Update the RPM (stable) packages
@@ -774,215 +279,10 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
-    - |
-      [ -f .skipci ] && exit 0
-
-    - echo "-- Installing dependencies"
-    - yum install -y createrepo rpm-build rpm-sign gpg wget yum-utils git python python-pip
-
-    - echo "-- Copying source directory"
-    - mkdir -p "$WORKDIR"
-    - export SRCDIR="$WORKDIR/src"
-    - cp -r "$PWD" "$SRCDIR"
-    - cd "$SRCDIR"
-    - mkdir -p "$RPM_ROOT"
-
-    - echo "--- Parsing metadata"
-    - git config --global --add safe.directory $PWD
-    - git pull --rebase origin master --tags
-    - export VERSION=$(python3 setup.py --version)
-    - export RELNUM="$(git log --pretty=oneline HEAD...v$VERSION | wc -l)"
-    - export SPECFILE="$WORKDIR/$PKG_NAME.spec"
-    - export BUILD_DIR="$WORKDIR/build"
-    - export TMP_RPM_ROOT="$WORKDIR/repo"
-    - export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/master.tar.gz"
-
-    - echo "--- Creating git package spec"
-    - |
-      cat <<EOF > $SPECFILE
-      Summary: Universal command executor and automation hub.
-      Name: $PKG_NAME-git
-      Version: $VERSION
-      Release: $RELNUM
-      URL: https://platypush.tech
-      Group: System
-      License: MIT
-      Packager: Fabio Manganiello <fabio@platypush.tech>
-      Source: $SRC_URL
-      Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ')
-      Conflicts: $PKG_NAME
-      Prefix: %{_prefix}
-      BuildRoot: %{_tmppath}/%{name}-root
-
-      %description
-      Universal command executor and automation hub.
-
-      %install
-      mkdir -p %{buildroot}/
-      cp -r "$BUILD_DIR"/* %{buildroot}/
-
-      %clean
-
-      %files
-      /usr/bin/*
-      /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush
-      /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info
-
-      %changelog
-      * $(date +'%a %b %d %Y') admin <admin@platypush.tech>
-      - [Automatic] Release $VERSION-$RELNUM
-      EOF
-
-    - echo "--- Building git package"
-    - mkdir -p "$BUILD_DIR"
-    - pip install --prefix="$BUILD_DIR/usr" --no-cache --no-deps .
-    - rpmbuild --target "noarch" -bb "$SPECFILE"
-
-    - echo "--- Copying the new RPM package"
-    - mkdir -p "$TMP_RPM_ROOT"
-    - cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-git-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT"
-
-    - echo "--- Checking the latest released stable version"
-    - export LATEST_STABLE_PKG=$(ls -rt "$RPM_ROOT/$PKG_NAME"*.rpm 2>/dev/null | grep -v "$PKG_NAME-git" | tail -1)
-    - |
-      if [ -z "$LATEST_STABLE_PKG" ]; then
-        # If not stable release is available, then create one
-        export UPDATE_STABLE_PKG=1
-      else
-        # Otherwise, create a new release if the reported version on the repo is different
-        # from the latest released version.
-        export LATEST_STABLE_VERSION=$(basename $LATEST_STABLE_PKG | cut -d- -f 2)
-        if [ "$VERSION" != "$LATEST_STABLE_VERSION" ]; then
-          export UPDATE_STABLE_PKG=1
-        else
-          # If the version has remained the same, then simply copy the existing RPM to the
-          # new repository directory.
-          echo "Copying the existing release $LATEST_STABLE_VERSION to the new repository"
-          cp "$LATEST_STABLE_PKG" "$TMP_RPM_ROOT"
-        fi
-      fi
-
-    # If a new stable release is required, build another RPM
-    - |
-      if [ -n "$UPDATE_STABLE_PKG" ]; then
-      export RELNUM=1
-      export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz"
-
-      cat <<EOF > $SPECFILE
-      Summary: Universal command executor and automation hub.
-      Name: $PKG_NAME
-      Version: $VERSION
-      Release: $RELNUM
-      URL: https://platypush.tech
-      Group: System
-      License: MIT
-      Packager: Fabio Manganiello <fabio@platypush.tech>
-      Source: $SRC_URL
-      Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ')
-      Conflicts: $PKG_NAME-git
-      Prefix: %{_prefix}
-      BuildRoot: %{_tmppath}/%{name}-root
-
-      %description
-      Universal command executor and automation hub.
-
-      %install
-      mkdir -p %{buildroot}/
-      cp -r "$BUILD_DIR"/* %{buildroot}/
-
-      %clean
-
-      %files
-      /usr/bin/*
-      /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush
-      /usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info
-
-      %changelog
-      * $(date +'%a %b %d %Y') admin <admin@platypush.tech>
-      - [Automatic] Release $VERSION-$RELNUM
-      EOF
-
-      echo "--- Building package for stable release $VERSION"
-      rpmbuild --target "noarch" -bb "$SPECFILE"
-      cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT"
-      fi
-
-    - echo "--- Importing the repository keys"
-    - |
-      cat <<EOF | gpg --import --armor
-      $PGP_PRIVKEY
-      EOF
-
-    - export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}')
-    - |
-      cat <<EOF > $HOME/.rpmmacros
-      %signature gpg
-      %_gpg_name $PGP_KEYID
-      EOF
-
-    - echo "--- Signing the new RPM packages"
-    - rpm --addsign "$TMP_RPM_ROOT"/*.rpm
-
-    - echo "--- Creating a new copy of the RPM repository"
-    - createrepo "$TMP_RPM_ROOT"
-    - gpg --detach-sign --armor "$TMP_RPM_ROOT/repodata/repomd.xml"
-
-    - |
-      cat <<EOF > "$TMP_RPM_ROOT/platypush.repo"
-      [platypush]
-      name=Platypush repository
-      baseurl=https://rpm.platypush.tech
-      enabled=1
-      type=rpm
-      gpgcheck=1
-      gpgkey=https://rpm.platypush.tech/pubkey.txt
-      EOF
-
-    - |
-      cat <<EOF > "$TMP_RPM_ROOT/index.txt"
-      Welcome to the Platypush RPM repository!
-
-      Project homepage: https://platypush.tech
-      Source code: https://git.platypush.tech/platypush/platypush
-      Documentation / API reference: https://docs.platypush.tech
-
-      You can use this RPM repository to install Platypush on Fedora or other
-      RPM-based distros - as long as they are compatible with the latest Fedora
-      release.
-
-      Steps:
-
-      1. Add the repository to your sources
-      =====================================
-
-      $ sudo yum config-manager --add-repo https://rpm.platypush.tech/platypush.repo
-
-      2. Install Platypush
-      ====================
-
-      $ sudo yum install platypush
-
-      Or, if you want to install a version always up-to-date with the git repo:
-
-      $ sudo yum install platypush-git
-      EOF
-
-    - |
-      cat <<EOF > "$TMP_RPM_ROOT/pubkey.txt"
-      $PGP_PUBKEY
-      EOF
-
-    - echo "--- Updating the repository"
-    - export NEW_RPM_ROOT="$REPOS_ROOT/rpm_new"
-    - export OLD_RPM_ROOT="$REPOS_ROOT/rpm_old"
-    - cp -r "$TMP_RPM_ROOT" "$NEW_RPM_ROOT"
-    - rm -rf "$TMP_RPM_ROOT"
-    - mv "$RPM_ROOT" "$OLD_RPM_ROOT"
-    - mv "$NEW_RPM_ROOT" "$RPM_ROOT"
-    - rm -rf "$OLD_RPM_ROOT"
+    - . .drone/update-rpm-repo.sh
 
 ###
 ### Updates the pip package upon new releases
@@ -1005,9 +305,7 @@ steps:
     - docs
 
   commands:
-    - apk add --update --no-cache py3-twine
-    - python setup.py sdist bdist_wheel
-    - twine upload dist/platypush-$(python setup.py --version).tar.gz
+    - . .drone/update-pip-package.sh
 
 ###
 ### Checkpoint step that waits for all the package update

.drone/build-ui.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+export SKIPCI="$PWD/.skipci"
+rm -rf "$SKIPCI"
+
+. .drone/macros/configure-git.sh
+
+cd platypush/backend/http/webapp
+if [ $(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE . | wc -l) -eq 0 ]; then
+  echo "No UI changes detected, skipping build"
+  exit 0
+fi
+
+
+if [ "$(git log --pretty=format:%s HEAD...HEAD~1 | head -1)" == "[Automatic] Updated UI files" ]; then
+  echo "UI changes have already been committed, skipping build"
+  exit 0
+fi
+
+rm -rf dist node_modules
+npm install
+npm run build
+
+if [ $(git status --porcelain dist | wc -l) -eq 0 ]; then
+  echo "No build files have been changed"
+  exit 0
+fi
+
+# Create a .skipci file to mark the fact that the next steps should be skipped
+# (we're going to do another push anyway, so another pipeline will be triggered)
+touch "$SKIPCI"
+
+. .drone/macros/configure-ssh.sh
+. .drone/macros/configure-gpg.sh
+
+git add dist
+git commit dist -S -m "[Automatic] Updated UI files" --no-verify
+git remote rm origin
+git remote add origin git@git.platypush.tech:platypush/platypush.git
+git push -f origin master
+
+# Restore the original git configuration
+mv "$TMP_GIT_CONF" "$GIT_CONF"

.drone/github-mirror.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. .drone/macros/configure-git.sh
+. .drone/macros/configure-ssh.sh
+
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+
+# Clone the repository
+git remote add github git@github.com:/BlackLight/platypush.git
+git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github"
+
+# Push the changes to the GitHub mirror
+git push --all -v github

.drone/macros/configure-git.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Install git
+if [ -z "$(which git)" ]; then
+  if [ -n "$(which apt-get)" ]; then
+    apt-get update
+    apt-get install -y git
+  elif [ -n "$(which apk)" ]; then
+    apk add --update --no-cache git
+  elif [ -n "$(which yum)" ]; then
+    yum install -y git
+  elif [ -n "$(which dnf)" ]; then
+    dnf install -y git
+  elif [ -n "$(which pacman)" ]; then
+    pacman -Sy --noconfirm git
+  else
+    echo "Could not find a package manager to install git"
+    exit 1
+  fi
+fi
+
+# Backup the original git configuration before changing attributes
+export GIT_CONF="$PWD/.git/config"
+export TMP_GIT_CONF=/tmp/git.config.orig
+cp "$GIT_CONF" "$TMP_GIT_CONF"
+
+git config --global --add safe.directory "$PWD"
+git config user.name "Platypush CI/CD Automation"
+git config user.email "admin@platypush.tech"

.drone/macros/configure-gpg.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+[ -z "$PGP_KEY" ] && echo "PGP_KEY is not set" && exit 1
+[ -z "$PGP_KEY_ID" ] && echo "PGP_KEY_ID is not set" && exit 1
+
+# Install gpg
+if [ -z "$(which gpg)" ]; then
+    if [ -n "$(which apt-get)" ]; then
+        apt-get update
+        apt-get install -y gnupg
+    elif [ -n "$(which apk)" ]; then
+        apk add --update --no-cache bash gnupg
+    elif [ -n "$(which yum)" ]; then
+        yum install -y gnupg
+    elif [ -n "$(which dnf)" ]; then
+        dnf install -y gnupg
+    elif [ -n "$(which pacman)" ]; then
+        pacman -Sy --noconfirm gnupg
+    else
+        echo "Could not find a package manager to install gnupg"
+        exit 1
+    fi
+fi
+
+cat <<EOF | gpg --import --armor
+$PGP_KEY
+EOF
+
+git config commit.gpgsign true
+git config user.signingkey "$PGP_KEY_ID"

.drone/macros/configure-ssh.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+if [ -z "$SSH_PUBKEY" ] || [ -z "$SSH_PRIVKEY" ]; then
+    echo "SSH_PUBKEY and SSH_PRIVKEY environment variables must be set"
+    exit 1
+fi
+
+# Install ssh
+if [ -z "$(which ssh)" ]; then
+    if [ -n "$(which apt-get)" ]; then
+        apt-get update
+        apt-get install -y openssh
+    elif [ -n "$(which apk)" ]; then
+        apk add --update --no-cache openssh
+    elif [ -n "$(which yum)" ]; then
+        yum install -y openssh
+    elif [ -n "$(which dnf)" ]; then
+        dnf install -y openssh
+    elif [ -n "$(which pacman)" ]; then
+        pacman -Sy --noconfirm openssh
+    else
+        echo "Could not find a package manager to install openssh"
+        exit 1
+    fi
+fi
+
+mkdir -p ~/.ssh
+echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
+
+cat <<EOF > ~/.ssh/id_rsa
+$SSH_PRIVKEY
+EOF
+
+chmod 0600 ~/.ssh/id_rsa
+ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null

.drone/rebuild-docs.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+echo "Installing required build dependencies"
+apk add --update --no-cache git make py3-sphinx py3-myst-parser py3-pip $(cat platypush/install/requirements/alpine.txt)
+pip install -U sphinx-rtd-theme sphinx-book-theme --break-system-packages
+pip install . --break-system-packages
+mkdir -p /docs/current
+export APPDIR="$PWD"
+rm -rf "$APPDIR/docs/build"
+
+echo "Building the updated documentation"
+cd "$APPDIR/docs/source"
+git clone 'https://git.platypush.tech/platypush/platypush.wiki.git' wiki
+
+echo "Linking the wiki to the Sphinx index"
+cd wiki
+ln -s Home.md index.md
+cd "$APPDIR/docs"
+make html
+rm -f config*.yaml
+cd "$APPDIR"
+
+echo "Copying the new documentation files to the target folder"
+mv -v "$APPDIR/docs/build" /docs/new
+cd /docs
+mv current old
+mv new current
+rm -rf old

.drone/run-tests.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
+pip install . --break-system-packages
+pip install -r requirements-tests.txt --break-system-packages
+pytest tests

.drone/sync-stable-branch.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. .drone/macros/configure-git.sh
+. .drone/macros/configure-ssh.sh
+. .drone/macros/configure-gpg.sh
+
+# Git configuration
+git remote rm origin
+git remote add origin git@git.platypush.tech:platypush/platypush.git
+
+# Merge and push to the `stable` branch
+git checkout stable
+git rebase master
+git push -u origin stable
+git checkout master
+
+# Restore the original git configuration
+mv "$TMP_GIT_CONF" "$GIT_CONF"

.drone/update-apt-repo.sh

@@ -0,0 +1,166 @@
+#!/bin/sh
+
+
+[ -f .skipci ] && exit 0
+
+echo "-- Installing dependencies"
+apt update
+apt install -y dpkg-dev gpg
+
+echo "-- Creating a new apt root folder"
+export TMP_APT_ROOT="/tmp/apt"
+cp -r "$APT_ROOT" "$TMP_APT_ROOT"
+
+echo "-- Cleaning up older apt releases"
+
+find "$TMP_APT_ROOT/pool" -mindepth 2 -maxdepth 2 -type d | while read reldir; do
+  pkg_to_remove=$(( $(ls "$reldir"/*.deb | wc -l) - 1 ))
+  [ $pkg_to_remove -le 0 ] && continue
+  ls "$reldir"/*.deb | sort -V | head -n$pkg_to_remove | xargs rm -f
+done
+
+echo "-- Updating Packages files"
+
+echo "stable\noldstable" | while read distro; do
+  echo "main\ndev" | while read branch; do
+    branch_dir="$TMP_APT_ROOT/pool/$distro/$branch"
+    echo "Checking pool folder: $branch_dir"
+    [ -d "$branch_dir" ] || mkdir -p "$branch_dir"
+    dist_dir="$TMP_APT_ROOT/dists/$distro/$branch/all"
+    mkdir -p "$dist_dir"
+    pkg_file="$dist_dir/Packages"
+    dpkg-scanpackages --arch all "$branch_dir" > "$pkg_file"
+    sed -i "$pkg_file" -re "s|^Filename: $TMP_APT_ROOT/|Filename: |"
+    cat "$pkg_file" | gzip -9 > "$pkg_file.gz"
+    echo "Generated Packages file: $pkg_file"
+    cat "$pkg_file"
+  done
+done
+
+echo "-- Updating Release files"
+
+add_hashes() {
+  dist_dir=$1
+  hash_cmd=$2
+  hash_label=$3
+
+  echo "$hash_label:"
+  find "$dist_dir" -name 'Packages*' | while read file; do
+    basename="$(echo "$file" | sed -r -e "s|^$dist_dir/||")"
+    hash="$($hash_cmd "$file" | cut -d" " -f1)"
+    size="$(wc -c < $file)"
+    echo " $hash $size $basename"
+    echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-i386/|')"
+    echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-amd64/|')"
+    echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armel/|')"
+    echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-armhf/|')"
+    echo " $hash $size $(echo $basename | sed -re 's|/all/|/binary-arm64/|')"
+  done
+}
+
+echo "stable\noldstable" | while read distro; do
+  dist_dir="$TMP_APT_ROOT/dists/$distro"
+  components=$(find "$dist_dir" -name Packages | awk -F '/' '{print $(NF-2)}' | uniq | tr '\n' ' ')
+  release_file="$dist_dir/Release"
+
+  cat <<EOF > "$release_file"
+Origin: Platypush repository
+Label: Platypush
+Suite: $distro
+Codename: $distro
+Architectures: i386 amd64 armel armhf arm64
+Components: $components
+Description: The official APT repository for Platypush
+Date: $(date -Ru)
+EOF
+
+  add_hashes "$dist_dir" "md5sum" "MD5Sum" >> "$release_file"
+  add_hashes "$dist_dir" "sha1sum" "SHA1" >> "$release_file"
+  add_hashes "$dist_dir" "sha256sum" "SHA256" >> "$release_file"
+done
+
+echo "-- Generating list files"
+mkdir -p "$TMP_APT_ROOT/lists"
+
+for distro in stable oldstable; do
+  for branch in main dev; do
+    echo "deb https://apt.platypush.tech/ $distro $branch" > "$TMP_APT_ROOT/lists/platypush-$distro-$branch.list"
+  done
+done
+
+echo "-- Updating index file"
+
+cat <<EOF > "$TMP_APT_ROOT/index.txt"
+Welcome to the Platypush APT repository!
+
+Project homepage: https://platypush.tech
+Source code: https://git.platypush.tech/platypush/platypush
+Documentation / API reference: https://docs.platypush.tech
+
+You can use this APT repository to install Platypush on Debian, Ubuntu or any
+Debian-based distro.
+
+Steps:
+
+1. Add this repository's PGP key to your apt keyring
+====================================================
+
+$ sudo wget -q -O \\\
+    /etc/apt/trusted.gpg.d/platypush.asc \\\
+    https://apt.platypush.tech/pubkey.txt
+
+2. Add the repository to your sources
+=====================================
+
+$ sudo wget -q -O \\\
+    /etc/apt/sources.list.d/platypush.list \\\
+    https://apt.platypush.tech/lists/platypush-<deb_version>-<branch>.list
+
+Where:
+
+- deb_version can be either *stable* (for the current Debian stable version) or
+  *oldstable* (for the previous Debian stable version)
+- branch can be either *main* (for the latest releases) or *dev* (for a package
+  that is always in sync with the git version)
+
+For example, to install the latest stable tags on Debian stable:
+
+$ sudo wget -q -O \\\
+    /etc/apt/sources.list.d/platypush.list \\\
+    https://apt.platypush.tech/lists/platypush-stable-main.list
+
+3. Update your repos
+====================
+
+$ sudo apt update
+
+4. Install Platypush
+====================
+
+$ sudo apt install platypush
+EOF
+
+echo "-- Importing and refreshing PGP key"
+echo "$PGP_PUBKEY" > "$TMP_APT_ROOT/pubkey.txt"
+export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}')
+
+cat <<EOF | gpg --import --armor
+$PGP_PRIVKEY
+EOF
+
+echo "-- Signing Release files"
+
+find "$TMP_APT_ROOT/dists" -type f -name Release | while read file; do
+  dirname="$(dirname "$file")"
+  cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs > "$file.gpg"
+  cat "$file" | gpg -q --default-key "$PGP_KEYID" -abs --clearsign > "$dirname/InRelease"
+done
+
+echo "-- Updating the apt repo root"
+export OLD_APT_ROOT="$REPOS_ROOT/oldapt"
+rm -rf "$OLD_APT_ROOT"
+mv "$APT_ROOT" "$OLD_APT_ROOT"
+mv "$TMP_APT_ROOT" "$APT_ROOT"
+
+chmod -R a+r "$APT_ROOT"
+chmod a+x "$APT_ROOT"

.drone/update-arch-packages.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+[ -f .skipci ] && exit 0
+
+apk add --update --no-cache curl pacman sudo
+
+. .drone/macros/configure-ssh.sh
+. .drone/macros/configure-git.sh
+
+git pull --rebase origin master --tags
+
+export VERSION=$(python setup.py --version)
+export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1)
+export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).g${HEAD}"
+export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz"
+
+ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null
+adduser -u 1000 -D build
+mkdir -p "$WORKDIR"
+
+echo "--- Updating Arch git version"
+export PKGDIR=$WORKDIR/git
+git clone ssh://aur@aur.archlinux.org/platypush-git.git "$PKGDIR"
+git config --global --add safe.directory "$PKGDIR"
+chown -R build "$PKGDIR"
+cd "$PKGDIR"
+
+sed -i 'PKGBUILD' -r \
+  -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \
+  -e "s/^pkgrel=.*/pkgrel=1/" \
+
+sudo -u build makepkg --printsrcinfo > .SRCINFO
+export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
+
+if [ $FILES_CHANGED -gt 0 ]; then
+  echo "--- Pushing git package version $GIT_VERSION"
+  git commit -a -m '[Automatic] Package update'
+  git push origin master
+fi
+
+echo "--- Updating Arch stable version"
+export PKGDIR="$WORKDIR/stable"
+git clone ssh://aur@aur.archlinux.org/platypush.git "$PKGDIR"
+git config --global --add safe.directory "$PKGDIR"
+chown -R build "$PKGDIR"
+cd "$PKGDIR"
+export RELEASED_VERSION=$(grep -e '^pkgver=' PKGBUILD | sed -r -e 's/^pkgver=(.*)\s*/\1/')
+
+if [ "$RELEASED_VERSION" == "$VERSION" ]; then
+  exit 0
+fi
+
+export TAG_CHECKSUM=$(curl --silent "$TAG_URL" | sha512sum | awk '{print $1}')
+
+sed -i 'PKGBUILD' -r \
+  -e "s/^pkgver=.*/pkgver=$VERSION/" \
+  -e "s/^pkgrel=.*/pkgrel=1/" \
+  -e "s/^sha512sums=.*/sha512sums=('$TAG_CHECKSUM')/"
+
+sudo -u build makepkg --printsrcinfo > .SRCINFO
+export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
+
+if [ $FILES_CHANGED -gt 0 ]; then
+  echo "--- Pushing stable package version $VERSION"
+  git commit -a -m '[Automatic] Package update'
+  git push origin master
+fi

.drone/update-components-cache.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+export SKIPCI="$PWD/.skipci"
+export CACHEFILE="$PWD/platypush/components.json.gz"
+
+[ -f "$SKIPCI" ] && exit 0
+
+# Backup the original git configuration before changing attributes
+export GIT_CONF="$PWD/.git/config"
+export TMP_GIT_CONF="/tmp/git.config.orig"
+cp "$GIT_CONF" "$TMP_GIT_CONF"
+
+. .drone/macros/configure-git.sh
+
+# Only regenerate the components cache if either the plugins, backends,
+# events or schemas folders have some changes (excluding the webapp files).
+if [ -z "$(git log --pretty=oneline $DRONE_COMMIT_AFTER...$DRONE_COMMIT_BEFORE -- platypush/backend platypush/plugins platypush/schemas platypush/message/event ':(exclude)platypush/backend/http/webapp')" ]; then
+  echo 'No changes to the components file'
+  exit 0
+fi
+
+. .drone/macros/configure-ssh.sh
+. .drone/macros/configure-gpg.sh
+
+echo 'Updating components cache'
+apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
+pip install . --break-system-packages
+
+python - <<EOF
+from platypush import get_plugin
+
+get_plugin('inspect').refresh_cache(force=True)
+EOF
+
+# Create a .skipci file to mark the fact that the next steps should be skipped
+# (we're going to do another push anyway, so another pipeline will be triggered)
+touch "$SKIPCI"
+
+git add "$CACHEFILE"
+git commit "$CACHEFILE" -S -m "[Automatic] Updated components cache" --no-verify
+git remote rm origin
+git remote add origin git@git.platypush.tech:platypush/platypush.git
+git push -f origin master
+
+# Restore the original git configuration
+mv "$TMP_GIT_CONF" "$GIT_CONF"

.drone/update-deb-packages.sh

@@ -0,0 +1,102 @@
+#!/bin/sh
+
+[ -f .skipci ] && exit 0
+
+echo "-- Copying source directory"
+mkdir -p "$WORKDIR/src"
+export SRCDIR="$WORKDIR/src/$DEB_VERSION"
+cp -r "$PWD" "$SRCDIR"
+cd "$SRCDIR"
+
+echo "-- Installing dependencies"
+apt update
+apt install -y curl dpkg-dev gpg git python3 python3-pip python3-setuptools
+
+echo "--- Parsing metadata"
+git config --global --add safe.directory "$PWD"
+git pull --rebase origin master --tags
+export VERSION=$(python3 setup.py --version)
+export GIT_VERSION="$VERSION-$(git log --pretty=oneline HEAD...v$VERSION | wc -l)"
+export GIT_BUILD_DIR="$WORKDIR/${PKG_NAME}_${GIT_VERSION}_all"
+export GIT_DEB="$WORKDIR/${PKG_NAME}_${GIT_VERSION}_all.deb"
+export POOL_PATH="$APT_ROOT/pool/$DEB_VERSION/dev"
+
+echo "--- Building git package"
+pip install --prefix="$GIT_BUILD_DIR/usr" --no-cache --no-deps .
+
+find "$GIT_BUILD_DIR" -name "site-packages" | while read dir; do
+  base="$(dirname "$dir")"
+  mv "$dir" "$base/dist-packages"
+done
+
+install -m755 -d "${GIT_BUILD_DIR}/usr/lib/systemd/system"
+install -m755 -d "${GIT_BUILD_DIR}/usr/lib/systemd/user"
+install -m750 -d "${GIT_BUILD_DIR}/var/lib/platypush"
+install -m750 -d "${GIT_BUILD_DIR}/etc/platypush/scripts"
+
+install -m644 "${SRCDIR}/platypush/config/config.yaml" "${GIT_BUILD_DIR}/etc/platypush/config.yaml"
+install -m644 "${SRCDIR}/platypush/config/systemd/platypush.service" "${GIT_BUILD_DIR}/usr/lib/systemd/user/platypush.service"
+install -m644 "${SRCDIR}/platypush/config/systemd/platypush.service" "${GIT_BUILD_DIR}/usr/lib/systemd/system/platypush.service"
+sed -i "${GIT_BUILD_DIR}/usr/lib/systemd/system/platypush.service" -r \
+    -e 's/^#\s*Requires=(.*)/Requires=\1/' \
+    -e 's/^\[Service\]$/\[Service\]\
+User=platypush\
+Group=platypush\
+WorkingDirectory=\/var\/lib\/platypush\
+Environment="PLATYPUSH_CONFIG=\/etc\/platypush\/config.yaml"\
+Environment="PLATYPUSH_WORKDIR=\/var\/lib\/platypush"/'
+
+mkdir -p "$GIT_BUILD_DIR/DEBIAN"
+
+cat <<EOF > "$GIT_BUILD_DIR/DEBIAN/control"
+Package: $PKG_NAME
+Version: $GIT_VERSION
+Maintainer: Fabio Manganiello <fabio@platypush.tech>
+Depends: $(cat platypush/install/requirements/debian.txt | tr '\n' ',' | sed -re 's/,$//' -e 's/,/, /g')
+Architecture: all
+Homepage: https://platypush.tech
+Description: Universal command executor and automation hub.
+EOF
+
+cat <<EOF > "$GIT_BUILD_DIR/DEBIAN/postinst" && chmod +x "$GIT_BUILD_DIR/DEBIAN/postinst"
+#!/bin/sh
+
+set -e
+
+if [ "\$1" = "configure" ]; then
+  grep -e '^platypush:' /etc/passwd 2>/dev/null || useradd -U -r -s /bin/false -d /var/lib/platypush platypush
+  mkdir -p /var/lib/platypush
+  chown -R platypush:platypush /var/lib/platypush
+  chown -R platypush:platypush /etc/platypush
+  if which systemctl; then systemctl daemon-reload; fi
+fi
+EOF
+
+mkdir -p "$POOL_PATH"
+rm -f "$POOL_PATH/"*.deb
+dpkg --build "$GIT_BUILD_DIR"
+
+echo "--- Copying $GIT_DEB to $POOL_PATH"
+cp "$GIT_DEB" "$POOL_PATH"
+
+# If main/all/Packages doesn't exist, then we should create the first main release
+[ $(ls "$APT_ROOT/pool/$DEB_VERSION/main/${PKG_NAME}_${VERSION}-"*"_all.deb" 2>/dev/null | wc -l) -eq 0 ] && export UPDATE_STABLE_PKG=1
+
+export PKGURL="https://apt.platypush.tech/dists/$DEB_VERSION/main/all/Packages"
+
+[ -z "$UPDATE_STABLE_PKG" ] &&
+  curl -ILs -o /dev/null -w "%{http_code}" "$PKGURL" |
+    grep -e '^4' >/dev/null && export UPDATE_STABLE_PKG=1
+
+# If the published release version differs from the current one, then we should publish a new main release
+if [ -z "$UPDATE_STABLE_PKG" ]; then
+  RELEASED_VERSION=$(curl -s "$PKGURL" | grep -e '^Version: ' | head -1 | awk '{print $2}' | cut -d- -f 1)
+  [ "$RELEASED_VERSION" != "$VERSION" ] && export UPDATE_STABLE_PKG=1
+fi
+
+# Proceed and update the main release if the version number has changed
+if [ -n "$UPDATE_STABLE_PKG" ]; then
+  echo "--- Updating main package"
+  mkdir -p "$APT_ROOT/pool/$DEB_VERSION/main"
+  cp "$GIT_DEB" "$APT_ROOT/pool/$DEB_VERSION/main/${PKG_NAME}_${VERSION}-1_all.deb"
+fi

.drone/update-pip-package.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+apk add --update --no-cache py3-twine
+python setup.py sdist bdist_wheel
+twine upload dist/platypush-$(python setup.py --version).tar.gz

.drone/update-rpm-repo.sh

@@ -0,0 +1,261 @@
+#!/bin/sh
+
+[ -f .skipci ] && exit 0
+
+echo "-- Installing dependencies"
+yum install -y \
+  createrepo \
+  git \
+  gpg \
+  python \
+  python-pip \
+  python-setuptools \
+  rpm-build \
+  rpm-sign \
+  systemd-rpm-macros \
+  wget \
+  yum-utils \
+
+echo "-- Copying source directory"
+mkdir -p "$WORKDIR"
+export SRCDIR="$WORKDIR/src"
+cp -r "$PWD" "$SRCDIR"
+cd "$SRCDIR"
+mkdir -p "$RPM_ROOT"
+
+echo "--- Parsing metadata"
+git config --global --add safe.directory $PWD
+git pull --rebase origin master --tags
+export VERSION=$(python3 setup.py --version)
+export RELNUM="$(git log --pretty=oneline HEAD...v$VERSION | wc -l)"
+export SPECFILE="$WORKDIR/$PKG_NAME.spec"
+export BUILD_DIR="$WORKDIR/build"
+export TMP_RPM_ROOT="$WORKDIR/repo"
+export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/master.tar.gz"
+
+echo "--- Creating git package spec"
+
+cat <<EOF > $SPECFILE
+Summary: Universal command executor and automation hub.
+Name: $PKG_NAME-git
+Version: $VERSION
+Release: $RELNUM
+URL: https://platypush.tech
+Group: System
+License: MIT
+Packager: Fabio Manganiello <fabio@platypush.tech>
+Source: $SRC_URL
+Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ')
+Conflicts: $PKG_NAME
+Prefix: %{_prefix}
+BuildRoot: %{_tmppath}/%{name}-root
+BuildRequires: systemd-rpm-macros
+%{?sysusers_requires_compat}
+
+%description
+Universal command executor and automation hub.
+
+%install
+mkdir -p %{buildroot}/
+cp -r "$BUILD_DIR"/* %{buildroot}/
+install -p -Dm0644 "${BUILD_DIR}/usr/lib/sysusers.d/platypush.conf" %{buildroot}%{_sysusersdir}/platypush.conf
+
+%pre
+%sysusers_create_compat "${BUILD_DIR}/usr/lib/sysusers.d/platypush.conf"
+
+%clean
+
+%files
+%defattr(750,platypush,platypush,750)
+%dir /etc/platypush
+/etc/platypush/*
+/usr/bin/*
+/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush
+/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info
+/usr/lib/systemd/system/*
+/usr/lib/systemd/user/*
+%defattr(750,platypush,platypush,750)
+%dir /var/lib/platypush
+%{_sysusersdir}/platypush.conf
+
+%changelog
+* $(date +'%a %b %d %Y') admin <admin@platypush.tech>
+- [Automatic] Release $VERSION-$RELNUM
+EOF
+
+echo "--- Building git package"
+mkdir -p "$BUILD_DIR"
+
+pip install --prefix="$BUILD_DIR/usr" --no-cache --no-deps .
+
+install -m755 -d "${BUILD_DIR}/usr/lib/systemd/system"
+install -m755 -d "${BUILD_DIR}/usr/lib/systemd/user"
+install -m755 -d "${BUILD_DIR}/usr/lib/sysusers.d"
+install -m750 -d "${BUILD_DIR}/var/lib/platypush"
+install -m750 -d "${BUILD_DIR}/etc/platypush/scripts"
+
+install -m644 "${SRCDIR}/platypush/config/config.yaml" "${BUILD_DIR}/etc/platypush/config.yaml"
+install -Dm644 "${SRCDIR}/platypush/config/systemd/platypush-sysusers.conf" "${BUILD_DIR}/usr/lib/sysusers.d/platypush.conf"
+install -m644 "${SRCDIR}/platypush/config/systemd/platypush.service" "${BUILD_DIR}/usr/lib/systemd/user/platypush.service"
+install -m644 "${SRCDIR}/platypush/config/systemd/platypush.service" "${BUILD_DIR}/usr/lib/systemd/system/platypush.service"
+sed -i "${BUILD_DIR}/usr/lib/systemd/system/platypush.service" -r \
+    -e 's/^#\s*Requires=(.*)/Requires=\1/' \
+    -e 's/^\[Service\]$/\[Service\]\
+User=platypush\
+Group=platypush\
+WorkingDirectory=\/var\/lib\/platypush\
+Environment="PLATYPUSH_CONFIG=\/etc\/platypush\/config.yaml"\
+Environment="PLATYPUSH_WORKDIR=\/var\/lib\/platypush"/'
+
+rpmbuild --target "noarch" -bb "$SPECFILE"
+
+echo "--- Copying the new RPM package"
+mkdir -p "$TMP_RPM_ROOT"
+cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-git-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT"
+
+echo "--- Checking the latest released stable version"
+export LATEST_STABLE_PKG=$(ls -rt "$RPM_ROOT/$PKG_NAME"*.rpm 2>/dev/null | grep -v "$PKG_NAME-git" | tail -1)
+
+if [ -z "$LATEST_STABLE_PKG" ]; then
+  # If not stable release is available, then create one
+  export UPDATE_STABLE_PKG=1
+else
+  # Otherwise, create a new release if the reported version on the repo is different
+  # from the latest released version.
+  export LATEST_STABLE_VERSION=$(basename $LATEST_STABLE_PKG | cut -d- -f 2)
+  if [ "$VERSION" != "$LATEST_STABLE_VERSION" ]; then
+    export UPDATE_STABLE_PKG=1
+  else
+    # If the version has remained the same, then simply copy the existing RPM to the
+    # new repository directory.
+    echo "Copying the existing release $LATEST_STABLE_VERSION to the new repository"
+    cp "$LATEST_STABLE_PKG" "$TMP_RPM_ROOT"
+  fi
+fi
+
+# If a new stable release is required, build another RPM
+if [ -n "$UPDATE_STABLE_PKG" ]; then
+  export RELNUM=1
+  export SRC_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz"
+
+  cat <<EOF > $SPECFILE
+Summary: Universal command executor and automation hub.
+Name: $PKG_NAME
+Version: $VERSION
+Release: $RELNUM
+URL: https://platypush.tech
+Group: System
+License: MIT
+Packager: Fabio Manganiello <fabio@platypush.tech>
+Source: $SRC_URL
+Requires: $(cat platypush/install/requirements/fedora.txt | tr '\n' ' ')
+Conflicts: $PKG_NAME-git
+Prefix: %{_prefix}
+BuildRoot: %{_tmppath}/%{name}-root
+BuildRequires: systemd-rpm-macros
+%{?sysusers_requires_compat}
+
+%description
+Universal command executor and automation hub.
+
+%install
+mkdir -p %{buildroot}/
+cp -r "$BUILD_DIR"/* %{buildroot}/
+install -p -Dm0644 "${BUILD_DIR}/usr/lib/sysusers.d/platypush.conf" %{buildroot}%{_sysusersdir}/platypush.conf
+
+%pre
+%sysusers_create_compat "${BUILD_DIR}/usr/lib/sysusers.d/platypush.conf"
+
+%clean
+
+%files
+%defattr(750,platypush,platypush,750)
+%dir /etc/platypush
+/etc/platypush/*
+/usr/bin/*
+/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush
+/usr/lib/python$(python3 --version | awk '{print $2}' | cut -d. -f 1,2)/site-packages/platypush-$VERSION.dist-info
+/usr/lib/systemd/system/*
+/usr/lib/systemd/user/*
+%defattr(750,platypush,platypush,750)
+%dir /var/lib/platypush
+%{_sysusersdir}/platypush.conf
+
+%changelog
+* $(date +'%a %b %d %Y') admin <admin@platypush.tech>
+- [Automatic] Release $VERSION-$RELNUM
+EOF
+
+  echo "--- Building package for stable release $VERSION"
+  rpmbuild --target "noarch" -bb "$SPECFILE"
+  cp "$HOME/rpmbuild/RPMS/noarch/$PKG_NAME-$VERSION-$RELNUM.noarch.rpm" "$TMP_RPM_ROOT"
+fi
+
+echo "--- Importing the repository keys"
+cat <<EOF | gpg --import --armor
+$PGP_PRIVKEY
+EOF
+
+export PGP_KEYID=$(echo "$PGP_PUBKEY" | gpg --with-colons --import-options show-only --import --fingerprint | grep -e '^fpr:' | head -1 | awk -F ':' '{print $(NF - 1)}')
+cat <<EOF > $HOME/.rpmmacros
+%signature gpg
+%_gpg_name $PGP_KEYID
+EOF
+
+echo "--- Signing the new RPM packages"
+rpm --addsign "$TMP_RPM_ROOT"/*.rpm
+
+echo "--- Creating a new copy of the RPM repository"
+createrepo "$TMP_RPM_ROOT"
+gpg --detach-sign --armor "$TMP_RPM_ROOT/repodata/repomd.xml"
+
+cat <<EOF > "$TMP_RPM_ROOT/platypush.repo"
+[platypush]
+name=Platypush repository
+baseurl=https://rpm.platypush.tech
+enabled=1
+type=rpm
+gpgcheck=1
+gpgkey=https://rpm.platypush.tech/pubkey.txt
+EOF
+
+cat <<EOF > "$TMP_RPM_ROOT/index.txt"
+Welcome to the Platypush RPM repository!
+
+Project homepage: https://platypush.tech
+Source code: https://git.platypush.tech/platypush/platypush
+Documentation / API reference: https://docs.platypush.tech
+
+You can use this RPM repository to install Platypush on Fedora or other
+RPM-based distros - as long as they are compatible with the latest Fedora
+release.
+
+Steps:
+
+1. Add the repository to your sources
+=====================================
+
+$ sudo yum config-manager --add-repo https://rpm.platypush.tech/platypush.repo
+
+2. Install Platypush
+====================
+
+$ sudo yum install platypush
+
+Or, if you want to install a version always up-to-date with the git repo:
+
+$ sudo yum install platypush-git
+EOF
+
+cat <<EOF > "$TMP_RPM_ROOT/pubkey.txt"
+$PGP_PUBKEY
+EOF
+
+echo "--- Updating the repository"
+export NEW_RPM_ROOT="$REPOS_ROOT/rpm_new"
+export OLD_RPM_ROOT="$REPOS_ROOT/rpm_old"
+cp -r "$TMP_RPM_ROOT" "$NEW_RPM_ROOT"
+rm -rf "$TMP_RPM_ROOT"
+mv "$RPM_ROOT" "$OLD_RPM_ROOT"
+mv "$NEW_RPM_ROOT" "$RPM_ROOT"
+rm -rf "$OLD_RPM_ROOT"

.gitignore

@@ -27,3 +27,4 @@ Session.vim
 /Dockerfile
 /docs/source/wiki
 /.skipci
+dump.rdb

CONTRIBUTING.md

@@ -27,13 +27,9 @@ Guidelines:
   you are changing some of the core entities (e.g. requests, events, procedures, hooks, crons
   or the bus) then make sure to add tests and not to break the existing tests.
 
-- If the feature requires an optional dependency then make sure to document it:
-
-  - In the class docstring (see other plugins and backends for examples).
-  - In [`setup.py`](https://git.platypush.tech/platypush/platypush/-/blob/master/setup.py#L72) as
-    an `extras_require` entry.
-  - In the plugin/backend class pydoc string.
-  - In the `manifest.yaml` - refer to the Wiki (how to write
-    [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
-    and [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
-    for examples on how to write an extension manifest file.
+- If the feature requires an optional dependency then make sure to document it
+  in the `manifest.json` - refer to the Wiki (how to write
+  [plugins](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-plugins)
+  and
+  [backends](https://git.platypush.tech/platypush/platypush/wiki/Writing-your-own-backends))
+  for examples on how to write an extension manifest file.

MANIFEST.in

@@ -2,4 +2,6 @@ recursive-include platypush/backend/http/webapp/dist *
 recursive-include platypush/install *
 include platypush/plugins/http/webpage/mercury-parser.js
 include platypush/config/*.yaml
-global-include manifest.yaml
+include platypush/config/systemd/*
+global-include manifest.json
+global-include components.json.gz

README.md

@@ -22,7 +22,7 @@ Platypush
   * [Install from sources](#install-from-sources)
   * [Installing the dependencies for your extensions](#installing-the-dependencies-for-your-extensions)
     + [Install via `extras` name](#install-via-extras-name)
-    + [Install via `manifest.yaml`](#install-via-manifestyaml)
+    + [Install via `manifest.json`](#install-via-manifestjson)
     + [Check the instructions reported in the documentation](#check-the-instructions-reported-in-the-documentation)
   * [Virtual environment installation](#virtual-environment-installation)
   * [Docker installation](#docker-installation-1)
@@ -216,16 +216,27 @@ ways to check the dependencies required by an extension:
 
 #### Install via `extras` name
 
-All the extensions that require extra dependencies are listed in the
-[`extras_require` section under
-`setup.py`](https://git.platypush.tech/platypush/platypush/src/branch/master/setup.py#L84).
+You can install extra dependencies via pip extras:
 
-#### Install via `manifest.yaml`
+```shell
+pip install 'platypush[plugin1,plugin2,...]'
+```
 
-All the plugins and backends have a `manifest.yaml` file in their source folder.
+For example:
+
+```shell
+pip install 'platypush[light.hue,music.mpd,rss]'
+```
+
+Will install Platypush with the dependencies for the `light.hue`, `music.mpd`
+and `rss` plugins.
+
+#### Install via `manifest.json`
+
+All the plugins and backends have a `manifest.json` file in their source folder.
 Any extra dependencies are listed there
 
-If you followed the `extras` or `manifest.yaml` way to discover the
+If you followed the `extras` or `manifest.json` way to discover the
 dependencies, then you can install them in two ways:
 
 1. `pip` installation:
@@ -419,9 +430,7 @@ backend](https://docs.platypush.tech/en/latest/platypush/backend/http.html), an
 [MQTT
 instance](https://docs.platypush.tech/en/latest/platypush/backend/mqtt.html), a
 [Kafka
-instance](https://docs.platypush.tech/en/latest/platypush/backend/kafka.html),
-[Pushbullet](https://docs.platypush.tech/en/latest/platypush/backend/pushbullet.html)
-etc.).
+instance](https://docs.platypush.tech/en/latest/platypush/backend/kafka.html).
 
 If a backend supports the execution of requests (e.g. HTTP, MQTT, Kafka,
 Websocket and TCP) then you can send requests to these services in JSON format.
@@ -507,11 +516,10 @@ event.hook.SearchSongVoiceCommand:
   [Example](https://git.platypush.tech/platypush/platypush/src/branch/master/examples/conf/hook.py):
 
 ```python
-from platypush.event.hook import hook
-from platypush.utils import run
+from platypush import run, when
 from platypush.message.event.assistant import SpeechRecognizedEvent
 
-@hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
+@when(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
 def on_music_play_command(event, title=None, artist=None, **context):
   results = run('music.mpd.search', filter={
     'artist': artist,
@@ -529,22 +537,22 @@ against partial event arguments are also possible, and relational operators are
 supported as well. For example:
 
 ```python
-from platypush.event.hook import hook
+from platypush import when
 from platypush.message.event.sensor import SensorDataChangeEvent
 
-@hook(SensorDataChangeEvent, data=1):
+@when(SensorDataChangeEvent, data=1):
 def hook_1(event):
     """
     Triggered when event.data == 1
     """
 
-@hook(SensorDataChangeEvent, data={'state': 1}):
+@when(SensorDataChangeEvent, data={'state': 1}):
 def hook_2(event):
     """
     Triggered when event.data['state'] == 1
     """
 
-@hook(SensorDataChangeEvent, data={
+@when(SensorDataChangeEvent, data={
   'temperature': {'$gt': 25},
   'humidity': {'$le': 15}
 }):
@@ -565,9 +573,7 @@ Procedures are pieces of custom logic that can be executed as atomic actions
 using `procedure.<name>` as an action name.
 
 They can be defined either in the `config.yaml` or as Python scripts stored
-under `~/.config/platypush/scripts` - provided that the procedure is also
-imported in `~/.config/platypush/scripts/__init__.py` so it can be discovered
-by the service.
+under `~/.config/platypush/scripts`.
 
 YAML example for a procedure that can be executed when we arrive home and turns
 on the lights if the luminosity is lower that a certain thresholds, says a
@@ -707,8 +713,6 @@ groups together the controls for most of the plugins - e.g. sensors, switches,
 music controls and search, media library and torrent management, lights,
 Zigbee/Z-Wave devices and so on. The UI is responsive and mobile-friendly.
 
-Some screenshots:
-
 #### The main web panel
 
 This is the default panel available at `http://<host>:<port>` after

docs/source/_ext/add_dependencies.py

@@ -159,7 +159,7 @@ class IntegrationEnricher:
             base_path,
             *doc.split(os.sep)[:-1],
             *doc.split(os.sep)[-1].split('.'),
-            'manifest.yaml',
+            'manifest.json',
         )
 
         if not os.path.isfile(manifest_file):

docs/source/_static/scripts/custom.js

@@ -188,9 +188,62 @@ const renderActionsList = () => {
   })
 }
 
+const createFilterBar = () => {
+  const input = document.createElement('input')
+  const referenceSection = document.getElementById('reference')
+
+  input.type = 'text'
+  input.placeholder = 'Filter'
+  input.classList.add('filter-bar')
+  input.addEventListener('input', (event) => {
+    const filter = event.target.value.toLowerCase()
+    referenceSection.querySelectorAll('ul.grid li').forEach((li) => {
+      if (li.innerText.toLowerCase().includes(filter)) {
+        li.style.display = 'flex'
+      } else {
+        li.style.display = 'none'
+      }
+    })
+  })
+
+  return input
+}
+
+const addFilterBar = () => {
+  const container = document.querySelector('.bd-main')
+  if (!container)
+    return
+
+  const referenceSection = document.getElementById('reference')
+  if (!referenceSection)
+    return
+
+  const header = referenceSection.querySelector('h2')
+  if (!header)
+    return
+
+  const origInnerHTML = header.innerHTML
+  header.innerHTML = '<span class="header-content">' + origInnerHTML + '</span>'
+
+  const input = createFilterBar()
+  header.appendChild(input)
+
+  const headerOffsetTop = header.offsetTop
+
+  // Apply the fixed class if the header is above the viewport
+  document.addEventListener('scroll', () => {
+    if (headerOffsetTop < window.scrollY) {
+      header.classList.add('fixed')
+    } else {
+      header.classList.remove('fixed')
+    }
+  })
+}
+
 document.addEventListener("DOMContentLoaded", function() {
   generateComponentsGrid()
   convertDepsToTabs()
   addClipboardToCodeBlocks()
   renderActionsList()
+  addFilterBar()
 })

docs/source/_static/styles/custom.css

@@ -29,15 +29,18 @@ a.grid-title {
 
 ul.grid li {
   display: flex;
+  background: linear-gradient(0deg, #fff, #f9f9f9);
   align-items: center;
+  justify-content: space-between;
   margin: 0 10px 10px 0;
-  padding: 10px;
+  padding: 20px;
   border: 1px solid #ccc;
   border-radius: 15px;
+  flex-direction: column;
 }
 
 ul.grid img {
-  width: 32px;
+  width: 48px;
   margin-right: 5px;
 }
 
@@ -52,13 +55,19 @@ ul.grid li code .pre {
 }
 
 ul.grid li:hover {
-  background: linear-gradient(0deg, #e0ffe8, #e3ffff);
+  background: linear-gradient(0deg, #157765, #cbffd8) !important;
 }
 
 ul.grid li a {
-  width: calc(100% - 35px);
+  width: 100%;
   display: flex;
   justify-content: center;
+  text-align: center;
+  margin-top: 0.5em;
+}
+
+ul.grid li:hover a {
+  color: white !important;
 }
 
 ul.grid li a code {
@@ -128,3 +137,51 @@ ul.grid .icon {
   border-radius: 0 0 0.75em 0.75em;
 }
 
+.bd-article-container {
+  position: relative;
+}
+
+.filter-bar {
+  width: 100%;
+  display: block;
+  font-size: 0.6em;
+  border: 1px solid #ccc;
+  border-radius: 0.75em;
+  margin: 0.5em 0;
+  padding: 0.25em;
+}
+
+#reference h2.fixed {
+  position: fixed;
+  top: 0;
+  background: white;
+  z-index: 1;
+}
+
+#reference h2.fixed .header-content {
+  display: none;
+}
+
+@media screen and (max-width: 768px) {
+  #reference h2.fixed {
+    width: 100%;
+    margin-left: -0.5em;
+    padding: 0.5em 0.5em 0 0.5em;
+  }
+}
+
+@media screen and (max-width: 959px) {
+  #reference h2.fixed {
+    width: 100%;
+    margin-left: -1em;
+    padding: 0.5em 0.5em 0 0.5em;
+  }
+}
+
+@media screen and (min-width: 960px) {
+  #reference h2.fixed {
+    width: 75%;
+    max-width: 800px;
+    padding-top: 0.5em;
+  }
+}

docs/source/backends.rst

@@ -6,30 +6,7 @@ Backends
     :maxdepth: 1
     :caption: Backends:
 
-    platypush/backend/adafruit.io.rst
-    platypush/backend/button.flic.rst
-    platypush/backend/camera.pi.rst
-    platypush/backend/chat.telegram.rst
-    platypush/backend/google.fit.rst
-    platypush/backend/google.pubsub.rst
-    platypush/backend/gps.rst
     platypush/backend/http.rst
-    platypush/backend/kafka.rst
-    platypush/backend/mail.rst
-    platypush/backend/midi.rst
-    platypush/backend/music.mopidy.rst
-    platypush/backend/music.mpd.rst
-    platypush/backend/music.spotify.rst
-    platypush/backend/nextcloud.rst
-    platypush/backend/nfc.rst
     platypush/backend/nodered.rst
-    platypush/backend/pushbullet.rst
     platypush/backend/redis.rst
-    platypush/backend/scard.rst
-    platypush/backend/sensor.ir.zeroborg.rst
-    platypush/backend/sensor.leap.rst
-    platypush/backend/stt.deepspeech.rst
-    platypush/backend/stt.picovoice.hotword.rst
-    platypush/backend/stt.picovoice.speech.rst
     platypush/backend/tcp.rst
-    platypush/backend/wiimote.rst

docs/source/events.rst

@@ -11,21 +11,19 @@ Events
     platypush/events/application.rst
     platypush/events/assistant.rst
     platypush/events/bluetooth.rst
-    platypush/events/button.flic.rst
     platypush/events/camera.rst
     platypush/events/chat.slack.rst
-    platypush/events/chat.telegram.rst
     platypush/events/clipboard.rst
     platypush/events/custom.rst
     platypush/events/dbus.rst
     platypush/events/distance.rst
     platypush/events/entities.rst
     platypush/events/file.rst
+    platypush/events/flic.rst
     platypush/events/foursquare.rst
     platypush/events/geo.rst
     platypush/events/github.rst
     platypush/events/google.rst
-    platypush/events/google.fit.rst
     platypush/events/google.pubsub.rst
     platypush/events/gotify.rst
     platypush/events/gpio.rst
@@ -66,6 +64,7 @@ Events
     platypush/events/sound.rst
     platypush/events/stt.rst
     platypush/events/sun.rst
+    platypush/events/telegram.rst
     platypush/events/tensorflow.rst
     platypush/events/torrent.rst
     platypush/events/trello.rst
@@ -74,7 +73,6 @@ Events
     platypush/events/web.rst
     platypush/events/web.widget.rst
     platypush/events/websocket.rst
-    platypush/events/wiimote.rst
     platypush/events/xmpp.rst
     platypush/events/zeroborg.rst
     platypush/events/zeroconf.rst

docs/source/index.rst

@@ -39,6 +39,7 @@ Wiki
     wiki/Configuration
     wiki/Installing-extensions
     wiki/A-configuration-example
+    wiki/The-Web-interface
 
 Reference
 =========
@@ -49,7 +50,6 @@ Reference
     backends
     plugins
     events
-    responses
 
 Indices and tables
 ==================

docs/source/platypush/backend/adafruit.io.rst

@@ -1,6 +0,0 @@
-``adafruit.io``
-=================================
-
-.. automodule:: platypush.backend.adafruit.io
-	:members:
-

docs/source/platypush/backend/button.flic.rst

@@ -1,6 +0,0 @@
-``button.flic``
-=================================
-
-.. automodule:: platypush.backend.button.flic
-    :members:
-

docs/source/platypush/backend/camera.pi.rst

@@ -1,6 +0,0 @@
-``camera.pi``
-===============================
-
-.. automodule:: platypush.backend.camera.pi
-    :members:
-

docs/source/platypush/backend/chat.telegram.rst

@@ -1,5 +0,0 @@
-``chat.telegram``
-===================================
-
-.. automodule:: platypush.backend.chat.telegram
-    :members:

docs/source/platypush/backend/google.fit.rst

@@ -1,6 +0,0 @@
-``google.fit``
-================================
-
-.. automodule:: platypush.backend.google.fit
-    :members:
-

docs/source/platypush/backend/google.pubsub.rst

@@ -1,5 +0,0 @@
-``google.pubsub``
-===================================
-
-.. automodule:: platypush.backend.google.pubsub
-    :members:

docs/source/platypush/backend/gps.rst

@@ -1,6 +0,0 @@
-``gps``
-=========================
-
-.. automodule:: platypush.backend.gps
-    :members:
-

docs/source/platypush/backend/kafka.rst

@@ -1,6 +0,0 @@
-``kafka``
-===========================
-
-.. automodule:: platypush.backend.kafka
-    :members:
-

docs/source/platypush/backend/mail.rst

@@ -1,5 +0,0 @@
-``mail``
-==========================
-
-.. automodule:: platypush.backend.mail
-    :members:

docs/source/platypush/backend/midi.rst

@@ -1,6 +0,0 @@
-``midi``
-==========================
-
-.. automodule:: platypush.backend.midi
-    :members:
-

docs/source/platypush/backend/music.mopidy.rst

@@ -1,6 +0,0 @@
-``music.mopidy``
-==================================
-
-.. automodule:: platypush.backend.music.mopidy
-	:members:
-

docs/source/platypush/backend/music.mpd.rst

@@ -1,6 +0,0 @@
-``music.mpd``
-===============================
-
-.. automodule:: platypush.backend.music.mpd
-    :members:
-

docs/source/platypush/backend/music.spotify.rst

@@ -1,5 +0,0 @@
-``music.spotify``
-===================================
-
-.. automodule:: platypush.backend.music.spotify
-    :members:

docs/source/platypush/backend/nextcloud.rst

@@ -1,5 +0,0 @@
-``nextcloud``
-===============================
-
-.. automodule:: platypush.backend.nextcloud
-    :members:

docs/source/platypush/backend/nfc.rst

@@ -1,6 +0,0 @@
-``nfc``
-=========================
-
-.. automodule:: platypush.backend.nfc
-    :members:
-

docs/source/platypush/backend/pushbullet.rst

@@ -1,6 +0,0 @@
-``pushbullet``
-================================
-
-.. automodule:: platypush.backend.pushbullet
-    :members:
-

docs/source/platypush/backend/scard.rst

@@ -1,6 +0,0 @@
-``scard``
-===========================
-
-.. automodule:: platypush.backend.scard
-    :members:
-

docs/source/platypush/backend/sensor.ir.zeroborg.rst

@@ -1,6 +0,0 @@
-``sensor.ir.zeroborg``
-========================================
-
-.. automodule:: platypush.backend.sensor.ir.zeroborg
-    :members:
-

docs/source/platypush/backend/sensor.leap.rst

@@ -1,7 +0,0 @@
-``sensor.leap``
-=================================
-
-.. automodule:: platypush.backend.sensor.leap
-    :members:
-
-

docs/source/platypush/backend/stt.deepspeech.rst

@@ -1,5 +0,0 @@
-``stt.deepspeech``
-====================================
-
-.. automodule:: platypush.backend.stt.deepspeech
-    :members:

docs/source/platypush/backend/stt.picovoice.hotword.rst

@@ -1,5 +0,0 @@
-``stt.picovoice.hotword``
-===========================================
-
-.. automodule:: platypush.backend.stt.picovoice.hotword
-    :members:

docs/source/platypush/backend/stt.picovoice.speech.rst

@@ -1,5 +0,0 @@
-``stt.picovoice.speech``
-==========================================
-
-.. automodule:: platypush.backend.stt.picovoice.speech
-    :members:

docs/source/platypush/backend/wiimote.rst

@@ -1,6 +0,0 @@
-``wiimote``
-=============================
-
-.. automodule:: platypush.backend.wiimote
-    :members:
-

docs/source/platypush/events/button.flic.rst

@@ -1,6 +0,0 @@
-``button.flic``
-=======================================
-
-.. automodule:: platypush.message.event.button.flic
-    :members:
-

docs/source/platypush/events/chat.telegram.rst

@@ -1,5 +0,0 @@
-``chat.telegram``
-=========================================
-
-.. automodule:: platypush.message.event.chat.telegram
-    :members:

docs/source/platypush/events/flic.rst

@@ -0,0 +1,5 @@
+``flic``
+========
+
+.. automodule:: platypush.message.event.flic
+    :members:

docs/source/platypush/events/google.fit.rst

@@ -1,6 +0,0 @@
-``google.fit``
-======================================
-
-.. automodule:: platypush.message.event.google.fit
-	:members:
-

docs/source/platypush/events/telegram.rst

@@ -0,0 +1,5 @@
+``telegram``
+============
+
+.. automodule:: platypush.message.event.telegram
+    :members:

docs/source/platypush/events/wiimote.rst

@@ -1,6 +0,0 @@
-``wiimote``
-===================================
-
-.. automodule:: platypush.message.event.wiimote
-	:members:
-

docs/source/platypush/plugins/assistant.picovoice.rst

@@ -0,0 +1,5 @@
+``assistant.picovoice``
+=======================
+
+.. automodule:: platypush.plugins.assistant.picovoice
+    :members:

docs/source/platypush/plugins/camera.pi.legacy.rst

@@ -0,0 +1,5 @@
+``camera.pi.legacy``
+====================
+
+.. automodule:: platypush.plugins.camera.pi.legacy
+    :members:

docs/source/platypush/plugins/chat.irc.rst

@@ -1,5 +0,0 @@
-``chat.irc``
-============
-
-.. automodule:: platypush.plugins.chat.irc
-    :members:

docs/source/platypush/plugins/chat.telegram.rst

@@ -1,5 +0,0 @@
-``chat.telegram``
-===================================
-
-.. automodule:: platypush.plugins.chat.telegram
-    :members:

docs/source/platypush/plugins/cups.rst

@@ -0,0 +1,5 @@
+``cups``
+========
+
+.. automodule:: platypush.plugins.cups
+    :members:

docs/source/platypush/plugins/flic.rst

@@ -0,0 +1,5 @@
+``flic``
+========
+
+.. automodule:: platypush.plugins.flic
+    :members:

docs/source/platypush/plugins/google.fit.rst

@@ -1,6 +0,0 @@
-``google.fit``
-================================
-
-.. automodule:: platypush.plugins.google.fit
-    :members:
-

docs/source/platypush/plugins/gps.rst

@@ -0,0 +1,5 @@
+``gps``
+=======
+
+.. automodule:: platypush.plugins.gps
+    :members:

docs/source/platypush/plugins/http.request.rst

@@ -1,7 +0,0 @@
-``http.request``
-==================================
-
-.. automodule:: platypush.plugins.http.request
-    :members:
-
-

docs/source/platypush/plugins/http.rst

@@ -0,0 +1,5 @@
+``http``
+========
+
+.. automodule:: platypush.plugins.http
+    :members:

docs/source/platypush/plugins/irc.rst

@@ -0,0 +1,5 @@
+``irc``
+=======
+
+.. automodule:: platypush.plugins.irc
+    :members:

docs/source/platypush/plugins/leap.rst

@@ -0,0 +1,5 @@
+``leap``
+========
+
+.. automodule:: platypush.plugins.leap
+    :members:

docs/source/platypush/plugins/mail.imap.rst

@@ -1,5 +0,0 @@
-``mail.imap``
-===============================
-
-.. automodule:: platypush.plugins.mail.imap
-    :members:

docs/source/platypush/plugins/mail.rst

@@ -0,0 +1,5 @@
+``mail``
+========
+
+.. automodule:: platypush.plugins.mail
+    :members:

docs/source/platypush/plugins/mail.smtp.rst

@@ -1,5 +0,0 @@
-``mail.smtp``
-===============================
-
-.. automodule:: platypush.plugins.mail.smtp
-    :members:

docs/source/platypush/plugins/music.mopidy.rst

@@ -0,0 +1,5 @@
+``music.mopidy``
+================
+
+.. automodule:: platypush.plugins.music.mopidy
+    :members:

docs/source/platypush/plugins/nfc.rst

@@ -0,0 +1,5 @@
+``nfc``
+=======
+
+.. automodule:: platypush.plugins.nfc
+    :members:

docs/source/platypush/plugins/openai.rst

@@ -0,0 +1,5 @@
+``openai``
+==========
+
+.. automodule:: platypush.plugins.openai
+    :members:

docs/source/platypush/plugins/printer.cups.rst

@@ -1,5 +0,0 @@
-``printer.cups``
-==================================
-
-.. automodule:: platypush.plugins.printer.cups
-    :members:

docs/source/platypush/plugins/stt.deepspeech.rst

@@ -1,5 +0,0 @@
-``stt.deepspeech``
-====================================
-
-.. automodule:: platypush.plugins.stt.deepspeech
-    :members:

docs/source/platypush/plugins/stt.picovoice.hotword.rst

@@ -1,5 +0,0 @@
-``stt.picovoice.hotword``
-===========================================
-
-.. automodule:: platypush.plugins.stt.picovoice.hotword
-    :members:

docs/source/platypush/plugins/stt.picovoice.speech.rst

@@ -1,5 +0,0 @@
-``stt.picovoice.speech``
-==========================================
-
-.. automodule:: platypush.plugins.stt.picovoice.speech
-    :members:

docs/source/platypush/plugins/telegram.rst

@@ -0,0 +1,5 @@
+``telegram``
+============
+
+.. automodule:: platypush.plugins.telegram
+    :members:

docs/source/platypush/plugins/tts.picovoice.rst

@@ -0,0 +1,5 @@
+``tts.picovoice``
+=================
+
+.. automodule:: platypush.plugins.tts.picovoice
+    :members:

docs/source/platypush/plugins/wiimote.rst

@@ -1,6 +0,0 @@
-``wiimote``
-=============================
-
-.. automodule:: platypush.plugins.wiimote
-	:members:
-

docs/source/platypush/responses/camera.android.rst

@@ -1,5 +0,0 @@
-``camera.android``
-=============================================
-
-.. automodule:: platypush.message.response.camera.android
-    :members:

docs/source/platypush/responses/camera.rst

@@ -1,5 +0,0 @@
-``camera``
-=====================================
-
-.. automodule:: platypush.message.response.camera
-    :members:

docs/source/platypush/responses/chat.telegram.rst

@@ -1,5 +0,0 @@
-``chat.telegram``
-============================================
-
-.. automodule:: platypush.message.response.chat.telegram
-    :members:

docs/source/platypush/responses/google.drive.rst

@@ -1,5 +0,0 @@
-``google.drive``
-===========================================
-
-.. automodule:: platypush.message.response.google.drive
-    :members:

docs/source/platypush/responses/pihole.rst

@@ -1,5 +0,0 @@
-``pihole``
-=====================================
-
-.. automodule:: platypush.message.response.pihole
-    :members:

docs/source/platypush/responses/printer.cups.rst

@@ -1,5 +0,0 @@
-``printer.cups``
-===========================================
-
-.. automodule:: platypush.message.response.printer.cups
-    :members:

docs/source/platypush/responses/qrcode.rst

@@ -1,5 +0,0 @@
-``qrcode``
-=====================================
-
-.. automodule:: platypush.message.response.qrcode
-    :members:

docs/source/platypush/responses/ssh.rst

@@ -1,5 +0,0 @@
-``ssh``
-==================================
-
-.. automodule:: platypush.message.response.ssh
-    :members:

docs/source/platypush/responses/stt.rst

@@ -1,5 +0,0 @@
-``stt``
-==================================
-
-.. automodule:: platypush.message.response.stt
-    :members:

docs/source/platypush/responses/tensorflow.rst

@@ -1,5 +0,0 @@
-``tensorflow``
-=========================================
-
-.. automodule:: platypush.message.response.tensorflow
-    :members:

docs/source/platypush/responses/translate.rst

@@ -1,5 +0,0 @@
-``translate``
-========================================
-
-.. automodule:: platypush.message.response.translate
-    :members:

docs/source/plugins.rst

@@ -11,6 +11,7 @@ Plugins
     platypush/plugins/application.rst
     platypush/plugins/arduino.rst
     platypush/plugins/assistant.google.rst
+    platypush/plugins/assistant.picovoice.rst
     platypush/plugins/autoremote.rst
     platypush/plugins/bluetooth.rst
     platypush/plugins/calendar.rst
@@ -21,11 +22,11 @@ Plugins
     platypush/plugins/camera.gstreamer.rst
     platypush/plugins/camera.ir.mlx90640.rst
     platypush/plugins/camera.pi.rst
-    platypush/plugins/chat.irc.rst
-    platypush/plugins/chat.telegram.rst
+    platypush/plugins/camera.pi.legacy.rst
     platypush/plugins/clipboard.rst
     platypush/plugins/config.rst
     platypush/plugins/csv.rst
+    platypush/plugins/cups.rst
     platypush/plugins/db.rst
     platypush/plugins/dbus.rst
     platypush/plugins/dropbox.rst
@@ -34,11 +35,11 @@ Plugins
     platypush/plugins/ffmpeg.rst
     platypush/plugins/file.rst
     platypush/plugins/file.monitor.rst
+    platypush/plugins/flic.rst
     platypush/plugins/foursquare.rst
     platypush/plugins/github.rst
     platypush/plugins/google.calendar.rst
     platypush/plugins/google.drive.rst
-    platypush/plugins/google.fit.rst
     platypush/plugins/google.mail.rst
     platypush/plugins/google.maps.rst
     platypush/plugins/google.pubsub.rst
@@ -46,24 +47,26 @@ Plugins
     platypush/plugins/gotify.rst
     platypush/plugins/gpio.rst
     platypush/plugins/gpio.zeroborg.rst
+    platypush/plugins/gps.rst
     platypush/plugins/graphite.rst
     platypush/plugins/hid.rst
-    platypush/plugins/http.request.rst
+    platypush/plugins/http.rst
     platypush/plugins/http.webpage.rst
     platypush/plugins/ifttt.rst
     platypush/plugins/inspect.rst
+    platypush/plugins/irc.rst
     platypush/plugins/joystick.rst
     platypush/plugins/kafka.rst
     platypush/plugins/lastfm.rst
     platypush/plugins/lcd.gpio.rst
     platypush/plugins/lcd.i2c.rst
+    platypush/plugins/leap.rst
     platypush/plugins/light.hue.rst
     platypush/plugins/linode.rst
     platypush/plugins/log.http.rst
     platypush/plugins/logger.rst
     platypush/plugins/luma.oled.rst
-    platypush/plugins/mail.imap.rst
-    platypush/plugins/mail.smtp.rst
+    platypush/plugins/mail.rst
     platypush/plugins/mailgun.rst
     platypush/plugins/mastodon.rst
     platypush/plugins/matrix.rst
@@ -82,18 +85,20 @@ Plugins
     platypush/plugins/ml.cv.rst
     platypush/plugins/mobile.join.rst
     platypush/plugins/mqtt.rst
+    platypush/plugins/music.mopidy.rst
     platypush/plugins/music.mpd.rst
     platypush/plugins/music.snapcast.rst
     platypush/plugins/music.spotify.rst
     platypush/plugins/music.tidal.rst
     platypush/plugins/nextcloud.rst
+    platypush/plugins/nfc.rst
     platypush/plugins/ngrok.rst
     platypush/plugins/nmap.rst
     platypush/plugins/ntfy.rst
+    platypush/plugins/openai.rst
     platypush/plugins/otp.rst
     platypush/plugins/pihole.rst
     platypush/plugins/ping.rst
-    platypush/plugins/printer.cups.rst
     platypush/plugins/pushbullet.rst
     platypush/plugins/pwm.pca9685.rst
     platypush/plugins/qrcode.rst
@@ -115,15 +120,13 @@ Plugins
     platypush/plugins/smartthings.rst
     platypush/plugins/sound.rst
     platypush/plugins/ssh.rst
-    platypush/plugins/stt.deepspeech.rst
-    platypush/plugins/stt.picovoice.hotword.rst
-    platypush/plugins/stt.picovoice.speech.rst
     platypush/plugins/sun.rst
     platypush/plugins/switch.tplink.rst
     platypush/plugins/switch.wemo.rst
     platypush/plugins/switchbot.rst
     platypush/plugins/system.rst
     platypush/plugins/tcp.rst
+    platypush/plugins/telegram.rst
     platypush/plugins/tensorflow.rst
     platypush/plugins/todoist.rst
     platypush/plugins/torrent.rst
@@ -131,6 +134,7 @@ Plugins
     platypush/plugins/tts.rst
     platypush/plugins/tts.google.rst
     platypush/plugins/tts.mimic3.rst
+    platypush/plugins/tts.picovoice.rst
     platypush/plugins/tv.samsung.ws.rst
     platypush/plugins/twilio.rst
     platypush/plugins/udp.rst
@@ -141,7 +145,6 @@ Plugins
     platypush/plugins/weather.buienradar.rst
     platypush/plugins/weather.openweathermap.rst
     platypush/plugins/websocket.rst
-    platypush/plugins/wiimote.rst
     platypush/plugins/xmpp.rst
     platypush/plugins/youtube.rst
     platypush/plugins/zeroconf.rst

docs/source/responses.rst

@@ -1,19 +0,0 @@
-
-Responses
-=========
-
-.. toctree::
-    :maxdepth: 1
-    :caption: Responses:
-
-    platypush/responses/camera.rst
-    platypush/responses/camera.android.rst
-    platypush/responses/chat.telegram.rst
-    platypush/responses/google.drive.rst
-    platypush/responses/pihole.rst
-    platypush/responses/printer.cups.rst
-    platypush/responses/qrcode.rst
-    platypush/responses/ssh.rst
-    platypush/responses/stt.rst
-    platypush/responses/tensorflow.rst
-    platypush/responses/translate.rst

examples/config/hook.py

@@ -1,15 +1,16 @@
-# A more versatile way to define event hooks than the YAML format of `config.yaml` is through native Python scripts.
-# You can define hooks as simple Python functions that use the `platypush.event.hook.hook` decorator to specify on
-# which event type they should be called, and optionally on which event attribute values.
+# A more versatile way to define event hooks than the YAML format of
+# `config.yaml` is through native Python scripts. You can define hooks as simple
+# Python functions that use the `platypush.event.hook.hook` decorator to specify
+# on which event type they should be called, and optionally on which event
+# attribute values.
 #
-# Event hooks should be stored in Python files under `~/.config/platypush/scripts`. All the functions that use the
-# @hook decorator will automatically be discovered and imported as event hooks into the platform at runtime.
+# Event hooks should be stored in Python files under
+# `~/.config/platypush/scripts`. All the functions that use the @when decorator
+# will automatically be discovered and imported as event hooks into the platform
+# at runtime.
 
 # `run` is a utility function that runs a request by name (e.g. `light.hue.on`).
-from platypush.utils import run
-
-# @hook decorator
-from platypush.event.hook import hook
+from platypush import when, run
 
 # Event types that you want to react to
 from platypush.message.event.assistant import (
@@ -18,13 +19,15 @@ from platypush.message.event.assistant import (
 )
 
 
-@hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
-def on_music_play_command(event, title=None, artist=None, **context):
+@when(SpeechRecognizedEvent, phrase='play ${title} by ${artist}')
+def on_music_play_command(event, title=None, artist=None):
     """
-    This function will be executed when a SpeechRecognizedEvent with `phrase="play the music"` is triggered.
-    `event` contains the event object and `context` any key-value info from the running context.
-    Note that in this specific case we can leverage the token-extraction feature of SpeechRecognizedEvent through
-    ${} that operates on regex-like principles to extract any text that matches the pattern into context variables.
+    This function will be executed when a SpeechRecognizedEvent with
+    `phrase="play the music"` is triggered. `event` contains the event object
+    and `context` any key-value info from the running context. Note that in this
+    specific case we can leverage the token-extraction feature of
+    SpeechRecognizedEvent through ${} that operates on regex-like principles to
+    extract any text that matches the pattern into context variables.
     """
     results = run(
         'music.mpd.search',
@@ -34,16 +37,17 @@ def on_music_play_command(event, title=None, artist=None, **context):
         },
     )
 
-    if results:
+    if results and results[0]:
         run('music.mpd.play', results[0]['file'])
     else:
         run('tts.say', "I can't find any music matching your query")
 
 
-@hook(ConversationStartEvent)
-def on_conversation_start(event, **context):
+@when(ConversationStartEvent)
+def on_conversation_start():
     """
-    A simple hook that gets invoked when a new conversation starts with a voice assistant and simply pauses the music
-    to make sure that your speech is properly detected.
+    A simple hook that gets invoked when a new conversation starts with a voice
+    assistant and simply pauses the music to make sure that your speech is
+    properly detected.
     """
     run('music.mpd.pause_if_playing')

examples/systemd/platypush.service

@@ -1,22 +0,0 @@
-# platypush systemd service example.
-# Edit and copy this file to your systemd folder. It's usually
-# /usr/lib/systemd/user for global installation or
-# ~/.config/systemd/user for user installation. You can
-# then control and monitor the service through
-# systemd [--user] [start|stop|restart|status] platypush.service
-
-[Unit]
-Description=Platypush daemon
-After=network.target redis.service
-
-[Service]
-# platypush installation path
-ExecStart=/usr/bin/platypush
-Restart=always
-# How long should be waited before restarting the service
-# in case of failure.
-RestartSec=10
-
-[Install]
-WantedBy=default.target
-

examples/systemd/platypush.service

@@ -0,0 +1 @@
+../../platypush/config/systemd/platypush.service

generate_missing_docs.py

@@ -8,7 +8,6 @@ import pkgutil
 
 from platypush.backend import Backend
 from platypush.message.event import Event
-from platypush.message.response import Response
 from platypush.plugins import Plugin
 from platypush.utils.manifest import Manifests
 from platypush.utils.mock import auto_mocks
@@ -26,10 +25,6 @@ def get_all_events():
     return _get_modules(Event)
 
 
-def get_all_responses():
-    return _get_modules(Response)
-
-
 def _get_modules(base_type: type):
     ret = set()
     base_dir = os.path.dirname(inspect.getfile(base_type))
@@ -151,20 +146,11 @@ def generate_events_doc():
     )
 
 
-def generate_responses_doc():
-    _generate_components_doc(
-        index_name='responses',
-        package_name='message.response',
-        components=sorted(response for response in get_all_responses() if response),
-    )
-
-
 def main():
     with auto_mocks():
         generate_plugins_doc()
         generate_backends_doc()
         generate_events_doc()
-        generate_responses_doc()
 
 
 if __name__ == '__main__':

platypush/__init__.py

@@ -5,29 +5,42 @@ Platypush
 .. license: MIT
 """
 
-from .app import Application
+from .app import Application, app
 from .config import Config
-from .context import get_backend, get_bus, get_plugin
+from .context import Variable, get_backend, get_bus, get_plugin
+from .cron import cron
+from .event.hook import hook
 from .message.event import Event
 from .message.request import Request
 from .message.response import Response
+from .procedure import procedure
 from .runner import main
 from .utils import run
 
+# Alias for platypush.event.hook.hook,
+# see https://git.platypush.tech/platypush/platypush/issues/399
+when = hook
+
 
 __author__ = 'Fabio Manganiello <fabio@manganiello.tech>'
 __version__ = '0.50.3'
 __all__ = [
     'Application',
+    'Variable',
     'Config',
     'Event',
     'Request',
     'Response',
+    'app',
+    'cron',
     'get_backend',
     'get_bus',
     'get_plugin',
+    'hook',
     'main',
+    'procedure',
     'run',
+    'when',
 ]
 
 

platypush/app/__init__.py

@@ -1,4 +1,4 @@
-from ._app import Application, main
+from ._app import Application, app, main
 
 
-__all__ = ["Application", "main"]
+__all__ = ["Application", "app", "main"]

platypush/app/_app.py

@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 import logging
+import pathlib
 import os
 import signal
 import subprocess
@@ -41,6 +42,7 @@ class Application:
         self,
         config_file: Optional[str] = None,
         workdir: Optional[str] = None,
+        db: Optional[str] = None,
         logsdir: Optional[str] = None,
         cachedir: Optional[str] = None,
         device_id: Optional[str] = None,
@@ -56,30 +58,87 @@ class Application:
         ctrl_sock: Optional[str] = None,
     ):
         """
-        :param config_file: Configuration file override (default: None).
-        :param workdir: Overrides the ``workdir`` setting in the configuration
-            file (default: None).
-        :param logsdir: Set logging directory. If not specified, the
-            ``filename`` setting under the ``logging`` section of the
-            configuration file is used. If not set, logging will be sent to
-            stdout and stderr.
-        :param cachedir: Overrides the ``cachedir`` setting in the configuration
-            file (default: None).
-        :param device_id: Override the device ID used to identify this
-            instance. If not passed here, it is inferred from the configuration
-            (device_id field). If not present there either, it is inferred from
-            the hostname.
+        :param config_file: Configuration file. The order of precedence is:
+
+            - The ``config_file`` parameter (or the ``-c``/``--config`` command
+              line argument).
+            - The ``PLATYPUSH_CONFIG`` environment variable.
+            - ``./config.yaml``
+            - ``~/.config/platypush/config.yaml``
+            - ``/etc/platypush/config.yaml``
+
+        :param workdir: Working directory where the application will store its
+            data and integration plugins will store their data. The order of
+            precedence is:
+
+                - The ``workdir`` parameter (or the ``-w``/``--workdir`` command
+                  line argument).
+                - The ``PLATYPUSH_WORKDIR`` environment variable.
+                - The ``workdir`` field in the configuration file.
+                - ``~/.local/share/platypush``
+
+        :param db: Main database engine for the application. Supports SQLAlchemy
+            engine strings. The order of precedence is:
+
+                - The ``db`` parameter (or the ``--main-db``/``--db`` command
+                  line argument).
+                - The ``PLATYPUSH_DB`` environment variable.
+                - The ``db`` field in the configuration file.
+                - ``sqlite:///<workdir>/main.db``
+
+        :param logsdir: Logs directory where the application will store its logs.
+            The order of precedence is:
+
+                - The ``logsdir`` parameter (or the ``-l``/``--logsdir`` command
+                  line argument).
+                - The ``PLATYPUSH_LOGSDIR`` environment variable.
+                - The ``logging`` -> ``filename`` field in the configuration
+                  file (the ``format`` and ``level`` fields can be set as well
+                  using Python logging configuration).
+                - stdout and stderr
+
+        :param cachedir: Directory where the application and the plugins will store
+            their cache data. The order of precedence is:
+
+                - The ``cachedir`` parameter (or the ``--cachedir`` command line
+                  argument).
+                - The ``PLATYPUSH_CACHEDIR`` environment variable.
+                - The ``cachedir`` field in the configuration file.
+                - ``~/.cache/platypush``
+
+        :param device_id: Device ID used to identify this instance. The order
+            of precedence is:
+
+              - The ``device_id`` parameter (or the ``--device-id`` command
+                line argument).
+              - The ``PLATYPUSH_DEVICE_ID`` environment variable.
+              - The ``device_id`` field in the configuration file.
+              - The hostname of the machine.
+
         :param pidfile: File where platypush will store its PID upon launch,
            useful if you're planning to integrate the application within a
-           service or a launcher script (default: None).
+           service or a launcher script. Order of precedence:
+
+               - The ``pidfile`` parameter (or the ``-P``/``--pidfile`` command
+                 line argument).
+               - The ``PLATYPUSH_PIDFILE`` environment variable.
+               - No PID file.
+
         :param requests_to_process: Exit after processing the specified
-            number of requests (default: None, loop forever).
+            number of requests (default: None, loop forever). This is usually
+            useful for testing purposes.
         :param no_capture_stdout: Set to true if you want to disable the
             stdout capture by the logging system (default: False).
         :param no_capture_stderr: Set to true if you want to disable the
             stderr capture by the logging system (default: False).
         :param redis_queue: Name of the (Redis) queue used for dispatching
-            messages (default: platypush/bus).
+            messages. Order of precedence:
+
+                - The ``redis_queue`` parameter (or the ``--redis-queue`` command
+                  line argument).
+                - The ``PLATYPUSH_REDIS_QUEUE`` environment variable.
+                - ``platypush/bus``
+
         :param verbose: Enable debug/verbose logging, overriding the stored
             configuration (default: False).
         :param start_redis: If set, it starts a managed Redis instance upon
@@ -87,34 +146,57 @@ class Application:
             server). This is particularly useful when running the application
             inside of Docker containers, without relying on ``docker-compose``
             to start multiple containers, and in tests (default: False).
-        :param redis_host: Host of the Redis server to be used. It overrides
-            the settings in the ``redis`` section of the configuration file.
-        :param redis_port: Port of the local Redis server. It overrides the
-            settings in the ``redis`` section of the configuration file.
+        :param redis_host: Host of the Redis server to be used. The order of
+            precedence is:
+
+                - The ``redis_host`` parameter (or the ``--redis-host`` command
+                  line argument).
+                - The ``PLATYPUSH_REDIS_HOST`` environment variable.
+                - The ``redis`` -> ``host`` field in the configuration file.
+                - The ``backend.redis`` -> ``redis_args`` -> ``host`` field in
+                  the configuration file.
+                - ``localhost``
+
+        :param redis_port: Port of the Redis server to be used. The order of
+            precedence is:
+
+                - The ``redis_port`` parameter (or the ``--redis-port`` command
+                  line argument).
+                - The ``PLATYPUSH_REDIS_PORT`` environment variable.
+                - The ``redis`` -> ``port`` field in the configuration file.
+                - The ``backend.redis`` -> ``redis_args`` -> ``port`` field in
+                  the configuration file.
+                - ``6379``
+
         :param ctrl_sock: If set, it identifies a path to a UNIX domain socket
             that the application can use to send control messages (e.g. STOP
             and RESTART) to its parent.
         """
 
-        self.pidfile = pidfile
+        self.pidfile = pidfile or os.environ.get('PLATYPUSH_PIDFILE')
         self.bus: Optional[Bus] = None
-        self.redis_queue = redis_queue or RedisBus.DEFAULT_REDIS_QUEUE
-        self.config_file = config_file
-        self._verbose = verbose
-        self._logsdir = (
-            os.path.abspath(os.path.expanduser(logsdir)) if logsdir else None
+        self.redis_queue = (
+            redis_queue
+            or os.environ.get('PLATYPUSH_REDIS_QUEUE')
+            or RedisBus.DEFAULT_REDIS_QUEUE
         )
+        self.config_file = config_file or os.environ.get('PLATYPUSH_CONFIG')
+        self.verbose = verbose
+        self.db_engine = db or os.environ.get('PLATYPUSH_DB')
+        self.device_id = device_id or os.environ.get('PLATYPUSH_DEVICE_ID')
 
+        self.logsdir = self.expand_path(logsdir or os.environ.get('PLATYPUSH_LOGSDIR'))
+        self.workdir = self.expand_path(workdir or os.environ.get('PLATYPUSH_WORKDIR'))
+        self.cachedir = self.expand_path(
+            cachedir or os.environ.get('PLATYPUSH_CACHEDIR')
+        )
         Config.init(
             self.config_file,
-            device_id=device_id,
-            workdir=os.path.abspath(os.path.expanduser(workdir)) if workdir else None,
-            cachedir=os.path.abspath(os.path.expanduser(cachedir))
-            if cachedir
-            else None,
-            ctrl_sock=os.path.abspath(os.path.expanduser(ctrl_sock))
-            if ctrl_sock
-            else None,
+            device_id=self.device_id,
+            workdir=self.workdir,
+            db=self.db_engine,
+            cachedir=self.cachedir,
+            ctrl_sock=self.expand_path(ctrl_sock),
         )
 
         self.no_capture_stdout = no_capture_stdout
@@ -125,24 +207,25 @@ class Application:
         self.processed_requests = 0
         self.cron_scheduler = None
         self.start_redis = start_redis
-        self.redis_host = redis_host
-        self.redis_port = redis_port
-        self.redis_conf = {}
+        self.redis_host = redis_host or os.environ.get('PLATYPUSH_REDIS_HOST')
+        self.redis_port = redis_port or os.environ.get('PLATYPUSH_REDIS_PORT')
+        self._redis_conf = {
+            'host': self.redis_host or 'localhost',
+            'port': self.redis_port or self._default_redis_port,
+        }
+
         self._redis_proc: Optional[subprocess.Popen] = None
         self.cmd_stream = CommandStream(ctrl_sock)
 
         self._init_bus()
         self._init_logging()
 
+    @staticmethod
+    def expand_path(path: Optional[str]) -> Optional[str]:
+        return os.path.abspath(os.path.expanduser(path)) if path else None
+
     def _init_bus(self):
-        self._redis_conf = get_redis_conf()
-        self._redis_conf['port'] = self.redis_port or self._redis_conf.get(
-            'port', self._default_redis_port
-        )
-
-        if self.redis_host:
-            self._redis_conf['host'] = self.redis_host
-
+        self._redis_conf = {**self._redis_conf, **get_redis_conf()}
         Config.set('redis', self._redis_conf)
         self.bus = RedisBus(
             redis_queue=self.redis_queue,
@@ -152,12 +235,18 @@ class Application:
 
     def _init_logging(self):
         logging_conf = Config.get('logging') or {}
-        if self._verbose:
+        if self.verbose:
             logging_conf['level'] = logging.DEBUG
-        if self._logsdir:
-            logging_conf['filename'] = os.path.join(self._logsdir, 'platypush.log')
+
+        if self.logsdir:
+            logging_conf['filename'] = os.path.join(self.logsdir, 'platypush.log')
             logging_conf.pop('stream', None)
 
+        if logging_conf.get('filename'):
+            pathlib.Path(os.path.dirname(logging_conf['filename'])).mkdir(
+                parents=True, exist_ok=True
+            )
+
         Config.set('logging', logging_conf)
         logging.basicConfig(**logging_conf)
 
@@ -214,6 +303,7 @@ class Application:
             workdir=opts.workdir,
             cachedir=opts.cachedir,
             logsdir=opts.logsdir,
+            db=opts.db_engine,
             device_id=opts.device_id,
             pidfile=opts.pidfile,
             no_capture_stdout=opts.no_capture_stdout,
@@ -274,6 +364,10 @@ class Application:
             backend.stop()
 
         for plugin in runnable_plugins:
+            # This is required because some plugins may redefine the `stop` method.
+            # In that case, at the very least the _should_stop event should be
+            # set to notify the plugin to stop.
+            plugin._should_stop.set()  # pylint: disable=protected-access
             plugin.stop()
 
         for backend in backends:
@@ -370,10 +464,15 @@ class Application:
             self._run()
 
 
+app: Optional[Application] = None
+
+
 def main(*args: str):
     """
     Application entry point.
     """
+    global app
+
     app = Application.from_cmdline(args)
 
     try:

platypush/backend/__init__.py

@@ -402,6 +402,13 @@ class Backend(Thread, EventGenerator, ExtensionWithManifest):
             )
             return
 
+        if self.zeroconf:
+            self.logger.info(
+                'Zeroconf service already registered for %s, removing the previous instance',
+                self.__class__.__name__,
+            )
+            self.unregister_service()
+
         self.zeroconf = Zeroconf()
         srv_desc = {
             'name': 'Platypush',