Merge branch 'master' into snyk-fix-be2eea233b2d14c94dc846778f7390d9

.drone.yml

@@ -114,10 +114,38 @@ steps:
     - . .drone/update-components-cache.sh
 
 ###
-### Update the Arch packages
+### Update the Arch git package
+### Note: This is probably no longer required, as the platypush-git PKGBUILD
+### #now includes a dynamic pkgver() function.
+### See https://aur.archlinux.org/packages/platypush-git#comment-982845
 ###
 
-- name: update-arch-packages
+# - name: update-arch-git-package
+#   image: python:3.11-alpine
+#   environment:
+#     WORKDIR: /tmp/workdir
+#     SSH_PUBKEY:
+#       from_secret: ssh_pubkey
+#     SSH_PRIVKEY:
+#       from_secret: ssh_privkey
+# 
+#   when:
+#     branch:
+#       - master
+#     event:
+#       - push
+# 
+#   depends_on:
+#     - update-components-cache
+# 
+#   commands:
+#     - . .drone/update-arch-git-package.sh
+
+###
+### Update the Arch stable package
+###
+
+- name: update-arch-stable-package
   image: python:3.11-alpine
   environment:
     WORKDIR: /tmp/workdir
@@ -127,16 +155,11 @@ steps:
       from_secret: ssh_privkey
 
   when:
-    branch:
-      - master
     event:
-      - push
-
-  depends_on:
-    - update-components-cache
+      - tag
 
   commands:
-    - . .drone/update-arch-packages.sh
+    - . .drone/update-arch-stable-package.sh
 
 ###
 ### Update the Debian (stable) packages
@@ -325,7 +348,7 @@ steps:
       - push
 
   depends_on:
-    - update-arch-packages
+    # - update-arch-git-package
     - update-rpm-repo
     - update-apt-repo
 
@@ -348,6 +371,7 @@ steps:
 
   depends_on:
     - update-pip-package
+    - update-arch-stable-package
 
   commands:
     - apk add --update --no-cache curl

.drone/update-arch-git-package.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+[ -f .skipci ] && exit 0
+
+apk add --update --no-cache curl pacman sudo
+
+. .drone/macros/configure-ssh.sh
+. .drone/macros/configure-git.sh
+
+git pull --rebase origin master --tags
+
+export VERSION=$(python setup.py --version)
+export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1)
+export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).g${HEAD}"
+
+ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null
+adduser -u 1000 -D build
+mkdir -p "$WORKDIR"
+
+echo "--- Updating Arch git version"
+export PKGDIR=$WORKDIR/git
+git clone ssh://aur@aur.archlinux.org/platypush-git.git "$PKGDIR"
+git config --global --add safe.directory "$PKGDIR"
+chown -R build "$PKGDIR"
+cd "$PKGDIR"
+
+sed -i 'PKGBUILD' -r \
+  -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \
+  -e "s/^pkgrel=.*/pkgrel=1/" \
+
+sudo -u build makepkg --printsrcinfo > .SRCINFO
+export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
+
+if [ $FILES_CHANGED -gt 0 ]; then
+  echo "--- Pushing git package version $GIT_VERSION"
+  git commit -a -m '[Automatic] Package update'
+  git push origin master
+fi

.drone/update-arch-stable-package.sh

@@ -10,34 +10,12 @@ apk add --update --no-cache curl pacman sudo
 git pull --rebase origin master --tags
 
 export VERSION=$(python setup.py --version)
-export HEAD=$(git log --pretty=format:%h HEAD...HEAD~1 | head -1)
-export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).g${HEAD}"
 export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz"
 
 ssh-keyscan aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null
 adduser -u 1000 -D build
 mkdir -p "$WORKDIR"
 
-echo "--- Updating Arch git version"
-export PKGDIR=$WORKDIR/git
-git clone ssh://aur@aur.archlinux.org/platypush-git.git "$PKGDIR"
-git config --global --add safe.directory "$PKGDIR"
-chown -R build "$PKGDIR"
-cd "$PKGDIR"
-
-sed -i 'PKGBUILD' -r \
-  -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \
-  -e "s/^pkgrel=.*/pkgrel=1/" \
-
-sudo -u build makepkg --printsrcinfo > .SRCINFO
-export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l)
-
-if [ $FILES_CHANGED -gt 0 ]; then
-  echo "--- Pushing git package version $GIT_VERSION"
-  git commit -a -m '[Automatic] Package update'
-  git push origin master
-fi
-
 echo "--- Updating Arch stable version"
 export PKGDIR="$WORKDIR/stable"
 git clone ssh://aur@aur.archlinux.org/platypush.git "$PKGDIR"
@@ -47,6 +25,7 @@ cd "$PKGDIR"
 export RELEASED_VERSION=$(grep -e '^pkgver=' PKGBUILD | sed -r -e 's/^pkgver=(.*)\s*/\1/')
 
 if [ "$RELEASED_VERSION" == "$VERSION" ]; then
+  echo "--- No changes in the stable package version"
   exit 0
 fi
 

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     - id: check-xml
     - id: check-symlinks
     - id: check-added-large-files
-      args: ['--maxkb=1500']
+      args: ['--maxkb=3000']
 
 -   repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs
     rev: v1.1.2

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+
+## [1.1.3] - 2024-07-16
+
+- [`core`]: New architecture for the Redis bus - now leveraging pub/sub with a
+  connection pool instead of a single-connection queue. It makes the
+  application much faster and less prone to Redis deadlocks.
+
+- [`youtube`]:
+  [#391](https://git.platypush.tech/platypush/platypush/issues/391): added
+  support for:
+
+  - Add/remove playlists (UI)
+  - Add to/remove from playlist (UI)
+  - Subscribe/unsubscribe from channels (UI)
+  - Browse channels and playlists directly in the UI
+  - Download media and audio
+
 ## [1.1.1] - 2024-06-24
 
 - [`torrent`]: [[#263](https://git.platypush.tech/platypush/platypush/issues/263)], [[#375](https://git.platypush.tech/platypush/platypush/issues/375)],

MANIFEST.in

@@ -5,3 +5,13 @@ include platypush/config/*.yaml
 include platypush/config/systemd/*
 global-include manifest.json
 global-include components.json.gz
+
+global-exclude __pycache__/*
+global-exclude *.pyc
+
+# Exclude symlinks to avoid issues with setuptools
+exclude platypush/backend/http/webapp/public/icons/openweathermap/black
+exclude platypush/backend/http/webapp/public/icons/openweathermap/white
+exclude platypush/backend/http/webapp/src/**
+exclude platypush/events
+exclude platypush/install/scripts/ubuntu

README.md

@@ -9,6 +9,7 @@
 [![Join chat on Matrix](https://img.shields.io/matrix/platypush:matrix.platypush.tech.svg?server_fqdn=matrix.platypush.tech&label=chat&logo=matrix)](https://matrix.to/#/#platypush:matrix.platypush.tech)
 
 [![pip version](https://img.shields.io/pypi/v/platypush.svg?style=flat)](https://pypi.python.org/pypi/platypush/)
+[![CodeFactor](https://www.codefactor.io/repository/github/blacklight/platypush/badge)](https://www.codefactor.io/repository/github/blacklight/platypush)
 [![Contributions](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://git.platypush.tech/platypush/platypush/src/branch/master/CONTRIBUTING.md)
 [![License](https://img.shields.io/github/license/BlackLight/platypush.svg)](https://git.platypush.tech/platypush/platypush/src/branch/master/LICENSE.txt)
 [![Sponsor](https://img.shields.io/github/sponsors/blacklight)](https://github.com/sponsors/blacklight)
@@ -1178,6 +1179,15 @@ redis:
   password: redis-pass
 ```
 
+If `--start-redis` is set, the application can be configured to start a custom
+`redis-server` executable through the:
+
+1. `--redis-bin` command-line option.
+2. `PLATYPUSH_REDIS_BIN` environment variable.
+
+Alternative drop-in implementations such as `keydb-server`, `valkey` or
+`redict` are also supported.
+
 ### nginx
 
 If you want to access your Platypush web panel outside your home network, it may

platypush/__init__.py

@@ -23,7 +23,7 @@ when = hook
 
 
 __author__ = 'Fabio Manganiello <fabio@manganiello.tech>'
-__version__ = '1.1.1'
+__version__ = '1.1.3'
 __all__ = [
     'Application',
     'Variable',

platypush/app/_app.py

@@ -19,7 +19,6 @@ from platypush.entities import init_entities_engine, EntitiesEngine
 from platypush.event.processor import EventProcessor
 from platypush.logger import Logger
 from platypush.message.event import Event
-from platypush.message.event.application import ApplicationStartedEvent
 from platypush.message.request import Request
 from platypush.message.response import Response
 from platypush.utils import get_enabled_plugins, get_redis_conf
@@ -33,6 +32,9 @@ class Application:
     # Default Redis port
     _default_redis_port = 6379
 
+    # Default Redis binary, if --start-redis is set
+    _default_redis_bin = 'redis-server'
+
     # backend_name => backend_obj map
     backends = None
 
@@ -56,6 +58,7 @@ class Application:
         start_redis: bool = False,
         redis_host: Optional[str] = None,
         redis_port: Optional[int] = None,
+        redis_bin: Optional[str] = None,
         ctrl_sock: Optional[str] = None,
     ):
         """
@@ -143,10 +146,11 @@ class Application:
         :param verbose: Enable debug/verbose logging, overriding the stored
             configuration (default: False).
         :param start_redis: If set, it starts a managed Redis instance upon
-            boot (it requires the ``redis-server`` executable installed on the
-            server). This is particularly useful when running the application
-            inside of Docker containers, without relying on ``docker-compose``
-            to start multiple containers, and in tests (default: False).
+            boot (it requires Redis installed on the server, see
+            ``redis_bin``). This is particularly useful when running the
+            application inside of Docker containers, without relying on
+            ``docker-compose`` to start multiple containers, and in tests
+            (default: False).
         :param redis_host: Host of the Redis server to be used. The order of
             precedence is:
 
@@ -169,6 +173,16 @@ class Application:
                   the configuration file.
                 - ``6379``
 
+        :param redis_bin: Path to the Redis server executable, if ``start_redis``
+            is set. Alternative drop-in Redis implementations such as
+            ``keydb-server``, ``valkey``, ``redict`` can be used. The order of
+            precedence is:
+
+                - The ``redis_bin`` parameter (or the ``--redis-bin`` command
+                  line argument).
+                - The ``PLATYPUSH_REDIS_BIN`` environment variable.
+                - ``redis-server``
+
         :param ctrl_sock: If set, it identifies a path to a UNIX domain socket
             that the application can use to send control messages (e.g. STOP
             and RESTART) to its parent.
@@ -181,6 +195,8 @@ class Application:
             or os.environ.get('PLATYPUSH_REDIS_QUEUE')
             or RedisBus.DEFAULT_REDIS_QUEUE
         )
+
+        os.environ['PLATYPUSH_REDIS_QUEUE'] = self.redis_queue
         self.config_file = config_file or os.environ.get('PLATYPUSH_CONFIG')
         self.verbose = verbose
         self.db_engine = db or os.environ.get('PLATYPUSH_DB')
@@ -210,6 +226,11 @@ class Application:
         self.start_redis = start_redis
         self.redis_host = redis_host or os.environ.get('PLATYPUSH_REDIS_HOST')
         self.redis_port = redis_port or os.environ.get('PLATYPUSH_REDIS_PORT')
+        self.redis_bin = (
+            redis_bin
+            or os.environ.get('PLATYPUSH_REDIS_BIN')
+            or self._default_redis_bin
+        )
         self._redis_conf = {
             'host': self.redis_host or 'localhost',
             'port': self.redis_port or self._default_redis_port,
@@ -261,7 +282,7 @@ class Application:
         port = self._redis_conf['port']
         log.info('Starting local Redis instance on %s', port)
         redis_cmd_args = [
-            'redis-server',
+            self.redis_bin,
             '--bind',
             'localhost',
             '--port',
@@ -460,7 +481,6 @@ class Application:
             self.cron_scheduler.start()
 
         assert self.bus, 'The bus is not running'
-        self.bus.post(ApplicationStartedEvent())
 
         # Poll for messages on the bus
         try:

platypush/backend/http/__init__.py

@@ -13,7 +13,7 @@ from typing import Mapping, Optional
 import psutil
 
 from tornado.httpserver import HTTPServer
-from tornado.netutil import bind_sockets
+from tornado.netutil import bind_sockets, bind_unix_socket
 from tornado.process import cpu_count, fork_processes
 from tornado.wsgi import WSGIContainer
 from tornado.web import Application, FallbackHandler
@@ -200,7 +200,8 @@ class HttpBackend(Backend):
     def __init__(
         self,
         port: int = DEFAULT_HTTP_PORT,
-        bind_address: str = '0.0.0.0',
+        bind_address: Optional[str] = '0.0.0.0',
+        bind_socket: Optional[str] = None,
         resource_dirs: Optional[Mapping[str, str]] = None,
         secret_key_file: Optional[str] = None,
         num_workers: Optional[int] = None,
@@ -209,7 +210,16 @@ class HttpBackend(Backend):
     ):
         """
         :param port: Listen port for the web server (default: 8008)
-        :param bind_address: Address/interface to bind to (default: 0.0.0.0, accept connection from any IP)
+        :param bind_address: Address/interface to bind to (default: 0.0.0.0,
+            accept connection from any IP). You can set it to null to disable
+            the network interface binding, but then you must set ``bind_socket``
+            as an alternative.
+        :param bind_socket: Path to the Unix socket to bind to. If set, the
+            server will bind to the path of the specified Unix socket. If set to
+            ``true``, then a socket will be automatically initialized on
+            ``<workdir>/platypush-<device_id>.sock``. If not set, the server will
+            only listen on the specified bind address and port. Note that either
+            ``bind_socket`` or ``socket_path`` must be set.
         :param resource_dirs: Static resources directories that will be
             accessible through ``/resources/<path>``. It is expressed as a map
             where the key is the relative path under ``/resources`` to expose and
@@ -231,11 +241,23 @@ class HttpBackend(Backend):
 
         super().__init__(**kwargs)
 
+        assert (
+            bind_address or bind_socket
+        ), 'Either bind_address or bind_socket must be set'
         self.port = port
         self._server_proc: Optional[Process] = None
         self._service_registry_thread = None
         self.bind_address = bind_address
 
+        if bind_socket is True:
+            bind_socket = os.path.join(
+                Config.get_workdir(), f'platypush-{Config.get_device_id()}.sock'
+            )
+
+        self.socket_path = None
+        if bind_socket:
+            self.socket_path = os.path.expanduser(bind_socket)
+
         if resource_dirs:
             self.resource_dirs = {
                 name: os.path.abspath(os.path.expanduser(d))
@@ -260,8 +282,8 @@ class HttpBackend(Backend):
         super().on_stop()
         self.logger.info('Received STOP event on HttpBackend')
         start = time()
-        remaining_time: partial[float] = partial(  # type: ignore
-            get_remaining_timeout, timeout=self._STOP_TIMEOUT, start=start
+        remaining_time: partial[float] = partial(
+            get_remaining_timeout, timeout=self._STOP_TIMEOUT, start=start  # type: ignore
         )
 
         if self._server_proc:
@@ -364,6 +386,7 @@ class HttpBackend(Backend):
         )
 
         if self.use_werkzeug_server:
+            assert self.bind_address, 'bind_address must be set when using Werkzeug'
             application.config['redis_queue'] = self.bus.redis_queue  # type: ignore
             application.run(
                 host=self.bind_address,
@@ -372,9 +395,13 @@ class HttpBackend(Backend):
                 debug=True,
             )
         else:
-            sockets = bind_sockets(
-                self.port, address=self.bind_address, reuse_port=True
-            )
+            sockets = []
+
+            if self.bind_address:
+                sockets.extend(bind_sockets(self.port, address=self.bind_address))
+
+            if self.socket_path:
+                sockets.append(bind_unix_socket(self.socket_path))
 
             try:
                 fork_processes(self.num_workers)
@@ -421,8 +448,8 @@ class HttpBackend(Backend):
 
         # Initialize the timeout
         start = time()
-        remaining_time: partial[int] = partial(  # type: ignore
-            get_remaining_timeout, timeout=self._STOP_TIMEOUT, start=start, cls=int
+        remaining_time: partial[int] = partial(
+            get_remaining_timeout, timeout=self._STOP_TIMEOUT, start=start, cls=int  # type: ignore
         )
 
         # Wait for all children to terminate (with timeout)

platypush/backend/http/app/routes/pwa.py

@@ -1,4 +1,7 @@
-from flask import Blueprint, jsonify, send_from_directory
+from typing import Optional
+from urllib.parse import urlparse
+
+from flask import Blueprint, jsonify, request, send_from_directory
 
 from platypush.config import Config
 from platypush.backend.http.app import template_folder
@@ -11,13 +14,37 @@ __routes__ = [
 ]
 
 
+def _get_plugin(url: Optional[str] = None) -> Optional[str]:
+    if not url:
+        return None
+
+    path = urlparse(url).path.lstrip('/').split('/')
+    if len(path) > 1 and path[0] == 'plugin':
+        return path[1]
+
+    return None
+
+
 @pwa.route('/manifest.json', methods=['GET'])
 def manifest_json():
     """Generated manifest file for the PWA"""
+
+    device_id = Config.get_device_id()
+    referer = request.headers.get('Referer')
+    plugin = _get_plugin(referer)
+    start_url = '/'
+    name = f'Platypush @ {device_id}'
+    short_name = device_id
+
+    if plugin:
+        start_url = f'/plugin/{plugin}'
+        name = f'{plugin} @ {device_id}'
+        short_name = plugin
+
     return jsonify(
         {
-            "name": f'Platypush @ {Config.get("device_id")}',
-            "short_name": Config.get('device_id'),
+            "name": name,
+            "short_name": short_name,
             "icons": [
                 {
                     "src": "/img/icons/favicon-16x16.png",
@@ -94,9 +121,9 @@ def manifest_json():
             ],
             "gcm_sender_id": "",
             "gcm_user_visible_only": True,
-            "start_url": "/",
+            "start_url": start_url,
             "permissions": ["gcm"],
-            "orientation": "portrait",
+            "orientation": "any",
             "display": "standalone",
             "theme_color": "#000000",
             "background_color": "#ffffff",

platypush/backend/http/app/utils/bus.py

@@ -1,24 +1,57 @@
+from multiprocessing import Lock
+
 from platypush.bus.redis import RedisBus
+from platypush.context import get_bus
 from platypush.config import Config
-from platypush.context import get_backend
 from platypush.message import Message
 from platypush.message.request import Request
-from platypush.utils import get_redis_conf, get_message_response
+from platypush.utils import get_message_response
 
 from .logger import logger
 
-_bus = None
+
+class BusWrapper:  # pylint: disable=too-few-public-methods
+    """
+    Lazy singleton wrapper for the bus object.
+    """
+
+    def __init__(self):
+        self._redis_queue = None
+        self._bus = None
+        self._bus_lock = Lock()
+
+    @property
+    def bus(self) -> RedisBus:
+        """
+        Lazy getter/initializer for the bus object.
+        """
+        with self._bus_lock:
+            if not self._bus:
+                self._bus = get_bus()
+
+        bus_: RedisBus = self._bus  # type: ignore
+        return bus_
+
+    def post(self, msg):
+        """
+        Send a message to the bus.
+
+        :param msg: The message to send.
+        """
+        try:
+            self.bus.post(msg)
+        except Exception as e:
+            logger().exception(e)
+
+
+_bus = BusWrapper()
 
 
 def bus():
     """
     Lazy getter/initializer for the bus object.
     """
-    global _bus  # pylint: disable=global-statement
-    if _bus is None:
-        redis_queue = get_backend('http').bus.redis_queue  # type: ignore
-        _bus = RedisBus(**get_redis_conf(), redis_queue=redis_queue)
-    return _bus
+    return _bus.bus
 
 
 def send_message(msg, wait_for_response=True):

platypush/backend/http/webapp/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.344d61f4.js"></script><script defer="defer" src="/static/js/app.6887e078.js"></script><link href="/static/css/chunk-vendors.a2412607.css" rel="stylesheet"><link href="/static/css/app.200dd9cf.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.525de538.js"></script><script defer="defer" src="/static/js/app.e68dae80.js"></script><link href="/static/css/chunk-vendors.d510eff2.css" rel="stylesheet"><link href="/static/css/app.34a0a3ba.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>