Merge branch 'master' into snyk-fix-5ef3afe3fbfad34ca892e17f8d68fd7a

.drone/github-mirror.sh

@@ -6,9 +6,18 @@
 ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
 
 # Clone the repository
+branch=$(git rev-parse --abbrev-ref HEAD)
+if [ -z "${branch}" ]; then
+  echo "No branch checked out"
+  exit 1
+fi
+
 git remote add github git@github.com:/blacklight/platypush.git
-git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github"
+
+if (( "$branch" == "master" )); then
+  git pull --rebase github "${branch}" || echo "No such branch on Github"
+fi
 
 # Push the changes to the GitHub mirror
-git push --all -v github
+git push -f --all -v github
 git push --tags -v github

.ignore

@@ -0,0 +1 @@
+dist/

CHANGELOG.md

@@ -1,5 +1,65 @@
 # Changelog
 
+## [1.3.1]
+
+- [[#344](https://git.platypush.tech/platypush/platypush/issues/344)]: removed
+  `marshmallow_dataclass` dependency. That package isn't included in the
+  package managers of any supported distros and requires to be installed via
+  pip. Making the Platypush' system packages depend on a pip-only package is
+  not a good idea. Plus, the library seems to be still in heavy development and
+  it has already broken compatibility with at least the `system` package.
+
+## [1.3.0]
+
+- [[#333](https://git.platypush.tech/platypush/platypush/issues/333)]: new file
+  browser UI/component. It includes custom MIME type support, a file editor
+  with syntax highlight, file download and file upload.
+
+- [[#341](https://git.platypush.tech/platypush/platypush/issues/341)]:
+  procedures are now native entities that can be managed from the entities panel.
+  A new versatile procedure editor has also been added, with support for nested
+  blocks, conditions, loops, variables, context autocomplete, and more.
+
+- [`procedure`]: Added the following features to YAML/structured procedures:
+
+  - `set`: to set variables whose scope is limited to the procedure / code
+    block where they are created. `variable.set` is useful to permanently
+    store variables on the db, `variable.mset` is useful to set temporary
+    global variables in memory through Redis, but sometimes you may just want
+    to assign a value to a variable that only needs to live within a procedure,
+    event hook or cron.
+
+    ```yaml
+    - set:
+        foo: bar
+        temperature: ${output.get('temperature')}
+    ```
+
+  - `return` can now return values too when invoked within a procedure:
+
+      ```yaml
+      - return: something
+      # Or
+      - return: "Result: ${output.get('response')}"
+      ```
+
+- The default logging format is now much more compact. The full body of events
+  and requests is no longer included by default in `info` mode - instead, a
+  summary with the message type, ID and response time is logged. The full
+  payloads can still be logged by enabling `debug` logs through e.g. `-v`.
+
+## [1.2.3]
+
+- [[#422](https://git.platypush.tech/platypush/platypush/issues/422)]: adapted
+  media plugins to support streaming from the yt-dlp process. This allows
+  videos to have merged audio+video even if they had separate tracks upstream.
+
+- [`media.*`] Many improvements on the media UI.
+
+- [`zigbee.mqtt`] Removed synchronous logic from `zigbee.mqtt.device_set`. It
+  was prone to timeouts as well as pointless - the updated device state will
+  anyway be received as an event.
+
 ## [1.2.2]
 
 - Fixed regression on older version of Python that don't fully support

docker-compose.yml

@@ -1,24 +1,9 @@
 services:
   platypush:
-    restart: "always"
+    # Replace the build section with the next line if instead of building the
-    command:
+    # image from a local checkout you want to pull the latest base
-      - platypush
+    # (Alpine-based) image from the remote registry
-      # Comment --start-redis if you want to run an external Redis service
+    # image: "registry.platypush.tech/platypush:latest"
-      # In such case you'll also have to ensure that the appropriate Redis
-      # variables are set in the .env file, or the Redis configuration is
-      # passed in the config.yaml, or use the --redis-host and --redis-port
-      # command-line options
-      - --start-redis
-
-    # Custom list of host devices that should be accessible to the container -
-    # e.g. an Arduino, an ESP-compatible microcontroller, a joystick etc.
-    # devices:
-    #   - /dev/ttyUSB0
-
-    # Uncomment if you need plugins that require access to low-level hardware
-    # (e.g. Bluetooth BLE or GPIO/SPI/I2C) if access to individual devices is
-    # not enough or isn't practical
-    # privileged: true
 
     build:
       context: .
@@ -31,6 +16,25 @@ services:
       # Fedora base image
       # dockerfile: ./platypush/install/docker/fedora.Dockerfile
 
+    restart: "always"
+    command:
+      - platypush
+      - --redis-host
+      - redis
+      # Or, if you want to run Redis from the same container as Platypush,
+      # replace --redis-host redis with the line below
+      # - --start-redis
+
+    # Custom list of host devices that should be accessible to the container -
+    # e.g. an Arduino, an ESP-compatible microcontroller, a joystick etc.
+    # devices:
+    #   - /dev/ttyUSB0
+
+    # Uncomment if you need plugins that require access to low-level hardware
+    # (e.g. Bluetooth BLE or GPIO/SPI/I2C) if access to individual devices is
+    # not enough or isn't practical
+    # privileged: true
+    
     # Copy .env.example to .env and modify as needed
     # env_file:
     #   - .env
@@ -40,7 +44,13 @@ services:
       # expose it
       - "8008:8008"
 
-    volumes:
+    # volumes:
-      - /path/to/your/config.yaml:/etc/platypush
+      # Replace with a path that contains/will contain your config.yaml file
-      - /path/to/a/workdir:/var/lib/platypush
+      # - /path/to/your/config:/etc/platypush
+      # Replace with a path that contains/will contain your working directory
+      # - /path/to/a/workdir:/var/lib/platypush
+      # Optionally, use an external volume for the cache
       # - /path/to/a/cachedir:/var/cache/platypush
+
+  redis:
+    image: redis

docs/source/platypush/plugins/media.omxplayer.rst

@@ -1,6 +0,0 @@
-``media.omxplayer``
-=====================================
-
-.. automodule:: platypush.plugins.media.omxplayer
-    :members:
-

docs/source/platypush/plugins/procedures.rst

@@ -0,0 +1,5 @@
+``procedures``
+==============
+
+.. automodule:: platypush.plugins.procedures
+    :members:

docs/source/plugins.rst

@@ -77,7 +77,6 @@ Plugins
     platypush/plugins/media.kodi.rst
     platypush/plugins/media.mplayer.rst
     platypush/plugins/media.mpv.rst
-    platypush/plugins/media.omxplayer.rst
     platypush/plugins/media.plex.rst
     platypush/plugins/media.subtitles.rst
     platypush/plugins/media.vlc.rst
@@ -100,6 +99,7 @@ Plugins
     platypush/plugins/otp.rst
     platypush/plugins/pihole.rst
     platypush/plugins/ping.rst
+    platypush/plugins/procedures.rst
     platypush/plugins/pushbullet.rst
     platypush/plugins/pwm.pca9685.rst
     platypush/plugins/qrcode.rst

platypush/__init__.py

@@ -21,7 +21,7 @@ from .utils import run
 # see https://git.platypush.tech/platypush/platypush/issues/399
 when = hook
 
-__version__ = '1.2.2'
+__version__ = '1.3.1'
 __author__ = 'Fabio Manganiello <fabio@manganiello.tech>'
 __all__ = [
     'Application',

platypush/app/_app.py

@@ -365,7 +365,13 @@ class Application:
             elif isinstance(msg, Response):
                 msg.log()
             elif isinstance(msg, Event):
-                msg.log()
+                log.info(
+                    'Received event: %s.%s[id=%s]',
+                    msg.__class__.__module__,
+                    msg.__class__.__name__,
+                    msg.id,
+                )
+                msg.log(level=logging.DEBUG)
                 self.event_processor.process_event(msg)
 
         return _f

platypush/backend/http/app/streaming/plugins/file.py

@@ -1,7 +1,8 @@
 import os
+import pathlib
 from contextlib import contextmanager
 from datetime import datetime as dt
-from typing import Optional, Tuple
+from typing import IO, Optional, Tuple
 
 from tornado.web import stream_request_body
 
@@ -17,6 +18,8 @@ class FileRoute(StreamingRoute):
     """
 
     BUFSIZE = 1024
+    _bytes_written = 0
+    _out_f: Optional[IO[bytes]] = None
 
     @classmethod
     def path(cls) -> str:
@@ -39,6 +42,10 @@ class FileRoute(StreamingRoute):
     def file_size(self) -> int:
         return os.path.getsize(self.file_path)
 
+    @property
+    def _content_length(self) -> int:
+        return int(self.request.headers.get('Content-Length', 0))
+
     @property
     def range(self) -> Tuple[Optional[int], Optional[int]]:
         range_hdr = self.request.headers.get('Range')
@@ -105,6 +112,77 @@ class FileRoute(StreamingRoute):
 
         self.finish()
 
+    def on_finish(self) -> None:
+        if self._out_f:
+            try:
+                if not (self._out_f and self._out_f.closed):
+                    self._out_f.close()
+            except Exception as e:
+                self.logger.warning('Error while closing the output file: %s', e)
+
+            self._out_f = None
+
+        return super().on_finish()
+
+    def _validate_upload(self, force: bool = False) -> bool:
+        if not self.file_path:
+            self.write_error(400, 'Missing path argument')
+            return False
+
+        if not self._out_f:
+            if not force and os.path.exists(self.file_path):
+                self.write_error(409, f'{self.file_path} already exists')
+                return False
+
+            self._bytes_written = 0
+            dir_path = os.path.dirname(self.file_path)
+
+            try:
+                pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
+                self._out_f = open(  # pylint: disable=consider-using-with
+                    self.file_path, 'wb'
+                )
+            except PermissionError:
+                self.write_error(403, 'Permission denied')
+                return False
+
+        return True
+
+    def finish(self, *args, **kwargs):  # type: ignore
+        try:
+            return super().finish(*args, **kwargs)
+        except Exception as e:
+            self.logger.warning('Error while finishing the request: %s', e)
+
+    def data_received(self, chunk: bytes):
+        # Ignore unless we're in POST/PUT mode
+        if self.request.method not in ('POST', 'PUT'):
+            return
+
+        force = self.request.method == 'PUT'
+        if not self._validate_upload(force=force):
+            self.finish()
+            return
+
+        if not chunk:
+            self.logger.debug('Received EOF from client')
+            self.finish()
+            return
+
+        assert self._out_f
+        self._out_f.write(chunk)
+        self._out_f.flush()
+        self._bytes_written += len(chunk)
+        self.logger.debug(
+            'Written chunk of size %d to %s, progress: %d/%d',
+            len(chunk),
+            self.file_path,
+            self._bytes_written,
+            self._content_length,
+        )
+
+        self.flush()
+
     def get(self) -> None:
         with self._serve() as f:
             if f:
@@ -119,3 +197,9 @@ class FileRoute(StreamingRoute):
     def head(self) -> None:
         with self._serve():
             pass
+
+    def post(self) -> None:
+        self.logger.info('Receiving file POST upload request for %r', self.file_path)
+
+    def put(self) -> None:
+        self.logger.info('Receiving file PUT upload request for %r', self.file_path)

platypush/backend/http/app/streaming/plugins/media/_register.py

@@ -3,7 +3,9 @@ from typing import Optional
 from platypush.backend.http.app.utils import logger, send_request
 from platypush.backend.http.media.handlers import MediaHandler
 
-from ._registry import load_media_map, save_media_map
+from ._registry import clear_media_map, load_media_map, save_media_map
+
+_init = False
 
 
 def get_media_url(media_id: str) -> str:
@@ -17,6 +19,12 @@ def register_media(source: str, subtitles: Optional[str] = None) -> MediaHandler
     """
     Registers a media file and returns its associated media handler.
     """
+    global _init
+
+    if not _init:
+        clear_media_map()
+        _init = True
+
     media_id = MediaHandler.get_media_id(source)
     media_url = get_media_url(media_id)
     media_map = load_media_map()

platypush/backend/http/app/streaming/plugins/media/_registry.py

@@ -25,10 +25,15 @@ def load_media_map() -> MediaMap:
             logger().warning('Could not load media map: %s', e)
             return {}
 
-    return {
+    parsed_map = {}
-        media_id: MediaHandler.build(**media_info)
+    for media_id, media_info in media_map.items():
-        for media_id, media_info in media_map.items()
+        try:
-    }
+            parsed_map[media_id] = MediaHandler.build(**media_info)
+        except Exception as e:
+            logger().debug('Could not load media %s: %s', media_id, e)
+            continue
+
+    return parsed_map
 
 
 def save_media_map(new_map: MediaMap):
@@ -38,3 +43,12 @@ def save_media_map(new_map: MediaMap):
     with media_map_lock:
         redis = get_redis()
         redis.mset({MEDIA_MAP_VAR: json.dumps(new_map, cls=Message.Encoder)})
+
+
+def clear_media_map():
+    """
+    Clears the media map from the server.
+    """
+    with media_map_lock:
+        redis = get_redis()
+        redis.delete(MEDIA_MAP_VAR)

platypush/backend/http/app/streaming/plugins/media/_stream.py

@@ -17,7 +17,7 @@ class MediaStreamRoute(StreamingRoute):
     Route for media streams.
     """
 
-    SUPPORTED_METHODS = ['GET', 'PUT', 'DELETE']
+    SUPPORTED_METHODS = ['GET', 'HEAD', 'PUT', 'DELETE']
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -50,6 +50,23 @@ class MediaStreamRoute(StreamingRoute):
         except Exception as e:
             self._on_error(e)
 
+    def head(self, media_id: Optional[str] = None):
+        """
+        Streams a media resource by ID.
+        """
+
+        if not media_id:
+            self.finish()
+            return
+
+        # Strip the extension
+        media_id = '.'.join(media_id.split('.')[:-1])
+
+        try:
+            self.stream_media(media_id, head=True)
+        except Exception as e:
+            self._on_error(e)
+
     def put(self, *_, **__):
         """
         The `PUT` route is used to prepare a new media resource for streaming.
@@ -93,10 +110,10 @@ class MediaStreamRoute(StreamingRoute):
         """
         Returns the list of registered media resources.
         """
-        self.add_header('Content-Type', 'application/json')
+        self.set_header('Content-Type', 'application/json')
         self.finish(json.dumps([dict(media) for media in load_media_map().values()]))
 
-    def stream_media(self, media_id: str):
+    def stream_media(self, media_id: str, head: bool = False):
         """
         Route to stream a media file given its ID.
         """
@@ -107,11 +124,11 @@ class MediaStreamRoute(StreamingRoute):
         range_hdr = self.request.headers.get('Range')
         content_length = media_hndl.content_length
 
-        self.add_header('Accept-Ranges', 'bytes')
+        self.set_header('Accept-Ranges', 'bytes')
-        self.add_header('Content-Type', media_hndl.mime_type)
+        self.set_header('Content-Type', media_hndl.mime_type)
 
         if 'download' in self.request.arguments:
-            self.add_header(
+            self.set_header(
                 'Content-Disposition',
                 'attachment'
                 + ('; filename="{media_hndl.filename}"' if media_hndl.filename else ''),
@@ -129,7 +146,7 @@ class MediaStreamRoute(StreamingRoute):
                 content_length = to_bytes - from_bytes
 
             self.set_status(206)
-            self.add_header(
+            self.set_header(
                 'Content-Range',
                 f'bytes {from_bytes}-{to_bytes}/{media_hndl.content_length}',
             )
@@ -137,7 +154,13 @@ class MediaStreamRoute(StreamingRoute):
             from_bytes = 0
             to_bytes = STREAMING_BLOCK_SIZE
 
-        self.add_header('Content-Length', str(content_length))
+        self.set_header('Content-Length', str(content_length))
+
+        if head:
+            self.flush()
+            self.finish()
+            return
+
         for chunk in media_hndl.get_data(
             from_bytes=from_bytes,
             to_bytes=to_bytes,

platypush/backend/http/media/handlers/__init__.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 import hashlib
 import logging
-import os
 from typing import Generator, Optional
 
 from platypush.message import JSONAble
@@ -57,9 +56,6 @@ class MediaHandler(JSONAble, ABC):
                 logging.exception(e)
                 errors[hndl_class.__name__] = str(e)
 
-        if os.path.exists(source):
-            source = f'file://{source}'
-
         raise AttributeError(
             f'The source {source} has no handlers associated. Errors: {errors}'
         )

platypush/backend/http/media/handlers/file.py

@@ -15,6 +15,9 @@ class FileHandler(MediaHandler):
     prefix_handlers = ['file://']
 
     def __init__(self, source, *args, **kwargs):
+        if isinstance(source, str) and os.path.exists(source):
+            source = f'file://{source}'
+
         super().__init__(source, *args, **kwargs)
 
         self.path = os.path.abspath(
@@ -33,7 +36,7 @@ class FileHandler(MediaHandler):
         ), f'{source} is not a valid media file (detected format: {self.mime_type})'
 
         self.extension = mimetypes.guess_extension(self.mime_type)
-        if self.url and self.extension:
+        if self.url and self.extension and not self.url.endswith(self.extension):
             self.url += self.extension
         self.content_length = os.path.getsize(self.path)
 

platypush/backend/http/webapp/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.05911ac4.js"></script><script defer="defer" src="/static/js/app.1f786d8c.js"></script><link href="/static/css/chunk-vendors.d510eff2.css" rel="stylesheet"><link href="/static/css/app.d1412c5b.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.5645dfdc.js"></script><script defer="defer" src="/static/js/app.d90fb573.js"></script><link href="/static/css/chunk-vendors.d510eff2.css" rel="stylesheet"><link href="/static/css/app.70fb1f4a.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>