[#341] Backend implementation of the new procedure entities architecture.

Closes: #333

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [1.2.3]
+
+- [[#422](https://git.platypush.tech/platypush/platypush/issues/422)]: adapted
+  media plugins to support streaming from the yt-dlp process. This allows
+  videos to have merged audio+video even if they had separate tracks upstream.
+
+- [`media.*`] Many improvements on the media UI.
+
+- [`zigbee.mqtt`] Removed synchronous logic from `zigbee.mqtt.device_set`. It
+  was prone to timeouts as well as pointless - the updated device state will
+  anyway be received as an event.
+
 ## [1.2.2]
 
 - Fixed regression on older version of Python that don't fully support

docs/source/platypush/plugins/media.omxplayer.rst

@@ -1,6 +0,0 @@
-``media.omxplayer``
-=====================================
-
-.. automodule:: platypush.plugins.media.omxplayer
-    :members:
-

docs/source/plugins.rst

@@ -77,7 +77,6 @@ Plugins
     platypush/plugins/media.kodi.rst
     platypush/plugins/media.mplayer.rst
     platypush/plugins/media.mpv.rst
-    platypush/plugins/media.omxplayer.rst
     platypush/plugins/media.plex.rst
     platypush/plugins/media.subtitles.rst
     platypush/plugins/media.vlc.rst

platypush/__init__.py

@@ -21,7 +21,7 @@ from .utils import run
 # see https://git.platypush.tech/platypush/platypush/issues/399
 when = hook
 
-__version__ = '1.2.2'
+__version__ = '1.2.3'
 __author__ = 'Fabio Manganiello <fabio@manganiello.tech>'
 __all__ = [
     'Application',

platypush/backend/http/app/streaming/plugins/file.py

@@ -1,7 +1,8 @@
 import os
+import pathlib
 from contextlib import contextmanager
 from datetime import datetime as dt
-from typing import Optional, Tuple
+from typing import IO, Optional, Tuple
 
 from tornado.web import stream_request_body
 
@@ -17,6 +18,8 @@ class FileRoute(StreamingRoute):
     """
 
     BUFSIZE = 1024
+    _bytes_written = 0
+    _out_f: Optional[IO[bytes]] = None
 
     @classmethod
     def path(cls) -> str:
@@ -39,6 +42,10 @@ class FileRoute(StreamingRoute):
     def file_size(self) -> int:
         return os.path.getsize(self.file_path)
 
+    @property
+    def _content_length(self) -> int:
+        return int(self.request.headers.get('Content-Length', 0))
+
     @property
     def range(self) -> Tuple[Optional[int], Optional[int]]:
         range_hdr = self.request.headers.get('Range')
@@ -105,6 +112,77 @@ class FileRoute(StreamingRoute):
 
         self.finish()
 
+    def on_finish(self) -> None:
+        if self._out_f:
+            try:
+                if not (self._out_f and self._out_f.closed):
+                    self._out_f.close()
+            except Exception as e:
+                self.logger.warning('Error while closing the output file: %s', e)
+
+            self._out_f = None
+
+        return super().on_finish()
+
+    def _validate_upload(self, force: bool = False) -> bool:
+        if not self.file_path:
+            self.write_error(400, 'Missing path argument')
+            return False
+
+        if not self._out_f:
+            if not force and os.path.exists(self.file_path):
+                self.write_error(409, f'{self.file_path} already exists')
+                return False
+
+            self._bytes_written = 0
+            dir_path = os.path.dirname(self.file_path)
+
+            try:
+                pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
+                self._out_f = open(  # pylint: disable=consider-using-with
+                    self.file_path, 'wb'
+                )
+            except PermissionError:
+                self.write_error(403, 'Permission denied')
+                return False
+
+        return True
+
+    def finish(self, *args, **kwargs):  # type: ignore
+        try:
+            return super().finish(*args, **kwargs)
+        except Exception as e:
+            self.logger.warning('Error while finishing the request: %s', e)
+
+    def data_received(self, chunk: bytes):
+        # Ignore unless we're in POST/PUT mode
+        if self.request.method not in ('POST', 'PUT'):
+            return
+
+        force = self.request.method == 'PUT'
+        if not self._validate_upload(force=force):
+            self.finish()
+            return
+
+        if not chunk:
+            self.logger.debug('Received EOF from client')
+            self.finish()
+            return
+
+        assert self._out_f
+        self._out_f.write(chunk)
+        self._out_f.flush()
+        self._bytes_written += len(chunk)
+        self.logger.debug(
+            'Written chunk of size %d to %s, progress: %d/%d',
+            len(chunk),
+            self.file_path,
+            self._bytes_written,
+            self._content_length,
+        )
+
+        self.flush()
+
     def get(self) -> None:
         with self._serve() as f:
             if f:
@@ -119,3 +197,9 @@ class FileRoute(StreamingRoute):
     def head(self) -> None:
         with self._serve():
             pass
+
+    def post(self) -> None:
+        self.logger.info('Receiving file POST upload request for %r', self.file_path)
+
+    def put(self) -> None:
+        self.logger.info('Receiving file PUT upload request for %r', self.file_path)

platypush/backend/http/app/streaming/plugins/media/_registry.py

@@ -25,10 +25,15 @@ def load_media_map() -> MediaMap:
             logger().warning('Could not load media map: %s', e)
             return {}
 
-    return {
-        media_id: MediaHandler.build(**media_info)
-        for media_id, media_info in media_map.items()
-    }
+    parsed_map = {}
+    for media_id, media_info in media_map.items():
+        try:
+            parsed_map[media_id] = MediaHandler.build(**media_info)
+        except Exception as e:
+            logger().debug('Could not load media %s: %s', media_id, e)
+            continue
+
+    return parsed_map
 
 
 def save_media_map(new_map: MediaMap):

platypush/backend/http/media/handlers/__init__.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 import hashlib
 import logging
-import os
 from typing import Generator, Optional
 
 from platypush.message import JSONAble
@@ -57,9 +56,6 @@ class MediaHandler(JSONAble, ABC):
                 logging.exception(e)
                 errors[hndl_class.__name__] = str(e)
 
-        if os.path.exists(source):
-            source = f'file://{source}'
-
         raise AttributeError(
             f'The source {source} has no handlers associated. Errors: {errors}'
         )

platypush/backend/http/media/handlers/file.py

@@ -15,6 +15,9 @@ class FileHandler(MediaHandler):
     prefix_handlers = ['file://']
 
     def __init__(self, source, *args, **kwargs):
+        if isinstance(source, str) and os.path.exists(source):
+            source = f'file://{source}'
+
         super().__init__(source, *args, **kwargs)
 
         self.path = os.path.abspath(

platypush/backend/http/webapp/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.05911ac4.js"></script><script defer="defer" src="/static/js/app.1f786d8c.js"></script><link href="/static/css/chunk-vendors.d510eff2.css" rel="stylesheet"><link href="/static/css/app.d1412c5b.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><!--[if IE]><link rel="icon" href="/favicon.ico"><![endif]--><link rel="stylesheet" href="/fonts/poppins.css"><title>platypush</title><script defer="defer" src="/static/js/chunk-vendors.d9b38fb8.js"></script><script defer="defer" src="/static/js/app.81716459.js"></script><link href="/static/css/chunk-vendors.d510eff2.css" rel="stylesheet"><link href="/static/css/app.92a2d867.css" rel="stylesheet"><link rel="icon" type="image/svg+xml" href="/img/icons/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/img/icons/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/img/icons/favicon-16x16.png"><link rel="manifest" href="/manifest.json"><meta name="theme-color" content="#ffffff"><meta name="apple-mobile-web-app-capable" content="no"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="apple-mobile-web-app-title" content="Platypush"><link rel="apple-touch-icon" href="/img/icons/apple-touch-icon-152x152.png"><link rel="mask-icon" href="/img/icons/safari-pinned-tab.svg" color="#ffffff"><meta name="msapplication-TileImage" content="/img/icons/msapplication-icon-144x144.png"><meta name="msapplication-TileColor" content="#000000"></head><body><noscript><strong>We're sorry but platypush doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>