Added file.monitor backend [closes #172]

The file.monitor backend leverages watchdog instead of the Linux-only
inotify API and it replaces the inotify backend.

CHANGELOG.md

@@ -5,6 +5,11 @@ Given the high speed of development in the first phase, changes are being report
 
 ## [Unreleased]
 
+### Added
+
+- Added `file.monitor` backend, which replaces the `inotify` backend
+  (see [#172](https://git.platypush.tech/platypush/platypush/-/issues/172)).
+
 ### Removed
 
 - Removed legacy `pusher` script and `local` backend.

docs/source/backends.rst

@@ -22,6 +22,7 @@ Backends
     platypush/backend/clipboard.rst
     platypush/backend/covid19.rst
     platypush/backend/dbus.rst
+    platypush/backend/file.monitor.rst
     platypush/backend/foursquare.rst
     platypush/backend/github.rst
     platypush/backend/google.fit.rst

docs/source/conf.py

@@ -23,7 +23,7 @@ import sys
 # -- Project information -----------------------------------------------------
 
 project = 'platypush'
-copyright = '2017-2020, Fabio Manganiello'
+copyright = '2017-2021, Fabio Manganiello'
 author = 'Fabio Manganiello'
 
 # The short X.Y version
@@ -265,6 +265,7 @@ autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers',
                         'imapclient',
                         'pysmartthings',
                         'aiohttp',
+                        'watchdog',
                         ]
 
 sys.path.insert(0, os.path.abspath('../..'))

docs/source/events.rst

@@ -18,6 +18,7 @@ Events
     platypush/events/covid19.rst
     platypush/events/custom.rst
     platypush/events/distance.rst
+    platypush/events/file.rst
     platypush/events/foursquare.rst
     platypush/events/geo.rst
     platypush/events/github.rst

docs/source/platypush/backend/file.monitor.rst

@@ -0,0 +1,5 @@
+``platypush.backend.file.monitor``
+==================================
+
+.. automodule:: platypush.backend.file.monitor
+    :members:

docs/source/platypush/events/file.rst

@@ -0,0 +1,5 @@
+``platypush.message.event.file``
+================================
+
+.. automodule:: platypush.message.event.file
+    :members:

platypush/backend/file/monitor/__init__.py

@@ -0,0 +1,108 @@
+from typing import List, Dict, Union, Any
+
+from watchdog.observers import Observer
+
+from platypush.backend import Backend
+from .entities.handlers import EventHandlerFactory
+from .entities.resources import MonitoredResource
+
+
+class FileMonitorBackend(Backend):
+    """
+    This backend monitors changes to local files and directories using the Watchdog API.
+
+    Triggers:
+
+        * :class:`platypush.message.event.file.FileSystemCreateEvent` if a resource is created.
+        * :class:`platypush.message.event.file.FileSystemDeleteEvent` if a resource is removed.
+        * :class:`platypush.message.event.file.FileSystemModifyEvent` if a resource is modified.
+
+    Requires:
+
+        * **watchdog** (``pip install watchdog``)
+
+    """
+
+    def __init__(self, paths: List[Union[str, Dict[str, Any], MonitoredResource]], **kwargs):
+        """
+        :param paths: List of paths to monitor. Paths can either be expressed in any of the following ways:
+
+            - Simple strings. In this case, paths will be interpreted as absolute references to a file or a directory
+              to monitor. Example:
+
+              .. code-block:: yaml
+
+                backend.file.monitor:
+                    paths:
+                        # Monitor changes on the /tmp folder
+                        - /tmp
+                        # Monitor changes on /etc/passwd
+                        - /etc/passwd
+
+            - Path with monitoring properties expressed as a key-value object. Example showing the supported attributes:
+
+                .. code-block:: yaml
+
+                    backend.file.monitor:
+                        paths:
+                            # Monitor changes on the /tmp folder and its subfolders
+                            - path: /tmp
+                              recursive: True
+
+            - Path with pattern-based search criteria for the files to monitor and exclude. Example:
+
+                .. code-block:: yaml
+
+                    backend.file.monitor:
+                        paths:
+                            # Recursively monitor changes on the ~/my-project folder that include all
+                            # *.py files, excluding those whose name starts with tmp_ and
+                            # all the files contained in the __pycache__ folders
+                            - path: ~/my-project
+                              recursive: True
+                              patterns:
+                                - "*.py"
+                              ignore_patterns:
+                                - "tmp_*"
+                              ignore_directories:
+                                - "__pycache__"
+
+            - Path with regex-based search criteria for the files to monitor and exclude. Example:
+
+                .. code-block:: yaml
+
+                    backend.file.monitor:
+                        paths:
+                            # Recursively monitor changes on the ~/my-images folder that include all
+                            # the files matching the "\.jpe?g$" pattern in case-insensitive mode,
+                            # excluding those whose name starts with tmp_ and
+                            # all the files contained in the __MACOSX folders
+                            - path: ~/my-images
+                              recursive: True
+                              regexes:
+                                - ".*\.jpe?g$"
+                              ignore_patterns:
+                                - "^tmp_.*"
+                              ignore_directories:
+                                - "__MACOSX"
+
+        """
+
+        super().__init__(**kwargs)
+        self._observer = Observer()
+
+        for path in paths:
+            handler = EventHandlerFactory.from_resource(path)
+            self._observer.schedule(handler, handler.resource.path, recursive=handler.resource.recursive)
+
+    def run(self):
+        super().run()
+        self.logger.info('Initializing file monitor backend')
+        self._observer.start()
+        self.wait_stop()
+
+    def on_stop(self):
+        self.logger.info('Stopping file monitor backend')
+        self._observer.stop()
+        self._observer.join()
+        self.logger.info('Stopped file monitor backend')

platypush/backend/file/monitor/entities/handlers.py

@@ -0,0 +1,78 @@
+import os
+from typing import Dict, Union, Any
+
+from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler, RegexMatchingEventHandler
+
+from platypush.backend.file.monitor.entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex
+from platypush.context import get_bus
+from platypush.message.event.file import FileSystemModifyEvent, FileSystemCreateEvent, FileSystemDeleteEvent
+
+
+class EventHandler(FileSystemEventHandler):
+    """
+    Base class for Watchdog event handlers.
+    """
+    def __init__(self, resource: MonitoredResource, **kwargs):
+        super().__init__(**kwargs)
+        resource.path = os.path.expanduser(resource.path)
+        self.resource = resource
+
+    def on_created(self, event):
+        get_bus().post(FileSystemCreateEvent(path=event.src_path, is_directory=event.is_directory))
+
+    def on_deleted(self, event):
+        get_bus().post(FileSystemDeleteEvent(path=event.src_path, is_directory=event.is_directory))
+
+    def on_modified(self, event):
+        get_bus().post(FileSystemModifyEvent(path=event.src_path, is_directory=event.is_directory))
+
+    @classmethod
+    def from_resource(cls, resource: MonitoredResource):
+        if isinstance(resource, MonitoredPattern):
+            return PatternEventHandler(resource)
+        if isinstance(resource, MonitoredRegex):
+            return RegexEventHandler(resource)
+        return cls(resource)
+
+
+class PatternEventHandler(EventHandler, PatternMatchingEventHandler):
+    """
+    Event handler for file patterns.
+    """
+    def __init__(self, resource: MonitoredPattern):
+        super().__init__(resource=resource,
+                         patterns=resource.patterns,
+                         ignore_patterns=resource.ignore_patterns,
+                         ignore_directories=resource.ignore_directories,
+                         case_sensitive=resource.case_sensitive)
+
+
+class RegexEventHandler(EventHandler, RegexMatchingEventHandler):
+    """
+    Event handler for regex-based file patterns.
+    """
+    def __init__(self, resource: MonitoredRegex):
+        super().__init__(resource=resource,
+                         regexes=resource.regexes,
+                         ignore_regexes=resource.ignore_regexes,
+                         ignore_directories=resource.ignore_directories,
+                         case_sensitive=resource.case_sensitive)
+
+
+class EventHandlerFactory:
+    """
+    Create a file system event handler from a string, dictionary or ``MonitoredResource`` resource.
+    """
+    @staticmethod
+    def from_resource(resource: Union[str, Dict[str, Any], MonitoredResource]) -> EventHandler:
+        if isinstance(resource, str):
+            resource = MonitoredResource(resource)
+        elif isinstance(resource, dict):
+            if 'patterns' in resource or 'ignore_patterns' in resource:
+                resource = MonitoredPattern(**resource)
+            elif 'regexes' in resource or 'ignore_regexes' in resource:
+                resource = MonitoredRegex(**resource)
+            else:
+                resource = MonitoredResource(**resource)
+
+        return EventHandler.from_resource(resource)

platypush/backend/file/monitor/entities/resources.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import Optional, List
+
+
+@dataclass
+class MonitoredResource:
+    path: str
+    recursive: bool = False
+
+
+@dataclass
+class MonitoredPattern(MonitoredResource):
+    patterns: Optional[List[str]] = None
+    ignore_patterns: Optional[List[str]] = None
+    ignore_directories: Optional[List[str]] = None
+    case_sensitive: bool = True
+
+
+@dataclass
+class MonitoredRegex(MonitoredResource):
+    regexes: Optional[List[str]] = None
+    ignore_regexes: Optional[List[str]] = None
+    ignore_directories: Optional[List[str]] = None
+    case_sensitive: bool = True

platypush/backend/inotify.py

@@ -7,6 +7,8 @@ from platypush.message.event.inotify import InotifyCreateEvent, InotifyDeleteEve
 
 class InotifyBackend(Backend):
     """
+    **NOTE**: This backend is *deprecated* in favour of :class:`platypush.backend.file.monitor.FileMonitorBackend`.
+
     (Linux only) This backend will listen for events on the filesystem (whether
     a file/directory on a watch list is opened, modified, created, deleted,
     closed or had its permissions changed) and will trigger a relevant event.

platypush/message/event/file.py

@@ -0,0 +1,27 @@
+from platypush.message.event import Event
+
+
+class FileSystemEvent(Event):
+    """
+    Base class for file system events - namely, file/directory creation, deletion and modification.
+    """
+    def __init__(self, path: str, *, is_directory: bool, **kwargs):
+        super().__init__(path=path, is_directory=is_directory, **kwargs)
+
+
+class FileSystemCreateEvent(FileSystemEvent):
+    """
+    Event triggered when a monitored file or directory is created.
+    """
+
+
+class FileSystemDeleteEvent(FileSystemEvent):
+    """
+    Event triggered when a monitored file or directory is deleted.
+    """
+
+
+class FileSystemModifyEvent(FileSystemEvent):
+    """
+    Event triggered when a monitored file or directory is modified.
+    """

requirements.txt

@@ -311,3 +311,6 @@ croniter
 # SmartThings integration
 # pysmartthings
 # aiohttp
+
+# Support for file.monitor backend
+#watchdog

setup.py

@@ -246,5 +246,7 @@ setup(
         'vlc': ['python-vlc'],
         # Support for SmartThings integration
         'smartthings': ['pysmartthings', 'aiohttp'],
+        # Support for file.monitor backend
+        'filemonitor': ['watchdog'],
     },
 )