From 352d421e619045eccaeb869867d647f964f665bd Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 14 Mar 2021 00:07:17 +0100 Subject: [PATCH] Added file.monitor backend [closes #172] The file.monitor backend leverages watchdog instead of the Linux-only inotify API and it replaces the inotify backend. --- CHANGELOG.md | 5 + docs/source/backends.rst | 1 + docs/source/conf.py | 3 +- docs/source/events.rst | 1 + .../source/platypush/backend/file.monitor.rst | 5 + docs/source/platypush/events/file.rst | 5 + platypush/backend/file/__init__.py | 0 platypush/backend/file/monitor/__init__.py | 108 ++++++++++++++++++ .../backend/file/monitor/entities/__init__.py | 0 .../backend/file/monitor/entities/handlers.py | 78 +++++++++++++ .../file/monitor/entities/resources.py | 24 ++++ platypush/backend/inotify.py | 2 + platypush/message/event/file.py | 27 +++++ requirements.txt | 5 +- setup.py | 2 + 15 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 docs/source/platypush/backend/file.monitor.rst create mode 100644 docs/source/platypush/events/file.rst create mode 100644 platypush/backend/file/__init__.py create mode 100644 platypush/backend/file/monitor/__init__.py create mode 100644 platypush/backend/file/monitor/entities/__init__.py create mode 100644 platypush/backend/file/monitor/entities/handlers.py create mode 100644 platypush/backend/file/monitor/entities/resources.py create mode 100644 platypush/message/event/file.py diff --git a/CHANGELOG.md b/CHANGELOG.md index df0850f58d..23b17a124c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Given the high speed of development in the first phase, changes are being report ## [Unreleased] +### Added + +- Added `file.monitor` backend, which replaces the `inotify` backend + (see [#172](https://git.platypush.tech/platypush/platypush/-/issues/172)). + ### Removed - Removed legacy `pusher` script and `local` backend. diff --git a/docs/source/backends.rst b/docs/source/backends.rst index 00d8fcb44d..e532c7b3be 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -22,6 +22,7 @@ Backends platypush/backend/clipboard.rst platypush/backend/covid19.rst platypush/backend/dbus.rst + platypush/backend/file.monitor.rst platypush/backend/foursquare.rst platypush/backend/github.rst platypush/backend/google.fit.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index cd0ba60eb3..ebb1eae65c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,7 @@ import sys # -- Project information ----------------------------------------------------- project = 'platypush' -copyright = '2017-2020, Fabio Manganiello' +copyright = '2017-2021, Fabio Manganiello' author = 'Fabio Manganiello' # The short X.Y version @@ -265,6 +265,7 @@ autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers', 'imapclient', 'pysmartthings', 'aiohttp', + 'watchdog', ] sys.path.insert(0, os.path.abspath('../..')) diff --git a/docs/source/events.rst b/docs/source/events.rst index 4e1ce08a67..fd5507531b 100644 --- a/docs/source/events.rst +++ b/docs/source/events.rst @@ -18,6 +18,7 @@ Events platypush/events/covid19.rst platypush/events/custom.rst platypush/events/distance.rst + platypush/events/file.rst platypush/events/foursquare.rst platypush/events/geo.rst platypush/events/github.rst diff --git a/docs/source/platypush/backend/file.monitor.rst b/docs/source/platypush/backend/file.monitor.rst new file mode 100644 index 0000000000..98f330bf38 --- /dev/null +++ b/docs/source/platypush/backend/file.monitor.rst @@ -0,0 +1,5 @@ +``platypush.backend.file.monitor`` +================================== + +.. automodule:: platypush.backend.file.monitor + :members: diff --git a/docs/source/platypush/events/file.rst b/docs/source/platypush/events/file.rst new file mode 100644 index 0000000000..7438ac6711 --- /dev/null +++ b/docs/source/platypush/events/file.rst @@ -0,0 +1,5 @@ +``platypush.message.event.file`` +================================ + +.. automodule:: platypush.message.event.file + :members: diff --git a/platypush/backend/file/__init__.py b/platypush/backend/file/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/platypush/backend/file/monitor/__init__.py b/platypush/backend/file/monitor/__init__.py new file mode 100644 index 0000000000..961e845164 --- /dev/null +++ b/platypush/backend/file/monitor/__init__.py @@ -0,0 +1,108 @@ +from typing import List, Dict, Union, Any + +from watchdog.observers import Observer + +from platypush.backend import Backend +from .entities.handlers import EventHandlerFactory +from .entities.resources import MonitoredResource + + +class FileMonitorBackend(Backend): + """ + This backend monitors changes to local files and directories using the Watchdog API. + + Triggers: + + * :class:`platypush.message.event.file.FileSystemCreateEvent` if a resource is created. + * :class:`platypush.message.event.file.FileSystemDeleteEvent` if a resource is removed. + * :class:`platypush.message.event.file.FileSystemModifyEvent` if a resource is modified. + + Requires: + + * **watchdog** (``pip install watchdog``) + + """ + + def __init__(self, paths: List[Union[str, Dict[str, Any], MonitoredResource]], **kwargs): + """ + :param paths: List of paths to monitor. Paths can either be expressed in any of the following ways: + + - Simple strings. In this case, paths will be interpreted as absolute references to a file or a directory + to monitor. Example: + + .. code-block:: yaml + + backend.file.monitor: + paths: + # Monitor changes on the /tmp folder + - /tmp + # Monitor changes on /etc/passwd + - /etc/passwd + + - Path with monitoring properties expressed as a key-value object. Example showing the supported attributes: + + .. code-block:: yaml + + backend.file.monitor: + paths: + # Monitor changes on the /tmp folder and its subfolders + - path: /tmp + recursive: True + + - Path with pattern-based search criteria for the files to monitor and exclude. Example: + + .. code-block:: yaml + + backend.file.monitor: + paths: + # Recursively monitor changes on the ~/my-project folder that include all + # *.py files, excluding those whose name starts with tmp_ and + # all the files contained in the __pycache__ folders + - path: ~/my-project + recursive: True + patterns: + - "*.py" + ignore_patterns: + - "tmp_*" + ignore_directories: + - "__pycache__" + + - Path with regex-based search criteria for the files to monitor and exclude. Example: + + .. code-block:: yaml + + backend.file.monitor: + paths: + # Recursively monitor changes on the ~/my-images folder that include all + # the files matching the "\.jpe?g$" pattern in case-insensitive mode, + # excluding those whose name starts with tmp_ and + # all the files contained in the __MACOSX folders + - path: ~/my-images + recursive: True + regexes: + - ".*\.jpe?g$" + ignore_patterns: + - "^tmp_.*" + ignore_directories: + - "__MACOSX" + + """ + + super().__init__(**kwargs) + self._observer = Observer() + + for path in paths: + handler = EventHandlerFactory.from_resource(path) + self._observer.schedule(handler, handler.resource.path, recursive=handler.resource.recursive) + + def run(self): + super().run() + self.logger.info('Initializing file monitor backend') + self._observer.start() + self.wait_stop() + + def on_stop(self): + self.logger.info('Stopping file monitor backend') + self._observer.stop() + self._observer.join() + self.logger.info('Stopped file monitor backend') diff --git a/platypush/backend/file/monitor/entities/__init__.py b/platypush/backend/file/monitor/entities/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/platypush/backend/file/monitor/entities/handlers.py b/platypush/backend/file/monitor/entities/handlers.py new file mode 100644 index 0000000000..7fb8ab256b --- /dev/null +++ b/platypush/backend/file/monitor/entities/handlers.py @@ -0,0 +1,78 @@ +import os +from typing import Dict, Union, Any + +from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler, RegexMatchingEventHandler + +from platypush.backend.file.monitor.entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex +from platypush.context import get_bus +from platypush.message.event.file import FileSystemModifyEvent, FileSystemCreateEvent, FileSystemDeleteEvent + + +class EventHandler(FileSystemEventHandler): + """ + Base class for Watchdog event handlers. + """ + def __init__(self, resource: MonitoredResource, **kwargs): + super().__init__(**kwargs) + resource.path = os.path.expanduser(resource.path) + self.resource = resource + + def on_created(self, event): + get_bus().post(FileSystemCreateEvent(path=event.src_path, is_directory=event.is_directory)) + + def on_deleted(self, event): + get_bus().post(FileSystemDeleteEvent(path=event.src_path, is_directory=event.is_directory)) + + def on_modified(self, event): + get_bus().post(FileSystemModifyEvent(path=event.src_path, is_directory=event.is_directory)) + + @classmethod + def from_resource(cls, resource: MonitoredResource): + if isinstance(resource, MonitoredPattern): + return PatternEventHandler(resource) + if isinstance(resource, MonitoredRegex): + return RegexEventHandler(resource) + return cls(resource) + + +class PatternEventHandler(EventHandler, PatternMatchingEventHandler): + """ + Event handler for file patterns. + """ + def __init__(self, resource: MonitoredPattern): + super().__init__(resource=resource, + patterns=resource.patterns, + ignore_patterns=resource.ignore_patterns, + ignore_directories=resource.ignore_directories, + case_sensitive=resource.case_sensitive) + + +class RegexEventHandler(EventHandler, RegexMatchingEventHandler): + """ + Event handler for regex-based file patterns. + """ + def __init__(self, resource: MonitoredRegex): + super().__init__(resource=resource, + regexes=resource.regexes, + ignore_regexes=resource.ignore_regexes, + ignore_directories=resource.ignore_directories, + case_sensitive=resource.case_sensitive) + + +class EventHandlerFactory: + """ + Create a file system event handler from a string, dictionary or ``MonitoredResource`` resource. + """ + @staticmethod + def from_resource(resource: Union[str, Dict[str, Any], MonitoredResource]) -> EventHandler: + if isinstance(resource, str): + resource = MonitoredResource(resource) + elif isinstance(resource, dict): + if 'patterns' in resource or 'ignore_patterns' in resource: + resource = MonitoredPattern(**resource) + elif 'regexes' in resource or 'ignore_regexes' in resource: + resource = MonitoredRegex(**resource) + else: + resource = MonitoredResource(**resource) + + return EventHandler.from_resource(resource) diff --git a/platypush/backend/file/monitor/entities/resources.py b/platypush/backend/file/monitor/entities/resources.py new file mode 100644 index 0000000000..2595cfcce2 --- /dev/null +++ b/platypush/backend/file/monitor/entities/resources.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass +from typing import Optional, List + + +@dataclass +class MonitoredResource: + path: str + recursive: bool = False + + +@dataclass +class MonitoredPattern(MonitoredResource): + patterns: Optional[List[str]] = None + ignore_patterns: Optional[List[str]] = None + ignore_directories: Optional[List[str]] = None + case_sensitive: bool = True + + +@dataclass +class MonitoredRegex(MonitoredResource): + regexes: Optional[List[str]] = None + ignore_regexes: Optional[List[str]] = None + ignore_directories: Optional[List[str]] = None + case_sensitive: bool = True diff --git a/platypush/backend/inotify.py b/platypush/backend/inotify.py index da865d7c8a..d8d47bd6e6 100644 --- a/platypush/backend/inotify.py +++ b/platypush/backend/inotify.py @@ -7,6 +7,8 @@ from platypush.message.event.inotify import InotifyCreateEvent, InotifyDeleteEve class InotifyBackend(Backend): """ + **NOTE**: This backend is *deprecated* in favour of :class:`platypush.backend.file.monitor.FileMonitorBackend`. + (Linux only) This backend will listen for events on the filesystem (whether a file/directory on a watch list is opened, modified, created, deleted, closed or had its permissions changed) and will trigger a relevant event. diff --git a/platypush/message/event/file.py b/platypush/message/event/file.py new file mode 100644 index 0000000000..3aabe08740 --- /dev/null +++ b/platypush/message/event/file.py @@ -0,0 +1,27 @@ +from platypush.message.event import Event + + +class FileSystemEvent(Event): + """ + Base class for file system events - namely, file/directory creation, deletion and modification. + """ + def __init__(self, path: str, *, is_directory: bool, **kwargs): + super().__init__(path=path, is_directory=is_directory, **kwargs) + + +class FileSystemCreateEvent(FileSystemEvent): + """ + Event triggered when a monitored file or directory is created. + """ + + +class FileSystemDeleteEvent(FileSystemEvent): + """ + Event triggered when a monitored file or directory is deleted. + """ + + +class FileSystemModifyEvent(FileSystemEvent): + """ + Event triggered when a monitored file or directory is modified. + """ diff --git a/requirements.txt b/requirements.txt index 51debc4f3b..cc70ede2d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -310,4 +310,7 @@ croniter # SmartThings integration # pysmartthings -# aiohttp \ No newline at end of file +# aiohttp + +# Support for file.monitor backend +#watchdog diff --git a/setup.py b/setup.py index f61ab6ca82..4a595bd501 100755 --- a/setup.py +++ b/setup.py @@ -246,5 +246,7 @@ setup( 'vlc': ['python-vlc'], # Support for SmartThings integration 'smartthings': ['pysmartthings', 'aiohttp'], + # Support for file.monitor backend + 'filemonitor': ['watchdog'], }, )