From 1843ab224b3d8f0f6b1b3c69cc7b9f476b78bfd1 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Dec 2023 02:42:51 +0100 Subject: [PATCH] [#289] Converted `backend.file.monitor` into a runnable plugin. Closes: #289 --- docs/source/backends.rst | 2 - .../source/platypush/backend/file.monitor.rst | 5 - docs/source/platypush/backend/ping.rst | 5 - .../source/platypush/plugins/file.monitor.rst | 5 + docs/source/platypush/responses/ping.rst | 5 - docs/source/plugins.rst | 1 + docs/source/responses.rst | 1 - platypush/backend/file/monitor/__init__.py | 127 ---------------- .../backend/file/monitor/entities/__init__.py | 0 platypush/backend/file/monitor/manifest.yaml | 18 --- platypush/config/config.yaml | 24 +++ platypush/plugins/file/monitor/__init__.py | 140 ++++++++++++++++++ .../file/monitor/entities}/__init__.py | 0 .../file/monitor/entities/handlers.py | 64 +++++--- .../file/monitor/entities/resources.py | 0 platypush/plugins/file/monitor/manifest.yaml | 18 +++ 16 files changed, 231 insertions(+), 184 deletions(-) delete mode 100644 docs/source/platypush/backend/file.monitor.rst delete mode 100644 docs/source/platypush/backend/ping.rst create mode 100644 docs/source/platypush/plugins/file.monitor.rst delete mode 100644 docs/source/platypush/responses/ping.rst delete mode 100644 platypush/backend/file/monitor/__init__.py delete mode 100644 platypush/backend/file/monitor/entities/__init__.py delete mode 100644 platypush/backend/file/monitor/manifest.yaml create mode 100644 platypush/plugins/file/monitor/__init__.py rename platypush/{backend/file => plugins/file/monitor/entities}/__init__.py (100%) rename platypush/{backend => plugins}/file/monitor/entities/handlers.py (58%) rename platypush/{backend => plugins}/file/monitor/entities/resources.py (100%) create mode 100644 platypush/plugins/file/monitor/manifest.yaml diff --git a/docs/source/backends.rst b/docs/source/backends.rst index f56aa4f3b5..2ac97b1c4c 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -11,7 +11,6 @@ Backends platypush/backend/button.flic.rst platypush/backend/camera.pi.rst platypush/backend/chat.telegram.rst - platypush/backend/file.monitor.rst platypush/backend/foursquare.rst platypush/backend/github.rst platypush/backend/google.fit.rst @@ -28,7 +27,6 @@ Backends platypush/backend/nextcloud.rst platypush/backend/nfc.rst platypush/backend/nodered.rst - platypush/backend/ping.rst platypush/backend/pushbullet.rst platypush/backend/redis.rst platypush/backend/scard.rst diff --git a/docs/source/platypush/backend/file.monitor.rst b/docs/source/platypush/backend/file.monitor.rst deleted file mode 100644 index 2ad678c654..0000000000 --- a/docs/source/platypush/backend/file.monitor.rst +++ /dev/null @@ -1,5 +0,0 @@ -``file.monitor`` -================================== - -.. automodule:: platypush.backend.file.monitor - :members: diff --git a/docs/source/platypush/backend/ping.rst b/docs/source/platypush/backend/ping.rst deleted file mode 100644 index 7733d6d8d1..0000000000 --- a/docs/source/platypush/backend/ping.rst +++ /dev/null @@ -1,5 +0,0 @@ -``ping`` -========================== - -.. automodule:: platypush.backend.ping - :members: diff --git a/docs/source/platypush/plugins/file.monitor.rst b/docs/source/platypush/plugins/file.monitor.rst new file mode 100644 index 0000000000..6e07eb588b --- /dev/null +++ b/docs/source/platypush/plugins/file.monitor.rst @@ -0,0 +1,5 @@ +``file.monitor`` +================ + +.. automodule:: platypush.plugins.file.monitor + :members: diff --git a/docs/source/platypush/responses/ping.rst b/docs/source/platypush/responses/ping.rst deleted file mode 100644 index 5437922238..0000000000 --- a/docs/source/platypush/responses/ping.rst +++ /dev/null @@ -1,5 +0,0 @@ -``ping`` -=================================== - -.. automodule:: platypush.message.response.ping - :members: diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index d4f63b1b02..1b9a0cd8b9 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -33,6 +33,7 @@ Plugins platypush/plugins/esp.rst platypush/plugins/ffmpeg.rst platypush/plugins/file.rst + platypush/plugins/file.monitor.rst platypush/plugins/foursquare.rst platypush/plugins/google.calendar.rst platypush/plugins/google.drive.rst diff --git a/docs/source/responses.rst b/docs/source/responses.rst index fcae85e46e..0b929f6b49 100644 --- a/docs/source/responses.rst +++ b/docs/source/responses.rst @@ -11,7 +11,6 @@ Responses platypush/responses/chat.telegram.rst platypush/responses/google.drive.rst platypush/responses/pihole.rst - platypush/responses/ping.rst platypush/responses/printer.cups.rst platypush/responses/qrcode.rst platypush/responses/ssh.rst diff --git a/platypush/backend/file/monitor/__init__.py b/platypush/backend/file/monitor/__init__.py deleted file mode 100644 index b52ad8df01..0000000000 --- a/platypush/backend/file/monitor/__init__.py +++ /dev/null @@ -1,127 +0,0 @@ -from typing import Iterable, Dict, Union, Any - -from watchdog.observers import Observer - -from platypush.backend import Backend -from .entities.handlers import EventHandler -from .entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex - - -class FileMonitorBackend(Backend): - """ - This backend monitors changes to local files and directories using the Watchdog API. - """ - - class EventHandlerFactory: - """ - Create a file system event handler from a string, dictionary or ``MonitoredResource`` resource. - """ - - @staticmethod - def from_resource( - resource: Union[str, Dict[str, Any], MonitoredResource] - ) -> EventHandler: - if isinstance(resource, str): - resource = MonitoredResource(resource) - elif isinstance(resource, dict): - if 'regexes' in resource or 'ignore_regexes' in resource: - resource = MonitoredRegex(**resource) - elif ( - 'patterns' in resource - or 'ignore_patterns' in resource - or 'ignore_directories' in resource - ): - resource = MonitoredPattern(**resource) - else: - resource = MonitoredResource(**resource) - - return EventHandler.from_resource(resource) - - def __init__( - self, paths: Iterable[Union[str, Dict[str, Any], MonitoredResource]], **kwargs - ): - """ - :param paths: List of paths to monitor. Paths can either be expressed in any of the following ways: - - - Simple strings. In this case, paths will be interpreted as absolute references to a file or a directory - to monitor. Example: - - .. code-block:: yaml - - backend.file.monitor: - paths: - # Monitor changes on the /tmp folder - - /tmp - # Monitor changes on /etc/passwd - - /etc/passwd - - - Path with monitoring properties expressed as a key-value object. Example showing the supported attributes: - - .. code-block:: yaml - - backend.file.monitor: - paths: - # Monitor changes on the /tmp folder and its subfolders - - path: /tmp - recursive: True - - - Path with pattern-based search criteria for the files to monitor and exclude. Example: - - .. code-block:: yaml - - backend.file.monitor: - paths: - # Recursively monitor changes on the ~/my-project folder that include all - # *.py files, excluding those whose name starts with tmp_ and - # all the files contained in the __pycache__ folders - - path: ~/my-project - recursive: True - patterns: - - "*.py" - ignore_patterns: - - "tmp_*" - ignore_directories: - - "__pycache__" - - - Path with regex-based search criteria for the files to monitor and exclude. Example: - - .. code-block:: yaml - - backend.file.monitor: - paths: - # Recursively monitor changes on the ~/my-images folder that include all - # the files matching a JPEG extension in case-insensitive mode, - # excluding those whose name starts with tmp_ and - # all the files contained in the __MACOSX folders - - path: ~/my-images - recursive: True - case_sensitive: False - regexes: - - '.*\\.jpe?g$' - ignore_patterns: - - '^tmp_.*' - ignore_directories: - - '__MACOSX' - - """ - - super().__init__(**kwargs) - self._observer = Observer() - - for path in paths: - handler = self.EventHandlerFactory.from_resource(path) - self._observer.schedule( - handler, handler.resource.path, recursive=handler.resource.recursive - ) - - def run(self): - super().run() - self.logger.info('Initializing file monitor backend') - self._observer.start() - self.wait_stop() - - def on_stop(self): - self.logger.info('Stopping file monitor backend') - self._observer.stop() - self._observer.join() - self.logger.info('Stopped file monitor backend') diff --git a/platypush/backend/file/monitor/entities/__init__.py b/platypush/backend/file/monitor/entities/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/platypush/backend/file/monitor/manifest.yaml b/platypush/backend/file/monitor/manifest.yaml deleted file mode 100644 index b504d5fe74..0000000000 --- a/platypush/backend/file/monitor/manifest.yaml +++ /dev/null @@ -1,18 +0,0 @@ -manifest: - events: - platypush.message.event.file.FileSystemCreateEvent: if a resource is created. - platypush.message.event.file.FileSystemDeleteEvent: if a resource is removed. - platypush.message.event.file.FileSystemModifyEvent: if a resource is modified. - install: - apk: - - py3-watchdog - apt: - - python3-watchdog - dnf: - - python-watchdog - pacman: - - python-watchdog - pip: - - watchdog - package: platypush.backend.file.monitor - type: backend diff --git a/platypush/config/config.yaml b/platypush/config/config.yaml index 335698bf86..c3f295a0b0 100644 --- a/platypush/config/config.yaml +++ b/platypush/config/config.yaml @@ -382,6 +382,30 @@ backend.http: # - https://api.quantamagazine.org/feed/ ### +### +# # The file monitor plugin can be used to track modifications to the +# # filesystem - for example, when a file or a directory is modified, created +# # or removed. +# +# file.monitor: +# paths: +# # Recursively monitor changes on the +# # ~/my-images folder that include all the files +# # matching a JPEG extension in case-insensitive +# # mode, excluding those whose name starts with +# # tmp_ and all the files contained in the +# # __MACOSX folders +# - path: ~/my-images +# recursive: true +# case_sensitive: false +# regexes: +# - '.*\\.jpe?g$' +# ignore_patterns: +# - '^tmp_.*' +# ignore_directories: +# - '__MACOSX' +### + ### # # Example configuration of a weather plugin # diff --git a/platypush/plugins/file/monitor/__init__.py b/platypush/plugins/file/monitor/__init__.py new file mode 100644 index 0000000000..f8b6411c0d --- /dev/null +++ b/platypush/plugins/file/monitor/__init__.py @@ -0,0 +1,140 @@ +from typing import Iterable, Dict, Optional, Union, Any + +from watchdog.observers import Observer + +from platypush.plugins import RunnablePlugin + +from .entities.handlers import EventHandler +from .entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex + + +def event_handler_from_resource( + resource: Union[str, Dict[str, Any], MonitoredResource] +) -> Optional[EventHandler]: + """ + Create a file system event handler from a string, dictionary or + ``MonitoredResource`` resource. + """ + + if isinstance(resource, str): + res = MonitoredResource(resource) + elif isinstance(resource, dict): + if 'regexes' in resource or 'ignore_regexes' in resource: + res = MonitoredRegex(**resource) + elif ( + 'patterns' in resource + or 'ignore_patterns' in resource + or 'ignore_directories' in resource + ): + res = MonitoredPattern(**resource) + else: + res = MonitoredResource(**resource) + else: + return None + + return EventHandler.from_resource(res) + + +class FileMonitorPlugin(RunnablePlugin): + """ + A plugin to monitor changes to files and directories. + """ + + def __init__( + self, paths: Iterable[Union[str, Dict[str, Any], MonitoredResource]], **kwargs + ): + """ + :param paths: List of paths to monitor. Paths can either be expressed + in any of the following ways: + + - Simple strings. In this case, paths will be interpreted as + absolute references to a file or a directory to monitor. + Example: + + .. code-block:: yaml + + file.monitor: + paths: + # Monitor changes on the /tmp folder + - /tmp + # Monitor changes on /etc/passwd + - /etc/passwd + + - Path with monitoring properties expressed as a key-value + object. Example showing the supported attributes: + + .. code-block:: yaml + + file.monitor: + paths: + # Monitor changes on the /tmp folder and its + # subfolders + - path: /tmp + recursive: True + + - Path with pattern-based search criteria for the files to monitor + and exclude. Example: + + .. code-block:: yaml + + file.monitor: + paths: + # Recursively monitor changes on the + # ~/my-project folder that include all *.py + # files, excluding those whose name starts with + # tmp_ and all the files contained in the + # __pycache__ folders + - path: ~/my-project + recursive: True + patterns: + - "*.py" + ignore_patterns: + - "tmp_*" + ignore_directories: + - "__pycache__" + + - Path with regex-based search criteria for the files to + monitor and exclude. Example: + + .. code-block:: yaml + + file.monitor: + paths: + # Recursively monitor changes on the + # ~/my-images folder that include all the files + # matching a JPEG extension in case-insensitive + # mode, excluding those whose name starts with + # tmp_ and all the files contained in the + # __MACOSX folders + - path: ~/my-images + recursive: True + case_sensitive: False + regexes: + - '.*\\.jpe?g$' + ignore_patterns: + - '^tmp_.*' + ignore_directories: + - '__MACOSX' + + """ + + super().__init__(**kwargs) + self._observer = Observer() + + for path in paths: + handler = event_handler_from_resource(path) + if not handler: + continue + + self._observer.schedule( + handler, handler.resource.path, recursive=handler.resource.recursive + ) + + def stop(self): + self._observer.stop() + self._observer.join() + super().stop() + + def main(self): + self._observer.start() + self.wait_stop() diff --git a/platypush/backend/file/__init__.py b/platypush/plugins/file/monitor/entities/__init__.py similarity index 100% rename from platypush/backend/file/__init__.py rename to platypush/plugins/file/monitor/entities/__init__.py diff --git a/platypush/backend/file/monitor/entities/handlers.py b/platypush/plugins/file/monitor/entities/handlers.py similarity index 58% rename from platypush/backend/file/monitor/entities/handlers.py rename to platypush/plugins/file/monitor/entities/handlers.py index ef7930317f..d49a1a2709 100644 --- a/platypush/backend/file/monitor/entities/handlers.py +++ b/platypush/plugins/file/monitor/entities/handlers.py @@ -1,17 +1,31 @@ import os import re -from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler, RegexMatchingEventHandler +from watchdog.events import ( + FileSystemEventHandler, + PatternMatchingEventHandler, + RegexMatchingEventHandler, +) -from platypush.backend.file.monitor.entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex from platypush.context import get_bus -from platypush.message.event.file import FileSystemModifyEvent, FileSystemCreateEvent, FileSystemDeleteEvent +from platypush.message.event.file import ( + FileSystemModifyEvent, + FileSystemCreateEvent, + FileSystemDeleteEvent, +) + +from .resources import ( + MonitoredResource, + MonitoredPattern, + MonitoredRegex, +) class EventHandler(FileSystemEventHandler): """ Base class for Watchdog event handlers. """ + def __init__(self, resource: MonitoredResource, **kwargs): super().__init__(**kwargs) resource.path = os.path.expanduser(resource.path) @@ -20,7 +34,8 @@ class EventHandler(FileSystemEventHandler): def _should_ignore_event(self, event) -> bool: ignore_dirs = [ os.path.expanduser( - _dir if os.path.expanduser(_dir).strip('/').startswith(self.resource.path) + _dir + if os.path.expanduser(_dir).strip('/').startswith(self.resource.path) else os.path.join(self.resource.path, _dir) ) for _dir in getattr(self.resource, 'ignore_directories', []) @@ -30,19 +45,18 @@ class EventHandler(FileSystemEventHandler): ignore_regexes = getattr(self.resource, 'ignore_regexes', None) if ignore_dirs and any( - event.src_path.startswith(ignore_dir) for ignore_dir in ignore_dirs + event.src_path.startswith(ignore_dir) for ignore_dir in ignore_dirs ): return True if ignore_patterns and any( - re.match(r'^{}$'.format(pattern.replace('*', '.*')), event.src_path) - for pattern in ignore_patterns + re.match(r'^{}$'.format(pattern.replace('*', '.*')), event.src_path) + for pattern in ignore_patterns ): return True if ignore_regexes and any( - re.match(regex, event.src_path) - for regex in ignore_patterns + re.match(regex, event.src_path) for regex in (ignore_patterns or []) ): return True @@ -51,7 +65,9 @@ class EventHandler(FileSystemEventHandler): def _on_event(self, event, output_event_type): if self._should_ignore_event(event): return - get_bus().post(output_event_type(path=event.src_path, is_directory=event.is_directory)) + get_bus().post( + output_event_type(path=event.src_path, is_directory=event.is_directory) + ) def on_created(self, event): self._on_event(event, FileSystemCreateEvent) @@ -62,7 +78,7 @@ class EventHandler(FileSystemEventHandler): def on_modified(self, event): self._on_event(event, FileSystemModifyEvent) - def on_moved(self, event): + def on_moved(self, _): pass @classmethod @@ -78,21 +94,27 @@ class PatternEventHandler(EventHandler, PatternMatchingEventHandler): """ Event handler for file patterns. """ + def __init__(self, resource: MonitoredPattern): - super().__init__(resource=resource, - patterns=resource.patterns, - ignore_patterns=resource.ignore_patterns, - ignore_directories=resource.ignore_directories, - case_sensitive=resource.case_sensitive) + super().__init__( + resource=resource, + patterns=resource.patterns, + ignore_patterns=resource.ignore_patterns, + ignore_directories=resource.ignore_directories, + case_sensitive=resource.case_sensitive, + ) class RegexEventHandler(EventHandler, RegexMatchingEventHandler): """ Event handler for regex-based file patterns. """ + def __init__(self, resource: MonitoredRegex): - super().__init__(resource=resource, - regexes=resource.regexes, - ignore_regexes=resource.ignore_regexes, - ignore_directories=resource.ignore_directories, - case_sensitive=resource.case_sensitive) + super().__init__( + resource=resource, + regexes=resource.regexes, + ignore_regexes=resource.ignore_regexes, + ignore_directories=resource.ignore_directories, + case_sensitive=resource.case_sensitive, + ) diff --git a/platypush/backend/file/monitor/entities/resources.py b/platypush/plugins/file/monitor/entities/resources.py similarity index 100% rename from platypush/backend/file/monitor/entities/resources.py rename to platypush/plugins/file/monitor/entities/resources.py diff --git a/platypush/plugins/file/monitor/manifest.yaml b/platypush/plugins/file/monitor/manifest.yaml new file mode 100644 index 0000000000..bcaf299c5a --- /dev/null +++ b/platypush/plugins/file/monitor/manifest.yaml @@ -0,0 +1,18 @@ +manifest: + events: + - platypush.message.event.file.FileSystemCreateEvent + - platypush.message.event.file.FileSystemDeleteEvent + - platypush.message.event.file.FileSystemModifyEvent + install: + apk: + - py3-watchdog + apt: + - python3-watchdog + dnf: + - python-watchdog + pacman: + - python-watchdog + pip: + - watchdog + package: platypush.plugins.file.monitor + type: plugin