platypush/platypush
platypush
/
platypush
2
2
Fork 4
Code Issues 46 Pull Requests Packages Projects 4 Releases Wiki Activity

[#289] Converted `backend.file.monitor` into a runnable plugin. 

Closes: #289
This commit is contained in:
Fabio Manganiello 2023-12-04 02:42:51 +01:00
parent d484a34c00
commit 1843ab224b
Signed by: blacklight
GPG Key ID: D90FBA7F76362774
16 changed files with 231 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/source/backends.rst
View File

@ -11,7 +11,6 @@ Backends
platypush/backend/button.flic.rst
platypush/backend/camera.pi.rst
platypush/backend/chat.telegram.rst
platypush/backend/file.monitor.rst
platypush/backend/foursquare.rst
platypush/backend/github.rst
platypush/backend/google.fit.rst
@ -28,7 +27,6 @@ Backends
platypush/backend/nextcloud.rst
platypush/backend/nfc.rst
platypush/backend/nodered.rst
platypush/backend/ping.rst
platypush/backend/pushbullet.rst
platypush/backend/redis.rst
platypush/backend/scard.rst

5
docs/source/platypush/backend/file.monitor.rst
View File

@ -1,5 +0,0 @@
``file.monitor``
==================================
.. automodule:: platypush.backend.file.monitor
:members:

5
docs/source/platypush/backend/ping.rst
View File

@ -1,5 +0,0 @@
``ping``
==========================
.. automodule:: platypush.backend.ping
:members:

5
docs/source/platypush/plugins/file.monitor.rst Normal file
View File

@ -0,0 +1,5 @@
``file.monitor``
================
.. automodule:: platypush.plugins.file.monitor
:members:

5
docs/source/platypush/responses/ping.rst
View File

@ -1,5 +0,0 @@
``ping``
===================================
.. automodule:: platypush.message.response.ping
:members:

1
docs/source/plugins.rst
View File

@ -33,6 +33,7 @@ Plugins
platypush/plugins/esp.rst
platypush/plugins/ffmpeg.rst
platypush/plugins/file.rst
platypush/plugins/file.monitor.rst
platypush/plugins/foursquare.rst
platypush/plugins/google.calendar.rst
platypush/plugins/google.drive.rst

1
docs/source/responses.rst
View File

@ -11,7 +11,6 @@ Responses
platypush/responses/chat.telegram.rst
platypush/responses/google.drive.rst
platypush/responses/pihole.rst
platypush/responses/ping.rst
platypush/responses/printer.cups.rst
platypush/responses/qrcode.rst
platypush/responses/ssh.rst

127
platypush/backend/file/monitor/__init__.py
View File

@ -1,127 +0,0 @@
from typing import Iterable, Dict, Union, Any
from watchdog.observers import Observer
from platypush.backend import Backend
from .entities.handlers import EventHandler
from .entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex
class FileMonitorBackend(Backend):
"""
This backend monitors changes to local files and directories using the Watchdog API.
"""
class EventHandlerFactory:
"""
Create a file system event handler from a string, dictionary or ``MonitoredResource`` resource.
"""
@staticmethod
def from_resource(
resource: Union[str, Dict[str, Any], MonitoredResource]
) -> EventHandler:
if isinstance(resource, str):
resource = MonitoredResource(resource)
elif isinstance(resource, dict):
if 'regexes' in resource or 'ignore_regexes' in resource:
resource = MonitoredRegex(**resource)
elif (
'patterns' in resource
or 'ignore_patterns' in resource
or 'ignore_directories' in resource
):
resource = MonitoredPattern(**resource)
else:
resource = MonitoredResource(**resource)
return EventHandler.from_resource(resource)
def __init__(
self, paths: Iterable[Union[str, Dict[str, Any], MonitoredResource]], **kwargs
):
"""
:param paths: List of paths to monitor. Paths can either be expressed in any of the following ways:
- Simple strings. In this case, paths will be interpreted as absolute references to a file or a directory
to monitor. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Monitor changes on the /tmp folder
- /tmp
# Monitor changes on /etc/passwd
- /etc/passwd
- Path with monitoring properties expressed as a key-value object. Example showing the supported attributes:
.. code-block:: yaml
backend.file.monitor:
paths:
# Monitor changes on the /tmp folder and its subfolders
- path: /tmp
recursive: True
- Path with pattern-based search criteria for the files to monitor and exclude. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Recursively monitor changes on the ~/my-project folder that include all
# *.py files, excluding those whose name starts with tmp_ and
# all the files contained in the __pycache__ folders
- path: ~/my-project
recursive: True
patterns:
- "*.py"
ignore_patterns:
- "tmp_*"
ignore_directories:
- "__pycache__"
- Path with regex-based search criteria for the files to monitor and exclude. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Recursively monitor changes on the ~/my-images folder that include all
# the files matching a JPEG extension in case-insensitive mode,
# excluding those whose name starts with tmp_ and
# all the files contained in the __MACOSX folders
- path: ~/my-images
recursive: True
case_sensitive: False
regexes:
- '.*\\.jpe?g$'
ignore_patterns:
- '^tmp_.*'
ignore_directories:
- '__MACOSX'
"""
super().__init__(**kwargs)
self._observer = Observer()
for path in paths:
handler = self.EventHandlerFactory.from_resource(path)
self._observer.schedule(
handler, handler.resource.path, recursive=handler.resource.recursive
)
def run(self):
super().run()
self.logger.info('Initializing file monitor backend')
self._observer.start()
self.wait_stop()
def on_stop(self):
self.logger.info('Stopping file monitor backend')
self._observer.stop()
self._observer.join()
self.logger.info('Stopped file monitor backend')

0
platypush/backend/file/monitor/entities/__init__.py
View File

18
platypush/backend/file/monitor/manifest.yaml
View File

@ -1,18 +0,0 @@
manifest:
events:
platypush.message.event.file.FileSystemCreateEvent: if a resource is created.
platypush.message.event.file.FileSystemDeleteEvent: if a resource is removed.
platypush.message.event.file.FileSystemModifyEvent: if a resource is modified.
install:
apk:
- py3-watchdog
apt:
- python3-watchdog
dnf:
- python-watchdog
pacman:
- python-watchdog
pip:
- watchdog
package: platypush.backend.file.monitor
type: backend

24
platypush/config/config.yaml
View File

@ -382,6 +382,30 @@ backend.http:
# - https://api.quantamagazine.org/feed/
###
###
# # The file monitor plugin can be used to track modifications to the
# # filesystem - for example, when a file or a directory is modified, created
# # or removed.
#
# file.monitor:
# paths:
# # Recursively monitor changes on the
# # ~/my-images folder that include all the files
# # matching a JPEG extension in case-insensitive
# # mode, excluding those whose name starts with
# # tmp_ and all the files contained in the
# # __MACOSX folders
# - path: ~/my-images
# recursive: true
# case_sensitive: false
# regexes:
# - '.*\\.jpe?g$'
# ignore_patterns:
# - '^tmp_.*'
# ignore_directories:
# - '__MACOSX'
###
###
# # Example configuration of a weather plugin
#

140
platypush/plugins/file/monitor/__init__.py Normal file
View File

@ -0,0 +1,140 @@
from typing import Iterable, Dict, Optional, Union, Any
from watchdog.observers import Observer
from platypush.plugins import RunnablePlugin
from .entities.handlers import EventHandler
from .entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex
def event_handler_from_resource(
resource: Union[str, Dict[str, Any], MonitoredResource]
) -> Optional[EventHandler]:
"""
Create a file system event handler from a string, dictionary or
``MonitoredResource`` resource.
"""
if isinstance(resource, str):
res = MonitoredResource(resource)
elif isinstance(resource, dict):
if 'regexes' in resource or 'ignore_regexes' in resource:
res = MonitoredRegex(**resource)
elif (
'patterns' in resource
or 'ignore_patterns' in resource
or 'ignore_directories' in resource
):
res = MonitoredPattern(**resource)
else:
res = MonitoredResource(**resource)
else:
return None
return EventHandler.from_resource(res)
class FileMonitorPlugin(RunnablePlugin):
"""
A plugin to monitor changes to files and directories.
"""
def __init__(
self, paths: Iterable[Union[str, Dict[str, Any], MonitoredResource]], **kwargs
):
"""
:param paths: List of paths to monitor. Paths can either be expressed
in any of the following ways:
- Simple strings. In this case, paths will be interpreted as
absolute references to a file or a directory to monitor.
Example:
.. code-block:: yaml
file.monitor:
paths:
# Monitor changes on the /tmp folder
- /tmp
# Monitor changes on /etc/passwd
- /etc/passwd
- Path with monitoring properties expressed as a key-value
object. Example showing the supported attributes:
.. code-block:: yaml
file.monitor:
paths:
# Monitor changes on the /tmp folder and its
# subfolders
- path: /tmp
recursive: True
- Path with pattern-based search criteria for the files to monitor
and exclude. Example:
.. code-block:: yaml
file.monitor:
paths:
# Recursively monitor changes on the
# ~/my-project folder that include all *.py
# files, excluding those whose name starts with
# tmp_ and all the files contained in the
# __pycache__ folders
- path: ~/my-project
recursive: True
patterns:
- "*.py"
ignore_patterns:
- "tmp_*"
ignore_directories:
- "__pycache__"
- Path with regex-based search criteria for the files to
monitor and exclude. Example:
.. code-block:: yaml
file.monitor:
paths:
# Recursively monitor changes on the
# ~/my-images folder that include all the files
# matching a JPEG extension in case-insensitive
# mode, excluding those whose name starts with
# tmp_ and all the files contained in the
# __MACOSX folders
- path: ~/my-images
recursive: True
case_sensitive: False
regexes:
- '.*\\.jpe?g$'
ignore_patterns:
- '^tmp_.*'
ignore_directories:
- '__MACOSX'
"""
super().__init__(**kwargs)
self._observer = Observer()
for path in paths:
handler = event_handler_from_resource(path)
if not handler:
continue
self._observer.schedule(
handler, handler.resource.path, recursive=handler.resource.recursive
)
def stop(self):
self._observer.stop()
self._observer.join()
super().stop()
def main(self):
self._observer.start()
self.wait_stop()

0
platypush/backend/file/__init__.py → platypush/plugins/file/monitor/entities/__init__.py
View File

64
platypush/backend/file/monitor/entities/handlers.py → platypush/plugins/file/monitor/entities/handlers.py
View File

@ -1,17 +1,31 @@
import os
import re
from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler, RegexMatchingEventHandler
from watchdog.events import (
FileSystemEventHandler,
PatternMatchingEventHandler,
RegexMatchingEventHandler,
)
from platypush.backend.file.monitor.entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex
from platypush.context import get_bus
from platypush.message.event.file import FileSystemModifyEvent, FileSystemCreateEvent, FileSystemDeleteEvent
from platypush.message.event.file import (
FileSystemModifyEvent,
FileSystemCreateEvent,
FileSystemDeleteEvent,
)
from .resources import (
MonitoredResource,
MonitoredPattern,
MonitoredRegex,
)
class EventHandler(FileSystemEventHandler):
"""
Base class for Watchdog event handlers.
"""
def __init__(self, resource: MonitoredResource, **kwargs):
super().__init__(**kwargs)
resource.path = os.path.expanduser(resource.path)
@ -20,7 +34,8 @@ class EventHandler(FileSystemEventHandler):
def _should_ignore_event(self, event) -> bool:
ignore_dirs = [
os.path.expanduser(
_dir if os.path.expanduser(_dir).strip('/').startswith(self.resource.path)
_dir
if os.path.expanduser(_dir).strip('/').startswith(self.resource.path)
else os.path.join(self.resource.path, _dir)
)
for _dir in getattr(self.resource, 'ignore_directories', [])
@ -30,19 +45,18 @@ class EventHandler(FileSystemEventHandler):
ignore_regexes = getattr(self.resource, 'ignore_regexes', None)
if ignore_dirs and any(
event.src_path.startswith(ignore_dir) for ignore_dir in ignore_dirs
event.src_path.startswith(ignore_dir) for ignore_dir in ignore_dirs
):
return True
if ignore_patterns and any(
re.match(r'^{}$'.format(pattern.replace('*', '.*')), event.src_path)
for pattern in ignore_patterns
re.match(r'^{}$'.format(pattern.replace('*', '.*')), event.src_path)
for pattern in ignore_patterns
):
return True
if ignore_regexes and any(
re.match(regex, event.src_path)
for regex in ignore_patterns
re.match(regex, event.src_path) for regex in (ignore_patterns or [])
):
return True
@ -51,7 +65,9 @@ class EventHandler(FileSystemEventHandler):
def _on_event(self, event, output_event_type):
if self._should_ignore_event(event):
return
get_bus().post(output_event_type(path=event.src_path, is_directory=event.is_directory))
get_bus().post(
output_event_type(path=event.src_path, is_directory=event.is_directory)
)
def on_created(self, event):
self._on_event(event, FileSystemCreateEvent)
@ -62,7 +78,7 @@ class EventHandler(FileSystemEventHandler):
def on_modified(self, event):
self._on_event(event, FileSystemModifyEvent)
def on_moved(self, event):
def on_moved(self, _):
pass
@classmethod
@ -78,21 +94,27 @@ class PatternEventHandler(EventHandler, PatternMatchingEventHandler):
"""
Event handler for file patterns.
"""
def __init__(self, resource: MonitoredPattern):
super().__init__(resource=resource,
patterns=resource.patterns,
ignore_patterns=resource.ignore_patterns,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive)
super().__init__(
resource=resource,
patterns=resource.patterns,
ignore_patterns=resource.ignore_patterns,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive,
)
class RegexEventHandler(EventHandler, RegexMatchingEventHandler):
"""
Event handler for regex-based file patterns.
"""
def __init__(self, resource: MonitoredRegex):
super().__init__(resource=resource,
regexes=resource.regexes,
ignore_regexes=resource.ignore_regexes,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive)
super().__init__(
resource=resource,
regexes=resource.regexes,
ignore_regexes=resource.ignore_regexes,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive,
)

0
platypush/backend/file/monitor/entities/resources.py → platypush/plugins/file/monitor/entities/resources.py
View File

18
platypush/plugins/file/monitor/manifest.yaml Normal file
View File

@ -0,0 +1,18 @@
manifest:
events:
- platypush.message.event.file.FileSystemCreateEvent
- platypush.message.event.file.FileSystemDeleteEvent
- platypush.message.event.file.FileSystemModifyEvent
install:
apk:
- py3-watchdog
apt:
- python3-watchdog
dnf:
- python-watchdog
pacman:
- python-watchdog
pip:
- watchdog
package: platypush.plugins.file.monitor
type: plugin