Added file.monitor backend [closes #172]

The file.monitor backend leverages watchdog instead of the Linux-only
inotify API and it replaces the inotify backend.
This commit is contained in:
Fabio Manganiello 2021-03-14 00:07:17 +01:00
parent 6f224cbda9
commit 352d421e61
15 changed files with 264 additions and 2 deletions

View file

@ -5,6 +5,11 @@ Given the high speed of development in the first phase, changes are being report
## [Unreleased] ## [Unreleased]
### Added
- Added `file.monitor` backend, which replaces the `inotify` backend
(see [#172](https://git.platypush.tech/platypush/platypush/-/issues/172)).
### Removed ### Removed
- Removed legacy `pusher` script and `local` backend. - Removed legacy `pusher` script and `local` backend.

View file

@ -22,6 +22,7 @@ Backends
platypush/backend/clipboard.rst platypush/backend/clipboard.rst
platypush/backend/covid19.rst platypush/backend/covid19.rst
platypush/backend/dbus.rst platypush/backend/dbus.rst
platypush/backend/file.monitor.rst
platypush/backend/foursquare.rst platypush/backend/foursquare.rst
platypush/backend/github.rst platypush/backend/github.rst
platypush/backend/google.fit.rst platypush/backend/google.fit.rst

View file

@ -23,7 +23,7 @@ import sys
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------
project = 'platypush' project = 'platypush'
copyright = '2017-2020, Fabio Manganiello' copyright = '2017-2021, Fabio Manganiello'
author = 'Fabio Manganiello' author = 'Fabio Manganiello'
# The short X.Y version # The short X.Y version
@ -265,6 +265,7 @@ autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers',
'imapclient', 'imapclient',
'pysmartthings', 'pysmartthings',
'aiohttp', 'aiohttp',
'watchdog',
] ]
sys.path.insert(0, os.path.abspath('../..')) sys.path.insert(0, os.path.abspath('../..'))

View file

@ -18,6 +18,7 @@ Events
platypush/events/covid19.rst platypush/events/covid19.rst
platypush/events/custom.rst platypush/events/custom.rst
platypush/events/distance.rst platypush/events/distance.rst
platypush/events/file.rst
platypush/events/foursquare.rst platypush/events/foursquare.rst
platypush/events/geo.rst platypush/events/geo.rst
platypush/events/github.rst platypush/events/github.rst

View file

@ -0,0 +1,5 @@
``platypush.backend.file.monitor``
==================================
.. automodule:: platypush.backend.file.monitor
:members:

View file

@ -0,0 +1,5 @@
``platypush.message.event.file``
================================
.. automodule:: platypush.message.event.file
:members:

View file

View file

@ -0,0 +1,108 @@
from typing import List, Dict, Union, Any
from watchdog.observers import Observer
from platypush.backend import Backend
from .entities.handlers import EventHandlerFactory
from .entities.resources import MonitoredResource
class FileMonitorBackend(Backend):
"""
This backend monitors changes to local files and directories using the Watchdog API.
Triggers:
* :class:`platypush.message.event.file.FileSystemCreateEvent` if a resource is created.
* :class:`platypush.message.event.file.FileSystemDeleteEvent` if a resource is removed.
* :class:`platypush.message.event.file.FileSystemModifyEvent` if a resource is modified.
Requires:
* **watchdog** (``pip install watchdog``)
"""
def __init__(self, paths: List[Union[str, Dict[str, Any], MonitoredResource]], **kwargs):
"""
:param paths: List of paths to monitor. Paths can either be expressed in any of the following ways:
- Simple strings. In this case, paths will be interpreted as absolute references to a file or a directory
to monitor. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Monitor changes on the /tmp folder
- /tmp
# Monitor changes on /etc/passwd
- /etc/passwd
- Path with monitoring properties expressed as a key-value object. Example showing the supported attributes:
.. code-block:: yaml
backend.file.monitor:
paths:
# Monitor changes on the /tmp folder and its subfolders
- path: /tmp
recursive: True
- Path with pattern-based search criteria for the files to monitor and exclude. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Recursively monitor changes on the ~/my-project folder that include all
# *.py files, excluding those whose name starts with tmp_ and
# all the files contained in the __pycache__ folders
- path: ~/my-project
recursive: True
patterns:
- "*.py"
ignore_patterns:
- "tmp_*"
ignore_directories:
- "__pycache__"
- Path with regex-based search criteria for the files to monitor and exclude. Example:
.. code-block:: yaml
backend.file.monitor:
paths:
# Recursively monitor changes on the ~/my-images folder that include all
# the files matching the "\.jpe?g$" pattern in case-insensitive mode,
# excluding those whose name starts with tmp_ and
# all the files contained in the __MACOSX folders
- path: ~/my-images
recursive: True
regexes:
- ".*\.jpe?g$"
ignore_patterns:
- "^tmp_.*"
ignore_directories:
- "__MACOSX"
"""
super().__init__(**kwargs)
self._observer = Observer()
for path in paths:
handler = EventHandlerFactory.from_resource(path)
self._observer.schedule(handler, handler.resource.path, recursive=handler.resource.recursive)
def run(self):
super().run()
self.logger.info('Initializing file monitor backend')
self._observer.start()
self.wait_stop()
def on_stop(self):
self.logger.info('Stopping file monitor backend')
self._observer.stop()
self._observer.join()
self.logger.info('Stopped file monitor backend')

View file

@ -0,0 +1,78 @@
import os
from typing import Dict, Union, Any
from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler, RegexMatchingEventHandler
from platypush.backend.file.monitor.entities.resources import MonitoredResource, MonitoredPattern, MonitoredRegex
from platypush.context import get_bus
from platypush.message.event.file import FileSystemModifyEvent, FileSystemCreateEvent, FileSystemDeleteEvent
class EventHandler(FileSystemEventHandler):
"""
Base class for Watchdog event handlers.
"""
def __init__(self, resource: MonitoredResource, **kwargs):
super().__init__(**kwargs)
resource.path = os.path.expanduser(resource.path)
self.resource = resource
def on_created(self, event):
get_bus().post(FileSystemCreateEvent(path=event.src_path, is_directory=event.is_directory))
def on_deleted(self, event):
get_bus().post(FileSystemDeleteEvent(path=event.src_path, is_directory=event.is_directory))
def on_modified(self, event):
get_bus().post(FileSystemModifyEvent(path=event.src_path, is_directory=event.is_directory))
@classmethod
def from_resource(cls, resource: MonitoredResource):
if isinstance(resource, MonitoredPattern):
return PatternEventHandler(resource)
if isinstance(resource, MonitoredRegex):
return RegexEventHandler(resource)
return cls(resource)
class PatternEventHandler(EventHandler, PatternMatchingEventHandler):
"""
Event handler for file patterns.
"""
def __init__(self, resource: MonitoredPattern):
super().__init__(resource=resource,
patterns=resource.patterns,
ignore_patterns=resource.ignore_patterns,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive)
class RegexEventHandler(EventHandler, RegexMatchingEventHandler):
"""
Event handler for regex-based file patterns.
"""
def __init__(self, resource: MonitoredRegex):
super().__init__(resource=resource,
regexes=resource.regexes,
ignore_regexes=resource.ignore_regexes,
ignore_directories=resource.ignore_directories,
case_sensitive=resource.case_sensitive)
class EventHandlerFactory:
"""
Create a file system event handler from a string, dictionary or ``MonitoredResource`` resource.
"""
@staticmethod
def from_resource(resource: Union[str, Dict[str, Any], MonitoredResource]) -> EventHandler:
if isinstance(resource, str):
resource = MonitoredResource(resource)
elif isinstance(resource, dict):
if 'patterns' in resource or 'ignore_patterns' in resource:
resource = MonitoredPattern(**resource)
elif 'regexes' in resource or 'ignore_regexes' in resource:
resource = MonitoredRegex(**resource)
else:
resource = MonitoredResource(**resource)
return EventHandler.from_resource(resource)

View file

@ -0,0 +1,24 @@
from dataclasses import dataclass
from typing import Optional, List
@dataclass
class MonitoredResource:
path: str
recursive: bool = False
@dataclass
class MonitoredPattern(MonitoredResource):
patterns: Optional[List[str]] = None
ignore_patterns: Optional[List[str]] = None
ignore_directories: Optional[List[str]] = None
case_sensitive: bool = True
@dataclass
class MonitoredRegex(MonitoredResource):
regexes: Optional[List[str]] = None
ignore_regexes: Optional[List[str]] = None
ignore_directories: Optional[List[str]] = None
case_sensitive: bool = True

View file

@ -7,6 +7,8 @@ from platypush.message.event.inotify import InotifyCreateEvent, InotifyDeleteEve
class InotifyBackend(Backend): class InotifyBackend(Backend):
""" """
**NOTE**: This backend is *deprecated* in favour of :class:`platypush.backend.file.monitor.FileMonitorBackend`.
(Linux only) This backend will listen for events on the filesystem (whether (Linux only) This backend will listen for events on the filesystem (whether
a file/directory on a watch list is opened, modified, created, deleted, a file/directory on a watch list is opened, modified, created, deleted,
closed or had its permissions changed) and will trigger a relevant event. closed or had its permissions changed) and will trigger a relevant event.

View file

@ -0,0 +1,27 @@
from platypush.message.event import Event
class FileSystemEvent(Event):
"""
Base class for file system events - namely, file/directory creation, deletion and modification.
"""
def __init__(self, path: str, *, is_directory: bool, **kwargs):
super().__init__(path=path, is_directory=is_directory, **kwargs)
class FileSystemCreateEvent(FileSystemEvent):
"""
Event triggered when a monitored file or directory is created.
"""
class FileSystemDeleteEvent(FileSystemEvent):
"""
Event triggered when a monitored file or directory is deleted.
"""
class FileSystemModifyEvent(FileSystemEvent):
"""
Event triggered when a monitored file or directory is modified.
"""

View file

@ -311,3 +311,6 @@ croniter
# SmartThings integration # SmartThings integration
# pysmartthings # pysmartthings
# aiohttp # aiohttp
# Support for file.monitor backend
#watchdog

View file

@ -246,5 +246,7 @@ setup(
'vlc': ['python-vlc'], 'vlc': ['python-vlc'],
# Support for SmartThings integration # Support for SmartThings integration
'smartthings': ['pysmartthings', 'aiohttp'], 'smartthings': ['pysmartthings', 'aiohttp'],
# Support for file.monitor backend
'filemonitor': ['watchdog'],
}, },
) )