From edfdf06b2e28c24691317dc4bab8a4b8b628d150 Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Wed, 16 Apr 2025 07:10:20 +0000
Subject: [PATCH 01/18] fix: upgrade axios from 1.8.3 to 1.8.4
Snyk has created this PR to upgrade axios from 1.8.3 to 1.8.4.
See this package in npm:
axios
See this project in Snyk:
https://app.snyk.io/org/blacklight/project/96bfd125-5816-4d9e-83c6-94d1569ab0f1?utm_source=github&utm_medium=referral&page=upgrade-pr
---
platypush/backend/http/webapp/package-lock.json | 8 ++++----
platypush/backend/http/webapp/package.json | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/platypush/backend/http/webapp/package-lock.json b/platypush/backend/http/webapp/package-lock.json
index 210d7dc54..5309b6ff3 100644
--- a/platypush/backend/http/webapp/package-lock.json
+++ b/platypush/backend/http/webapp/package-lock.json
@@ -9,7 +9,7 @@
"version": "0.1.0",
"dependencies": {
"@fortawesome/fontawesome-free": "^6.7.2",
- "axios": "^1.8.3",
+ "axios": "^1.8.4",
"core-js": "^3.42.0",
"cronstrue": "^2.56.0",
"highlight.js": "^11.11.1",
@@ -3776,9 +3776,9 @@
}
},
"node_modules/axios": {
- "version": "1.8.3",
- "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz",
- "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==",
+ "version": "1.8.4",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz",
+ "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.15.6",
diff --git a/platypush/backend/http/webapp/package.json b/platypush/backend/http/webapp/package.json
index ca9bf3934..5d0e6a207 100644
--- a/platypush/backend/http/webapp/package.json
+++ b/platypush/backend/http/webapp/package.json
@@ -9,7 +9,7 @@
},
"dependencies": {
"@fortawesome/fontawesome-free": "^6.7.2",
- "axios": "^1.8.3",
+ "axios": "^1.8.4",
"core-js": "^3.42.0",
"cronstrue": "^2.56.0",
"highlight.js": "^11.11.1",
From ee17c6e35ed79483a17435ab956b8dc342ce1de2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 16 Apr 2025 15:31:03 +0000
Subject: [PATCH 02/18] Bump http-proxy-middleware in
/platypush/backend/http/webapp
Bumps [http-proxy-middleware](https://github.com/chimurai/http-proxy-middleware) from 2.0.6 to 2.0.9.
- [Release notes](https://github.com/chimurai/http-proxy-middleware/releases)
- [Changelog](https://github.com/chimurai/http-proxy-middleware/blob/v2.0.9/CHANGELOG.md)
- [Commits](https://github.com/chimurai/http-proxy-middleware/compare/v2.0.6...v2.0.9)
---
updated-dependencies:
- dependency-name: http-proxy-middleware
dependency-version: 2.0.9
dependency-type: indirect
...
Signed-off-by: dependabot[bot] <support@github.com>
---
platypush/backend/http/webapp/package-lock.json | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/platypush/backend/http/webapp/package-lock.json b/platypush/backend/http/webapp/package-lock.json
index 5309b6ff3..6a0fc9d43 100644
--- a/platypush/backend/http/webapp/package-lock.json
+++ b/platypush/backend/http/webapp/package-lock.json
@@ -7243,10 +7243,11 @@
}
},
"node_modules/http-proxy-middleware": {
- "version": "2.0.6",
- "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz",
- "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==",
+ "version": "2.0.9",
+ "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz",
+ "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==",
"dev": true,
+ "license": "MIT",
"dependencies": {
"@types/http-proxy": "^1.17.8",
"http-proxy": "^1.18.1",
From d0416ca571502718e0aeaeeae802e033da898007 Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Wed, 23 Apr 2025 07:11:00 +0000
Subject: [PATCH 03/18] fix: upgrade cronstrue from 2.56.0 to 2.57.0
Snyk has created this PR to upgrade cronstrue from 2.56.0 to 2.57.0.
See this package in npm:
cronstrue
See this project in Snyk:
https://app.snyk.io/org/blacklight/project/96bfd125-5816-4d9e-83c6-94d1569ab0f1?utm_source=github&utm_medium=referral&page=upgrade-pr
---
platypush/backend/http/webapp/package-lock.json | 8 ++++----
platypush/backend/http/webapp/package.json | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/platypush/backend/http/webapp/package-lock.json b/platypush/backend/http/webapp/package-lock.json
index 6a0fc9d43..2c1fd2214 100644
--- a/platypush/backend/http/webapp/package-lock.json
+++ b/platypush/backend/http/webapp/package-lock.json
@@ -11,7 +11,7 @@
"@fortawesome/fontawesome-free": "^6.7.2",
"axios": "^1.8.4",
"core-js": "^3.42.0",
- "cronstrue": "^2.56.0",
+ "cronstrue": "^2.57.0",
"highlight.js": "^11.11.1",
"lato-font": "^3.0.0",
"mitt": "^2.1.0",
@@ -4712,9 +4712,9 @@
}
},
"node_modules/cronstrue": {
- "version": "2.56.0",
- "resolved": "https://registry.npmjs.org/cronstrue/-/cronstrue-2.56.0.tgz",
- "integrity": "sha512-/YC3b4D/E/S8ToQ7f676A2fqoC3vVpXKjJ4SMsP0jYsvRYJdZ6h9+Fq/Y7FoFDEUFCqLTca+G2qTV227lyyFZg==",
+ "version": "2.57.0",
+ "resolved": "https://registry.npmjs.org/cronstrue/-/cronstrue-2.57.0.tgz",
+ "integrity": "sha512-gQOfxJa1RA9uDT4hx37NshhX4dW9t9zTCtIYu15LUziH+mkpuLXYcSEyM8ZewMJ2p8UVuHGjI3n4hGpu3HtCbg==",
"license": "MIT",
"bin": {
"cronstrue": "bin/cli.js"
diff --git a/platypush/backend/http/webapp/package.json b/platypush/backend/http/webapp/package.json
index 5d0e6a207..6ee3d4315 100644
--- a/platypush/backend/http/webapp/package.json
+++ b/platypush/backend/http/webapp/package.json
@@ -11,7 +11,7 @@
"@fortawesome/fontawesome-free": "^6.7.2",
"axios": "^1.8.4",
"core-js": "^3.42.0",
- "cronstrue": "^2.56.0",
+ "cronstrue": "^2.57.0",
"highlight.js": "^11.11.1",
"lato-font": "^3.0.0",
"mitt": "^2.1.0",
From b97fed4ffe8a4182fce6e8cc58f7ca404e6b6aa2 Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Wed, 21 May 2025 09:17:26 +0000
Subject: [PATCH 04/18] fix: upgrade vue-router from 4.5.0 to 4.5.1
Snyk has created this PR to upgrade vue-router from 4.5.0 to 4.5.1.
See this package in npm:
vue-router
See this project in Snyk:
https://app.snyk.io/org/blacklight/project/96bfd125-5816-4d9e-83c6-94d1569ab0f1?utm_source=github&utm_medium=referral&page=upgrade-pr
---
platypush/backend/http/webapp/package-lock.json | 8 ++++----
platypush/backend/http/webapp/package.json | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/platypush/backend/http/webapp/package-lock.json b/platypush/backend/http/webapp/package-lock.json
index 2c1fd2214..d0e2c27d9 100644
--- a/platypush/backend/http/webapp/package-lock.json
+++ b/platypush/backend/http/webapp/package-lock.json
@@ -19,7 +19,7 @@
"sass": "^1.77.6",
"sass-loader": "^10.5.2",
"vue": "^3.5.13",
- "vue-router": "^4.5.0",
+ "vue-router": "^4.5.1",
"vue-skycons": "^4.3.4",
"w3css": "^2.7.0"
},
@@ -12313,9 +12313,9 @@
}
},
"node_modules/vue-router": {
- "version": "4.5.0",
- "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.5.0.tgz",
- "integrity": "sha512-HDuk+PuH5monfNuY+ct49mNmkCRK4xJAV9Ts4z9UFc4rzdDnxQLyCMGGc8pKhZhHTVzfanpNwB/lwqevcBwI4w==",
+ "version": "4.5.1",
+ "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.5.1.tgz",
+ "integrity": "sha512-ogAF3P97NPm8fJsE4by9dwSYtDwXIY1nFY9T6DyQnGHd1E2Da94w9JIolpe42LJGIl0DwOHBi8TcRPlPGwbTtw==",
"license": "MIT",
"dependencies": {
"@vue/devtools-api": "^6.6.4"
diff --git a/platypush/backend/http/webapp/package.json b/platypush/backend/http/webapp/package.json
index 6ee3d4315..8cd833494 100644
--- a/platypush/backend/http/webapp/package.json
+++ b/platypush/backend/http/webapp/package.json
@@ -19,7 +19,7 @@
"sass": "^1.77.6",
"sass-loader": "^10.5.2",
"vue": "^3.5.13",
- "vue-router": "^4.5.0",
+ "vue-router": "^4.5.1",
"vue-skycons": "^4.3.4",
"w3css": "^2.7.0"
},
From 9d8f4bbcadc0533dd2a9cf0226a42db08a7e99c4 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sat, 10 May 2025 21:31:40 +0200
Subject: [PATCH 05/18] Added stack trace in debug mode when a message fails to
be serialized
---
platypush/message/__init__.py | 22 ++++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/platypush/message/__init__.py b/platypush/message/__init__.py
index a1f295ff0..615bce232 100644
--- a/platypush/message/__init__.py
+++ b/platypush/message/__init__.py
@@ -8,12 +8,14 @@ import logging
import inspect
import json
import time
+import traceback as tb
from typing import Optional, Union
from uuid import UUID
_logger = logging.getLogger('platypush')
+# pylint: disable=too-few-public-methods
class JSONAble(ABC):
"""
Generic interface for JSON-able objects.
@@ -38,6 +40,7 @@ class Message:
"""
@staticmethod
+ # pylint: disable=too-many-return-statements
def parse_numpy(obj):
try:
import numpy as np
@@ -57,13 +60,16 @@ class Message:
except (AttributeError, ImportError, TypeError):
pass
- return
+ return None
@staticmethod
def parse_datetime(obj):
if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
return obj.isoformat()
+ return None
+
+ # pylint: disable=too-many-return-statements
def default(self, o):
from platypush.procedure import Procedure
@@ -94,7 +100,7 @@ class Message:
return f'<{o.__class__.__name__}>' + (f' {o}' if o else '')
if is_dataclass(o):
- return asdict(o)
+ return asdict(o) # type: ignore
if isinstance(o, Message):
return o.to_dict(o)
@@ -106,10 +112,20 @@ class Message:
try:
return super().default(o)
except Exception as e:
+ _logger.debug(
+ 'Could not serialize object type %s: %s: %s\n%s',
+ type(o),
+ e,
+ o,
+ tb.format_exc(),
+ )
+
_logger.warning(
'Could not serialize object type %s: %s: %s', type(o), e, o
)
+ return None
+
def __init__(self, *_, timestamp=None, logging_level=logging.INFO, **__):
self.timestamp = timestamp or time.time()
self.logging_level = logging_level
@@ -212,5 +228,7 @@ class Message:
if msgtype != cls:
return msgtype.build(msg)
+ return None
+
# vim:sw=4:ts=4:et:
From e67f5016158a64f748fe73b877b32890df3fb057 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Wed, 18 Jun 2025 22:33:53 +0200
Subject: [PATCH 06/18] [#449] Added Joplin integration + general-purpose
template for note plugins
---
platypush/common/notes.py | 143 ++++
platypush/message/event/notes.py | 120 ++++
platypush/plugins/_notes.py | 932 +++++++++++++++++++++++++
platypush/plugins/joplin/__init__.py | 441 ++++++++++++
platypush/plugins/joplin/manifest.json | 17 +
platypush/schemas/notes.py | 237 +++++++
6 files changed, 1890 insertions(+)
create mode 100644 platypush/common/notes.py
create mode 100644 platypush/message/event/notes.py
create mode 100644 platypush/plugins/_notes.py
create mode 100644 platypush/plugins/joplin/__init__.py
create mode 100644 platypush/plugins/joplin/manifest.json
create mode 100644 platypush/schemas/notes.py
diff --git a/platypush/common/notes.py b/platypush/common/notes.py
new file mode 100644
index 000000000..1e8a5da0c
--- /dev/null
+++ b/platypush/common/notes.py
@@ -0,0 +1,143 @@
+from abc import abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from hashlib import sha256
+from typing import Any, Dict, List, Optional, Set
+
+from platypush.message import JSONAble
+from platypush.schemas.notes import NoteCollectionSchema, NoteItemSchema
+
+
+class Serializable(JSONAble):
+ """
+ Base class for serializable objects.
+ """
+
+ @abstractmethod
+ def to_dict(self) -> dict:
+ """
+ Convert the object to a dictionary representation.
+ """
+
+ def to_json(self) -> dict:
+ return self.to_dict()
+
+
+@dataclass
+class NoteSource(Serializable):
+ """
+ Represents a source for a note, such as a URL or file path.
+ """
+
+ name: Optional[str] = None
+ url: Optional[str] = None
+ app: Optional[str] = None
+
+ def to_dict(self) -> dict:
+ return self.__dict__
+
+
+@dataclass
+class Note(Serializable):
+ """
+ Represents a note with a title and content.
+ """
+
+ id: Any
+ title: str
+ description: Optional[str] = None
+ content: Optional[str] = None
+ parent: Optional['NoteCollection'] = None
+ tags: Set[str] = field(default_factory=set)
+ created_at: Optional[datetime] = None
+ updated_at: Optional[datetime] = None
+ digest: Optional[str] = field(default=None)
+ latitude: Optional[float] = None
+ longitude: Optional[float] = None
+ altitude: Optional[float] = None
+ author: Optional[str] = None
+ source: Optional[NoteSource] = None
+
+ def __post_init__(self):
+ """
+ Post-initialization to update the digest if content is provided.
+ """
+ self.digest = self._update_digest()
+
+ def _update_digest(self) -> Optional[str]:
+ if self.content and not self.digest:
+ self.digest = sha256(self.content.encode('utf-8')).hexdigest()
+ return self.digest
+
+ def to_dict(self) -> dict:
+ return NoteItemSchema().dump( # type: ignore
+ {
+ **{
+ field: getattr(self, field)
+ for field in self.__dataclass_fields__
+ if not field.startswith('_') and field != 'parent'
+ },
+ 'parent': (
+ {
+ 'id': self.parent.id if self.parent else None,
+ 'title': self.parent.title if self.parent else None,
+ }
+ if self.parent
+ else None
+ ),
+ },
+ )
+
+
+@dataclass
+class NoteCollection(Serializable):
+ """
+ Represents a collection of notes.
+ """
+
+ id: Any
+ title: str
+ description: Optional[str] = None
+ parent: Optional['NoteCollection'] = None
+ created_at: Optional[datetime] = None
+ updated_at: Optional[datetime] = None
+ _notes: Dict[Any, Note] = field(default_factory=dict)
+ _collections: Dict[Any, 'NoteCollection'] = field(default_factory=dict)
+
+ @property
+ def notes(self) -> List[Note]:
+ return list(self._notes.values())
+
+ @property
+ def collections(self) -> List['NoteCollection']:
+ return list(self._collections.values())
+
+ def to_dict(self) -> dict:
+ return NoteCollectionSchema().dump( # type: ignore
+ {
+ **{
+ field: getattr(self, field)
+ for field in self.__dataclass_fields__
+ if not field.startswith('_') and field != 'parent'
+ },
+ 'parent': (
+ {
+ 'id': self.parent.id,
+ 'title': self.parent.title,
+ }
+ if self.parent
+ else None
+ ),
+ 'collections': [
+ collection.to_dict() for collection in self.collections
+ ],
+ 'notes': [
+ {
+ field: getattr(note, field)
+ for field in [*note.__dataclass_fields__, 'digest']
+ if field not in ['parent', 'content']
+ }
+ for note in self.notes
+ ],
+ }
+ )
diff --git a/platypush/message/event/notes.py b/platypush/message/event/notes.py
new file mode 100644
index 000000000..db9ead988
--- /dev/null
+++ b/platypush/message/event/notes.py
@@ -0,0 +1,120 @@
+import datetime
+from abc import ABC
+from typing import Optional, Union
+
+from dateutil.parser import isoparse
+
+from platypush.common.notes import Note, NoteCollection
+from platypush.message.event import Event
+
+DateLike = Optional[Union[float, str, datetime.datetime]]
+
+
+class BaseNoteEvent(Event, ABC):
+ """
+ Base class for note events.
+ """
+
+ def __init__(
+ self,
+ *args,
+ plugin: str,
+ **kwargs,
+ ):
+ """
+ :param plugin: The name of the plugin that triggered the event.
+ """
+ super().__init__(*args, plugin=plugin, **kwargs)
+
+ def _parse_timestamp(
+ self, timestamp: DateLike = None
+ ) -> Optional[datetime.datetime]:
+ """
+ Parse a timestamp string into a datetime object.
+ """
+ if timestamp is None:
+ return None
+
+ if isinstance(timestamp, datetime.datetime):
+ return timestamp
+
+ if isinstance(timestamp, (int, float)):
+ return datetime.datetime.fromtimestamp(timestamp)
+
+ try:
+ return isoparse(timestamp)
+ except ValueError:
+ return None
+
+
+class NoteItemEvent(BaseNoteEvent, ABC):
+ """
+ Base class for note item events.
+ """
+
+ def __init__( # pylint: disable=useless-parent-delegation
+ self, *args, note: Note, **kwargs
+ ):
+ """
+ :param note: Format:
+
+ .. schema:: notes.NoteItemSchema
+
+ """
+ super().__init__(*args, note=note, **kwargs)
+
+
+class NoteCollectionEvent(BaseNoteEvent, ABC):
+ """
+ Base class for note collection events.
+ """
+
+ def __init__( # pylint: disable=useless-parent-delegation
+ self, *args, collection: NoteCollection, **kwargs
+ ):
+ """
+ :param collection: Format:
+
+ .. schema:: notes.NoteCollectionSchema
+
+ """
+ super().__init__(*args, collection=collection, **kwargs)
+
+
+class NoteCreatedEvent(NoteItemEvent):
+ """
+ Event triggered when a note is created
+ """
+
+
+class NoteUpdatedEvent(NoteItemEvent):
+ """
+ Event triggered when a note is updated.
+ """
+
+
+class NoteDeletedEvent(NoteItemEvent):
+ """
+ Event triggered when a note is deleted.
+ """
+
+
+class CollectionCreatedEvent(NoteCollectionEvent):
+ """
+ Event triggered when a note collection is created.
+ """
+
+
+class CollectionUpdatedEvent(NoteCollectionEvent):
+ """
+ Event triggered when a note collection is updated.
+ """
+
+
+class CollectionDeletedEvent(NoteCollectionEvent):
+ """
+ Event triggered when a note collection is deleted.
+ """
+
+
+# vim:sw=4:ts=4:et:
diff --git a/platypush/plugins/_notes.py b/platypush/plugins/_notes.py
new file mode 100644
index 000000000..2b8682cc1
--- /dev/null
+++ b/platypush/plugins/_notes.py
@@ -0,0 +1,932 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from threading import RLock
+from time import time
+from typing import Any, Dict, Iterable, Optional, Type
+
+from platypush.common.notes import Note, NoteCollection, NoteSource
+from platypush.context import Variable
+from platypush.message.event.notes import (
+ BaseNoteEvent,
+ NoteCreatedEvent,
+ NoteUpdatedEvent,
+ NoteDeletedEvent,
+ CollectionCreatedEvent,
+ CollectionUpdatedEvent,
+ CollectionDeletedEvent,
+)
+from platypush.plugins import RunnablePlugin, action
+from platypush.utils import get_plugin_name_by_class
+
+
+class BaseNotePlugin(RunnablePlugin, ABC):
+ """
+ Base class for note-taking plugins.
+ """
+
+ def __init__(self, *args, poll_interval: float = 300, **kwargs):
+ """
+ :param poll_interval: Poll interval in seconds to check for updates (default: 300).
+ If set to zero or null, the plugin will not poll for updates,
+ and events will be generated only when you manually call :meth:`.sync`.
+ """
+ super().__init__(*args, poll_interval=poll_interval, **kwargs)
+ self._notes: Dict[Any, Note] = {}
+ self._collections: Dict[Any, NoteCollection] = {}
+ self._notes_lock = RLock()
+ self._collections_lock = RLock()
+ self._sync_lock = RLock()
+ self.__last_sync_time: Optional[datetime] = None
+
+ @property
+ def _last_sync_time_var(self) -> Variable:
+ """
+ Variable name for the last sync time.
+ """
+ return Variable(
+ f'_LAST_ITEMS_SYNC_TIME[{get_plugin_name_by_class(self.__class__)}]'
+ )
+
+ @property
+ def _last_sync_time(self) -> Optional[datetime]:
+ """
+ Get the last sync time from the variable.
+ """
+ if not self.__last_sync_time:
+ t = self._last_sync_time_var.get()
+ self.__last_sync_time = datetime.fromtimestamp(float(t)) if t else None
+
+ return self.__last_sync_time
+
+ @_last_sync_time.setter
+ def _last_sync_time(self, value: Optional[datetime]):
+ """
+ Set the last sync time to the variable.
+ """
+ with self._sync_lock:
+ self.__last_sync_time = value
+ if value is None:
+ self._last_sync_time_var.set(None)
+ else:
+ self._last_sync_time_var.set(value.timestamp())
+
+ @abstractmethod
+ def _fetch_note(self, note_id: Any, *args, **kwargs) -> Optional[Note]:
+ """
+ Don't call this directly if possible.
+ Instead, use :meth:`.get_note` method to retrieve a note and update the cache
+ in a thread-safe manner.
+
+ :param note_id: The ID of the note to fetch.
+ :return: The latest note from the backend.
+ """
+
+ @abstractmethod
+ def _fetch_notes(self, *args, **kwargs) -> Iterable[Note]:
+ """
+ Don't call this directly if possible.
+ Instead, use :meth:`.get_notes` method to retrieve notes and update the cache
+ in a thread-safe manner.
+
+ :return: The latest list of notes from the backend.
+ """
+
+ @abstractmethod
+ def _create_note(
+ self,
+ title: str,
+ content: str,
+ *args,
+ description: Optional[str] = None,
+ collection: Optional[Any] = None,
+ geo: Optional[Dict[str, Optional[float]]] = None,
+ author: Optional[str] = None,
+ source: Optional[NoteSource] = None,
+ **kwargs,
+ ) -> Note:
+ """
+ Create a new note with the specified title and content.
+ """
+
+ @abstractmethod
+ def _edit_note(
+ self,
+ note_id: Any,
+ *args,
+ title: Optional[str] = None,
+ content: Optional[str] = None,
+ collection: Optional[Any] = None,
+ geo: Optional[Dict[str, Optional[float]]] = None,
+ author: Optional[str] = None,
+ source: Optional[NoteSource] = None,
+ **kwargs,
+ ) -> Note:
+ """
+ Edit an existing note by ID.
+ """
+
+ @abstractmethod
+ def _delete_note(self, note_id: Any, *args, **kwargs):
+ """
+ Delete a note by ID.
+ """
+
+ @abstractmethod
+ def _fetch_collection(
+ self, collection_id: Any, *args, **kwargs
+ ) -> Optional[NoteCollection]:
+ """
+ Don't call this directly if possible.
+ Instead, use :meth:`.get_collection` to retrieve a collection and update the cache
+ in a thread-safe manner.
+
+ :param collection_id: The ID of the collection to fetch.
+ :return: The latest collection from the backend.
+ """
+
+ @abstractmethod
+ def _fetch_collections(self, *args, **kwargs) -> Iterable[NoteCollection]:
+ """
+ Don't call this directly if possible.
+ Instead, use :meth:`.get_collections` to retrieve collections and update the cache
+ in a thread-safe manner.
+
+ :return: The latest list of note collections from the backend.
+ """
+
+ @abstractmethod
+ def _create_collection(
+ self,
+ title: str,
+ *args,
+ description: Optional[str] = None,
+ parent: Optional[Any] = None,
+ **kwargs,
+ ) -> NoteCollection:
+ """
+ Create a new note collection with the specified title and description.
+ """
+
+ @abstractmethod
+ def _edit_collection(
+ self,
+ collection_id: Any,
+ *args,
+ title: Optional[str] = None,
+ description: Optional[str] = None,
+ parent: Optional[Any] = None,
+ **kwargs,
+ ) -> NoteCollection:
+ """
+ Edit an existing note collection by ID.
+ """
+
+ @abstractmethod
+ def _delete_collection(self, collection_id: Any, *args, **kwargs):
+ """
+ Delete a note collection by ID.
+ This method should be implemented by subclasses.
+ """
+
+ def _process_results( # pylint: disable=too-many-positional-arguments
+ self,
+ items: Iterable[Any],
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ sort: Optional[Dict[str, bool]] = None,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ ) -> Iterable[Any]:
+ if not sort:
+ sort = {'created_at': True}
+
+ if filter:
+ items = [
+ item
+ for item in items
+ if all(getattr(item, k) == v for k, v in filter.items())
+ ]
+
+ items = sorted(
+ items,
+ key=lambda item: tuple(getattr(item, field) for field in sort.keys()),
+ reverse=any(not ascending for ascending in sort.values()),
+ )
+
+ if offset is not None:
+ items = items[offset:]
+ if limit is not None:
+ items = items[:limit]
+
+ return items
+
+ def _dispatch_events(self, *events):
+ """
+ Dispatch the given events to the event bus.
+ """
+ if not self.bus:
+ self.logger.warning(
+ 'Event bus not available. Events will not be dispatched.'
+ )
+ return
+
+ for event in events:
+ self.bus.post(event)
+
+ def _process_events(
+ self,
+ notes: Iterable[Note],
+ prev_notes: Dict[Any, Note],
+ collections: Iterable[NoteCollection],
+ prev_collections: Dict[Any, NoteCollection],
+ ):
+ most_recent_note = list(notes)[0] if notes else None
+ most_recent_collection = list(collections)[0] if collections else None
+ max_updated_at = max(
+ (
+ most_recent_note.updated_at.timestamp()
+ if most_recent_note and most_recent_note.updated_at
+ else 0
+ ),
+ (
+ most_recent_collection.updated_at.timestamp()
+ if most_recent_collection and most_recent_collection.updated_at
+ else 0
+ ),
+ )
+
+ with self._sync_lock:
+ self._process_collections_events(collections, prev_collections)
+ self._process_notes_events(notes, prev_notes)
+ self._last_sync_time = (
+ datetime.fromtimestamp(max_updated_at) if max_updated_at > 0 else None
+ )
+
+ @classmethod
+ def _make_event(
+ cls, evt_type: Type[BaseNoteEvent], *args, **kwargs
+ ) -> BaseNoteEvent:
+ """
+ Create a note event of the specified type.
+ """
+ return evt_type(*args, plugin=get_plugin_name_by_class(cls), **kwargs)
+
+ def _process_notes_events(
+ self,
+ notes: Iterable[Note],
+ prev_notes: Dict[Any, Note],
+ ):
+ last_sync_time = self._last_sync_time.timestamp() if self._last_sync_time else 0
+ new_notes_by_id = {note.id: note for note in notes}
+
+ removed_note_events = [
+ self._make_event(NoteDeletedEvent, note=note)
+ for note_id, note in prev_notes.items()
+ if note_id not in new_notes_by_id
+ ]
+
+ created_note_events = []
+ updated_note_events = []
+
+ for note in notes:
+ created_at = note.created_at.timestamp() if note.created_at else 0
+ updated_at = note.updated_at.timestamp() if note.updated_at else 0
+
+ if created_at > last_sync_time:
+ created_note_events.append(
+ self._make_event(NoteCreatedEvent, note=note)
+ )
+ elif updated_at > last_sync_time:
+ updated_note_events.append(
+ self._make_event(NoteUpdatedEvent, note=note)
+ )
+ else:
+ # Assuming that the list of notes is sorted by updated_at
+ break
+
+ self._dispatch_events(
+ *removed_note_events,
+ *created_note_events,
+ *updated_note_events,
+ )
+
+ def _process_collections_events(
+ self,
+ collections: Iterable[NoteCollection],
+ prev_collections: Dict[Any, NoteCollection],
+ ):
+ last_sync_time = self._last_sync_time.timestamp() if self._last_sync_time else 0
+ new_collections_by_id = {
+ collection.id: collection for collection in collections
+ }
+
+ removed_collection_events = [
+ self._make_event(CollectionDeletedEvent, collection=collection)
+ for collection_id, collection in prev_collections.items()
+ if collection_id not in new_collections_by_id
+ ]
+
+ created_collection_events = []
+ updated_collection_events = []
+
+ for collection in collections:
+ created_at = (
+ collection.created_at.timestamp() if collection.created_at else 0
+ )
+ updated_at = (
+ collection.updated_at.timestamp() if collection.updated_at else 0
+ )
+
+ if created_at > last_sync_time:
+ created_collection_events.append(
+ self._make_event(CollectionCreatedEvent, collection=collection)
+ )
+ elif updated_at > last_sync_time:
+ updated_collection_events.append(
+ self._make_event(CollectionUpdatedEvent, collection=collection)
+ )
+ else:
+ # Assuming that the list of collections is sorted by updated_at
+ break
+
+ self._dispatch_events(
+ *removed_collection_events,
+ *created_collection_events,
+ *updated_collection_events,
+ )
+
+ def _get_note(self, note_id: Any, *args, **kwargs) -> Note:
+ note = self._fetch_note(note_id, *args, **kwargs)
+ assert note, f'Note with ID {note_id} not found'
+ with self._notes_lock:
+ # Always overwrite the note in the cache,
+ # as this is the most up-to-date complete version
+ self._notes[note.id] = note
+ if note.parent and note.parent.id in self._collections:
+ self._collections[ # pylint: disable=protected-access
+ note.parent.id
+ ]._notes[ # pylint: disable=protected-access
+ note.id
+ ] = note
+
+ return self._notes[note.id]
+
+ def _get_notes(
+ self,
+ *args,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ sort: Optional[Dict[str, bool]] = None,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ fetch: bool = False,
+ **kwargs,
+ ) -> Iterable[Note]:
+ # Always fetch if background polling is disabled
+ fetch = fetch or not self.poll_interval
+ if fetch:
+ with self._notes_lock:
+ self._notes = {
+ note.id: note for note in self._fetch_notes(*args, **kwargs)
+ }
+ self._refresh_notes_cache()
+
+ return self._process_results(
+ self._notes.values(),
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ )
+
+ def _get_collection(self, collection_id: Any, *args, **kwargs) -> NoteCollection:
+ collection = self._fetch_collection(collection_id, *args, **kwargs)
+ assert collection, f'Collection with ID {collection_id} not found'
+ with self._collections_lock:
+ # Always overwrite the collection in the cache,
+ # as this is the most up-to-date complete version
+ self._collections[collection.id] = collection
+ self._refresh_collections_cache()
+
+ return self._collections[collection.id]
+
+ def _get_collections(
+ self,
+ *args,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ sort: Optional[Dict[str, bool]] = None,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ fetch: bool = False,
+ **kwargs,
+ ) -> Iterable[NoteCollection]:
+ """
+ Get the note collections from the cache or fetch them from the backend if needed.
+
+ :return: An iterable of NoteCollection objects.
+ """
+ # Always fetch if background polling is disabled
+ fetch = fetch or not self.poll_interval
+ if fetch:
+ with self._collections_lock:
+ self._collections = {
+ collection.id: collection
+ for collection in self._fetch_collections(*args, **kwargs)
+ }
+ self._refresh_collections_cache()
+
+ return self._process_results(
+ self._collections.values(),
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ )
+
+ def _refresh_notes_cache(self):
+ for note in list(self._notes.values()):
+ if note.parent and note.parent.id in self._collections:
+ note.parent = self._collections[note.parent.id]
+ self._collections[ # pylint: disable=protected-access
+ note.parent.id
+ ]._notes[ # pylint: disable=protected-access
+ note.id
+ ] = note
+
+ def _refresh_collections_cache(self):
+ for collection in list(self._collections.values()):
+ # Link the notes to their parent collections
+ for note in list(collection.notes):
+ if note.id in self._notes:
+ collection._notes[note.id] = self._notes[
+ note.id
+ ] # pylint: disable=protected-access
+
+ # Link the child collections to their parent collections
+ tmp_collections = list(collection.collections)
+ for collection in tmp_collections:
+ if collection.id not in self._collections:
+ collection._collections[
+ collection.id
+ ] = self._collections[ # pylint: disable=protected-access
+ collection.id
+ ]
+
+ # Link the parent collections to their child collections
+ tmp_collections = list(collection.collections)
+ for collection in tmp_collections:
+ if collection.parent and collection.parent.id in self._collections:
+ collection.parent = self._collections[collection.parent.id]
+
+ @staticmethod
+ def _parse_geo(data: dict) -> Dict[str, Optional[float]]:
+ return {
+ key: value or None
+ for key, value in {
+ key: float(data.get(key, 0))
+ for key in ('latitude', 'longitude', 'altitude')
+ }.items()
+ }
+
+ @action
+ def get_note(self, note_id: Any, *args, **kwargs) -> dict:
+ """
+ Get a single note and its full content by ID.
+
+ :param note_id: The ID of the note to retrieve.
+ :return: The note as a dictionary, format:
+
+ .. schema:: notes.NoteItemSchema
+
+ """
+ return self._get_note(note_id, *args, **kwargs).to_dict()
+
+ @action
+ def get_notes(
+ self,
+ *args,
+ limit: Optional[int] = None,
+ offset: Optional[int] = 0,
+ sort: Optional[Dict[str, bool]] = None,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ fetch: bool = False,
+ **kwargs,
+ ) -> Iterable[dict]:
+ """
+ Get notes from the cache or fetch them from the backend.
+
+ :param limit: Maximum number of collections to retrieve (default: None, meaning no limit).
+ :param offset: Offset to start retrieving collections from (default: 0).
+ :param sort: A dictionary specifying the fields to sort by and their order.
+ Example: {'created_at': True} sorts by creation date in ascending
+ order, while {'created_at': False} sorts in descending order.
+ :param filter: A dictionary specifying filters to apply to the collections.
+ :param fetch: If True, always fetch the latest collections from the backend,
+ regardless of the cache state (default: False).
+ :param kwargs: Additional keyword arguments to pass to the fetch method.
+ :return: An iterable of notes, format:
+
+ .. schema:: notes.NoteItemSchema(many=True)
+
+ """
+ return [
+ note.to_dict()
+ for note in self._get_notes(
+ *args,
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ fetch=fetch,
+ **kwargs,
+ )
+ ]
+
+ @action
+ def create_note(
+ self,
+ title: str,
+ content: str,
+ *args,
+ description: Optional[str] = None,
+ collection: Optional[Any] = None,
+ geo: Optional[Dict[str, Optional[float]]] = None,
+ source: Optional[dict] = None,
+ author: Optional[str] = None,
+ **kwargs,
+ ) -> dict:
+ """
+ Create a new note with the specified title and content.
+
+ :param title: The title of the new note.
+ :param content: The content of the new note.
+ :param description: Optional description for the note.
+ :param collection: Optional collection ID to add the note to.
+ :param geo: Optional geographical coordinates as a dictionary with the
+ fields ``latitude``, ``longitude``, and ``altitude``.
+ :param source: Optional source information for the note, with at least
+ one of the fields ``url``, ``name`` or ``app``. By default, the
+ source is ``platypush`` and the app is ``tech.platypush``.
+ :param author: Optional author of the note.
+ :return: The created note as a dictionary, format:
+
+ .. schema:: notes.NoteItemSchema
+ """
+ src = NoteSource(
+ **(
+ source
+ or {
+ 'name': 'platypush',
+ 'app': 'tech.platypush',
+ }
+ )
+ )
+
+ note = self._create_note(
+ title,
+ content,
+ *args,
+ description=description,
+ collection=collection,
+ geo=self._parse_geo(geo) if geo else None,
+ author=author,
+ source=src,
+ **kwargs,
+ )
+
+ with self._notes_lock:
+ # Add the new note to the cache
+ self._notes[note.id] = note
+
+ # If it has a parent collection, add it to the collection's notes
+ if note.parent and note.parent.id in self._collections:
+ self._collections[ # pylint: disable=protected-access
+ note.parent.id
+ ]._notes[note.id] = note
+
+ # Trigger the note created event
+ self._dispatch_events(self._make_event(NoteCreatedEvent, note=note))
+
+ return note.to_dict()
+
+ @action
+ def edit_note(
+ self,
+ note_id: Any,
+ *args,
+ title: Optional[str] = None,
+ content: Optional[str] = None,
+ description: Optional[str] = None,
+ collection: Optional[Any] = None,
+ geo: Optional[Dict[str, Optional[float]]] = None,
+ source: Optional[dict] = None,
+ author: Optional[str] = None,
+ **kwargs,
+ ) -> dict:
+ """
+ Edit an existing note by ID.
+
+ :param note_id: The ID of the note to edit.
+ :param title: New title for the note.
+ :param content: New content for the note.
+ :param description: Optional new description for the note.
+ :param collection: New collection ID to move the note under.
+ :param geo: Optional geographical coordinates as a dictionary with the
+ fields ``latitude``, ``longitude``, and ``altitude``.
+ :param source: Optional source information for the note, with at least
+ one of the fields ``url``, ``name`` or ``app``.
+ :param author: Optional author of the note.
+ :return: The updated note as a dictionary, format:
+
+ .. schema:: notes.NoteItemSchema
+ """
+ with self._notes_lock:
+ note = self._edit_note(
+ note_id,
+ *args,
+ title=title,
+ content=content,
+ description=description,
+ collection=collection,
+ geo=self._parse_geo(geo) if geo else None,
+ author=author,
+ source=NoteSource(**source) if source else None,
+ **kwargs,
+ )
+
+ # Update the cache with the edited note
+ self._notes[note.id] = note
+
+ # If it has a parent collection, update it in the collection's notes
+ if note.parent and note.parent.id in self._collections:
+ self._collections[ # pylint: disable=protected-access
+ note.parent.id
+ ]._notes[ # pylint: disable=protected-access
+ note.id
+ ] = note
+
+ # Trigger the note updated event
+ self._dispatch_events(self._make_event(NoteUpdatedEvent, note=note))
+
+ return note.to_dict()
+
+ @action
+ def delete_note(self, note_id: Any, *args, **kwargs):
+ """
+ Delete a note by ID.
+
+ :param note_id: The ID of the note to delete.
+ """
+ with self._notes_lock:
+ self._delete_note(note_id, *args, **kwargs)
+ note = self._notes.pop(note_id, None)
+ if not note:
+ note = Note(id=note_id, title='')
+
+ # Remove the note from its parent collection if it has one
+ if note.parent and note.parent.id in self._collections:
+ parent_collection = self._collections[note.parent.id]
+ parent_collection.notes.remove(note)
+
+ # Dispatch the deletion event
+ self._dispatch_events(self._make_event(NoteDeletedEvent, note=note))
+
+ @action
+ def get_collection(self, collection_id: Any, *args, **kwargs) -> dict:
+ """
+ Get a single note collection by ID.
+
+ :param collection_id: The ID of the collection to retrieve.
+ :return: The collection as a dictionary, format:
+
+ .. schema:: notes.NoteCollectionSchema
+
+ """
+ return self._get_collection(collection_id, *args, **kwargs).to_dict()
+
+ @action
+ def get_collections(
+ self,
+ *args,
+ limit: Optional[int] = None,
+ offset: Optional[int] = 0,
+ sort: Optional[Dict[str, bool]] = None,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ fetch: bool = False,
+ **kwargs,
+ ) -> Iterable[dict]:
+ """
+ Get note collections from the cache or fetch them from the backend.
+
+ :param limit: Maximum number of collections to retrieve (default: None, meaning no limit).
+ :param offset: Offset to start retrieving collections from (default: 0).
+ :param sort: A dictionary specifying the fields to sort by and their order.
+ Example: {'created_at': True} sorts by creation date in ascending
+ order, while {'created_at': False} sorts in descending order.
+ :param filter: A dictionary specifying filters to apply to the collections.
+ :param fetch: If True, always fetch the latest collections from the backend,
+ regardless of the cache state (default: False).
+ :param kwargs: Additional keyword arguments to pass to the fetch method.
+ :return: An iterable of note collections, format:
+
+ .. schema:: notes.NoteCollectionSchema(many=True)
+
+ """
+ return [
+ collection.to_dict()
+ for collection in self._get_collections(
+ *args,
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ fetch=fetch,
+ **kwargs,
+ )
+ ]
+
+ @action
+ def create_collection(
+ self,
+ title: str,
+ *args,
+ description: Optional[str] = None,
+ parent: Optional[Any] = None,
+ **kwargs,
+ ) -> dict:
+ """
+ Create a new note collection with the specified title and description.
+
+ :param title: The title of the new collection.
+ :param description: Optional description for the new collection.
+ :param parent: Optional parent collection ID to create a sub-collection.
+ :return: The created collection as a dictionary, format:
+
+ .. schema:: notes.NoteCollectionSchema
+
+ """
+ collection = self._create_collection(
+ title, *args, description=description, parent=parent, **kwargs
+ )
+
+ with self._collections_lock:
+ # Add the new collection to the cache
+ self._collections[collection.id] = collection
+
+ # If it has a parent, add it to the parent's collections
+ if collection.parent and collection.parent.id in self._collections:
+ parent_collection = self._collections[collection.parent.id]
+ parent_collection.collections.append(collection)
+
+ # Trigger the collection created event
+ self._dispatch_events(
+ self._make_event(CollectionCreatedEvent, collection=collection)
+ )
+
+ return collection.to_dict()
+
+ @action
+ def edit_collection(
+ self,
+ collection_id: Any,
+ *args,
+ title: Optional[str] = None,
+ description: Optional[str] = None,
+ parent: Optional[Any] = None,
+ **kwargs,
+ ) -> dict:
+ """
+ Edit an existing note collection by ID.
+
+ :param collection_id: The ID of the collection to edit.
+ :param title: New title for the collection.
+ :param description: New description for the collection.
+ :param parent: New parent collection ID to move the collection under.
+ :return: The updated collection as a dictionary, format:
+
+ .. schema:: notes.NoteCollectionSchema
+
+ """
+ with self._collections_lock:
+ collection = self._edit_collection(
+ collection_id,
+ *args,
+ title=title,
+ description=description,
+ parent=parent,
+ **kwargs,
+ )
+
+ # Update the cache with the edited collection
+ old_collection = self._collections.get(collection.id)
+ self._collections[collection.id] = collection
+
+ # If the parent has changed, remove it from the old parent's collections
+ if (
+ old_collection
+ and old_collection.parent
+ and old_collection.parent != collection.parent
+ and old_collection.parent.id in self._collections
+ ):
+ parent_collection = self._collections.get(old_collection.parent.id)
+ if (
+ parent_collection
+ and old_collection in parent_collection.collections
+ ):
+ parent_collection.collections.remove(old_collection)
+
+ # If it has a parent, update it in the parent's collections
+ if collection.parent and collection.parent.id in self._collections:
+ parent_collection = self._collections.get(collection.parent.id)
+ if (
+ parent_collection
+ and collection not in parent_collection.collections
+ ):
+ parent_collection.collections.append(collection)
+
+ # Trigger the collection updated event
+ self._dispatch_events(
+ self._make_event(CollectionUpdatedEvent, collection=collection)
+ )
+
+ return collection.to_dict()
+
+ @action
+ def delete_collection(self, collection_id: Any, *args, **kwargs):
+ """
+ Delete a note collection by ID.
+
+ :param collection_id: The ID of the collection to delete.
+ """
+ with self._collections_lock:
+ self._delete_collection(collection_id, *args, **kwargs)
+ collection = self._collections.pop(collection_id, None)
+ if not collection:
+ collection = NoteCollection(id=collection_id, title='')
+
+ # Remove the collection from its parent if it has one
+ if collection.parent and collection.parent.id in self._collections:
+ parent_collection = self._collections[collection.parent.id]
+ parent_collection.collections.remove(collection)
+
+ # Dispatch the deletion event
+ self._dispatch_events(
+ self._make_event(CollectionDeletedEvent, collection=collection)
+ )
+
+ @action
+ def sync(self, *args, **kwargs):
+ """
+ Synchronize the notes and collections with the backend.
+ This method is called periodically based on the ``poll_interval`` setting.
+ If ``poll_interval`` is zero or null, you can manually call this method
+ to synchronize the notes and collections.
+ """
+ t_start = time()
+ self.logger.info('Synchronizing notes and collections...')
+
+ with self._sync_lock:
+ prev_notes = self._notes.copy()
+ prev_collections = self._collections.copy()
+ notes = self._get_notes(
+ *args, fetch=True, sort={'updated_at': False}, **kwargs
+ )
+ collections = self._get_collections(
+ *args, fetch=True, sort={'updated_at': False}, **kwargs
+ )
+
+ self._process_events(
+ notes=notes,
+ prev_notes=prev_notes,
+ collections=collections,
+ prev_collections=prev_collections,
+ )
+
+ self.logger.info(
+ 'Synchronization completed in %.2f seconds',
+ time() - t_start,
+ )
+
+ @action
+ def reset_sync(self):
+ """
+ Reset the last sync time to None, forcing a full resync on the next call to
+ :meth:`.sync`, which in turn will re-trigger all notes and collections events.
+ """
+ self.logger.info('Resetting last sync time')
+ with self._sync_lock:
+ self._last_sync_time = None
+ self._notes.clear()
+ self._collections.clear()
+
+ def main(self):
+ if not self.poll_interval:
+ # If no poll interval is set then we won't poll for new check-ins
+ self.wait_stop()
+ return
+
+ while not self.should_stop():
+ try:
+ self.sync()
+ except Exception as e:
+ self.logger.error('Error during sync: %s', e)
+ finally:
+ self.wait_stop(self.poll_interval)
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
new file mode 100644
index 000000000..61e6e05d6
--- /dev/null
+++ b/platypush/plugins/joplin/__init__.py
@@ -0,0 +1,441 @@
+import datetime
+from typing import Any, List, Optional
+from urllib.parse import urljoin
+
+import requests
+
+from platypush.common.notes import Note, NoteCollection, NoteSource
+from platypush.plugins._notes import BaseNotePlugin
+
+
+class JoplinPlugin(BaseNotePlugin):
+ r"""
+ Plugin to interact with `Joplin <https://joplinapp.org/>`_, a free and
+ open-source note-taking application.
+
+ ## Advised setup
+
+ Joplin is mainly intended as a desktop application, with support for
+ synchronization across multiple devices via various backends.
+
+ This plugin is designed to interact with the Joplin API exposed by the
+ "desktop" application - the same one used by the `Joplin Web Clipper
+ extension <https://joplinapp.org/clipper/>`_.
+
+ This can be achieved in two ways:
+
+ 1. Using the Joplin desktop application
+ 2. Using Joplin in headless mode
+
+ ### Using the Joplin desktop application
+
+ To use the Joplin desktop application, you need to enable the Web Clipper
+ service in the Joplin settings. This will expose an HTTP API on the port
+ 41184 on your local machine.
+
+ Note that the Joplin desktop application must be running for this plugin to
+ work, and by default it will only accept requests from the local machine.
+
+ If you want to run the Platypush server on a different machine, you can
+ e.g. open an SSH tunnel to the Joplin machine:
+
+ .. code-block:: bash
+ ssh -L 41184:localhost:41184 user@joplin-machine
+
+ ### Using Joplin in headless mode
+
+ The downside of the Joplin desktop application is that it must be running
+ on a live desktop session for the plugin to work. Unfortunately, Joplin
+ does not provide an official headless mode, but there is `a community
+ project <https://github.com/jspiers/headless-joplin>`_ that allows you to
+ run Joplin in a Docker container in headless mode, exposing the same HTTP
+ API as the desktop application.
+
+ It's first advised to run and configure your Joplin on the desktop app.
+ Then locate the Joplin profile directory (usually
+ ``~/.config/joplin-desktop`` on Linux, but depending on the installed
+ version it could also be named ``Joplin`` or ``joplin``) and copy the
+ file named ``settings.json`` to ``~/.config/joplin/settings.json`` on the
+ machine where you run the headless Joplin container.
+
+ Optionally, you can also copy the ``database.sqlite`` file from the
+ desktop Joplin profile directory to the headless Joplin profile, but make
+ sure that the desktop instance and the headless instance run the same
+ version of Joplin, otherwise you might run into database incompatibility
+ issues.
+
+ #### Credentials
+
+ If you opt not to copy the database file to the headless Joplin profile,
+ then you may have to manually set up the credentials in a separate file
+ named ``secrets.json`` in the Joplin profile directory. This will contain
+ both the API token and the passwords for any connected synchronization
+ services. For example:
+
+ .. code-block:: json
+
+ {
+ "api.token": "your_api_token_here",
+ "sync.1.password": "your_sync_password_here"
+ }
+
+ Note:
+
+ 1. The ``api.token`` field is mandatory, and it should match the one
+ you configure in this plugin.
+
+ 2. ``sync.<n>.password`` should be such that ``<n>`` is the index of
+ the synchronization target configured in ``settings.json`` (e.g.
+ Dropbox, Nextcloud, S3, Joplin Cloud, a local Joplin server, etc.).
+
+ #### Running the service
+
+ .. code-block:: bash
+
+ docker run --rm \
+ --name joplin-headless \
+ -v ~/.config/joplin:/home/node/.config/joplin \
+ -v ~/.config/joplin/secrets.json:/run/secrets/joplin-config.json:ro \
+ -p 41184:80 \
+ jspiers/headless-joplin:2.13.2-node-20.11.1
+
+ #### Synchronization
+
+ Unlike the Joplin desktop application, the headless Joplin instance does
+ not provide a periodic synchronization mechanism. But you can schedule a
+ cronjob to periodically run synchronization each e.g. 5 minutes:
+
+ .. code-block:: bash
+
+ crontab -e
+ # Add the following line to the crontab file
+ */5 * * * * /usr/bin/docker exec joplin-headless joplin sync
+
+ It is advised to have at least a remote synchronization target, and have
+ the same target configured both in the Joplin desktop or mobile application
+ and in the headless instance, so that you can keep your notes in sync
+ across all of your devices, even though this plugin will only interact with
+ the headless instance.
+ """
+
+ _default_note_fields = (
+ 'id',
+ 'parent_id',
+ 'title',
+ 'created_time',
+ 'updated_time',
+ 'latitude',
+ 'longitude',
+ 'altitude',
+ 'author',
+ 'source',
+ 'source_url',
+ 'source_application',
+ )
+
+ _default_collection_fields = (
+ 'id',
+ 'title',
+ 'parent_id',
+ 'created_time',
+ 'updated_time',
+ )
+
+ def __init__(self, *args, host: str, port: int = 41184, token: str, **kwargs):
+ """
+ :param host: The hostname or IP address of your Joplin application.
+ :param port: The port number of your Joplin application (default: 41184).
+ :param token: The access token of your Joplin server.
+ """
+ super().__init__(*args, **kwargs)
+ self.host = host
+ self.port = port
+ self.token = token
+
+ def _base_url(self, path: str = '') -> str:
+ return urljoin(
+ f'http://{self.host}:{self.port}/',
+ path.lstrip('/'),
+ )
+
+ def _exec(self, method: str, path: str = '', **kwargs) -> Optional[dict]:
+ """
+ Execute a request to the Joplin API.
+ """
+ url = self._base_url(path)
+ params = kwargs.pop('params', {})
+ self.logger.debug(
+ 'Calling Joplin API: %s %s with params: %s',
+ method,
+ url,
+ params,
+ )
+
+ params['token'] = self.token
+ response = requests.request(method, url, params=params, timeout=10, **kwargs)
+
+ if not response.ok:
+ err = response.text
+ try:
+ rs = response.json()
+ err = rs.get('error', err).splitlines()[0]
+ except (TypeError, ValueError):
+ pass
+
+ raise RuntimeError(
+ f'Joplin API request failed with status {response.status_code}: {err}'
+ )
+
+ try:
+ return response.json()
+ except ValueError:
+ return None
+
+ def _parse_source(self, data: dict) -> Optional[NoteSource]:
+ has_source = any(
+ key in data for key in ('source', 'source_url', 'source_application')
+ )
+
+ if not has_source:
+ return None
+
+ return NoteSource(
+ name=data.get('source'),
+ url=data.get('source_url'),
+ app=data.get('source_application'),
+ )
+
+ @staticmethod
+ def _parse_time(t: Optional[int]) -> Optional[datetime.datetime]:
+ """
+ Parse a Joplin timestamp (in milliseconds) into a datetime object.
+ """
+ if t is None:
+ return None
+ return datetime.datetime.fromtimestamp(t / 1000)
+
+ def _to_note(self, data: dict) -> Note:
+ parent_id = data.get('parent_id')
+ parent = None
+
+ if parent_id:
+ parent = self._collections.get(
+ parent_id, NoteCollection(id=parent_id, title='')
+ )
+
+ return Note(
+ **{
+ 'id': data.get('id', ''),
+ 'title': data.get('title', ''),
+ 'description': data.get('description'),
+ 'content': data.get('body'),
+ 'parent': parent,
+ 'source': self._parse_source(data),
+ **self._parse_geo(data),
+ 'created_at': self._parse_time(data.get('created_time')),
+ 'updated_at': self._parse_time(data.get('updated_time')),
+ }
+ )
+
+ def _to_collection(self, data: dict) -> NoteCollection:
+ """
+ Convert a Joplin collection (folder) to a NoteCollection.
+ """
+ return NoteCollection(
+ id=data.get('id', ''),
+ title=data.get('title', ''),
+ description=data.get('description'),
+ created_at=self._parse_time(data.get('created_time')),
+ updated_at=self._parse_time(data.get('updated_time')),
+ )
+
+ def _fetch_note(self, note_id: Any, *_, **__) -> Optional[Note]:
+ note = None
+ err = None
+
+ try:
+ note = self._exec(
+ 'GET',
+ f'notes/{note_id}',
+ params={
+ 'fields': ','.join(
+ *[
+ self._default_note_fields,
+ 'body', # Include body content
+ ]
+ )
+ },
+ )
+ except RuntimeError as e:
+ err = e
+
+ if not note:
+ self.logger.warning(
+ 'Note with ID %s could not be fetched: %s',
+ note_id,
+ err if err else 'Unknown error',
+ )
+ return None
+
+ return self._to_note(note) # type: ignore[return-value]
+
+ def _fetch_notes(self, *_, **__) -> List[Note]:
+ """
+ Fetch notes from Joplin.
+ """
+ notes_data = (
+ self._exec(
+ 'GET', 'notes', params={'fields': ','.join(self._default_note_fields)}
+ )
+ or {}
+ ).get('items', [])
+ return [self._to_note(note) for note in notes_data]
+
+ def _create_note(
+ self,
+ title: str,
+ content: str,
+ *_,
+ parent: Optional[Any] = None,
+ geo: Optional[dict] = None,
+ source: Optional[NoteSource] = None,
+ author: Optional[str] = None,
+ **__,
+ ) -> Note:
+ data = {
+ 'title': title,
+ 'body': content,
+ 'parent_id': parent,
+ 'latitude': geo.get('latitude') if geo else None,
+ 'longitude': geo.get('longitude') if geo else None,
+ 'altitude': geo.get('altitude') if geo else None,
+ 'author': author or '',
+ }
+
+ if source:
+ data['source'] = source.name or ''
+ data['source_url'] = source.url or ''
+ data['source_application'] = source.app or ''
+
+ response = self._exec('POST', 'notes', json=data)
+ assert response, 'Failed to create note'
+ return self._to_note(response)
+
+ def _edit_note(
+ self,
+ note_id: Any,
+ *_,
+ title: Optional[str] = None,
+ content: Optional[str] = None,
+ parent: Optional[Any] = None,
+ geo: Optional[dict] = None,
+ source: Optional[NoteSource] = None,
+ author: Optional[str] = None,
+ **__,
+ ) -> Note:
+ data = {}
+
+ if title is not None:
+ data['title'] = title
+ if content is not None:
+ data['body'] = content
+ if parent is not None:
+ data['parent_id'] = parent
+ if geo:
+ data['latitude'] = geo.get('latitude')
+ data['longitude'] = geo.get('longitude')
+ data['altitude'] = geo.get('altitude')
+ if author is not None:
+ data['author'] = author
+ if source:
+ data['source'] = source.name or ''
+ data['source_url'] = source.url or ''
+ data['source_application'] = source.app or ''
+
+ response = self._exec('PUT', f'notes/{note_id}', json=data)
+ assert response, 'Failed to edit note'
+ return self._to_note(response)
+
+ def _delete_note(self, note_id: Any, *_, **__):
+ self._exec('DELETE', f'notes/{note_id}')
+
+ def _fetch_collection(
+ self, collection_id: Any, *_, **__
+ ) -> Optional[NoteCollection]:
+ """
+ Fetch a collection (folder) by its ID.
+ """
+ collection_data = self._exec(
+ 'GET',
+ f'folders/{collection_id}',
+ params={'fields': ','.join(self._default_collection_fields)},
+ )
+
+ if not collection_data:
+ self.logger.warning(
+ 'Collection with ID %s could not be fetched', collection_id
+ )
+ return None
+
+ return self._to_collection(collection_data)
+
+ def _fetch_collections(self, *_, **__) -> List[NoteCollection]:
+ """
+ Fetch collections (folders) from Joplin.
+ """
+ collections_data = (
+ self._exec(
+ 'GET',
+ 'folders',
+ params={'fields': ','.join(self._default_collection_fields)},
+ )
+ or {}
+ ).get('items', [])
+ return [self._to_collection(coll) for coll in collections_data]
+
+ def _create_collection(
+ self,
+ title: str,
+ *_,
+ parent: Optional[Any] = None,
+ **__,
+ ) -> NoteCollection:
+ response = self._exec(
+ 'POST',
+ 'folders',
+ json={
+ 'title': title,
+ 'parent_id': parent,
+ },
+ )
+
+ assert response, 'Failed to create collection'
+ return self._to_collection(response)
+
+ def _edit_collection(
+ self,
+ collection_id: Any,
+ *_,
+ title: Optional[str] = None,
+ parent: Optional[Any] = None,
+ **__,
+ ) -> NoteCollection:
+ data = {}
+
+ if title is not None:
+ data['title'] = title
+ if parent is not None:
+ data['parent_id'] = parent
+
+ response = self._exec('PUT', f'folders/{collection_id}', json=data)
+ assert response, 'Failed to edit collection'
+ return self._to_collection(response)
+
+ def _delete_collection(self, collection_id: Any, *_: Any, **__: Any):
+ """
+ Delete a collection (folder) by its ID.
+ """
+ self._exec('DELETE', f'folders/{collection_id}')
+
+
+# vim:sw=4:ts=4:et:
diff --git a/platypush/plugins/joplin/manifest.json b/platypush/plugins/joplin/manifest.json
new file mode 100644
index 000000000..cf4e652d3
--- /dev/null
+++ b/platypush/plugins/joplin/manifest.json
@@ -0,0 +1,17 @@
+{
+ "manifest": {
+ "events": [
+ "platypush.message.event.notes.NoteCreatedEvent",
+ "platypush.message.event.notes.NoteUpdatedEvent",
+ "platypush.message.event.notes.NoteDeletedEvent",
+ "platypush.message.event.notes.CollectionCreatedEvent",
+ "platypush.message.event.notes.CollectionUpdatedEvent",
+ "platypush.message.event.notes.CollectionDeletedEvent"
+ ],
+ "install": {
+ "pip": []
+ },
+ "package": "platypush.plugins.joplin",
+ "type": "plugin"
+ }
+}
diff --git a/platypush/schemas/notes.py b/platypush/schemas/notes.py
new file mode 100644
index 000000000..5a4bc1324
--- /dev/null
+++ b/platypush/schemas/notes.py
@@ -0,0 +1,237 @@
+from abc import ABC
+from marshmallow import INCLUDE, Schema, fields
+
+from platypush.schemas import DateTime
+
+
+class BaseNoteSchema(Schema, ABC):
+ """
+ Base schema for note objects.
+ """
+
+ # pylint: disable=too-few-public-methods
+ class Meta: # type: ignore
+ """
+ Meta class for the schema.
+ """
+
+ unknown = INCLUDE
+
+
+class NoteSourceSchema(BaseNoteSchema):
+ """
+ Schema for a note source, such as a URL or file path.
+ """
+
+ name = fields.String(
+ metadata={
+ 'description': 'Name of the source',
+ 'example': 'My Note Source',
+ },
+ )
+
+ url = fields.String(
+ metadata={
+ 'description': 'URL of the source',
+ 'example': 'https://example.com/note',
+ },
+ )
+
+ app = fields.String(
+ metadata={
+ 'description': 'Application associated with the source',
+ 'example': 'My Note App',
+ },
+ )
+
+
+class NoteItemSchema(BaseNoteSchema):
+ """
+ Schema for a note object.
+ """
+
+ id = fields.Raw(
+ required=True,
+ dump_only=True,
+ metadata={'example': 2345},
+ )
+
+ title = fields.String(
+ metadata={
+ 'description': 'Title of the note',
+ 'example': 'My Important Note',
+ },
+ )
+
+ content = fields.String(
+ metadata={
+ 'example': 'Note content goes here',
+ },
+ )
+
+ description = fields.String(
+ metadata={
+ 'description': 'Description of the note',
+ 'example': 'This note contains important information.',
+ },
+ )
+
+ digest = fields.String(
+ dump_only=True,
+ metadata={
+ 'description': 'Unique digest of the note content',
+ 'example': 'abc123def456',
+ },
+ )
+
+ parent = fields.Nested(
+ 'NoteCollectionSchema',
+ dump_only=True,
+ metadata={
+ 'description': 'Parent note collection',
+ 'example': {
+ 'id': 1234,
+ 'title': 'My Notes',
+ 'description': 'A collection of my important notes',
+ },
+ },
+ )
+
+ source = fields.Nested(
+ NoteSourceSchema,
+ metadata={
+ 'description': 'Source of the note, such as a URL or file path',
+ 'example': {
+ 'name': 'My Note Source',
+ 'url': 'https://example.com/note',
+ 'app': 'My Note App',
+ },
+ },
+ )
+
+ author = fields.String(
+ metadata={
+ 'description': 'Author of the note',
+ 'example': 'John Doe',
+ },
+ )
+
+ latitude = fields.Float(
+ metadata={
+ 'description': 'Latitude of the note location',
+ 'example': 37.7749,
+ },
+ )
+
+ longitude = fields.Float(
+ metadata={
+ 'description': 'Longitude of the note location',
+ 'example': -122.4194,
+ },
+ )
+
+ altitude = fields.Float(
+ metadata={
+ 'description': 'Altitude of the note location',
+ 'example': 30.0,
+ },
+ )
+
+ tags = fields.List(
+ fields.String(),
+ metadata={
+ 'description': 'List of tags associated with the note',
+ 'example': ['important', 'work'],
+ },
+ )
+
+ created_at = DateTime(
+ metadata={
+ 'description': 'When the note was created',
+ },
+ )
+
+ updated_at = DateTime(
+ metadata={
+ 'description': 'When the note was last updated',
+ },
+ )
+
+
+class NoteCollectionSchema(BaseNoteSchema):
+ """
+ Schema for a collection of notes.
+ """
+
+ id = fields.Raw(
+ dump_only=True,
+ metadata={'example': 1234},
+ )
+
+ title = fields.String(
+ metadata={
+ 'description': 'Title of the note collection',
+ 'example': 'My Notes',
+ },
+ )
+
+ description = fields.String(
+ metadata={
+ 'description': 'Description of the note collection',
+ 'example': 'A collection of my important notes',
+ },
+ )
+
+ parent = fields.Nested(
+ 'NoteCollectionSchema',
+ metadata={
+ 'description': 'Parent note collection',
+ 'example': {
+ 'id': 1233,
+ 'title': 'All Notes',
+ 'description': 'A top-level collection of all notes',
+ },
+ },
+ )
+
+ collections = fields.List(
+ fields.Nested('NoteCollectionSchema'),
+ metadata={
+ 'description': 'List of sub-collections',
+ 'example': [
+ {
+ 'id': 1235,
+ 'title': 'Work Notes',
+ 'description': 'Notes related to work projects',
+ },
+ ],
+ },
+ )
+
+ notes = fields.List(
+ fields.Nested(NoteItemSchema),
+ metadata={
+ 'description': 'List of notes',
+ 'example': [
+ {
+ 'id': 2345,
+ 'title': 'My Important Note',
+ 'content': 'Note content goes here',
+ 'created_at': '2023-10-01T12:00:00Z',
+ 'updated_at': '2023-10-02T12:00:00Z',
+ },
+ ],
+ },
+ )
+
+ created_at = DateTime(
+ metadata={
+ 'description': 'When the note collection was created',
+ },
+ )
+
+ updated_at = DateTime(
+ metadata={
+ 'description': 'When the note collection was last updated',
+ },
+ )
From f602d18099ca3c0b92fbedf8e0ed954c1eefd5d5 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Wed, 18 Jun 2025 22:35:45 +0200
Subject: [PATCH 07/18] Moved `plugins/_notes.py` to
`plugins/_notes/__init__.py`
---
platypush/plugins/{_notes.py => _notes/__init__.py} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename platypush/plugins/{_notes.py => _notes/__init__.py} (100%)
diff --git a/platypush/plugins/_notes.py b/platypush/plugins/_notes/__init__.py
similarity index 100%
rename from platypush/plugins/_notes.py
rename to platypush/plugins/_notes/__init__.py
From 70fa5e4343ddf5da5049bf184066451628d26ba9 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Fri, 20 Jun 2025 16:28:28 +0200
Subject: [PATCH 08/18] [`alarm`] Synchronize with the entities engine before
interacting with the db
Otherwise the plugin may fail on startup if the database hasn't been
initialized yet.
---
platypush/plugins/alarm/__init__.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/platypush/plugins/alarm/__init__.py b/platypush/plugins/alarm/__init__.py
index f98ebe53b..edacfb76f 100644
--- a/platypush/plugins/alarm/__init__.py
+++ b/platypush/plugins/alarm/__init__.py
@@ -6,7 +6,7 @@ from typing import Collection, Generator, Optional, Dict, Any, List, Union
from sqlalchemy.orm import Session
from platypush.context import get_plugin
-from platypush.entities import EntityManager
+from platypush.entities import EntityManager, get_entities_engine
from platypush.entities.alarm import Alarm as DbAlarm
from platypush.message.event.entities import EntityDeleteEvent
from platypush.plugins import RunnablePlugin, action
@@ -197,6 +197,9 @@ class AlarmPlugin(RunnablePlugin, EntityManager):
alarm.stop()
def _sync_alarms(self):
+ # Wait for the entities engine to start
+ get_entities_engine().wait_start()
+
with self._get_session() as session:
db_alarms = {
str(alarm.name): alarm for alarm in session.query(DbAlarm).all()
From 0fcaf3f87ea1e5e2d8eb8a12ea80e40bffcc52a3 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Fri, 20 Jun 2025 22:24:38 +0200
Subject: [PATCH 09/18] Added persistent storage+sync for notes
---
platypush/common/notes.py | 38 +-
...bc6eb9c81_added_tables_for_note_plugins.py | 203 +++++++++
platypush/plugins/_notes/__init__.py | 394 +++++++++++-------
platypush/plugins/_notes/_model.py | 55 +++
platypush/plugins/_notes/db/__init__.py | 5 +
platypush/plugins/_notes/db/_mixin.py | 221 ++++++++++
platypush/plugins/_notes/db/_model.py | 152 +++++++
platypush/plugins/joplin/__init__.py | 9 +-
8 files changed, 908 insertions(+), 169 deletions(-)
create mode 100644 platypush/migrations/alembic/versions/692bc6eb9c81_added_tables_for_note_plugins.py
create mode 100644 platypush/plugins/_notes/_model.py
create mode 100644 platypush/plugins/_notes/db/__init__.py
create mode 100644 platypush/plugins/_notes/db/_mixin.py
create mode 100644 platypush/plugins/_notes/db/_model.py
diff --git a/platypush/common/notes.py b/platypush/common/notes.py
index 1e8a5da0c..c3d9f624a 100644
--- a/platypush/common/notes.py
+++ b/platypush/common/notes.py
@@ -1,14 +1,15 @@
-from abc import abstractmethod
+from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime
-from hashlib import sha256
+from hashlib import md5, sha256
from typing import Any, Dict, List, Optional, Set
+from uuid import UUID
from platypush.message import JSONAble
from platypush.schemas.notes import NoteCollectionSchema, NoteItemSchema
-class Serializable(JSONAble):
+class Serializable(JSONAble, ABC):
"""
Base class for serializable objects.
"""
@@ -23,7 +24,26 @@ class Serializable(JSONAble):
return self.to_dict()
-@dataclass
+@dataclass(kw_only=True)
+class Storable(Serializable, ABC):
+ """
+ Base class for note objects that can be represented as databases entries.
+ """
+
+ id: Any
+ plugin: str
+
+ @property
+ def _db_id(self) -> UUID:
+ """
+ Generate a deterministic UUID based on the note's plugin and ID.
+ """
+ key = f'{self.plugin}:{self.id}'
+ digest = md5(key.encode()).digest()
+ return UUID(int=int.from_bytes(digest, 'little'))
+
+
+@dataclass(kw_only=True)
class NoteSource(Serializable):
"""
Represents a source for a note, such as a URL or file path.
@@ -37,13 +57,12 @@ class NoteSource(Serializable):
return self.__dict__
-@dataclass
-class Note(Serializable):
+@dataclass(kw_only=True)
+class Note(Storable):
"""
Represents a note with a title and content.
"""
- id: Any
title: str
description: Optional[str] = None
content: Optional[str] = None
@@ -89,13 +108,12 @@ class Note(Serializable):
)
-@dataclass
-class NoteCollection(Serializable):
+@dataclass(kw_only=True)
+class NoteCollection(Storable):
"""
Represents a collection of notes.
"""
- id: Any
title: str
description: Optional[str] = None
parent: Optional['NoteCollection'] = None
diff --git a/platypush/migrations/alembic/versions/692bc6eb9c81_added_tables_for_note_plugins.py b/platypush/migrations/alembic/versions/692bc6eb9c81_added_tables_for_note_plugins.py
new file mode 100644
index 000000000..3a5dbadd8
--- /dev/null
+++ b/platypush/migrations/alembic/versions/692bc6eb9c81_added_tables_for_note_plugins.py
@@ -0,0 +1,203 @@
+"""Added tables for note plugins
+
+Revision ID: 692bc6eb9c81
+Revises: 0876439530cb
+Create Date: 2025-06-20 12:27:54.533624
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+from platypush.plugins._notes.db._model import (
+ Note as DbNote,
+ NoteCollection as DbNoteCollection,
+ NoteNoteResource as DbNoteNoteResource,
+ NoteResource as DbNoteResource,
+ NoteTag as DbNoteTag,
+)
+
+
+# revision identifiers, used by Alembic.
+revision = '692bc6eb9c81'
+down_revision = '0876439530cb'
+branch_labels = None
+depends_on = None
+
+
+def _create_note_collection_table(metadata: sa.MetaData) -> sa.Table:
+ table_name = DbNoteCollection.__tablename__
+ existing_table = metadata.tables.get(table_name)
+ if existing_table is not None:
+ print(f'Table `{table_name}` already exists, skipping creation')
+ return existing_table
+
+ table = op.create_table(
+ table_name,
+ sa.Column('id', sa.UUID, primary_key=True),
+ sa.Column('external_id', sa.String, nullable=False),
+ sa.Column('plugin', sa.String, nullable=False),
+ sa.Column('title', sa.String, nullable=False),
+ sa.Column('description', sa.String, nullable=True),
+ sa.Column(
+ 'parent_id', sa.UUID, sa.ForeignKey(f'{table_name}.id'), nullable=True
+ ),
+ sa.Column('created_at', sa.DateTime, nullable=False),
+ sa.Column('updated_at', sa.DateTime, nullable=False),
+ )
+
+ sa.Index(
+ 'note_collection_external_id_plugin_idx',
+ table.c.external_id,
+ table.c.plugin,
+ unique=True,
+ )
+
+ return table
+
+
+def _create_note_resource_table(metadata: sa.MetaData) -> sa.Table:
+ table_name = DbNoteResource.__tablename__
+ existing_table = metadata.tables.get(table_name)
+ if existing_table is not None:
+ print(f'Table `{table_name}` already exists, skipping creation')
+ return existing_table
+
+ table = op.create_table(
+ table_name,
+ sa.Column('id', sa.UUID, primary_key=True),
+ sa.Column('external_id', sa.String, nullable=False),
+ sa.Column('plugin', sa.String, nullable=False),
+ sa.Column('title', sa.String, nullable=False),
+ sa.Column('filename', sa.String, nullable=False),
+ sa.Column('content_type', sa.String, nullable=True),
+ sa.Column('size', sa.Integer, nullable=True),
+ sa.Column('created_at', sa.DateTime, nullable=False),
+ sa.Column('updated_at', sa.DateTime, nullable=True),
+ )
+
+ sa.Index(
+ 'note_resource_external_id_plugin_idx',
+ table.c.external_id,
+ table.c.plugin,
+ unique=True,
+ )
+
+ return table
+
+
+def _create_note_table(metadata: sa.MetaData) -> sa.Table:
+ table_name = DbNote.__tablename__
+ existing_table = metadata.tables.get(table_name)
+ if existing_table is not None:
+ print(f'Table `{table_name}` already exists, skipping creation')
+ return existing_table
+
+ table = op.create_table(
+ table_name,
+ sa.Column('id', sa.UUID, primary_key=True),
+ sa.Column('external_id', sa.String, nullable=False),
+ sa.Column('plugin', sa.String, nullable=False),
+ sa.Column('title', sa.String, nullable=False),
+ sa.Column('description', sa.String, nullable=True),
+ sa.Column('content', sa.String, nullable=True),
+ sa.Column(
+ 'parent_id',
+ sa.UUID,
+ sa.ForeignKey(f'{DbNoteCollection.__tablename__}.id'),
+ nullable=True,
+ ),
+ sa.Column('digest', sa.String, nullable=True, index=True),
+ sa.Column('latitude', sa.Float, nullable=True),
+ sa.Column('longitude', sa.Float, nullable=True),
+ sa.Column('altitude', sa.Float, nullable=True),
+ sa.Column('author', sa.String, nullable=True),
+ sa.Column('source_name', sa.String, nullable=True, index=True),
+ sa.Column('source_url', sa.String, nullable=True, index=True),
+ sa.Column('source_app', sa.String, nullable=True, index=True),
+ sa.Column('created_at', sa.DateTime, nullable=False),
+ sa.Column('updated_at', sa.DateTime, nullable=False),
+ )
+
+ sa.Index(
+ 'note_external_id_plugin_idx',
+ table.c.external_id,
+ table.c.plugin,
+ unique=True,
+ )
+
+ return table
+
+
+def _create_note_note_resource_table(metadata: sa.MetaData) -> sa.Table:
+ table_name = DbNoteNoteResource.__tablename__
+ existing_table = metadata.tables.get(table_name)
+ if existing_table is not None:
+ print(f'Table `{table_name}` already exists, skipping creation')
+ return existing_table
+
+ return op.create_table(
+ table_name,
+ sa.Column('note_id', sa.UUID, sa.ForeignKey(f'{DbNote.__tablename__}.id')),
+ sa.Column(
+ 'resource_id',
+ sa.String,
+ sa.ForeignKey(f'{DbNoteResource.__tablename__}.id'),
+ ),
+ sa.PrimaryKeyConstraint('note_id', 'resource_id'),
+ )
+
+
+def _create_note_tag_table(metadata: sa.MetaData) -> sa.Table:
+ table_name = DbNoteTag.__tablename__
+ existing_table = metadata.tables.get(table_name)
+ if existing_table is not None:
+ print(f'Table `{table_name}` already exists, skipping creation')
+ return existing_table
+
+ return op.create_table(
+ table_name,
+ sa.Column(
+ 'note_id',
+ sa.UUID,
+ sa.ForeignKey(f'{DbNote.__tablename__}.id'),
+ nullable=False,
+ ),
+ sa.Column('tag', sa.String, nullable=False),
+ sa.PrimaryKeyConstraint('note_id', 'tag', name='note_tag_pkey'),
+ )
+
+
+def _drop_table_if_exists(table_name: str, metadata: sa.MetaData) -> None:
+ if table_name in metadata.tables:
+ op.drop_table(table_name)
+ print(f'Table `{table_name}` dropped successfully.')
+ else:
+ print(f'Table `{table_name}` does not exist, skipping drop.')
+
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ metadata = sa.MetaData()
+ metadata.reflect(bind=conn)
+
+ _create_note_collection_table(metadata)
+ _create_note_resource_table(metadata)
+ _create_note_table(metadata)
+ _create_note_note_resource_table(metadata)
+ _create_note_tag_table(metadata)
+
+
+def downgrade() -> None:
+ conn = op.get_bind()
+ metadata = sa.MetaData()
+ metadata.reflect(bind=conn)
+
+ for table_name in [
+ DbNoteTag.__tablename__,
+ DbNoteNoteResource.__tablename__,
+ DbNoteResource.__tablename__,
+ DbNote.__tablename__,
+ DbNoteCollection.__tablename__,
+ ]:
+ _drop_table_if_exists(table_name, metadata)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index 2b8682cc1..434edd31e 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -1,4 +1,5 @@
from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from threading import RLock
from time import time
@@ -6,6 +7,7 @@ from typing import Any, Dict, Iterable, Optional, Type
from platypush.common.notes import Note, NoteCollection, NoteSource
from platypush.context import Variable
+from platypush.entities import get_entities_engine
from platypush.message.event.notes import (
BaseNoteEvent,
NoteCreatedEvent,
@@ -16,10 +18,12 @@ from platypush.message.event.notes import (
CollectionDeletedEvent,
)
from platypush.plugins import RunnablePlugin, action
-from platypush.utils import get_plugin_name_by_class
+
+from .db import DbMixin
+from ._model import CollectionsDelta, NotesDelta, StateDelta
-class BaseNotePlugin(RunnablePlugin, ABC):
+class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
"""
Base class for note-taking plugins.
"""
@@ -30,11 +34,8 @@ class BaseNotePlugin(RunnablePlugin, ABC):
If set to zero or null, the plugin will not poll for updates,
and events will be generated only when you manually call :meth:`.sync`.
"""
- super().__init__(*args, poll_interval=poll_interval, **kwargs)
- self._notes: Dict[Any, Note] = {}
- self._collections: Dict[Any, NoteCollection] = {}
- self._notes_lock = RLock()
- self._collections_lock = RLock()
+ RunnablePlugin.__init__(self, *args, poll_interval=poll_interval, **kwargs)
+ DbMixin.__init__(self, *args, **kwargs)
self._sync_lock = RLock()
self.__last_sync_time: Optional[datetime] = None
@@ -43,9 +44,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
"""
Variable name for the last sync time.
"""
- return Variable(
- f'_LAST_ITEMS_SYNC_TIME[{get_plugin_name_by_class(self.__class__)}]'
- )
+ return Variable(f'_LAST_ITEMS_SYNC_TIME[{self._plugin_name}]')
@property
def _last_sync_time(self) -> Optional[datetime]:
@@ -232,76 +231,34 @@ class BaseNotePlugin(RunnablePlugin, ABC):
for event in events:
self.bus.post(event)
- def _process_events(
- self,
- notes: Iterable[Note],
- prev_notes: Dict[Any, Note],
- collections: Iterable[NoteCollection],
- prev_collections: Dict[Any, NoteCollection],
- ):
- most_recent_note = list(notes)[0] if notes else None
- most_recent_collection = list(collections)[0] if collections else None
- max_updated_at = max(
- (
- most_recent_note.updated_at.timestamp()
- if most_recent_note and most_recent_note.updated_at
- else 0
- ),
- (
- most_recent_collection.updated_at.timestamp()
- if most_recent_collection and most_recent_collection.updated_at
- else 0
- ),
- )
-
+ def _process_events(self, state_delta: StateDelta):
with self._sync_lock:
- self._process_collections_events(collections, prev_collections)
- self._process_notes_events(notes, prev_notes)
- self._last_sync_time = (
- datetime.fromtimestamp(max_updated_at) if max_updated_at > 0 else None
- )
+ self._process_collections_events(state_delta.collections)
+ self._process_notes_events(state_delta.notes)
- @classmethod
def _make_event(
- cls, evt_type: Type[BaseNoteEvent], *args, **kwargs
+ self, evt_type: Type[BaseNoteEvent], *args, **kwargs
) -> BaseNoteEvent:
"""
Create a note event of the specified type.
"""
- return evt_type(*args, plugin=get_plugin_name_by_class(cls), **kwargs)
-
- def _process_notes_events(
- self,
- notes: Iterable[Note],
- prev_notes: Dict[Any, Note],
- ):
- last_sync_time = self._last_sync_time.timestamp() if self._last_sync_time else 0
- new_notes_by_id = {note.id: note for note in notes}
+ return evt_type(*args, plugin=self._plugin_name, **kwargs)
+ def _process_notes_events(self, notes_delta: NotesDelta):
removed_note_events = [
self._make_event(NoteDeletedEvent, note=note)
- for note_id, note in prev_notes.items()
- if note_id not in new_notes_by_id
+ for note in notes_delta.deleted.values()
]
- created_note_events = []
- updated_note_events = []
+ created_note_events = [
+ self._make_event(NoteCreatedEvent, note=note)
+ for note in notes_delta.added.values()
+ ]
- for note in notes:
- created_at = note.created_at.timestamp() if note.created_at else 0
- updated_at = note.updated_at.timestamp() if note.updated_at else 0
-
- if created_at > last_sync_time:
- created_note_events.append(
- self._make_event(NoteCreatedEvent, note=note)
- )
- elif updated_at > last_sync_time:
- updated_note_events.append(
- self._make_event(NoteUpdatedEvent, note=note)
- )
- else:
- # Assuming that the list of notes is sorted by updated_at
- break
+ updated_note_events = [
+ self._make_event(NoteUpdatedEvent, note=note)
+ for note in notes_delta.updated.values()
+ ]
self._dispatch_events(
*removed_note_events,
@@ -309,44 +266,21 @@ class BaseNotePlugin(RunnablePlugin, ABC):
*updated_note_events,
)
- def _process_collections_events(
- self,
- collections: Iterable[NoteCollection],
- prev_collections: Dict[Any, NoteCollection],
- ):
- last_sync_time = self._last_sync_time.timestamp() if self._last_sync_time else 0
- new_collections_by_id = {
- collection.id: collection for collection in collections
- }
-
+ def _process_collections_events(self, collections_delta: CollectionsDelta):
removed_collection_events = [
self._make_event(CollectionDeletedEvent, collection=collection)
- for collection_id, collection in prev_collections.items()
- if collection_id not in new_collections_by_id
+ for collection in collections_delta.deleted.values()
]
- created_collection_events = []
- updated_collection_events = []
+ created_collection_events = [
+ self._make_event(CollectionCreatedEvent, collection=collection)
+ for collection in collections_delta.added.values()
+ ]
- for collection in collections:
- created_at = (
- collection.created_at.timestamp() if collection.created_at else 0
- )
- updated_at = (
- collection.updated_at.timestamp() if collection.updated_at else 0
- )
-
- if created_at > last_sync_time:
- created_collection_events.append(
- self._make_event(CollectionCreatedEvent, collection=collection)
- )
- elif updated_at > last_sync_time:
- updated_collection_events.append(
- self._make_event(CollectionUpdatedEvent, collection=collection)
- )
- else:
- # Assuming that the list of collections is sorted by updated_at
- break
+ updated_collection_events = [
+ self._make_event(CollectionUpdatedEvent, collection=collection)
+ for collection in collections_delta.updated.values()
+ ]
self._dispatch_events(
*removed_collection_events,
@@ -357,7 +291,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
def _get_note(self, note_id: Any, *args, **kwargs) -> Note:
note = self._fetch_note(note_id, *args, **kwargs)
assert note, f'Note with ID {note_id} not found'
- with self._notes_lock:
+ with self._sync_lock:
# Always overwrite the note in the cache,
# as this is the most up-to-date complete version
self._notes[note.id] = note
@@ -368,7 +302,33 @@ class BaseNotePlugin(RunnablePlugin, ABC):
note.id
] = note
- return self._notes[note.id]
+ return self._notes[note.id]
+
+ def _merge_note(self, note: Note, skip_content: bool = True) -> Note:
+ """
+ Merge the state of an incoming note with the existing one in the cache.
+ """
+
+ existing_note = self._notes.get(note.id)
+ if not existing_note:
+ # If the note doesn't exist, just return the new one
+ return note
+
+ for field in note.__dataclass_fields__:
+ existing_value = getattr(existing_note, field)
+ value = getattr(note, field)
+ # Don't overwrite content, digest and tags here unless they have been re-fetched
+ if (
+ skip_content
+ and field in ('content', 'digest', 'tags')
+ and existing_value
+ and not value
+ ):
+ continue
+
+ setattr(existing_note, field, value)
+
+ return existing_note
def _get_notes(
self,
@@ -383,9 +343,10 @@ class BaseNotePlugin(RunnablePlugin, ABC):
# Always fetch if background polling is disabled
fetch = fetch or not self.poll_interval
if fetch:
- with self._notes_lock:
+ with self._sync_lock:
self._notes = {
- note.id: note for note in self._fetch_notes(*args, **kwargs)
+ note.id: self._merge_note(note)
+ for note in self._fetch_notes(*args, **kwargs)
}
self._refresh_notes_cache()
@@ -400,7 +361,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
def _get_collection(self, collection_id: Any, *args, **kwargs) -> NoteCollection:
collection = self._fetch_collection(collection_id, *args, **kwargs)
assert collection, f'Collection with ID {collection_id} not found'
- with self._collections_lock:
+ with self._sync_lock:
# Always overwrite the collection in the cache,
# as this is the most up-to-date complete version
self._collections[collection.id] = collection
@@ -426,7 +387,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
# Always fetch if background polling is disabled
fetch = fetch or not self.poll_interval
if fetch:
- with self._collections_lock:
+ with self._sync_lock:
self._collections = {
collection.id: collection
for collection in self._fetch_collections(*args, **kwargs)
@@ -486,6 +447,92 @@ class BaseNotePlugin(RunnablePlugin, ABC):
}.items()
}
+ def _get_state_delta(
+ self,
+ previous_notes: Dict[Any, Note],
+ previous_collections: Dict[Any, NoteCollection],
+ notes: Dict[Any, Note],
+ collections: Dict[Any, NoteCollection],
+ ) -> StateDelta:
+ """
+ Get the state delta between the previous and current state of notes and collections.
+
+ :param previous_notes: Previous notes state.
+ :param previous_collections: Previous collections state.
+ :param notes: Current notes state.
+ :param collections: Current collections state.
+ :return: A StateDelta object containing the changes.
+ """
+ state_delta = StateDelta()
+ latest_updated_at = new_latest_updated_at = (
+ self._last_sync_time.timestamp() if self._last_sync_time else 0
+ )
+
+ # Get new and updated notes
+ for note in notes.values():
+ updated_at = note.updated_at.timestamp() if note.updated_at else 0
+ if updated_at > latest_updated_at:
+ if note.id not in previous_notes:
+ state_delta.notes.added[note.id] = note
+ else:
+ state_delta.notes.updated[note.id] = note
+
+ new_latest_updated_at = max(new_latest_updated_at, updated_at)
+
+ # Get deleted notes
+ for note_id in previous_notes:
+ if note_id not in notes:
+ state_delta.notes.deleted[note_id] = previous_notes[note_id]
+
+ # Get new and updated collections
+ for collection in collections.values():
+ updated_at = (
+ collection.updated_at.timestamp() if collection.updated_at else 0
+ )
+ if updated_at > latest_updated_at:
+ if collection.id not in previous_collections:
+ state_delta.collections.added[collection.id] = collection
+ else:
+ state_delta.collections.updated[collection.id] = collection
+
+ new_latest_updated_at = max(new_latest_updated_at, updated_at)
+
+ # Get deleted collections
+ for collection_id in previous_collections:
+ if collection_id not in collections:
+ state_delta.collections.deleted[collection_id] = previous_collections[
+ collection_id
+ ]
+
+ state_delta.latest_updated_at = new_latest_updated_at
+ return state_delta
+
+ def _refresh_notes(self, notes: Dict[Any, Note]):
+ """
+ Fetch the given notes from the backend and update the cache.
+ """
+ if not notes:
+ return
+
+ self.logger.info(
+ 'Refreshing the state for %d notes from the backend', len(notes)
+ )
+
+ with ThreadPoolExecutor(max_workers=5) as pool:
+ # Fetch notes in parallel
+ futures = [
+ pool.submit(self._fetch_note, note.id) for note in notes.values()
+ ]
+
+ # Wait for all futures to complete and collect the results
+ results = pool.map(lambda f: f.result(), futures)
+
+ with self._sync_lock:
+ self._notes.update(
+ {note.id: self._merge_note(note) for note in results if note}
+ )
+ self._refresh_notes_cache()
+
@action
def get_note(self, note_id: Any, *args, **kwargs) -> dict:
"""
@@ -592,7 +639,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
**kwargs,
)
- with self._notes_lock:
+ with self._sync_lock:
# Add the new note to the cache
self._notes[note.id] = note
@@ -638,20 +685,20 @@ class BaseNotePlugin(RunnablePlugin, ABC):
.. schema:: notes.NoteItemSchema
"""
- with self._notes_lock:
- note = self._edit_note(
- note_id,
- *args,
- title=title,
- content=content,
- description=description,
- collection=collection,
- geo=self._parse_geo(geo) if geo else None,
- author=author,
- source=NoteSource(**source) if source else None,
- **kwargs,
- )
+ note = self._edit_note(
+ note_id,
+ *args,
+ title=title,
+ content=content,
+ description=description,
+ collection=collection,
+ geo=self._parse_geo(geo) if geo else None,
+ author=author,
+ source=NoteSource(**source) if source else None,
+ **kwargs,
+ )
+ with self._sync_lock:
# Update the cache with the edited note
self._notes[note.id] = note
@@ -675,11 +722,12 @@ class BaseNotePlugin(RunnablePlugin, ABC):
:param note_id: The ID of the note to delete.
"""
- with self._notes_lock:
- self._delete_note(note_id, *args, **kwargs)
+ self._delete_note(note_id, *args, **kwargs)
+
+ with self._sync_lock:
note = self._notes.pop(note_id, None)
if not note:
- note = Note(id=note_id, title='')
+ note = Note(id=note_id, plugin=self._plugin_name, title='')
# Remove the note from its parent collection if it has one
if note.parent and note.parent.id in self._collections:
@@ -767,7 +815,7 @@ class BaseNotePlugin(RunnablePlugin, ABC):
title, *args, description=description, parent=parent, **kwargs
)
- with self._collections_lock:
+ with self._sync_lock:
# Add the new collection to the cache
self._collections[collection.id] = collection
@@ -776,10 +824,10 @@ class BaseNotePlugin(RunnablePlugin, ABC):
parent_collection = self._collections[collection.parent.id]
parent_collection.collections.append(collection)
- # Trigger the collection created event
- self._dispatch_events(
- self._make_event(CollectionCreatedEvent, collection=collection)
- )
+ # Trigger the collection created event
+ self._dispatch_events(
+ self._make_event(CollectionCreatedEvent, collection=collection)
+ )
return collection.to_dict()
@@ -805,16 +853,16 @@ class BaseNotePlugin(RunnablePlugin, ABC):
.. schema:: notes.NoteCollectionSchema
"""
- with self._collections_lock:
- collection = self._edit_collection(
- collection_id,
- *args,
- title=title,
- description=description,
- parent=parent,
- **kwargs,
- )
+ collection = self._edit_collection(
+ collection_id,
+ *args,
+ title=title,
+ description=description,
+ parent=parent,
+ **kwargs,
+ )
+ with self._sync_lock:
# Update the cache with the edited collection
old_collection = self._collections.get(collection.id)
self._collections[collection.id] = collection
@@ -842,10 +890,10 @@ class BaseNotePlugin(RunnablePlugin, ABC):
):
parent_collection.collections.append(collection)
- # Trigger the collection updated event
- self._dispatch_events(
- self._make_event(CollectionUpdatedEvent, collection=collection)
- )
+ # Trigger the collection updated event
+ self._dispatch_events(
+ self._make_event(CollectionUpdatedEvent, collection=collection)
+ )
return collection.to_dict()
@@ -856,11 +904,14 @@ class BaseNotePlugin(RunnablePlugin, ABC):
:param collection_id: The ID of the collection to delete.
"""
- with self._collections_lock:
- self._delete_collection(collection_id, *args, **kwargs)
+ self._delete_collection(collection_id, *args, **kwargs)
+
+ with self._sync_lock:
collection = self._collections.pop(collection_id, None)
if not collection:
- collection = NoteCollection(id=collection_id, title='')
+ collection = NoteCollection(
+ id=collection_id, plugin=self._plugin_name, title=''
+ )
# Remove the collection from its parent if it has one
if collection.parent and collection.parent.id in self._collections:
@@ -880,26 +931,52 @@ class BaseNotePlugin(RunnablePlugin, ABC):
If ``poll_interval`` is zero or null, you can manually call this method
to synchronize the notes and collections.
"""
+ # Wait for the entities engine to start
+ get_entities_engine().wait_start()
+
t_start = time()
self.logger.info('Synchronizing notes and collections...')
with self._sync_lock:
+ # Initialize the latest state from the database if not already done
+ self._db_init()
prev_notes = self._notes.copy()
prev_collections = self._collections.copy()
- notes = self._get_notes(
- *args, fetch=True, sort={'updated_at': False}, **kwargs
- )
- collections = self._get_collections(
- *args, fetch=True, sort={'updated_at': False}, **kwargs
+
+ # Fetch the latest version of the notes from the backend
+ notes = {
+ note.id: note
+ for note in self._get_notes(
+ *args, fetch=True, sort={'updated_at': False}, **kwargs
+ )
+ }
+
+ # Fetch the latest version of the collections from the backend
+ collections = {
+ collection.id: collection
+ for collection in self._get_collections(
+ *args, fetch=True, sort={'updated_at': False}, **kwargs
+ )
+ }
+
+ # Get the state delta between the previous and current state
+ state_delta = self._get_state_delta(
+ previous_notes=prev_notes,
+ previous_collections=prev_collections,
+ notes=notes,
+ collections=collections,
)
- self._process_events(
- notes=notes,
- prev_notes=prev_notes,
- collections=collections,
- prev_collections=prev_collections,
+ # Re-fetch any notes that have been updated since the last sync
+ self._refresh_notes(
+ {**state_delta.notes.added, **state_delta.notes.updated}
)
+ # Update the local cache with the latest notes and collections
+ self._db_sync(state_delta)
+ self._last_sync_time = datetime.fromtimestamp(state_delta.latest_updated_at)
+ self._process_events(state_delta)
+
self.logger.info(
'Synchronization completed in %.2f seconds',
time() - t_start,
@@ -908,11 +985,16 @@ class BaseNotePlugin(RunnablePlugin, ABC):
@action
def reset_sync(self):
"""
- Reset the last sync time to None, forcing a full resync on the next call to
- :meth:`.sync`, which in turn will re-trigger all notes and collections events.
+ Reset the sync state.
+
+ 1. Reset the last sync time to None, forcing a full resync on the
+ next call to :meth:`.sync`, which in turn will re-trigger all
+ notes and collections events.
+ 2. Clear the local notes and collections cache.
"""
self.logger.info('Resetting last sync time')
with self._sync_lock:
+ self._db_clear()
self._last_sync_time = None
self._notes.clear()
self._collections.clear()
diff --git a/platypush/plugins/_notes/_model.py b/platypush/plugins/_notes/_model.py
new file mode 100644
index 000000000..56069c71c
--- /dev/null
+++ b/platypush/plugins/_notes/_model.py
@@ -0,0 +1,55 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+from platypush.common.notes import Note, NoteCollection
+
+
+@dataclass
+class NotesDelta:
+ """
+ Represents a delta of changes in notes.
+ """
+
+ added: Dict[Any, Note] = field(default_factory=dict)
+ updated: Dict[Any, Note] = field(default_factory=dict)
+ deleted: Dict[Any, Note] = field(default_factory=dict)
+
+ def is_empty(self) -> bool:
+ """
+ Check if the delta is empty (no added, updated, or deleted notes).
+ """
+ return not (self.added or self.updated or self.deleted)
+
+
+@dataclass
+class CollectionsDelta:
+ """
+ Represents a delta of changes in note collections.
+ """
+
+ added: Dict[Any, NoteCollection] = field(default_factory=dict)
+ updated: Dict[Any, NoteCollection] = field(default_factory=dict)
+ deleted: Dict[Any, NoteCollection] = field(default_factory=dict)
+
+ def is_empty(self) -> bool:
+ """
+ Check if the delta is empty (no added, updated, or deleted collections).
+ """
+ return not (self.added or self.updated or self.deleted)
+
+
+@dataclass
+class StateDelta:
+ """
+ Represents a delta of changes in the state of notes and collections.
+ """
+
+ notes: NotesDelta = field(default_factory=NotesDelta)
+ collections: CollectionsDelta = field(default_factory=CollectionsDelta)
+ latest_updated_at: float = 0
+
+ def is_empty(self) -> bool:
+ """
+ Check if the state delta is empty (no changes in notes or collections).
+ """
+ return self.notes.is_empty() and self.collections.is_empty()
diff --git a/platypush/plugins/_notes/db/__init__.py b/platypush/plugins/_notes/db/__init__.py
new file mode 100644
index 000000000..06e9ae368
--- /dev/null
+++ b/platypush/plugins/_notes/db/__init__.py
@@ -0,0 +1,5 @@
+from ._mixin import DbMixin
+
+__all__ = [
+ "DbMixin",
+]
diff --git a/platypush/plugins/_notes/db/_mixin.py b/platypush/plugins/_notes/db/_mixin.py
new file mode 100644
index 000000000..01fda4b4f
--- /dev/null
+++ b/platypush/plugins/_notes/db/_mixin.py
@@ -0,0 +1,221 @@
+from contextlib import contextmanager
+from threading import Event, RLock
+from typing import Any, Dict, Generator
+
+from sqlalchemy.orm import Session
+
+from platypush.common.notes import Note, NoteCollection
+from platypush.context import get_plugin
+from platypush.plugins.db import DbPlugin
+from platypush.utils import get_plugin_name_by_class, utcnow
+
+from .._model import StateDelta
+from ._model import (
+ Note as DbNote,
+ NoteCollection as DbNoteCollection,
+)
+
+
+class DbMixin:
+ """
+ Mixin class for the database synchronization layer.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._notes: Dict[Any, Note] = {}
+ self._collections: Dict[Any, NoteCollection] = {}
+ self._sync_lock = RLock()
+ self._db_lock = RLock()
+ self._db_initialized = Event()
+
+ @property
+ def _db(self) -> DbPlugin:
+ """
+ Get the database plugin instance for the current context.
+ """
+ db = get_plugin(DbPlugin)
+ assert db is not None, 'Database plugin not found'
+ return db
+
+ @contextmanager
+ def _get_db_session(self, *args, **kwargs) -> Generator[Session, None, None]:
+ """
+ Context manager to get a database session.
+ """
+ with self._db_lock, self._db.get_session(*args, **kwargs) as session:
+ yield session
+
+ @property
+ def _plugin_name(self) -> str:
+ """
+ Get the plugin name for the current context.
+ """
+ return get_plugin_name_by_class(self.__class__)
+
+ def _to_db_collection(self, collection: NoteCollection) -> DbNoteCollection:
+ """
+ Convert a NoteCollection object to a DbNoteCollection object.
+ """
+ return DbNoteCollection(
+ id=collection._db_id, # pylint:disable=protected-access
+ external_id=collection.id,
+ plugin=self._plugin_name,
+ title=collection.title,
+ description=collection.description,
+ parent_id=collection.parent.id if collection.parent else None,
+ created_at=collection.created_at or utcnow(),
+ updated_at=collection.updated_at or utcnow(),
+ )
+
+ def _to_db_note(self, note: Note) -> DbNote:
+ """
+ Convert a Note object to a DbNote object.
+ """
+ return DbNote(
+ external_id=note.id,
+ plugin=self._plugin_name,
+ title=note.title,
+ description=note.description,
+ content=note.content,
+ parent_id=note.parent._db_id if note.parent else None,
+ digest=note.digest,
+ latitude=note.latitude,
+ longitude=note.longitude,
+ altitude=note.altitude,
+ author=note.author,
+ source_name=note.source.name if note.source else None,
+ source_url=note.source.url if note.source else None,
+ source_app=note.source.app if note.source else None,
+ created_at=note.created_at or utcnow(),
+ updated_at=note.updated_at or utcnow(),
+ )
+
+ def _from_db_note(self, db_note: DbNote) -> Note:
+ """
+ Convert a DbNote object to a Note object.
+ """
+ return Note(
+ id=db_note.external_id,
+ plugin=self._plugin_name,
+ title=db_note.title, # type: ignore[arg-type]
+ description=db_note.description, # type: ignore[arg-type]
+ content=db_note.content, # type: ignore[arg-type]
+ parent=(
+ self._from_db_collection(db_note.parent) # type: ignore[arg-type]
+ if db_note.parent
+ else None
+ ),
+ digest=db_note.digest, # type: ignore[arg-type]
+ latitude=db_note.latitude, # type: ignore[arg-type]
+ longitude=db_note.longitude, # type: ignore[arg-type]
+ altitude=db_note.altitude, # type: ignore[arg-type]
+ author=db_note.author, # type: ignore[arg-type]
+ source=(
+ { # type: ignore[arg-type]
+ 'name': db_note.source_name,
+ 'url': db_note.source_url,
+ 'app': db_note.source_app,
+ }
+ if db_note.source_name # type: ignore[arg-type]
+ else None
+ ),
+ created_at=db_note.created_at, # type: ignore[arg-type]
+ updated_at=db_note.updated_at, # type: ignore[arg-type]
+ )
+
+ def _from_db_collection(self, db_collection: DbNoteCollection) -> NoteCollection:
+ """
+ Convert a DbNoteCollection object to a NoteCollection object.
+ """
+ return NoteCollection(
+ id=db_collection.external_id,
+ plugin=self._plugin_name,
+ title=db_collection.title, # type: ignore[arg-type]
+ description=db_collection.description, # type: ignore[arg-type]
+ parent=(
+ self._from_db_collection(db_collection.parent) # type: ignore[arg-type]
+ if db_collection.parent
+ else None
+ ),
+ created_at=db_collection.created_at, # type: ignore[arg-type]
+ updated_at=db_collection.updated_at, # type: ignore[arg-type]
+ )
+
+ def _db_fetch_notes(self, session: Session) -> Dict[Any, Note]:
+ """
+ Fetch notes from the database.
+ """
+ return {
+ note.external_id: self._from_db_note(note)
+ for note in session.query(DbNote).filter_by(plugin=self._plugin_name).all()
+ }
+
+ def _db_fetch_collections(self, session: Session) -> Dict[Any, NoteCollection]:
+ """
+ Fetch collections from the database.
+ """
+ return {
+ collection.external_id: self._from_db_collection(collection)
+ for collection in session.query(DbNoteCollection)
+ .filter_by(plugin=self._plugin_name)
+ .all()
+ }
+
+ def _db_init(self, force: bool = False) -> None:
+ """
+ Initialize the database by fetching notes and collections.
+ """
+ with self._db_lock:
+ if self._db_initialized.is_set() and not force:
+ return
+
+ with self._get_db_session() as session:
+ notes = self._db_fetch_notes(session)
+ collections = self._db_fetch_collections(session)
+
+ with self._sync_lock:
+ self._notes = notes
+ self._collections = collections
+
+ def _db_sync(self, state: StateDelta):
+ if state.is_empty():
+ return
+
+ with self._get_db_session(autoflush=False) as session:
+ for collection in [
+ *state.collections.added.values(),
+ *state.collections.updated.values(),
+ ]:
+ db_collection = self._to_db_collection(collection)
+ session.merge(db_collection)
+
+ for collection in state.collections.deleted.values():
+ session.query(DbNoteCollection).filter_by(
+ id=collection._db_id # pylint:disable=protected-access
+ ).delete()
+
+ session.flush() # Ensure collections are saved before notes
+
+ for note in [*state.notes.added.values(), *state.notes.updated.values()]:
+ db_note = self._to_db_note(note)
+ session.merge(db_note)
+
+ for note in state.notes.deleted.values():
+ session.query(DbNote).filter_by(
+ id=note._db_id # pylint:disable=protected-access
+ ).delete()
+
+ session.commit()
+
+ def _db_clear(self) -> None:
+ """
+ Clear the database by removing all notes and collections.
+ """
+ with self._get_db_session() as session:
+ session.query(DbNote).filter_by(plugin=self._plugin_name).delete()
+ session.query(DbNoteCollection).filter_by(plugin=self._plugin_name).delete()
+
+ with self._sync_lock:
+ self._notes.clear()
+ self._collections.clear()
diff --git a/platypush/plugins/_notes/db/_model.py b/platypush/plugins/_notes/db/_model.py
new file mode 100644
index 000000000..6fcd72dc6
--- /dev/null
+++ b/platypush/plugins/_notes/db/_model.py
@@ -0,0 +1,152 @@
+from uuid import uuid4
+
+from sqlalchemy import (
+ UUID,
+ Column,
+ DateTime,
+ ForeignKey,
+ Float,
+ Index,
+ Integer,
+ String,
+)
+from sqlalchemy.orm import relationship
+from sqlalchemy.schema import PrimaryKeyConstraint
+
+from platypush.common.db import Base
+
+TABLE_PREFIX = 'notes_'
+
+
+class NoteCollection(Base):
+ """
+ Models the notes_collection table, which contains collections of notes.
+ """
+
+ __tablename__ = f'{TABLE_PREFIX}collection'
+
+ id = Column(UUID, primary_key=True, nullable=False, default=uuid4)
+ external_id = Column(String, nullable=False)
+ plugin = Column(String, nullable=False)
+ title = Column(String, nullable=False)
+ description = Column(String, nullable=True)
+ parent_id = Column(UUID, ForeignKey(f'{TABLE_PREFIX}collection.id'), nullable=True)
+ created_at = Column(DateTime, nullable=False)
+ updated_at = Column(DateTime, nullable=False)
+
+ parent = relationship(
+ 'NoteCollection', remote_side=[id], foreign_keys=[parent_id], backref='children'
+ )
+ Index('note_collection_external_id_plugin_idx', external_id, plugin, unique=True)
+
+
+class Note(Base):
+ """
+ Models the notes_note table, which contains user notes.
+ """
+
+ __tablename__ = f'{TABLE_PREFIX}note'
+
+ id = Column(UUID, primary_key=True, nullable=False, default=uuid4)
+ external_id = Column(String, nullable=False)
+ plugin = Column(String, nullable=False)
+ title = Column(String, nullable=False)
+ description = Column(String, nullable=True)
+ content = Column(String, nullable=True)
+ parent_id = Column(UUID, ForeignKey(f'{TABLE_PREFIX}collection.id'), nullable=True)
+ digest = Column(String, nullable=True, index=True)
+ latitude = Column(Float, nullable=True)
+ longitude = Column(Float, nullable=True)
+ altitude = Column(Float, nullable=True)
+ author = Column(String, nullable=True)
+ source_name = Column(String, nullable=True, index=True)
+ source_url = Column(String, nullable=True, index=True)
+ source_app = Column(String, nullable=True, index=True)
+ created_at = Column(DateTime, nullable=False)
+ updated_at = Column(DateTime, nullable=False)
+
+ parent = relationship(
+ 'NoteCollection',
+ remote_side=[NoteCollection.id],
+ foreign_keys=[parent_id],
+ backref='notes',
+ )
+
+ tags = relationship('NoteTag', backref='note', cascade='all, delete-orphan')
+ resources = relationship(
+ 'NoteNoteResource',
+ backref='note',
+ cascade='all, delete-orphan',
+ )
+
+ Index('note_external_id_plugin_idx', external_id, plugin, unique=True)
+
+
+class NoteTag(Base):
+ """
+ Models the notes_tag table, which contains tags for notes.
+ """
+
+ __tablename__ = f'{TABLE_PREFIX}tag'
+ __table_args__ = (PrimaryKeyConstraint('note_id', 'tag', name='note_tag_pkey'),)
+
+ note_id = Column(
+ UUID,
+ ForeignKey(f'{TABLE_PREFIX}note.id'),
+ primary_key=True,
+ nullable=False,
+ )
+ tag = Column(String, primary_key=True, nullable=False)
+
+
+class NoteResource(Base):
+ """
+ Models the notes_resource table, which contains resources associated with notes.
+ """
+
+ __tablename__ = f'{TABLE_PREFIX}resource'
+
+ id = Column(UUID, primary_key=True, nullable=False, default=uuid4)
+ external_id = Column(String, nullable=False)
+ plugin = Column(String, nullable=False)
+ title = Column(String, nullable=False)
+ filename = Column(String, nullable=False)
+ content_type = Column(String, nullable=True)
+ size = Column(Integer, nullable=True)
+ created_at = Column(DateTime)
+ updated_at = Column(DateTime)
+
+ notes = relationship(
+ 'NoteNoteResource',
+ backref='resource',
+ cascade='all, delete-orphan',
+ )
+
+ Index('note_resource_external_id_plugin_idx', external_id, plugin, unique=True)
+
+
+class NoteNoteResource(Base):
+ """
+ Models the notes_note_resource table, which associates notes with resources.
+ """
+
+ __tablename__ = f'{TABLE_PREFIX}note_resource'
+ __table_args__ = (
+ PrimaryKeyConstraint('note_id', 'resource_id', name='note_resource_pkey'),
+ )
+
+ note_id = Column(
+ UUID,
+ ForeignKey(f'{TABLE_PREFIX}note.id'),
+ primary_key=True,
+ nullable=False,
+ )
+ resource_id = Column(
+ UUID,
+ ForeignKey(f'{TABLE_PREFIX}resource.id'),
+ primary_key=True,
+ nullable=False,
+ )
+
+
+# vim:sw=4:ts=4:et:
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
index 61e6e05d6..d8dcb898e 100644
--- a/platypush/plugins/joplin/__init__.py
+++ b/platypush/plugins/joplin/__init__.py
@@ -220,12 +220,14 @@ class JoplinPlugin(BaseNotePlugin):
if parent_id:
parent = self._collections.get(
- parent_id, NoteCollection(id=parent_id, title='')
+ parent_id,
+ NoteCollection(id=parent_id, plugin=self._plugin_name, title=''),
)
return Note(
**{
'id': data.get('id', ''),
+ 'plugin': self._plugin_name,
'title': data.get('title', ''),
'description': data.get('description'),
'content': data.get('body'),
@@ -243,6 +245,7 @@ class JoplinPlugin(BaseNotePlugin):
"""
return NoteCollection(
id=data.get('id', ''),
+ plugin=self._plugin_name,
title=data.get('title', ''),
description=data.get('description'),
created_at=self._parse_time(data.get('created_time')),
@@ -259,8 +262,8 @@ class JoplinPlugin(BaseNotePlugin):
f'notes/{note_id}',
params={
'fields': ','.join(
- *[
- self._default_note_fields,
+ [
+ *self._default_note_fields,
'body', # Include body content
]
)
From 166ebdf6955c9a9514c3d74eb824e90f455c9f56 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Fri, 20 Jun 2025 23:01:54 +0200
Subject: [PATCH 10/18] `filter` on the notes APIs should support regular
expressions.
---
platypush/plugins/_notes/__init__.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index 434edd31e..9a9adeefa 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -1,3 +1,4 @@
+import re
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
@@ -202,7 +203,10 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
items = [
item
for item in items
- if all(getattr(item, k) == v for k, v in filter.items())
+ if all(
+ re.search(v, str(getattr(item, k, '')), re.IGNORECASE)
+ for k, v in filter.items()
+ )
]
items = sorted(
@@ -565,7 +569,9 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
:param sort: A dictionary specifying the fields to sort by and their order.
Example: {'created_at': True} sorts by creation date in ascending
order, while {'created_at': False} sorts in descending order.
- :param filter: A dictionary specifying filters to apply to the collections.
+ :param filter: A dictionary specifying filters to apply to the notes, in the form
+ of a dictionary where the keys are field names and the values are regular expressions
+ to match against the field values.
:param fetch: If True, always fetch the latest collections from the backend,
regardless of the cache state (default: False).
:param kwargs: Additional keyword arguments to pass to the fetch method.
@@ -769,7 +775,9 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
:param sort: A dictionary specifying the fields to sort by and their order.
Example: {'created_at': True} sorts by creation date in ascending
order, while {'created_at': False} sorts in descending order.
- :param filter: A dictionary specifying filters to apply to the collections.
+ :param filter: A dictionary specifying filters to apply to the collections, in the form
+ of a dictionary where the keys are field names and the values are regular expressions
+ to match against the field values.
:param fetch: If True, always fetch the latest collections from the backend,
regardless of the cache state (default: False).
:param kwargs: Additional keyword arguments to pass to the fetch method.
From 7db3e470b125ae674c1094a15e9a21ca0fa62210 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sat, 21 Jun 2025 00:26:24 +0200
Subject: [PATCH 11/18] [`notes`] Better sync delete logic
- Remove the note records in batch.
- Query by `<external_id, plugin>` rather than `id` alone - you never
know if the SQLAlchemy respects the UUID primary key override.
---
platypush/plugins/_notes/db/_mixin.py | 35 ++++++++++++++++++++-------
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/platypush/plugins/_notes/db/_mixin.py b/platypush/plugins/_notes/db/_mixin.py
index 01fda4b4f..a5179219c 100644
--- a/platypush/plugins/_notes/db/_mixin.py
+++ b/platypush/plugins/_notes/db/_mixin.py
@@ -2,6 +2,7 @@ from contextlib import contextmanager
from threading import Event, RLock
from typing import Any, Dict, Generator
+from sqlalchemy import and_
from sqlalchemy.orm import Session
from platypush.common.notes import Note, NoteCollection
@@ -183,6 +184,7 @@ class DbMixin:
return
with self._get_db_session(autoflush=False) as session:
+ # Add new/updated collections
for collection in [
*state.collections.added.values(),
*state.collections.updated.values(),
@@ -190,21 +192,36 @@ class DbMixin:
db_collection = self._to_db_collection(collection)
session.merge(db_collection)
- for collection in state.collections.deleted.values():
- session.query(DbNoteCollection).filter_by(
- id=collection._db_id # pylint:disable=protected-access
- ).delete()
+ # Delete removed collections
+ session.query(DbNoteCollection).filter(
+ and_(
+ DbNoteCollection.plugin == self._plugin_name,
+ DbNoteCollection.external_id.in_(
+ [
+ collection.id
+ for collection in state.collections.deleted.values()
+ ]
+ ),
+ )
+ ).delete()
- session.flush() # Ensure collections are saved before notes
+ # Ensure that collections are saved before notes
+ session.flush()
+ # Add new/updated notes
for note in [*state.notes.added.values(), *state.notes.updated.values()]:
db_note = self._to_db_note(note)
session.merge(db_note)
- for note in state.notes.deleted.values():
- session.query(DbNote).filter_by(
- id=note._db_id # pylint:disable=protected-access
- ).delete()
+ # Delete removed notes
+ session.query(DbNote).filter(
+ and_(
+ DbNote.plugin == self._plugin_name,
+ DbNote.external_id.in_(
+ [note.id for note in state.notes.deleted.values()]
+ ),
+ )
+ ).delete()
session.commit()
From 46873c317970931a1ea53958e91c99b87748f65f Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sat, 21 Jun 2025 01:09:14 +0200
Subject: [PATCH 12/18] [`notes`] Added `search` action
---
platypush/plugins/_notes/__init__.py | 99 +++++++++++++++++++++-
platypush/plugins/_notes/_model.py | 39 ++++++++-
platypush/plugins/joplin/__init__.py | 121 +++++++++++++++++++++++++--
platypush/utils/__init__.py | 2 +-
4 files changed, 252 insertions(+), 9 deletions(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index 9a9adeefa..d627cf6ed 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -4,7 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from threading import RLock
from time import time
-from typing import Any, Dict, Iterable, Optional, Type
+from typing import Any, Dict, Iterable, List, Optional, Type
from platypush.common.notes import Note, NoteCollection, NoteSource
from platypush.context import Variable
@@ -19,9 +19,10 @@ from platypush.message.event.notes import (
CollectionDeletedEvent,
)
from platypush.plugins import RunnablePlugin, action
+from platypush.utils import to_datetime
from .db import DbMixin
-from ._model import CollectionsDelta, NotesDelta, StateDelta
+from ._model import CollectionsDelta, Item, ItemType, NotesDelta, StateDelta
class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
@@ -537,6 +538,100 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
)
self._refresh_notes_cache()
+ @abstractmethod
+ def _search(
+ self,
+ query: str,
+ *args,
+ item_type: ItemType,
+ include_terms: Optional[Dict[str, Any]] = None,
+ exclude_terms: Optional[Dict[str, Any]] = None,
+ created_before: Optional[datetime] = None,
+ created_after: Optional[datetime] = None,
+ updated_before: Optional[datetime] = None,
+ updated_after: Optional[datetime] = None,
+ **kwargs,
+ ) -> List[Item]:
+ """
+ Search for notes or collections based on the provided query and filters.
+ """
+
+ @action
+ def search(
+ self,
+ *args,
+ query: str,
+ item_type: ItemType = ItemType.NOTE,
+ include_terms: Optional[Dict[str, Any]] = None,
+ exclude_terms: Optional[Dict[str, Any]] = None,
+ created_before: Optional[datetime] = None,
+ created_after: Optional[datetime] = None,
+ updated_before: Optional[datetime] = None,
+ updated_after: Optional[datetime] = None,
+ **kwargs,
+ ):
+ """
+ Search for notes or collections based on the provided query and filters.
+
+ In most of the cases (but it depends on the backend) double-quoted
+ search terms will match exact phrases, while unquoted queries will
+ match any of the words in the query.
+
+ Wildcards (again, depending on the backend) in the search terms are
+ also supported.
+
+ :param query: The search query string (it will be searched in all the
+ fields).
+ :param item_type: The type of items to search for - ``note``,
+ ``collection``, or ``tag`` (default: ``note``).
+ :param include_terms: Optional dictionary of terms to include in the search.
+ The keys are field names and the values are strings to match against.
+ :param exclude_terms: Optional dictionary of terms to exclude from the search.
+ The keys are field names and the values are strings to exclude from the results.
+ :param created_before: Optional datetime ISO string or UNIX timestamp
+ to filter items created before this date.
+ :param created_after: Optional datetime ISO string or UNIX timestamp
+ to filter items created after this date.
+ :param updated_before: Optional datetime ISO string or UNIX timestamp
+ to filter items updated before this date.
+ :param updated_after: Optional datetime ISO string or UNIX timestamp
+ to filter items updated after this date.
+ :return: An iterable of matching items, format:
+
+ .. code-block:: python
+
+ [
+ {
+ "type": "note",
+ "item": {
+ "id": "note-id",
+ "title": "Note Title",
+ "content": "Note content...",
+ "created_at": "2023-10-01T12:00:00Z",
+ "updated_at": "2023-10-01T12:00:00Z",
+ ...
+ }
+ }
+ ]
+
+ """
+ print('==== include_terms ===', include_terms)
+ return [
+ item.to_dict()
+ for item in self._search(
+ query,
+ *args,
+ item_type=item_type,
+ include_terms=include_terms,
+ exclude_terms=exclude_terms,
+ created_before=to_datetime(created_before) if created_before else None,
+ created_after=to_datetime(created_after) if created_after else None,
+ updated_before=to_datetime(updated_before) if updated_before else None,
+ updated_after=to_datetime(updated_after) if updated_after else None,
+ **kwargs,
+ )
+ ]
+
@action
def get_note(self, note_id: Any, *args, **kwargs) -> dict:
"""
diff --git a/platypush/plugins/_notes/_model.py b/platypush/plugins/_notes/_model.py
index 56069c71c..849c946eb 100644
--- a/platypush/plugins/_notes/_model.py
+++ b/platypush/plugins/_notes/_model.py
@@ -1,7 +1,8 @@
from dataclasses import dataclass, field
+from enum import Enum
from typing import Any, Dict
-from platypush.common.notes import Note, NoteCollection
+from platypush.common.notes import Note, NoteCollection, Serializable, Storable
@dataclass
@@ -53,3 +54,39 @@ class StateDelta:
Check if the state delta is empty (no changes in notes or collections).
"""
return self.notes.is_empty() and self.collections.is_empty()
+
+
+class ItemType(Enum):
+ """
+ Enum representing the type of item.
+ """
+
+ NOTE = 'note'
+ COLLECTION = 'collection'
+ TAG = 'tag'
+
+
+@dataclass
+class Item(Serializable):
+ """
+ Represents a generic note item.
+ """
+
+ type: ItemType
+ item: Storable
+
+ def __post_init__(self):
+ """
+ Validate the item type after initialization.
+ """
+ if not isinstance(self.type, ItemType):
+ raise ValueError(f'Invalid item type: {self.type}')
+
+ def to_dict(self) -> Dict[str, Any]:
+ """
+ Convert the item to a dictionary representation.
+ """
+ return {
+ 'type': self.type.value,
+ 'item': self.item.to_dict(),
+ }
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
index d8dcb898e..5c43eb808 100644
--- a/platypush/plugins/joplin/__init__.py
+++ b/platypush/plugins/joplin/__init__.py
@@ -1,11 +1,11 @@
-import datetime
-from typing import Any, List, Optional
+from datetime import datetime
+from typing import Any, Dict, List, Optional
from urllib.parse import urljoin
import requests
from platypush.common.notes import Note, NoteCollection, NoteSource
-from platypush.plugins._notes import BaseNotePlugin
+from platypush.plugins._notes import BaseNotePlugin, Item, ItemType
class JoplinPlugin(BaseNotePlugin):
@@ -141,6 +141,26 @@ class JoplinPlugin(BaseNotePlugin):
'updated_time',
)
+ # Mapping of the internal note fields to the Joplin API fields.
+ _joplin_search_fields = {
+ 'id': 'id',
+ 'title': 'title',
+ 'content': 'body',
+ 'type': 'type',
+ 'parent': 'notebook',
+ 'latitude': 'latitude',
+ 'longitude': 'longitude',
+ 'altitude': 'altitude',
+ 'source': 'sourceurl',
+ }
+
+ # Mapping of ItemType values to Joplin API item types.
+ _joplin_item_types = {
+ ItemType.NOTE: 'note',
+ ItemType.COLLECTION: 'folder',
+ ItemType.TAG: 'tag',
+ }
+
def __init__(self, *args, host: str, port: int = 41184, token: str, **kwargs):
"""
:param host: The hostname or IP address of your Joplin application.
@@ -206,13 +226,13 @@ class JoplinPlugin(BaseNotePlugin):
)
@staticmethod
- def _parse_time(t: Optional[int]) -> Optional[datetime.datetime]:
+ def _parse_time(t: Optional[int]) -> Optional[datetime]:
"""
Parse a Joplin timestamp (in milliseconds) into a datetime object.
"""
if t is None:
return None
- return datetime.datetime.fromtimestamp(t / 1000)
+ return datetime.fromtimestamp(t / 1000)
def _to_note(self, data: dict) -> Note:
parent_id = data.get('parent_id')
@@ -440,5 +460,96 @@ class JoplinPlugin(BaseNotePlugin):
"""
self._exec('DELETE', f'folders/{collection_id}')
+ def _build_search_query(
+ self,
+ query: str,
+ *,
+ include_terms: Optional[Dict[str, Any]] = None,
+ exclude_terms: Optional[Dict[str, Any]] = None,
+ created_before: Optional[datetime] = None,
+ created_after: Optional[datetime] = None,
+ updated_before: Optional[datetime] = None,
+ updated_after: Optional[datetime] = None,
+ ) -> str:
+ query += ' ' + ' '.join(
+ [
+ f'{self._joplin_search_fields.get(k, k)}:"{v}"'
+ for k, v in (include_terms or {}).items()
+ ]
+ )
+
+ query += ' ' + ' '.join(
+ [
+ f'-{self._joplin_search_fields.get(k, k)}:"{v}"'
+ for k, v in (exclude_terms or {}).items()
+ ]
+ )
+
+ if created_before:
+ query += f' -created:{created_before.strftime("%Y%m%d")}'
+ if created_after:
+ query += f' created:{created_after.strftime("%Y%m%d")}'
+ if updated_before:
+ query += f' -updated:{updated_before.strftime("%Y%m%d")}'
+ if updated_after:
+ query += f' updated:{updated_after.strftime("%Y%m%d")}'
+
+ return query.strip()
+
+ def _search(
+ self,
+ query: str,
+ *_,
+ item_type: ItemType,
+ include_terms: Optional[Dict[str, Any]] = None,
+ exclude_terms: Optional[Dict[str, Any]] = None,
+ created_before: Optional[datetime] = None,
+ created_after: Optional[datetime] = None,
+ updated_before: Optional[datetime] = None,
+ updated_after: Optional[datetime] = None,
+ **__,
+ ) -> List[Item]:
+ """
+ Search for notes or collections based on the provided query and filters.
+ """
+ api_item_type = self._joplin_item_types.get(item_type)
+ assert (
+ api_item_type
+ ), f'Invalid item type: {item_type}. Supported types: {list(self._joplin_item_types.keys())}'
+
+ results = self._exec(
+ 'GET',
+ 'search',
+ params={
+ 'type': api_item_type,
+ 'fields': ','.join(
+ self._default_note_fields
+ if item_type == ItemType.NOTE
+ else self._default_collection_fields
+ ),
+ 'query': self._build_search_query(
+ query,
+ include_terms=include_terms,
+ exclude_terms=exclude_terms,
+ created_before=created_before,
+ created_after=created_after,
+ updated_before=updated_before,
+ updated_after=updated_after,
+ ),
+ },
+ )
+
+ return [
+ Item(
+ type=item_type,
+ item=(
+ self._to_note(result)
+ if item_type == ItemType.NOTE
+ else self._to_collection(result)
+ ),
+ )
+ for result in (results or {}).get('items', [])
+ ]
+
# vim:sw=4:ts=4:et:
diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py
index ea5ef460f..6eea56dd0 100644
--- a/platypush/utils/__init__.py
+++ b/platypush/utils/__init__.py
@@ -700,7 +700,7 @@ def to_datetime(t: Union[str, int, float, datetime.datetime]) -> datetime.dateti
if isinstance(t, (int, float)):
return datetime.datetime.fromtimestamp(t, tz=tz.tzutc())
if isinstance(t, str):
- return parser.parse(t)
+ return parser.isoparse(t)
return t
From 5fa12bbf3945e0abf7f2d609c56a1b5e457e495e Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sat, 21 Jun 2025 01:58:42 +0200
Subject: [PATCH 13/18] [`notes`] Support for pagination on all GET endpoints
---
platypush/plugins/_notes/__init__.py | 137 +++++++++++++++++++--------
platypush/plugins/_notes/_model.py | 31 +++++-
platypush/plugins/joplin/__init__.py | 137 ++++++++++++++++++---------
3 files changed, 223 insertions(+), 82 deletions(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index d627cf6ed..3d3577e1f 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -4,7 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from threading import RLock
from time import time
-from typing import Any, Dict, Iterable, List, Optional, Type
+from typing import Any, Dict, Iterable, Optional, Type
from platypush.common.notes import Note, NoteCollection, NoteSource
from platypush.context import Variable
@@ -22,7 +22,15 @@ from platypush.plugins import RunnablePlugin, action
from platypush.utils import to_datetime
from .db import DbMixin
-from ._model import CollectionsDelta, Item, ItemType, NotesDelta, StateDelta
+from ._model import (
+ ApiSettings,
+ CollectionsDelta,
+ Item,
+ ItemType,
+ NotesDelta,
+ Results,
+ StateDelta,
+)
class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
@@ -83,7 +91,15 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
"""
@abstractmethod
- def _fetch_notes(self, *args, **kwargs) -> Iterable[Note]:
+ def _fetch_notes(
+ self,
+ *args,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ sort: Optional[Dict[str, bool]] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ **kwargs,
+ ) -> Iterable[Note]:
"""
Don't call this directly if possible.
Instead, use :meth:`.get_notes` method to retrieve notes and update the cache
@@ -146,7 +162,15 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
"""
@abstractmethod
- def _fetch_collections(self, *args, **kwargs) -> Iterable[NoteCollection]:
+ def _fetch_collections(
+ self,
+ *args,
+ filter: Optional[Dict[str, Any]] = None, # pylint: disable=redefined-builtin
+ sort: Optional[Dict[str, bool]] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ **kwargs,
+ ) -> Iterable[NoteCollection]:
"""
Don't call this directly if possible.
Instead, use :meth:`.get_collections` to retrieve collections and update the cache
@@ -216,13 +240,17 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
reverse=any(not ascending for ascending in sort.values()),
)
- if offset is not None:
+ if offset is not None and not self._api_settings.supports_offset:
items = items[offset:]
- if limit is not None:
+ if limit is not None and not self._api_settings.supports_limit:
items = items[:limit]
return items
+ @property
+ def _api_settings(self) -> ApiSettings:
+ return ApiSettings()
+
def _dispatch_events(self, *events):
"""
Dispatch the given events to the event bus.
@@ -351,7 +379,14 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
with self._sync_lock:
self._notes = {
note.id: self._merge_note(note)
- for note in self._fetch_notes(*args, **kwargs)
+ for note in self._fetch_notes(
+ *args,
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ **kwargs,
+ )
}
self._refresh_notes_cache()
@@ -395,7 +430,14 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
with self._sync_lock:
self._collections = {
collection.id: collection
- for collection in self._fetch_collections(*args, **kwargs)
+ for collection in self._fetch_collections(
+ *args,
+ limit=limit,
+ offset=offset,
+ sort=sort,
+ filter=filter,
+ **kwargs,
+ )
}
self._refresh_collections_cache()
@@ -550,8 +592,10 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
created_after: Optional[datetime] = None,
updated_before: Optional[datetime] = None,
updated_after: Optional[datetime] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = 0,
**kwargs,
- ) -> List[Item]:
+ ) -> Results:
"""
Search for notes or collections based on the provided query and filters.
"""
@@ -568,6 +612,8 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
created_after: Optional[datetime] = None,
updated_before: Optional[datetime] = None,
updated_after: Optional[datetime] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = 0,
**kwargs,
):
"""
@@ -596,41 +642,45 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
to filter items updated before this date.
:param updated_after: Optional datetime ISO string or UNIX timestamp
to filter items updated after this date.
+ :param limit: Maximum number of items to retrieve (default: None,
+ meaning no limit, or depending on the default limit of the backend).
+ :param offset: Offset to start retrieving items from (default: 0).
:return: An iterable of matching items, format:
- .. code-block:: python
+ .. code-block:: javascript
- [
- {
- "type": "note",
- "item": {
- "id": "note-id",
- "title": "Note Title",
- "content": "Note content...",
- "created_at": "2023-10-01T12:00:00Z",
- "updated_at": "2023-10-01T12:00:00Z",
- ...
+ {
+ "has_more": false
+ "results" [
+ {
+ "type": "note",
+ "item": {
+ "id": "note-id",
+ "title": "Note Title",
+ "content": "Note content...",
+ "created_at": "2023-10-01T12:00:00Z",
+ "updated_at": "2023-10-01T12:00:00Z",
+ // ...
+ }
}
- }
- ]
+ ]
+ }
"""
- print('==== include_terms ===', include_terms)
- return [
- item.to_dict()
- for item in self._search(
- query,
- *args,
- item_type=item_type,
- include_terms=include_terms,
- exclude_terms=exclude_terms,
- created_before=to_datetime(created_before) if created_before else None,
- created_after=to_datetime(created_after) if created_after else None,
- updated_before=to_datetime(updated_before) if updated_before else None,
- updated_after=to_datetime(updated_after) if updated_after else None,
- **kwargs,
- )
- ]
+ return self._search(
+ query,
+ *args,
+ item_type=item_type,
+ include_terms=include_terms,
+ exclude_terms=exclude_terms,
+ created_before=to_datetime(created_before) if created_before else None,
+ created_after=to_datetime(created_after) if created_after else None,
+ updated_before=to_datetime(updated_before) if updated_before else None,
+ updated_after=to_datetime(updated_after) if updated_after else None,
+ limit=limit,
+ offset=offset,
+ **kwargs,
+ ).to_dict()
@action
def get_note(self, note_id: Any, *args, **kwargs) -> dict:
@@ -1115,3 +1165,14 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
self.logger.error('Error during sync: %s', e)
finally:
self.wait_stop(self.poll_interval)
+
+
+__all__ = [
+ 'ApiSettings',
+ 'BaseNotePlugin',
+ 'Item',
+ 'ItemType',
+ 'Note',
+ 'NoteCollection',
+ 'NoteSource',
+]
diff --git a/platypush/plugins/_notes/_model.py b/platypush/plugins/_notes/_model.py
index 849c946eb..f00a63637 100644
--- a/platypush/plugins/_notes/_model.py
+++ b/platypush/plugins/_notes/_model.py
@@ -1,6 +1,6 @@
from dataclasses import dataclass, field
from enum import Enum
-from typing import Any, Dict
+from typing import Any, Dict, Iterable
from platypush.common.notes import Note, NoteCollection, Serializable, Storable
@@ -90,3 +90,32 @@ class Item(Serializable):
'type': self.type.value,
'item': self.item.to_dict(),
}
+
+
+@dataclass
+class Results(Serializable):
+ """
+ Represents a collection of results, which can include notes, collections, and tags.
+ """
+
+ items: Iterable[Item] = field(default_factory=list)
+ has_more: bool = False
+
+ def to_dict(self) -> Dict[str, Any]:
+ """
+ Convert the results to a dictionary representation.
+ """
+ return {
+ 'results': [item.to_dict() for item in self.items],
+ 'has_more': self.has_more,
+ }
+
+
+@dataclass
+class ApiSettings:
+ """
+ Represents plugin-specific API settings.
+ """
+
+ supports_limit: bool = False
+ supports_offset: bool = False
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
index 5c43eb808..349d4574d 100644
--- a/platypush/plugins/joplin/__init__.py
+++ b/platypush/plugins/joplin/__init__.py
@@ -5,7 +5,13 @@ from urllib.parse import urljoin
import requests
from platypush.common.notes import Note, NoteCollection, NoteSource
-from platypush.plugins._notes import BaseNotePlugin, Item, ItemType
+from platypush.plugins._notes import (
+ ApiSettings,
+ BaseNotePlugin,
+ Item,
+ ItemType,
+ Results,
+)
class JoplinPlugin(BaseNotePlugin):
@@ -272,6 +278,17 @@ class JoplinPlugin(BaseNotePlugin):
updated_at=self._parse_time(data.get('updated_time')),
)
+ def _offset_to_page(
+ self, offset: Optional[int], limit: Optional[int]
+ ) -> Optional[int]:
+ """
+ Convert an offset to a page number for Joplin API requests.
+ """
+ limit = limit or 100 # Default limit if not provided
+ if offset is None:
+ return None
+ return (offset // limit) + 1 if limit > 0 else 1
+
def _fetch_note(self, note_id: Any, *_, **__) -> Optional[Note]:
note = None
err = None
@@ -302,17 +319,27 @@ class JoplinPlugin(BaseNotePlugin):
return self._to_note(note) # type: ignore[return-value]
- def _fetch_notes(self, *_, **__) -> List[Note]:
+ def _fetch_notes(
+ self, *_, limit: Optional[int] = None, offset: Optional[int] = None, **__
+ ) -> List[Note]:
"""
Fetch notes from Joplin.
"""
- notes_data = (
- self._exec(
- 'GET', 'notes', params={'fields': ','.join(self._default_note_fields)}
- )
- or {}
- ).get('items', [])
- return [self._to_note(note) for note in notes_data]
+ return [
+ self._to_note(note)
+ for note in (
+ self._exec(
+ 'GET',
+ 'notes',
+ params={
+ 'fields': ','.join(self._default_note_fields),
+ 'limit': limit,
+ 'page': self._offset_to_page(offset=offset, limit=limit),
+ },
+ )
+ or {}
+ ).get('items', [])
+ ]
def _create_note(
self,
@@ -402,7 +429,9 @@ class JoplinPlugin(BaseNotePlugin):
return self._to_collection(collection_data)
- def _fetch_collections(self, *_, **__) -> List[NoteCollection]:
+ def _fetch_collections(
+ self, *_, limit: Optional[int] = None, offset: Optional[int] = None, **__
+ ) -> List[NoteCollection]:
"""
Fetch collections (folders) from Joplin.
"""
@@ -410,7 +439,11 @@ class JoplinPlugin(BaseNotePlugin):
self._exec(
'GET',
'folders',
- params={'fields': ','.join(self._default_collection_fields)},
+ params={
+ 'fields': ','.join(self._default_collection_fields),
+ 'limit': limit,
+ 'page': self._offset_to_page(offset=offset, limit=limit),
+ },
)
or {}
).get('items', [])
@@ -496,6 +529,13 @@ class JoplinPlugin(BaseNotePlugin):
return query.strip()
+ @property
+ def _api_settings(self) -> ApiSettings:
+ return ApiSettings(
+ supports_limit=True,
+ supports_offset=True,
+ )
+
def _search(
self,
query: str,
@@ -507,8 +547,10 @@ class JoplinPlugin(BaseNotePlugin):
created_after: Optional[datetime] = None,
updated_before: Optional[datetime] = None,
updated_after: Optional[datetime] = None,
+ limit: Optional[int] = None,
+ offset: Optional[int] = 0,
**__,
- ) -> List[Item]:
+ ) -> Results:
"""
Search for notes or collections based on the provided query and filters.
"""
@@ -517,39 +559,48 @@ class JoplinPlugin(BaseNotePlugin):
api_item_type
), f'Invalid item type: {item_type}. Supported types: {list(self._joplin_item_types.keys())}'
- results = self._exec(
- 'GET',
- 'search',
- params={
- 'type': api_item_type,
- 'fields': ','.join(
- self._default_note_fields
- if item_type == ItemType.NOTE
- else self._default_collection_fields
- ),
- 'query': self._build_search_query(
- query,
- include_terms=include_terms,
- exclude_terms=exclude_terms,
- created_before=created_before,
- created_after=created_after,
- updated_before=updated_before,
- updated_after=updated_after,
- ),
- },
+ limit = limit or 100
+ results = (
+ self._exec(
+ 'GET',
+ 'search',
+ params={
+ 'type': api_item_type,
+ 'limit': limit,
+ 'page': self._offset_to_page(offset=offset, limit=limit),
+ 'fields': ','.join(
+ self._default_note_fields
+ if item_type == ItemType.NOTE
+ else self._default_collection_fields
+ ),
+ 'query': self._build_search_query(
+ query,
+ include_terms=include_terms,
+ exclude_terms=exclude_terms,
+ created_before=created_before,
+ created_after=created_after,
+ updated_before=updated_before,
+ updated_after=updated_after,
+ ),
+ },
+ )
+ or {}
)
- return [
- Item(
- type=item_type,
- item=(
- self._to_note(result)
- if item_type == ItemType.NOTE
- else self._to_collection(result)
- ),
- )
- for result in (results or {}).get('items', [])
- ]
+ return Results(
+ has_more=bool(results.get('has_more')),
+ items=[
+ Item(
+ type=item_type,
+ item=(
+ self._to_note(result)
+ if item_type == ItemType.NOTE
+ else self._to_collection(result)
+ ),
+ )
+ for result in results.get('items', [])
+ ],
+ )
# vim:sw=4:ts=4:et:
From bddad6e2d034e46aa6d08d97ec12d90bf87ee4d7 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sun, 22 Jun 2025 14:38:47 +0200
Subject: [PATCH 14/18] [`notes`] Added `path` propery to `Note` objects
---
platypush/common/notes.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/platypush/common/notes.py b/platypush/common/notes.py
index c3d9f624a..050adc13a 100644
--- a/platypush/common/notes.py
+++ b/platypush/common/notes.py
@@ -76,6 +76,7 @@ class Note(Storable):
altitude: Optional[float] = None
author: Optional[str] = None
source: Optional[NoteSource] = None
+ _path: Optional[str] = None
def __post_init__(self):
"""
@@ -83,6 +84,28 @@ class Note(Storable):
"""
self.digest = self._update_digest()
+ @property
+ def path(self) -> str:
+ # If the path is already set, return it
+ if self._path:
+ return self._path
+
+ # Recursively build the path by expanding the parent collections
+ path = []
+ parent = self.parent
+ while parent:
+ path.append(parent.title)
+ parent = parent.parent
+
+ return '/'.join(reversed(path)) + f'/{self.title}.md'
+
+ @path.setter
+ def path(self, value: str):
+ """
+ Set the path for the note.
+ """
+ self._path = value
+
def _update_digest(self) -> Optional[str]:
if self.content and not self.digest:
self.digest = sha256(self.content.encode('utf-8')).hexdigest()
@@ -96,6 +119,7 @@ class Note(Storable):
for field in self.__dataclass_fields__
if not field.startswith('_') and field != 'parent'
},
+ 'path': self.path,
'parent': (
{
'id': self.parent.id if self.parent else None,
From f84b2358b3dd0451d5b336056296cf63c018dc2a Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sun, 22 Jun 2025 14:40:22 +0200
Subject: [PATCH 15/18] [`notes`] Added customizable `timeout` to note plugin
instances
---
platypush/plugins/_notes/__init__.py | 6 +++++-
platypush/plugins/joplin/__init__.py | 4 +++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index 3d3577e1f..fcdaf0692 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -38,15 +38,19 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
Base class for note-taking plugins.
"""
- def __init__(self, *args, poll_interval: float = 300, **kwargs):
+ def __init__(
+ self, *args, poll_interval: float = 300, timeout: Optional[int] = 60, **kwargs
+ ):
"""
:param poll_interval: Poll interval in seconds to check for updates (default: 300).
If set to zero or null, the plugin will not poll for updates,
and events will be generated only when you manually call :meth:`.sync`.
+ :param timeout: Timeout in seconds for the plugin operations (default: 60).
"""
RunnablePlugin.__init__(self, *args, poll_interval=poll_interval, **kwargs)
DbMixin.__init__(self, *args, **kwargs)
self._sync_lock = RLock()
+ self._timeout = timeout
self.__last_sync_time: Optional[datetime] = None
@property
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
index 349d4574d..267b8d3a7 100644
--- a/platypush/plugins/joplin/__init__.py
+++ b/platypush/plugins/joplin/__init__.py
@@ -198,7 +198,9 @@ class JoplinPlugin(BaseNotePlugin):
)
params['token'] = self.token
- response = requests.request(method, url, params=params, timeout=10, **kwargs)
+ response = requests.request(
+ method, url, params=params, timeout=self._timeout, **kwargs
+ )
if not response.ok:
err = response.text
From 82822e6870243e3a8706fcde4c6218ba438ce17f Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sun, 22 Jun 2025 14:41:38 +0200
Subject: [PATCH 16/18] [`notes`] Plugin model improvements
- Added `ResultsType` to note plugins results
- API settings should be more granular - not only a generic
`supports_limits` and `supports_offset`, but more granular flags on
`notes`, `collections` and `search`.
---
platypush/plugins/_notes/__init__.py | 21 +++++++++++++++++++--
platypush/plugins/_notes/_model.py | 19 +++++++++++++++++--
platypush/plugins/joplin/__init__.py | 9 +++++++--
3 files changed, 43 insertions(+), 6 deletions(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index fcdaf0692..233b8761d 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -29,6 +29,7 @@ from ._model import (
ItemType,
NotesDelta,
Results,
+ ResultsType,
StateDelta,
)
@@ -220,6 +221,7 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
def _process_results( # pylint: disable=too-many-positional-arguments
self,
items: Iterable[Any],
+ results_type: ResultsType,
limit: Optional[int] = None,
offset: Optional[int] = None,
sort: Optional[Dict[str, bool]] = None,
@@ -244,9 +246,22 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
reverse=any(not ascending for ascending in sort.values()),
)
- if offset is not None and not self._api_settings.supports_offset:
+ supports_limit = False
+ supports_offset = False
+
+ if results_type == ResultsType.NOTES:
+ supports_limit = self._api_settings.supports_notes_limit
+ supports_offset = self._api_settings.supports_notes_offset
+ elif results_type == ResultsType.COLLECTIONS:
+ supports_limit = self._api_settings.supports_collections_limit
+ supports_offset = self._api_settings.supports_collections_offset
+ elif results_type == ResultsType.SEARCH:
+ supports_limit = self._api_settings.supports_search_limit
+ supports_offset = self._api_settings.supports_search_offset
+
+ if offset is not None and not supports_offset:
items = items[offset:]
- if limit is not None and not self._api_settings.supports_limit:
+ if limit is not None and not supports_limit:
items = items[:limit]
return items
@@ -400,6 +415,7 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
offset=offset,
sort=sort,
filter=filter,
+ results_type=ResultsType.NOTES,
)
def _get_collection(self, collection_id: Any, *args, **kwargs) -> NoteCollection:
@@ -451,6 +467,7 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
offset=offset,
sort=sort,
filter=filter,
+ results_type=ResultsType.COLLECTIONS,
)
def _refresh_notes_cache(self):
diff --git a/platypush/plugins/_notes/_model.py b/platypush/plugins/_notes/_model.py
index f00a63637..035f7b74b 100644
--- a/platypush/plugins/_notes/_model.py
+++ b/platypush/plugins/_notes/_model.py
@@ -117,5 +117,20 @@ class ApiSettings:
Represents plugin-specific API settings.
"""
- supports_limit: bool = False
- supports_offset: bool = False
+ supports_notes_limit: bool = False
+ supports_notes_offset: bool = False
+ supports_collections_limit: bool = False
+ supports_collections_offset: bool = False
+ supports_search_limit: bool = False
+ supports_search_offset: bool = False
+ supports_search: bool = False
+
+
+class ResultsType(Enum):
+ """
+ Enum representing the type of results.
+ """
+
+ NOTES = 'notes'
+ COLLECTIONS = 'collections'
+ SEARCH = 'search'
diff --git a/platypush/plugins/joplin/__init__.py b/platypush/plugins/joplin/__init__.py
index 267b8d3a7..0fccad13f 100644
--- a/platypush/plugins/joplin/__init__.py
+++ b/platypush/plugins/joplin/__init__.py
@@ -534,8 +534,13 @@ class JoplinPlugin(BaseNotePlugin):
@property
def _api_settings(self) -> ApiSettings:
return ApiSettings(
- supports_limit=True,
- supports_offset=True,
+ supports_notes_limit=True,
+ supports_notes_offset=True,
+ supports_collections_limit=True,
+ supports_collections_offset=True,
+ supports_search_limit=True,
+ supports_search_offset=True,
+ supports_search=True,
)
def _search(
From b2de1cd6e7aec08564342050bf3810db62cfce6e Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sun, 22 Jun 2025 15:27:58 +0200
Subject: [PATCH 17/18] [`notes`] `notes._to_db_note` should always set the
deterministic UUID as the primary key
---
platypush/plugins/_notes/db/_mixin.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/platypush/plugins/_notes/db/_mixin.py b/platypush/plugins/_notes/db/_mixin.py
index a5179219c..c41294625 100644
--- a/platypush/plugins/_notes/db/_mixin.py
+++ b/platypush/plugins/_notes/db/_mixin.py
@@ -74,6 +74,7 @@ class DbMixin:
Convert a Note object to a DbNote object.
"""
return DbNote(
+ id=note._db_id, # pylint:disable=protected-access
external_id=note.id,
plugin=self._plugin_name,
title=note.title,
From d8cbff6a9d190cbabe7222bb74f64bd3c925b626 Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Sun, 22 Jun 2025 17:30:21 +0200
Subject: [PATCH 18/18] [`notes`] Log state delta on sync
---
platypush/plugins/_notes/__init__.py | 3 +++
platypush/plugins/_notes/_model.py | 30 +++++++++++++++++++++++++++
platypush/plugins/_notes/db/_mixin.py | 2 +-
3 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/platypush/plugins/_notes/__init__.py b/platypush/plugins/_notes/__init__.py
index 233b8761d..fa8c8fcbd 100644
--- a/platypush/plugins/_notes/__init__.py
+++ b/platypush/plugins/_notes/__init__.py
@@ -1147,6 +1147,9 @@ class BaseNotePlugin(RunnablePlugin, DbMixin, ABC):
)
# Update the local cache with the latest notes and collections
+ if not state_delta.is_empty():
+ self.logger.info('Synchronizing changes: %s', state_delta)
+
self._db_sync(state_delta)
self._last_sync_time = datetime.fromtimestamp(state_delta.latest_updated_at)
self._process_events(state_delta)
diff --git a/platypush/plugins/_notes/_model.py b/platypush/plugins/_notes/_model.py
index 035f7b74b..d4aef1f1f 100644
--- a/platypush/plugins/_notes/_model.py
+++ b/platypush/plugins/_notes/_model.py
@@ -21,6 +21,16 @@ class NotesDelta:
"""
return not (self.added or self.updated or self.deleted)
+ def __str__(self):
+ """
+ String representation of the NotesDelta.
+ """
+ return (
+ f'NotesDelta(added={len(self.added)}, '
+ f'updated={len(self.updated)}, '
+ f'deleted={len(self.deleted)})'
+ )
+
@dataclass
class CollectionsDelta:
@@ -38,6 +48,16 @@ class CollectionsDelta:
"""
return not (self.added or self.updated or self.deleted)
+ def __str__(self):
+ """
+ String representation of the CollectionsDelta.
+ """
+ return (
+ f'CollectionsDelta(added={len(self.added)}, '
+ f'updated={len(self.updated)}, '
+ f'deleted={len(self.deleted)})'
+ )
+
@dataclass
class StateDelta:
@@ -55,6 +75,16 @@ class StateDelta:
"""
return self.notes.is_empty() and self.collections.is_empty()
+ def __str__(self):
+ """
+ String representation of the StateDelta.
+ """
+ return (
+ f'StateDelta(notes={self.notes}, '
+ f'collections={self.collections}, '
+ f'latest_updated_at={self.latest_updated_at})'
+ )
+
class ItemType(Enum):
"""
diff --git a/platypush/plugins/_notes/db/_mixin.py b/platypush/plugins/_notes/db/_mixin.py
index c41294625..facacbde2 100644
--- a/platypush/plugins/_notes/db/_mixin.py
+++ b/platypush/plugins/_notes/db/_mixin.py
@@ -17,7 +17,7 @@ from ._model import (
)
-class DbMixin:
+class DbMixin: # pylint: disable=too-few-public-methods
"""
Mixin class for the database synchronization layer.
"""