From 5eb09eab1a798b754cf4c5517c63c0db2cf47162 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 30 Dec 2023 16:28:12 +0100 Subject: [PATCH] [#287] Migrated `github` from a backend to a runnable plugin. Closes: #287 --- docs/source/backends.rst | 1 - docs/source/platypush/backend/github.rst | 5 -- docs/source/platypush/plugins/github.rst | 5 ++ docs/source/plugins.rst | 1 + platypush/backend/github/manifest.yaml | 32 --------- .../{backend => plugins}/github/__init__.py | 70 ++++++++----------- platypush/plugins/github/manifest.yaml | 20 ++++++ 7 files changed, 57 insertions(+), 77 deletions(-) delete mode 100644 docs/source/platypush/backend/github.rst create mode 100644 docs/source/platypush/plugins/github.rst delete mode 100644 platypush/backend/github/manifest.yaml rename platypush/{backend => plugins}/github/__init__.py (81%) create mode 100644 platypush/plugins/github/manifest.yaml diff --git a/docs/source/backends.rst b/docs/source/backends.rst index 9e8464cf91..1347f2e3cc 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -11,7 +11,6 @@ Backends platypush/backend/camera.pi.rst platypush/backend/chat.telegram.rst platypush/backend/foursquare.rst - platypush/backend/github.rst platypush/backend/google.fit.rst platypush/backend/google.pubsub.rst platypush/backend/gps.rst diff --git a/docs/source/platypush/backend/github.rst b/docs/source/platypush/backend/github.rst deleted file mode 100644 index a299331d38..0000000000 --- a/docs/source/platypush/backend/github.rst +++ /dev/null @@ -1,5 +0,0 @@ -``github`` -============================ - -.. automodule:: platypush.backend.github - :members: diff --git a/docs/source/platypush/plugins/github.rst b/docs/source/platypush/plugins/github.rst new file mode 100644 index 0000000000..483d504066 --- /dev/null +++ b/docs/source/platypush/plugins/github.rst @@ -0,0 +1,5 @@ +``github`` +========== + +.. automodule:: platypush.plugins.github + :members: diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 8f39bf6efe..070ff26720 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -35,6 +35,7 @@ Plugins platypush/plugins/file.rst platypush/plugins/file.monitor.rst platypush/plugins/foursquare.rst + platypush/plugins/github.rst platypush/plugins/google.calendar.rst platypush/plugins/google.drive.rst platypush/plugins/google.fit.rst diff --git a/platypush/backend/github/manifest.yaml b/platypush/backend/github/manifest.yaml deleted file mode 100644 index 18eced00de..0000000000 --- a/platypush/backend/github/manifest.yaml +++ /dev/null @@ -1,32 +0,0 @@ -manifest: - events: - platypush.message.event.github.GithubCommitCommentEvent: when a new commit comment - is created. - platypush.message.event.github.GithubCreateEvent: when a tag or branch is created. - platypush.message.event.github.GithubDeleteEvent: when a tag or branch is deleted. - platypush.message.event.github.GithubEvent: for any event that doesn't fall in - the above categories(``event_type`` will be set accordingly). - platypush.message.event.github.GithubForkEvent: when a user forks a repository. - platypush.message.event.github.GithubIssueCommentEvent: when new activity happens - on an issue comment. - platypush.message.event.github.GithubIssueEvent: when new repository issue activity - happens. - platypush.message.event.github.GithubMemberEvent: when new repository collaborators - activity happens. - platypush.message.event.github.GithubPublicEvent: when a repository goes public. - platypush.message.event.github.GithubPullRequestEvent: when new pull request related - activity happens. - platypush.message.event.github.GithubPullRequestReviewCommentEvent: when activity - happens on a pullrequest commit. - platypush.message.event.github.GithubPushEvent: when a new push is created. - platypush.message.event.github.GithubReleaseEvent: when a new release happens. - platypush.message.event.github.GithubSponsorshipEvent: when new sponsorship related - activity happens. - platypush.message.event.github.GithubWatchEvent: when someone stars/starts watching - a repository. - platypush.message.event.github.GithubWikiEvent: when new activity happens on a - repository wiki. - install: - pip: [] - package: platypush.backend.github - type: backend diff --git a/platypush/backend/github/__init__.py b/platypush/plugins/github/__init__.py similarity index 81% rename from platypush/backend/github/__init__.py rename to platypush/plugins/github/__init__.py index a66b6364a4..1e3755e4e6 100644 --- a/platypush/backend/github/__init__.py +++ b/platypush/plugins/github/__init__.py @@ -8,7 +8,6 @@ import requests from sqlalchemy import create_engine, Column, String, DateTime from sqlalchemy.orm import sessionmaker, scoped_session -from platypush.backend import Backend from platypush.common.db import declarative_base from platypush.config import Config from platypush.message.event.github import ( @@ -29,11 +28,13 @@ from platypush.message.event.github import ( GithubSponsorshipEvent, GithubWatchEvent, ) +from platypush.plugins import RunnablePlugin Base = declarative_base() Session = scoped_session(sessionmaker()) +# pylint: disable=too-few-public-methods class GithubResource(Base): """ Models the GithubLastEvent table, containing the timestamp where a certain URL was last checked. @@ -44,9 +45,9 @@ class GithubResource(Base): last_updated_at = Column(DateTime) -class GithubBackend(Backend): +class GithubPlugin(RunnablePlugin): """ - This backend monitors for notifications and events either on Github user, organization or repository level. + This plugin monitors for notifications and events either on Github user, organization or repository level. You'll need a Github personal access token to use the service. To get one: - Access your Github profile settings @@ -54,7 +55,7 @@ class GithubBackend(Backend): - Select *Personal access tokens* - Click *Generate new token* - This backend requires the following permissions: + This plugin requires the following permissions: - ``repo`` - ``notifications`` @@ -72,22 +73,21 @@ class GithubBackend(Backend): org: Optional[str] = None, poll_seconds: int = 60, max_events_per_scan: Optional[int] = 10, - *args, - **kwargs + **kwargs, ): """ - If neither ``repos`` nor ``org`` is specified then the backend will monitor all new events on user level. + If neither ``repos`` nor ``org`` is specified then the plugin will monitor all new events on user level. :param user: Github username. :param user_token: Github personal access token. :param repos: List of repos to be monitored - if a list is provided then only these repositories will be monitored for events. Repositories should be passed in the format ``username/repository``. :param org: Organization to be monitored - if provided then only this organization will be monitored for events. - :param poll_seconds: How often the backend should check for new events, in seconds (default: 60). + :param poll_seconds: How often the plugin should check for new events, in seconds (default: 60). :param max_events_per_scan: Maximum number of events per resource that will be triggered if there is a large number of events/notification since the last check (default: 10). Specify 0 or null for no limit. """ - super().__init__(*args, **kwargs) + super().__init__(**kwargs) self._last_text: Optional[str] = None self.user = user self.user_token = user_token @@ -95,7 +95,7 @@ class GithubBackend(Backend): self.org = org self.poll_seconds = poll_seconds self.db_lock = threading.RLock() - self.workdir = os.path.join(os.path.expanduser(Config.get('workdir')), 'github') + self.workdir = os.path.join(os.path.expanduser(Config.get_workdir()), 'github') self.dbfile = os.path.join(self.workdir, 'github.db') self.max_events_per_scan = max_events_per_scan @@ -103,8 +103,8 @@ class GithubBackend(Backend): self._init_db() def _request(self, uri: str, method: str = 'get') -> dict: - method = getattr(requests, method.lower()) - return method( + func = getattr(requests, method.lower()) + return func( self._base_url + uri, auth=(self.user, self.user_token), headers={'Accept': 'application/vnd.github.v3+json'}, @@ -112,7 +112,7 @@ class GithubBackend(Backend): def _init_db(self): engine = create_engine( - 'sqlite:///{}'.format(self.dbfile), + f'sqlite:///{self.dbfile}', connect_args={'check_same_thread': False}, ) Base.metadata.create_all(engine) @@ -124,7 +124,7 @@ class GithubBackend(Backend): return datetime.datetime.fromisoformat(time_string[:-1] + '+00:00') @staticmethod - def _get_or_create_resource(uri: str, session: Session) -> GithubResource: + def _get_or_create_resource(uri: str, session: scoped_session) -> GithubResource: record = session.query(GithubResource).filter_by(uri=uri).first() if record is None: record = GithubResource(uri=uri) @@ -135,18 +135,18 @@ class GithubBackend(Backend): def _get_last_event_time(self, uri: str): with self.db_lock: - record = self._get_or_create_resource(uri=uri, session=Session()) + record = self._get_or_create_resource(uri=uri, session=Session()) # type: ignore return ( record.last_updated_at.replace(tzinfo=datetime.timezone.utc) - if record.last_updated_at + if record.last_updated_at # type: ignore else None ) def _update_last_event_time(self, uri: str, last_updated_at: datetime.datetime): with self.db_lock: session = Session() - record = self._get_or_create_resource(uri=uri, session=session) - record.last_updated_at = last_updated_at + record = self._get_or_create_resource(uri=uri, session=session) # type: ignore + record.last_updated_at = last_updated_at # type: ignore session.add(record) session.commit() @@ -211,16 +211,17 @@ class GithubBackend(Backend): fired_events.append(self._parse_event(event)) for event in fired_events: - self.bus.post(event) + self._bus.post(event) - self._update_last_event_time( - uri=uri, last_updated_at=new_last_event_time - ) + if new_last_event_time: + self._update_last_event_time( + uri=uri, last_updated_at=new_last_event_time + ) except Exception as e: self.logger.warning( - 'Encountered exception while fetching events from {}: {}'.format( - uri, str(e) - ) + 'Encountered exception while fetching events from %s: %s', + uri, + e, ) self.logger.exception(e) @@ -229,45 +230,36 @@ class GithubBackend(Backend): return thread - def run(self): - self.logger.info('Starting Github backend') + def main(self): monitors = [] if self.repos: for repo in self.repos: monitors.append( threading.Thread( - target=self._events_monitor( - '/networks/{repo}/events'.format(repo=repo) - ) + target=self._events_monitor(f'/networks/{repo}/events') ) ) + if self.org: monitors.append( threading.Thread( - target=self._events_monitor( - '/orgs/{org}/events'.format(org=self.org) - ) + target=self._events_monitor(f'/orgs/{self.org}/events') ) ) if not (self.repos or self.org): monitors.append( threading.Thread( - target=self._events_monitor( - '/users/{user}/events'.format(user=self.user) - ) + target=self._events_monitor(f'/users/{self.user}/events') ) ) for monitor in monitors: monitor.start() - self.logger.info('Started Github backend') for monitor in monitors: monitor.join() - self.logger.info('Github backend terminated') - # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/github/manifest.yaml b/platypush/plugins/github/manifest.yaml new file mode 100644 index 0000000000..7d6ff3e4ec --- /dev/null +++ b/platypush/plugins/github/manifest.yaml @@ -0,0 +1,20 @@ +manifest: + events: + - platypush.message.event.github.GithubCommitCommentEvent + - platypush.message.event.github.GithubCreateEvent + - platypush.message.event.github.GithubDeleteEvent + - platypush.message.event.github.GithubEvent + - platypush.message.event.github.GithubForkEvent + - platypush.message.event.github.GithubIssueCommentEvent + - platypush.message.event.github.GithubIssueEvent + - platypush.message.event.github.GithubMemberEvent + - platypush.message.event.github.GithubPublicEvent + - platypush.message.event.github.GithubPullRequestEvent + - platypush.message.event.github.GithubPullRequestReviewCommentEvent + - platypush.message.event.github.GithubPushEvent + - platypush.message.event.github.GithubReleaseEvent + - platypush.message.event.github.GithubSponsorshipEvent + - platypush.message.event.github.GithubWatchEvent + - platypush.message.event.github.GithubWikiEvent + package: platypush.plugin.github + type: plugin