From 2dc8fe94379421aa47be462212a888ef41f7da24 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 22 Aug 2020 12:09:24 +0200 Subject: [PATCH] Added Github backend [closes #95] --- docs/source/conf.py | 1 + platypush/backend/__init__.py | 4 +- platypush/backend/github.py | 226 ++++++++++++++++++++++++++++++ platypush/message/event/github.py | 146 +++++++++++++++++++ requirements.txt | 5 +- setup.py | 7 +- 6 files changed, 384 insertions(+), 5 deletions(-) create mode 100644 platypush/backend/github.py create mode 100644 platypush/message/event/github.py diff --git a/docs/source/conf.py b/docs/source/conf.py index ce661e6c3..0a64ec936 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -258,6 +258,7 @@ autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers', 'gi', 'gi.repository', 'twilio', + 'pytz', ] sys.path.insert(0, os.path.abspath('../..')) diff --git a/platypush/backend/__init__.py b/platypush/backend/__init__.py index 9c78e928f..dce170f97 100644 --- a/platypush/backend/__init__.py +++ b/platypush/backend/__init__.py @@ -280,8 +280,8 @@ class Backend(Thread, EventGenerator): def should_stop(self): return self._should_stop - def wait_stop(self, timeout=None): - self._stop_event.wait(timeout) + def wait_stop(self, timeout=None) -> bool: + return self._stop_event.wait(timeout) def _get_redis(self): import redis diff --git a/platypush/backend/github.py b/platypush/backend/github.py new file mode 100644 index 000000000..b1fd3dfab --- /dev/null +++ b/platypush/backend/github.py @@ -0,0 +1,226 @@ +import datetime +import os +import threading + +from typing import Optional, List + +import pytz +import requests +from sqlalchemy import create_engine, Column, String, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, scoped_session + +from platypush.backend import Backend +from platypush.config import Config +from platypush.message.event.github import GithubPushEvent, GithubCommitCommentEvent, GithubCreateEvent, \ + GithubDeleteEvent, GithubEvent, GithubForkEvent, GithubWikiEvent, GithubIssueCommentEvent, GithubIssueEvent, \ + GithubMemberEvent, GithubPublicEvent, GithubPullRequestEvent, GithubPullRequestReviewCommentEvent, \ + GithubReleaseEvent, GithubSponsorshipEvent, GithubWatchEvent + +Base = declarative_base() +Session = scoped_session(sessionmaker()) + + +class GithubResource(Base): + """ + Models the GithubLastEvent table, containing the timestamp where a certain URL was last checked. + """ + + __tablename__ = 'GithubLastEvent' + uri = Column(String, primary_key=True) + last_updated_at = Column(DateTime) + + +class GithubBackend(Backend): + """ + This backend monitors for notifications and events either on Github user, organization or repository level. + You'll need a Github personal access token to use the service. To get one: + + - Access your Github profile settings + - Select *Developer Settings* + - Select *Personal access tokens* + - Click *Generate new token* + + This backend requires the following permissions: + + - ``repo`` + - ``notifications`` + - ``read:org`` if you want to access repositories on organization level. + + Triggers: + + - :class:`platypush.message.event.github.GithubPushEvent` when a new push is created. + - :class:`platypush.message.event.github.GithubCommitCommentEvent` when a new commit comment is created. + - :class:`platypush.message.event.github.GithubCreateEvent` when a tag or branch is created. + - :class:`platypush.message.event.github.GithubDeleteEvent` when a tag or branch is deleted. + - :class:`platypush.message.event.github.GithubForkEvent` when a user forks a repository. + - :class:`platypush.message.event.github.GithubWikiEvent` when new activity happens on a repository wiki. + - :class:`platypush.message.event.github.GithubIssueCommentEvent` when new activity happens on an issue comment. + - :class:`platypush.message.event.github.GithubIssueEvent` when new repository issue activity happens. + - :class:`platypush.message.event.github.GithubMemberEvent` when new repository collaborators activity happens. + - :class:`platypush.message.event.github.GithubPublicEvent` when a repository goes public. + - :class:`platypush.message.event.github.GithubPullRequestEvent` when new pull request related activity happens. + - :class:`platypush.message.event.github.GithubPullRequestReviewCommentEvent` when activity happens on a pull + request commit. + - :class:`platypush.message.event.github.GithubReleaseEvent` when a new release happens. + - :class:`platypush.message.event.github.GithubSponsorshipEvent` when new sponsorship related activity happens. + - :class:`platypush.message.event.github.GithubWatchEvent` when someone stars/starts watching a repository. + - :class:`platypush.message.event.github.GithubEvent` for any event that doesn't fall in the above categories + (``event_type`` will be set accordingly). + + """ + + _base_url = 'https://api.github.com' + + def __init__(self, user: str, user_token: str, repos: Optional[List[str]] = None, org: Optional[str] = None, + poll_seconds: int = 60, max_events_per_scan: Optional[int] = 10, *args, **kwargs): + """ + If neither ``repos`` nor ``org`` is specified then the backend will monitor all new events on user level. + + :param user: Github username. + :param user_token: Github personal access token. + :param repos: List of repos to be monitored - if a list is provided then only these repositories will be + monitored for events. Repositories should be passed in the format ``username/repository``. + :param org: Organization to be monitored - if provided then only this organization will be monitored for events. + :param poll_seconds: How often the backend should check for new events, in seconds (default: 60). + :param max_events_per_scan: Maximum number of events per resource that will be triggered if there is a large + number of events/notification since the last check (default: 10). Specify 0 or null for no limit. + """ + super().__init__(*args, **kwargs) + self._last_text: Optional[str] = None + self.user = user + self.user_token = user_token + self.repos = repos or [] + self.org = org + self.poll_seconds = poll_seconds + self.db_lock = threading.RLock() + self.workdir = os.path.join(os.path.expanduser(Config.get('workdir')), 'github') + self.dbfile = os.path.join(self.workdir, 'github.db') + self.max_events_per_scan = max_events_per_scan + + os.makedirs(os.path.dirname(self.dbfile), exist_ok=True) + self._init_db() + + def _request(self, uri: str, method: str = 'get') -> dict: + method = getattr(requests, method.lower()) + return method(self._base_url + uri, auth=(self.user, self.user_token), + headers={'Accept': 'application/vnd.github.v3+json'}).json() + + def _init_db(self): + engine = create_engine('sqlite:///{}'.format(self.dbfile), connect_args={'check_same_thread': False}) + Base.metadata.create_all(engine) + Session.configure(bind=engine) + + @staticmethod + def _to_datetime(time_string: str) -> datetime.datetime: + """ Convert ISO 8061 string format with leading 'Z' into something understandable by Python """ + return datetime.datetime.fromisoformat(time_string[:-1] + '+00:00') + + @staticmethod + def _get_or_create_resource(uri: str, session: Session) -> GithubResource: + record = session.query(GithubResource).filter_by(uri=uri).first() + if record is None: + record = GithubResource(uri=uri) + session.add(record) + session.commit() + + return record + + def _get_last_event_time(self, uri: str): + with self.db_lock: + record = self._get_or_create_resource(uri=uri, session=Session()) + return record.last_updated_at.replace(tzinfo=pytz.UTC) if record.last_updated_at else None + + def _update_last_event_time(self, uri: str, last_updated_at: datetime.datetime): + with self.db_lock: + session = Session() + record = self._get_or_create_resource(uri=uri, session=session) + record.last_updated_at = last_updated_at + session.add(record) + session.commit() + + @classmethod + def _parse_event(cls, event: dict) -> GithubEvent: + event_mapping = { + 'PushEvent': GithubPushEvent, + 'CommitCommentEvent': GithubCommitCommentEvent, + 'CreateEvent': GithubCreateEvent, + 'DeleteEvent': GithubDeleteEvent, + 'ForkEvent': GithubForkEvent, + 'GollumEvent': GithubWikiEvent, + 'IssueCommentEvent': GithubIssueCommentEvent, + 'IssuesEvent': GithubIssueEvent, + 'MemberEvent': GithubMemberEvent, + 'PublicEvent': GithubPublicEvent, + 'PullRequestEvent': GithubPullRequestEvent, + 'PullRequestReviewCommentEvent': GithubPullRequestReviewCommentEvent, + 'ReleaseEvent': GithubReleaseEvent, + 'SponsorshipEvent': GithubSponsorshipEvent, + 'WatchEvent': GithubWatchEvent, + } + + event_type = event_mapping[event['type']] if event['type'] in event_mapping else GithubEvent + return event_type(event_type=event['type'], actor=event['actor'], repo=event.get('repo', {}), + payload=event['payload'], created_at=cls._to_datetime(event['created_at'])) + + def _events_monitor(self, uri: str, method: str = 'get'): + def thread(): + while True: + try: + events = self._request(uri, method) + if not events: + continue + + last_event_time = self._get_last_event_time(uri) + new_last_event_time = last_event_time + fired_events = [] + + for event in events: + if self.max_events_per_scan and len(fired_events) >= self.max_events_per_scan: + break + + event_time = self._to_datetime(event['created_at']) + if last_event_time and event_time <= last_event_time: + break + if not new_last_event_time or event_time > new_last_event_time: + new_last_event_time = event_time + + fired_events.append(self._parse_event(event)) + + for event in fired_events: + self.bus.post(event) + + self._update_last_event_time(uri=uri, last_updated_at=new_last_event_time) + except Exception as e: + self.logger.warning('Encountered exception while fetching events from {}: {}'.format( + uri, str(e))) + self.logger.exception(e) + finally: + if self.wait_stop(timeout=self.poll_seconds): + break + + return thread + + def run(self): + self.logger.info('Starting Github backend') + monitors = [] + + if self.repos: + for repo in self.repos: + monitors.append(threading.Thread(target=self._events_monitor('/networks/{repo}/events'.format(repo=repo)))) + if self.org: + monitors.append(threading.Thread(target=self._events_monitor('/orgs/{org}/events'.format(org=self.org)))) + + if not (self.repos or self.org): + monitors.append(threading.Thread(target=self._events_monitor('/users/{user}/events'.format(user=self.user)))) + + for monitor in monitors: + monitor.start() + + self.logger.info('Started Github backend') + for monitor in monitors: + monitor.join() + + self.logger.info('Github backend terminated') + +# vim:sw=4:ts=4:et: diff --git a/platypush/message/event/github.py b/platypush/message/event/github.py new file mode 100644 index 000000000..7fae3a843 --- /dev/null +++ b/platypush/message/event/github.py @@ -0,0 +1,146 @@ +from datetime import datetime +from dataclasses import dataclass +from typing import Optional, Dict + +from platypush.message.event import Event + + +@dataclass +class Actor: + id: str + login: str + display_login: str + url: str + gravatar_id: str + avatar_url: str + + +@dataclass +class Repo: + id: str + name: str + url: str + + +class GithubEvent(Event): + """ Generic Github event """ + + def __init__(self, + event_type: str, + created_at: datetime, + actor: Optional[Dict[str, str]] = None, + repo: Optional[Dict[str, str]] = None, + *args, **kwargs): + super().__init__(*args, actor=actor, event_type=event_type, created_at=created_at, **kwargs) + self.event_type = event_type + self.actor = Actor(**actor) if actor else None + self.repo = Repo(**repo) if repo else None + self.created_at = created_at + + +class GithubPushEvent(GithubEvent): + """ Github push event. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubCommitCommentEvent(GithubEvent): + """ A commit comment is created. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubCreateEvent(GithubEvent): + """ A git branch or tag is created. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubDeleteEvent(GithubEvent): + """ A git branch or tag is deleted. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubForkEvent(GithubEvent): + """ A user forks a watched repository. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubWikiEvent(GithubEvent): + """ A wiki page is created or updated on a watched repository. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubIssueCommentEvent(GithubEvent): + """ A comment is added or updated on an issue. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubIssueEvent(GithubEvent): + """ A new activity is registered on an issue. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubMemberEvent(GithubEvent): + """ New activity related to repository collaborators. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubPublicEvent(GithubEvent): + """ A private repository is made public. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubPullRequestEvent(GithubEvent): + """ New activity related to a pull request. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubPullRequestReviewCommentEvent(GithubEvent): + """ New activity related to comments of a pull request. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubReleaseEvent(GithubEvent): + """ New activity related to the release of a repository. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubSponsorshipEvent(GithubEvent): + """ New activity related to the sponsorship of a repository. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +class GithubWatchEvent(GithubEvent): + """ Event triggered when someone stars or starts watching a repository. """ + + def __init__(self, payload: dict, *args, **kwargs): + super().__init__(*args, payload=payload, **kwargs) + + +# vim:sw=4:ts=4:et: diff --git a/requirements.txt b/requirements.txt index 176242e2b..c3b64de82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -281,4 +281,7 @@ croniter # python-dbus # Support for Twilio -# twilio \ No newline at end of file +# twilio + +# Support for Github +# pytz diff --git a/setup.py b/setup.py index 475ce4d8d..75b2f71d7 100755 --- a/setup.py +++ b/setup.py @@ -162,6 +162,7 @@ setup( 'websocket-client', 'wheel', 'zeroconf>=0.27.0', + 'tz', ], extras_require={ @@ -172,9 +173,9 @@ setup( # Support for Pushbullet backend and plugin 'pushbullet': ['pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'], # Support for HTTP backend - 'http': ['flask', 'python-dateutil', 'tz', 'frozendict', 'bcrypt'], + 'http': ['flask', 'python-dateutil', 'frozendict', 'bcrypt'], # Support for uWSGI HTTP backend - 'uwsgi': ['flask', 'python-dateutil', 'tz', 'frozendict', 'uwsgi', 'bcrypt'], + 'uwsgi': ['flask', 'python-dateutil', 'frozendict', 'uwsgi', 'bcrypt'], # Support for MQTT backends 'mqtt': ['paho-mqtt'], # Support for RSS feeds parser @@ -323,5 +324,7 @@ setup( 'dbus': ['dbus-python'], # Support for Twilio integration 'twilio': ['twilio'], + # Support for Github integration + 'github': ['pytz'], }, )