forked from platypush/platypush
Added Github backend [closes #95]
This commit is contained in:
parent
a0d97c0f18
commit
2dc8fe9437
6 changed files with 384 additions and 5 deletions
|
@ -258,6 +258,7 @@ autodoc_mock_imports = ['googlesamples.assistant.grpc.audio_helpers',
|
||||||
'gi',
|
'gi',
|
||||||
'gi.repository',
|
'gi.repository',
|
||||||
'twilio',
|
'twilio',
|
||||||
|
'pytz',
|
||||||
]
|
]
|
||||||
|
|
||||||
sys.path.insert(0, os.path.abspath('../..'))
|
sys.path.insert(0, os.path.abspath('../..'))
|
||||||
|
|
|
@ -280,8 +280,8 @@ class Backend(Thread, EventGenerator):
|
||||||
def should_stop(self):
|
def should_stop(self):
|
||||||
return self._should_stop
|
return self._should_stop
|
||||||
|
|
||||||
def wait_stop(self, timeout=None):
|
def wait_stop(self, timeout=None) -> bool:
|
||||||
self._stop_event.wait(timeout)
|
return self._stop_event.wait(timeout)
|
||||||
|
|
||||||
def _get_redis(self):
|
def _get_redis(self):
|
||||||
import redis
|
import redis
|
||||||
|
|
226
platypush/backend/github.py
Normal file
226
platypush/backend/github.py
Normal file
|
@ -0,0 +1,226 @@
|
||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from typing import Optional, List
|
||||||
|
|
||||||
|
import pytz
|
||||||
|
import requests
|
||||||
|
from sqlalchemy import create_engine, Column, String, DateTime
|
||||||
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
|
from sqlalchemy.orm import sessionmaker, scoped_session
|
||||||
|
|
||||||
|
from platypush.backend import Backend
|
||||||
|
from platypush.config import Config
|
||||||
|
from platypush.message.event.github import GithubPushEvent, GithubCommitCommentEvent, GithubCreateEvent, \
|
||||||
|
GithubDeleteEvent, GithubEvent, GithubForkEvent, GithubWikiEvent, GithubIssueCommentEvent, GithubIssueEvent, \
|
||||||
|
GithubMemberEvent, GithubPublicEvent, GithubPullRequestEvent, GithubPullRequestReviewCommentEvent, \
|
||||||
|
GithubReleaseEvent, GithubSponsorshipEvent, GithubWatchEvent
|
||||||
|
|
||||||
|
Base = declarative_base()
|
||||||
|
Session = scoped_session(sessionmaker())
|
||||||
|
|
||||||
|
|
||||||
|
class GithubResource(Base):
|
||||||
|
"""
|
||||||
|
Models the GithubLastEvent table, containing the timestamp where a certain URL was last checked.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = 'GithubLastEvent'
|
||||||
|
uri = Column(String, primary_key=True)
|
||||||
|
last_updated_at = Column(DateTime)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubBackend(Backend):
|
||||||
|
"""
|
||||||
|
This backend monitors for notifications and events either on Github user, organization or repository level.
|
||||||
|
You'll need a Github personal access token to use the service. To get one:
|
||||||
|
|
||||||
|
- Access your Github profile settings
|
||||||
|
- Select *Developer Settings*
|
||||||
|
- Select *Personal access tokens*
|
||||||
|
- Click *Generate new token*
|
||||||
|
|
||||||
|
This backend requires the following permissions:
|
||||||
|
|
||||||
|
- ``repo``
|
||||||
|
- ``notifications``
|
||||||
|
- ``read:org`` if you want to access repositories on organization level.
|
||||||
|
|
||||||
|
Triggers:
|
||||||
|
|
||||||
|
- :class:`platypush.message.event.github.GithubPushEvent` when a new push is created.
|
||||||
|
- :class:`platypush.message.event.github.GithubCommitCommentEvent` when a new commit comment is created.
|
||||||
|
- :class:`platypush.message.event.github.GithubCreateEvent` when a tag or branch is created.
|
||||||
|
- :class:`platypush.message.event.github.GithubDeleteEvent` when a tag or branch is deleted.
|
||||||
|
- :class:`platypush.message.event.github.GithubForkEvent` when a user forks a repository.
|
||||||
|
- :class:`platypush.message.event.github.GithubWikiEvent` when new activity happens on a repository wiki.
|
||||||
|
- :class:`platypush.message.event.github.GithubIssueCommentEvent` when new activity happens on an issue comment.
|
||||||
|
- :class:`platypush.message.event.github.GithubIssueEvent` when new repository issue activity happens.
|
||||||
|
- :class:`platypush.message.event.github.GithubMemberEvent` when new repository collaborators activity happens.
|
||||||
|
- :class:`platypush.message.event.github.GithubPublicEvent` when a repository goes public.
|
||||||
|
- :class:`platypush.message.event.github.GithubPullRequestEvent` when new pull request related activity happens.
|
||||||
|
- :class:`platypush.message.event.github.GithubPullRequestReviewCommentEvent` when activity happens on a pull
|
||||||
|
request commit.
|
||||||
|
- :class:`platypush.message.event.github.GithubReleaseEvent` when a new release happens.
|
||||||
|
- :class:`platypush.message.event.github.GithubSponsorshipEvent` when new sponsorship related activity happens.
|
||||||
|
- :class:`platypush.message.event.github.GithubWatchEvent` when someone stars/starts watching a repository.
|
||||||
|
- :class:`platypush.message.event.github.GithubEvent` for any event that doesn't fall in the above categories
|
||||||
|
(``event_type`` will be set accordingly).
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
_base_url = 'https://api.github.com'
|
||||||
|
|
||||||
|
def __init__(self, user: str, user_token: str, repos: Optional[List[str]] = None, org: Optional[str] = None,
|
||||||
|
poll_seconds: int = 60, max_events_per_scan: Optional[int] = 10, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
If neither ``repos`` nor ``org`` is specified then the backend will monitor all new events on user level.
|
||||||
|
|
||||||
|
:param user: Github username.
|
||||||
|
:param user_token: Github personal access token.
|
||||||
|
:param repos: List of repos to be monitored - if a list is provided then only these repositories will be
|
||||||
|
monitored for events. Repositories should be passed in the format ``username/repository``.
|
||||||
|
:param org: Organization to be monitored - if provided then only this organization will be monitored for events.
|
||||||
|
:param poll_seconds: How often the backend should check for new events, in seconds (default: 60).
|
||||||
|
:param max_events_per_scan: Maximum number of events per resource that will be triggered if there is a large
|
||||||
|
number of events/notification since the last check (default: 10). Specify 0 or null for no limit.
|
||||||
|
"""
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._last_text: Optional[str] = None
|
||||||
|
self.user = user
|
||||||
|
self.user_token = user_token
|
||||||
|
self.repos = repos or []
|
||||||
|
self.org = org
|
||||||
|
self.poll_seconds = poll_seconds
|
||||||
|
self.db_lock = threading.RLock()
|
||||||
|
self.workdir = os.path.join(os.path.expanduser(Config.get('workdir')), 'github')
|
||||||
|
self.dbfile = os.path.join(self.workdir, 'github.db')
|
||||||
|
self.max_events_per_scan = max_events_per_scan
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(self.dbfile), exist_ok=True)
|
||||||
|
self._init_db()
|
||||||
|
|
||||||
|
def _request(self, uri: str, method: str = 'get') -> dict:
|
||||||
|
method = getattr(requests, method.lower())
|
||||||
|
return method(self._base_url + uri, auth=(self.user, self.user_token),
|
||||||
|
headers={'Accept': 'application/vnd.github.v3+json'}).json()
|
||||||
|
|
||||||
|
def _init_db(self):
|
||||||
|
engine = create_engine('sqlite:///{}'.format(self.dbfile), connect_args={'check_same_thread': False})
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
Session.configure(bind=engine)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_datetime(time_string: str) -> datetime.datetime:
|
||||||
|
""" Convert ISO 8061 string format with leading 'Z' into something understandable by Python """
|
||||||
|
return datetime.datetime.fromisoformat(time_string[:-1] + '+00:00')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_or_create_resource(uri: str, session: Session) -> GithubResource:
|
||||||
|
record = session.query(GithubResource).filter_by(uri=uri).first()
|
||||||
|
if record is None:
|
||||||
|
record = GithubResource(uri=uri)
|
||||||
|
session.add(record)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
return record
|
||||||
|
|
||||||
|
def _get_last_event_time(self, uri: str):
|
||||||
|
with self.db_lock:
|
||||||
|
record = self._get_or_create_resource(uri=uri, session=Session())
|
||||||
|
return record.last_updated_at.replace(tzinfo=pytz.UTC) if record.last_updated_at else None
|
||||||
|
|
||||||
|
def _update_last_event_time(self, uri: str, last_updated_at: datetime.datetime):
|
||||||
|
with self.db_lock:
|
||||||
|
session = Session()
|
||||||
|
record = self._get_or_create_resource(uri=uri, session=session)
|
||||||
|
record.last_updated_at = last_updated_at
|
||||||
|
session.add(record)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _parse_event(cls, event: dict) -> GithubEvent:
|
||||||
|
event_mapping = {
|
||||||
|
'PushEvent': GithubPushEvent,
|
||||||
|
'CommitCommentEvent': GithubCommitCommentEvent,
|
||||||
|
'CreateEvent': GithubCreateEvent,
|
||||||
|
'DeleteEvent': GithubDeleteEvent,
|
||||||
|
'ForkEvent': GithubForkEvent,
|
||||||
|
'GollumEvent': GithubWikiEvent,
|
||||||
|
'IssueCommentEvent': GithubIssueCommentEvent,
|
||||||
|
'IssuesEvent': GithubIssueEvent,
|
||||||
|
'MemberEvent': GithubMemberEvent,
|
||||||
|
'PublicEvent': GithubPublicEvent,
|
||||||
|
'PullRequestEvent': GithubPullRequestEvent,
|
||||||
|
'PullRequestReviewCommentEvent': GithubPullRequestReviewCommentEvent,
|
||||||
|
'ReleaseEvent': GithubReleaseEvent,
|
||||||
|
'SponsorshipEvent': GithubSponsorshipEvent,
|
||||||
|
'WatchEvent': GithubWatchEvent,
|
||||||
|
}
|
||||||
|
|
||||||
|
event_type = event_mapping[event['type']] if event['type'] in event_mapping else GithubEvent
|
||||||
|
return event_type(event_type=event['type'], actor=event['actor'], repo=event.get('repo', {}),
|
||||||
|
payload=event['payload'], created_at=cls._to_datetime(event['created_at']))
|
||||||
|
|
||||||
|
def _events_monitor(self, uri: str, method: str = 'get'):
|
||||||
|
def thread():
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
events = self._request(uri, method)
|
||||||
|
if not events:
|
||||||
|
continue
|
||||||
|
|
||||||
|
last_event_time = self._get_last_event_time(uri)
|
||||||
|
new_last_event_time = last_event_time
|
||||||
|
fired_events = []
|
||||||
|
|
||||||
|
for event in events:
|
||||||
|
if self.max_events_per_scan and len(fired_events) >= self.max_events_per_scan:
|
||||||
|
break
|
||||||
|
|
||||||
|
event_time = self._to_datetime(event['created_at'])
|
||||||
|
if last_event_time and event_time <= last_event_time:
|
||||||
|
break
|
||||||
|
if not new_last_event_time or event_time > new_last_event_time:
|
||||||
|
new_last_event_time = event_time
|
||||||
|
|
||||||
|
fired_events.append(self._parse_event(event))
|
||||||
|
|
||||||
|
for event in fired_events:
|
||||||
|
self.bus.post(event)
|
||||||
|
|
||||||
|
self._update_last_event_time(uri=uri, last_updated_at=new_last_event_time)
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.warning('Encountered exception while fetching events from {}: {}'.format(
|
||||||
|
uri, str(e)))
|
||||||
|
self.logger.exception(e)
|
||||||
|
finally:
|
||||||
|
if self.wait_stop(timeout=self.poll_seconds):
|
||||||
|
break
|
||||||
|
|
||||||
|
return thread
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.logger.info('Starting Github backend')
|
||||||
|
monitors = []
|
||||||
|
|
||||||
|
if self.repos:
|
||||||
|
for repo in self.repos:
|
||||||
|
monitors.append(threading.Thread(target=self._events_monitor('/networks/{repo}/events'.format(repo=repo))))
|
||||||
|
if self.org:
|
||||||
|
monitors.append(threading.Thread(target=self._events_monitor('/orgs/{org}/events'.format(org=self.org))))
|
||||||
|
|
||||||
|
if not (self.repos or self.org):
|
||||||
|
monitors.append(threading.Thread(target=self._events_monitor('/users/{user}/events'.format(user=self.user))))
|
||||||
|
|
||||||
|
for monitor in monitors:
|
||||||
|
monitor.start()
|
||||||
|
|
||||||
|
self.logger.info('Started Github backend')
|
||||||
|
for monitor in monitors:
|
||||||
|
monitor.join()
|
||||||
|
|
||||||
|
self.logger.info('Github backend terminated')
|
||||||
|
|
||||||
|
# vim:sw=4:ts=4:et:
|
146
platypush/message/event/github.py
Normal file
146
platypush/message/event/github.py
Normal file
|
@ -0,0 +1,146 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Optional, Dict
|
||||||
|
|
||||||
|
from platypush.message.event import Event
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Actor:
|
||||||
|
id: str
|
||||||
|
login: str
|
||||||
|
display_login: str
|
||||||
|
url: str
|
||||||
|
gravatar_id: str
|
||||||
|
avatar_url: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Repo:
|
||||||
|
id: str
|
||||||
|
name: str
|
||||||
|
url: str
|
||||||
|
|
||||||
|
|
||||||
|
class GithubEvent(Event):
|
||||||
|
""" Generic Github event """
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
event_type: str,
|
||||||
|
created_at: datetime,
|
||||||
|
actor: Optional[Dict[str, str]] = None,
|
||||||
|
repo: Optional[Dict[str, str]] = None,
|
||||||
|
*args, **kwargs):
|
||||||
|
super().__init__(*args, actor=actor, event_type=event_type, created_at=created_at, **kwargs)
|
||||||
|
self.event_type = event_type
|
||||||
|
self.actor = Actor(**actor) if actor else None
|
||||||
|
self.repo = Repo(**repo) if repo else None
|
||||||
|
self.created_at = created_at
|
||||||
|
|
||||||
|
|
||||||
|
class GithubPushEvent(GithubEvent):
|
||||||
|
""" Github push event. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubCommitCommentEvent(GithubEvent):
|
||||||
|
""" A commit comment is created. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubCreateEvent(GithubEvent):
|
||||||
|
""" A git branch or tag is created. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubDeleteEvent(GithubEvent):
|
||||||
|
""" A git branch or tag is deleted. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubForkEvent(GithubEvent):
|
||||||
|
""" A user forks a watched repository. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubWikiEvent(GithubEvent):
|
||||||
|
""" A wiki page is created or updated on a watched repository. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubIssueCommentEvent(GithubEvent):
|
||||||
|
""" A comment is added or updated on an issue. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubIssueEvent(GithubEvent):
|
||||||
|
""" A new activity is registered on an issue. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubMemberEvent(GithubEvent):
|
||||||
|
""" New activity related to repository collaborators. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubPublicEvent(GithubEvent):
|
||||||
|
""" A private repository is made public. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubPullRequestEvent(GithubEvent):
|
||||||
|
""" New activity related to a pull request. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubPullRequestReviewCommentEvent(GithubEvent):
|
||||||
|
""" New activity related to comments of a pull request. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubReleaseEvent(GithubEvent):
|
||||||
|
""" New activity related to the release of a repository. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubSponsorshipEvent(GithubEvent):
|
||||||
|
""" New activity related to the sponsorship of a repository. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GithubWatchEvent(GithubEvent):
|
||||||
|
""" Event triggered when someone stars or starts watching a repository. """
|
||||||
|
|
||||||
|
def __init__(self, payload: dict, *args, **kwargs):
|
||||||
|
super().__init__(*args, payload=payload, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
# vim:sw=4:ts=4:et:
|
|
@ -282,3 +282,6 @@ croniter
|
||||||
|
|
||||||
# Support for Twilio
|
# Support for Twilio
|
||||||
# twilio
|
# twilio
|
||||||
|
|
||||||
|
# Support for Github
|
||||||
|
# pytz
|
||||||
|
|
7
setup.py
7
setup.py
|
@ -162,6 +162,7 @@ setup(
|
||||||
'websocket-client',
|
'websocket-client',
|
||||||
'wheel',
|
'wheel',
|
||||||
'zeroconf>=0.27.0',
|
'zeroconf>=0.27.0',
|
||||||
|
'tz',
|
||||||
],
|
],
|
||||||
|
|
||||||
extras_require={
|
extras_require={
|
||||||
|
@ -172,9 +173,9 @@ setup(
|
||||||
# Support for Pushbullet backend and plugin
|
# Support for Pushbullet backend and plugin
|
||||||
'pushbullet': ['pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'],
|
'pushbullet': ['pushbullet.py @ https://github.com/rbrcsk/pushbullet.py/tarball/master'],
|
||||||
# Support for HTTP backend
|
# Support for HTTP backend
|
||||||
'http': ['flask', 'python-dateutil', 'tz', 'frozendict', 'bcrypt'],
|
'http': ['flask', 'python-dateutil', 'frozendict', 'bcrypt'],
|
||||||
# Support for uWSGI HTTP backend
|
# Support for uWSGI HTTP backend
|
||||||
'uwsgi': ['flask', 'python-dateutil', 'tz', 'frozendict', 'uwsgi', 'bcrypt'],
|
'uwsgi': ['flask', 'python-dateutil', 'frozendict', 'uwsgi', 'bcrypt'],
|
||||||
# Support for MQTT backends
|
# Support for MQTT backends
|
||||||
'mqtt': ['paho-mqtt'],
|
'mqtt': ['paho-mqtt'],
|
||||||
# Support for RSS feeds parser
|
# Support for RSS feeds parser
|
||||||
|
@ -323,5 +324,7 @@ setup(
|
||||||
'dbus': ['dbus-python'],
|
'dbus': ['dbus-python'],
|
||||||
# Support for Twilio integration
|
# Support for Twilio integration
|
||||||
'twilio': ['twilio'],
|
'twilio': ['twilio'],
|
||||||
|
# Support for Github integration
|
||||||
|
'github': ['pytz'],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue