diff --git a/platypush/backend/http/request/rss/__init__.py b/platypush/backend/http/request/rss/__init__.py index 515f32a7d6..412e289287 100644 --- a/platypush/backend/http/request/rss/__init__.py +++ b/platypush/backend/http/request/rss/__init__.py @@ -1,4 +1,5 @@ import datetime +import enum import feedparser import logging import os @@ -6,7 +7,7 @@ import requests import time from sqlalchemy import create_engine, Column, Integer, String, DateTime, \ - UniqueConstraint, ForeignKey + Enum, UniqueConstraint, ForeignKey from sqlalchemy.orm import sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base @@ -19,18 +20,21 @@ Base = declarative_base() Session = scoped_session(sessionmaker()) -class GetRssUpdates(HttpRequest): +class RssUpdates(HttpRequest): """ Gets new items in an RSS feed """ - dbfile = os.path.join(os.path.expanduser(Config.get('workdir')), 'feeds', 'rss.db') - - def __init__(self, url, headers=None, params=None, dbfile=None, - mercury_api_key=None, *args, **kwargs): + workdir = os.path.join(os.path.expanduser(Config.get('workdir')), 'feeds') + dbfile = os.path.join(workdir, 'rss.db') + user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36' + def __init__(self, url, title=None, headers=None, params=None, max_entries=None, + mercury_api_key=None, digest_format='html', *args, **kwargs): self.url = url + self.title = title + self.max_entries = max_entries self.mercury_api_key = mercury_api_key # Mercury Reader API used to parse the content of the link + self.digest_format = digest_format.lower() if digest_format else 'html' - if dbfile: self.dbfile = dbfile os.makedirs(os.path.expanduser(os.path.dirname(self.dbfile)), exist_ok=True) self.engine = create_engine('sqlite:///{}'.format(self.dbfile)) @@ -38,10 +42,13 @@ class GetRssUpdates(HttpRequest): Session.configure(bind=self.engine) self._get_or_create_source(session=Session()) + if headers is None: headers = {} + headers['User-Agent'] = self.user_agent + request_args = { 'method': 'get', 'url': self.url, - 'headers': headers or {}, + 'headers': headers, 'params': params or {}, } @@ -79,22 +86,36 @@ class GetRssUpdates(HttpRequest): parse_start_time = datetime.datetime.utcnow() entries = [] - if source_record.title != feed.feed['title']: - source_record.title = feed.feed['title'] + if not self.title and 'title' in feed.feed: + self.title = feed.feed['title'] + source_record.title = self.title + + digest = u''' +

{}

+

+ Feeds digest generated on {}

'''.format(self.title, + datetime.datetime.now().strftime('%d %B %Y, %H:%M') + ) for entry in feed.entries: entry_timestamp = datetime.datetime(*entry.published_parsed[:6]) if source_record.last_updated_at is None \ or entry_timestamp > source_record.last_updated_at: - entry.content = self._parse_entry_content(entry.link) \ - if self.mercury_api_key else None + if self.mercury_api_key: + entry.content = self._parse_entry_content(entry.link) + elif hasattr(entry, 'summary'): + entry.content = entry.summary + else: + entry.content = None + + digest += '

{}

{}' \ + .format(entry.title, entry.content) e = { 'entry_id': entry.id, 'title': entry.title, 'link': entry.link, - 'summary': entry.summary, 'content': entry.content, 'source_id': source_record.id, 'published': entry_timestamp, @@ -102,18 +123,48 @@ class GetRssUpdates(HttpRequest): entries.append(e) session.add(FeedEntry(**e)) + if self.max_entries and len(entries) > self.max_entries: break source_record.last_updated_at = parse_start_time - session.commit() + digest_filename = None if entries: logging.info('Parsed {} new entries from the RSS feed {}'.format( - len(entries), source_record.title)) + len(entries), self.title)) - return NewFeedEvent(dict(self), entries) + digest_filename = os.path.join(self.workdir, 'cache', '{}_{}.{}'.format( + datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), + self.title, self.digest_format)) + + os.makedirs(os.path.dirname(digest_filename), exist_ok=True) + + if self.digest_format == 'html': + with open(digest_filename, 'w', encoding='utf-8') as f: + f.write(digest) + elif self.digest_format == 'pdf': + import weasyprint + weasyprint.HTML(string=digest).write_pdf(digest_filename) + else: + raise RuntimeError('Unsupported format: {}. Supported formats: ' + + 'html or pdf'.format(self.digest_format)) + + digest_entry = FeedDigest(source_id=source_record.id, + format=self.digest_format, + filename=digest_filename) + + session.add(digest_entry) + + session.commit() + + return NewFeedEvent(request=dict(self), response=entries, + source_id=source_record.id, title=self.title, + digest_format=self.digest_format, + digest_filename=digest_filename) class FeedSource(Base): + """ Models the FeedSource table, containing RSS sources to be parsed """ + __tablename__ = 'FeedSource' __table_args__ = ({ 'sqlite_autoincrement': True }) @@ -124,6 +175,8 @@ class FeedSource(Base): class FeedEntry(Base): + """ Models the FeedEntry table, which contains RSS entries """ + __tablename__ = 'FeedEntry' __table_args__ = ({ 'sqlite_autoincrement': True }) @@ -132,10 +185,27 @@ class FeedEntry(Base): source_id = Column(Integer, ForeignKey('FeedSource.id'), nullable=False) title = Column(String) link = Column(String) - summary = Column(String) content = Column(String) published = Column(DateTime) +class FeedDigest(Base): + """ Models the FeedDigest table, containing feed digests either in HTML + or PDF format """ + + class DigestFormat(enum.Enum): + html = 1 + pdf = 2 + + __tablename__ = 'FeedDigest' + __table_args__ = ({ 'sqlite_autoincrement': True }) + + id = Column(Integer, primary_key=True) + source_id = Column(Integer, ForeignKey('FeedSource.id'), nullable=False) + format = Column(Enum(DigestFormat), nullable=False) + filename = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) + + # vim:sw=4:ts=4:et: diff --git a/platypush/message/event/http/rss.py b/platypush/message/event/http/rss.py index ebb5549647..93d80a1bd7 100644 --- a/platypush/message/event/http/rss.py +++ b/platypush/message/event/http/rss.py @@ -1,8 +1,12 @@ from platypush.message.event.http import HttpEvent class NewFeedEvent(HttpEvent): - def __init__(self, request, response, *args, **kwargs): - super().__init__(request=request, response=response, *args, **kwargs) + def __init__(self, request, response, source_id=None, title=None, + digest_format=None, digest_filename=None, *args, **kwargs): + + super().__init__(request=request, response=response, source_id=source_id, + digest_format=digest_format, title=title, + digest_filename=digest_filename, *args, **kwargs) # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/google/mail.py b/platypush/plugins/google/mail.py index 1240912ad2..59489a64a6 100644 --- a/platypush/plugins/google/mail.py +++ b/platypush/plugins/google/mail.py @@ -1,6 +1,7 @@ import base64 import httplib2 import mimetypes +import os from apiclient import discovery diff --git a/requirements.txt b/requirements.txt index e4b08745e6..5c4bfadd8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,9 @@ python-dateutil # RSS feeds support feedparser +# PDF generation support +weasyprint + # Philips Hue plugin support phue diff --git a/setup.py b/setup.py index b1a02c5176..367f2b28e1 100755 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ setup( 'Support for HTTP poll backend': ['frozendict'], 'Support for database plugin': ['sqlalchemy'], 'Support for RSS feeds': ['feedparser'], + 'Support for PDF generation': ['weasyprint'], 'Support for Philips Hue plugin': ['phue'], 'Support for MPD/Mopidy music server plugin': ['python-mpd2'], 'Support for Belkin WeMo Switch plugin': ['ouimeaux'],