#73: Implemented wrapper plugin for the new Node.js mercury-parser.

As the Mercury reader web API is deprecated, and the only available
implementation is the open source mercury-parser, node, npm and
@postlight/mercury-parser have to be added as dependencies for the
http.webpage plugin (or at least for the `simplify` action).

platypush/backend/http/request/rss/__init__.py

@@ -2,11 +2,9 @@ import datetime
 import enum
 import feedparser
 import os
-import requests
-import time
 
 from sqlalchemy import create_engine, Column, Integer, String, DateTime, \
-    Enum, UniqueConstraint, ForeignKey
+    Enum, ForeignKey
 
 from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
@@ -14,6 +12,7 @@ from sqlalchemy.sql.expression import func
 
 from platypush.backend.http.request import HttpRequest
 from platypush.config import Config
+from platypush.context import get_plugin
 from platypush.message.event.http.rss import NewFeedEvent
 
 Base = declarative_base()
@@ -23,21 +22,26 @@ Session = scoped_session(sessionmaker())
 class RssUpdates(HttpRequest):
     """ Gets new items in an RSS feed """
 
-    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
+    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' + \
+                 'Chrome/62.0.3202.94 Safari/537.36'
 
     def __init__(self, url, title=None, headers=None, params=None, max_entries=None,
-                 mercury_api_key=None, digest_format=None, *argv, **kwargs):
+                 extract_content=None, digest_format=None, *argv, **kwargs):
         self.workdir = os.path.join(os.path.expanduser(Config.get('workdir')), 'feeds')
         self.dbfile = os.path.join(self.workdir, 'rss.db')
         self.url = url
         self.title = title
         self.max_entries = max_entries
-        self.mercury_api_key = mercury_api_key  # Mercury Reader API used to parse the content of the link
+
+        # If true, then the http.webpage plugin will be used to parse the content
+        self.extract_content = extract_content
+
         self.digest_format = digest_format.lower() if digest_format else None  # Supported formats: html, pdf
 
         os.makedirs(os.path.expanduser(os.path.dirname(self.dbfile)), exist_ok=True)
 
-        if headers is None: headers = {}
+        if headers is None:
+            headers = {}
         headers['User-Agent'] = self.user_agent
 
         request_args = {
@@ -58,55 +62,24 @@ class RssUpdates(HttpRequest):
         session.commit()
         return record
 
-
+    @staticmethod
-    def _get_latest_update(self, session, source_id):
+    def _get_latest_update(session, source_id):
         return session.query(func.max(FeedEntry.published)).filter_by(source_id=source_id).scalar()
 
-
     def _parse_entry_content(self, link):
-        response = None
+        parser = get_plugin('http.webpage')
-        err = None
+        response = parser.simplify(link).output
-        n_tries = 5
+        errors = parser.simplify(link).errors
 
-        for _ in range(0, n_tries):
+        if not response:
-            try:
+            self.logger.warning('Mercury parser error: '.format(errors or '[unknown error]'))
-                self.logger.info('Parsing content for {}'.format(link))
-                response = requests.get('https://mercury.postlight.com/parser',
-                                        params = {'url': link},
-                                        headers = {'x-api-key': self.mercury_api_key })
-            except Exception as e:
-                err = e
-
-            if response.text:
-                err = None
-                break
-            else:
-                time.sleep(1)
-
-        if err:
-            raise err
-
-        if not response.text:
-            self.logger.warning('No response from Mercury API for URL {} after {} tries'.format(link, n_tries))
-            return
-
-        if not response.ok:
-            self.logger.warning('Mercury API call failed with status {}'.format(response.status_code))
-            return
-
-        response = response.json()
-        error = response.get('error')
-
-        if error:
-            self.logger.warning('Mercury API error: {}'.format(error))
             return
 
         return response.get('content')
 
-
     def get_new_items(self, response):
         engine = create_engine('sqlite:///{}'.format(self.dbfile),
-                               connect_args = { 'check_same_thread': False })
+                               connect_args={'check_same_thread': False})
 
         Base.metadata.create_all(engine)
         Session.configure(bind=engine)
@@ -128,11 +101,10 @@ class RssUpdates(HttpRequest):
             <h1 style="margin-top: 30px">{}</h1>
             <h2 style="margin-top: 10px; page-break-after: always">
                 Feeds digest generated on {} </h2>'''.format(self.title,
-                datetime.datetime.now().strftime('%d %B %Y, %H:%M')
+                                                             datetime.datetime.now().strftime('%d %B %Y, %H:%M'))
-            )
 
         self.logger.info('Parsed {:d} items from RSS feed <{}>'
-                     .format(len(feed.entries), self.url))
+                         .format(len(feed.entries), self.url))
 
         for entry in feed.entries:
             if not entry.published_parsed:
@@ -146,7 +118,7 @@ class RssUpdates(HttpRequest):
                     self.logger.info('Processed new item from RSS feed <{}>'.format(self.url))
                     entry.summary = entry.summary if hasattr(entry, 'summary') else None
 
-                    if self.mercury_api_key:
+                    if self.extract_content:
                         entry.content = self._parse_entry_content(entry.link)
                     elif hasattr(entry, 'summary'):
                         entry.content = entry.summary
@@ -168,7 +140,8 @@ class RssUpdates(HttpRequest):
 
                     entries.append(e)
                     session.add(FeedEntry(**e))
-                    if self.max_entries and len(entries) > self.max_entries: break
+                    if self.max_entries and len(entries) > self.max_entries:
+                        break
             except Exception as e:
                 self.logger.warning('Exception encountered while parsing RSS ' +
                                     'RSS feed {}: {}'.format(self.url, str(e)))
@@ -196,11 +169,11 @@ class RssUpdates(HttpRequest):
                     weasyprint.HTML(string=digest).write_pdf(digest_filename)
                 else:
                     raise RuntimeError('Unsupported format: {}. Supported formats: ' +
-                                    'html or pdf'.format(self.digest_format))
+                                       'html or pdf'.format(self.digest_format))
 
                 digest_entry = FeedDigest(source_id=source_record.id,
-                                        format=self.digest_format,
+                                          format=self.digest_format,
-                                        filename=digest_filename)
+                                          filename=digest_filename)
 
                 session.add(digest_entry)
                 self.logger.info('{} digest ready: {}'.format(self.digest_format, digest_filename))
@@ -218,7 +191,7 @@ class FeedSource(Base):
     """ Models the FeedSource table, containing RSS sources to be parsed """
 
     __tablename__ = 'FeedSource'
-    __table_args__ = ({ 'sqlite_autoincrement': True })
+    __table_args__ = ({'sqlite_autoincrement': True})
 
     id = Column(Integer, primary_key=True)
     title = Column(String)
@@ -230,7 +203,7 @@ class FeedEntry(Base):
     """ Models the FeedEntry table, which contains RSS entries """
 
     __tablename__ = 'FeedEntry'
-    __table_args__ = ({ 'sqlite_autoincrement': True })
+    __table_args__ = ({'sqlite_autoincrement': True})
 
     id = Column(Integer, primary_key=True)
     entry_id = Column(String)
@@ -251,7 +224,7 @@ class FeedDigest(Base):
         pdf = 2
 
     __tablename__ = 'FeedDigest'
-    __table_args__ = ({ 'sqlite_autoincrement': True })
+    __table_args__ = ({'sqlite_autoincrement': True})
 
     id = Column(Integer, primary_key=True)
     source_id = Column(Integer, ForeignKey('FeedSource.id'), nullable=False)
@@ -259,6 +232,4 @@ class FeedDigest(Base):
     filename = Column(String, nullable=False)
     created_at = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
 
-
 # vim:sw=4:ts=4:et:
-

platypush/message/event/http/__init__.py

@@ -1,5 +1,6 @@
 from platypush.message.event import Event
 
+
 class HttpEvent(Event):
     """
     Event triggered upon HTTP request/response cycle completion
@@ -11,7 +12,7 @@ class HttpEvent(Event):
         :type request: dict
 
         :param response: The server response
-        :type response: dict
+        :type response: dict or list
         """
 
         super().__init__(request=request, response=response, *args, **kwargs)

platypush/message/event/http/rss.py

@@ -1,11 +1,12 @@
 from platypush.message.event.http import HttpEvent
 
+
 class NewFeedEvent(HttpEvent):
     """
     Event triggered when a monitored RSS feed has some new content
     """
 
-    def __init__(self, request, response, source_id=None, title=None,
+    def __init__(self, request, response: list, source_id=None, title=None,
                  digest_format=None, digest_filename=None, *args, **kwargs):
         """
         :param request: Original request
@@ -22,4 +23,3 @@ class NewFeedEvent(HttpEvent):
 
 
 # vim:sw=4:ts=4:et:
-

platypush/plugins/http/webpage/__init__.py

@@ -1,6 +1,7 @@
+import datetime
+import json
 import os
-import requests
+import subprocess
-import time
 
 from platypush.plugins import action
 from platypush.plugins.http.request import Plugin
@@ -8,22 +9,19 @@ from platypush.plugins.http.request import Plugin
 
 class HttpWebpagePlugin(Plugin):
     """
-    Plugin to handle and parse/simplify web pages
+    Plugin to handle and parse/simplify web pages.
+    It used to use the Mercury Reader web API, but now that the API is discontinued this plugin is basically a
+    wrapper around the `mercury-parser <https://github.com/postlight/mercury-parser>`_ JavaScript library.
 
     Requires:
 
         * **requests** (``pip install requests``)
         * **weasyprint** (``pip install weasyprint``), optional, for HTML->PDF conversion
+        * **node** and **npm** installed on your system (to use the mercury-parser interface)
+        * The mercury-parser library installed (``npm install @postlight/mercury-parser``)
     """
 
-    def __init__(self, mercury_api_key=None, **kwargs):
+    _mercury_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js')
-        """
-        :param mercury_api_key: If set then Mercury will be used to parse web pages content
-        :type mercury_api_key: str
-        """
-
-        super().__init__(**kwargs)
-        self.mercury_api_key = mercury_api_key
 
     @action
     def simplify(self, url, outfile=None):
@@ -56,24 +54,17 @@ class HttpWebpagePlugin(Plugin):
 
         """
 
-        if not self.mercury_api_key:
-            raise RuntimeError("mercury_api_key not set")
-
         self.logger.info('Parsing URL {}'.format(url))
-        response = requests.get('https://mercury.postlight.com/parser',
+        parser = subprocess.Popen(['node', self._mercury_script, url], stdout=subprocess.PIPE)
-                                params={'url': url},
+        response = json.loads(parser.stdout.read().decode())
-                                headers={'x-api-key': self.mercury_api_key})
 
-        if not response or not response.ok:
+        self.logger.info('Got response from Mercury API: {}'.format(response))
-            raise RuntimeError("Unable to parse content for {}: {}".format(url, response.reason))
+        title = response.get('title', '{} on {}'.format(
+            'Published' if response.get('date_published') else 'Generated',
+            response.get('date_published', datetime.datetime.now().isoformat())))
 
-        if not len(response.text):
-            raise RuntimeError("Empty response from Mercury API for URL {}".format(url))
-
-        self.logger.info('Got response from Mercury API: {}'.format(response.json()))
-        title = response.json().get('title', 'No_title_{}'.format(int(time.time())))
         content = '<body style="{body_style}"><h1>{title}</h1>{content}</body>'.\
-            format(title=title, content=response.json()['content'],
+            format(title=title, content=response.get('content', '[No content available]'),
                    body_style='font-size: 22px; font-family: Tahoma, Geneva, sans-serif')
 
         if not outfile:
@@ -85,7 +76,7 @@ class HttpWebpagePlugin(Plugin):
 
         outfile = os.path.abspath(os.path.expanduser(outfile))
 
-        if outfile.endswith('.pdf'):
+        if outfile.lower().endswith('.pdf'):
             import weasyprint
             weasyprint.HTML(string=content).write_pdf(outfile)
         else:

platypush/plugins/http/webpage/mercury-parser.js

@@ -0,0 +1,20 @@
+#!node
+
+// This script will parse the content and title of a webpage using the
+// mercury-parser JavaScript library (https://github.com/postlight/mercury-parser)
+// and print a JSON object with the extracted information.
+
+'use strict';
+
+const parser = require('@postlight/mercury-parser');
+
+if (process.argv.length < 3) {
+    console.error('Usage: ' + process.argv[1] + ' <url to parse>');
+    process.exit(1);
+}
+
+const url = process.argv[2];
+parser.parse(url).then(result => {
+    console.log(JSON.stringify(result));
+});
+