From d7405ad05d2eacd555d46700f1194330a7722b9d Mon Sep 17 00:00:00 2001
From: Fabio Manganiello <fabio@manganiello.tech>
Date: Mon, 22 May 2023 02:20:58 +0200
Subject: [PATCH] Added multiple parsers for the entities referenced in
 docstrings.

The `inspect` plugin can now detect references to plugins, backends,
events, responses and schemas in docstrings and replace them either with
links to the documentation or auto-generated examples.
---
 .../http/webapp/src/components/Nav.vue        |   2 +
 .../src/components/panels/Execute/Index.vue   |  35 +++-
 .../http/webapp/src/style/components.scss     |   7 +-
 platypush/plugins/inspect/_model.py           | 158 ++++++++++--------
 .../plugins/inspect/_parsers/__init__.py      |  16 ++
 .../plugins/inspect/_parsers/_backend.py      |  34 ++++
 platypush/plugins/inspect/_parsers/_base.py   |  12 ++
 platypush/plugins/inspect/_parsers/_event.py  |  34 ++++
 platypush/plugins/inspect/_parsers/_method.py |  35 ++++
 platypush/plugins/inspect/_parsers/_plugin.py |  34 ++++
 .../plugins/inspect/_parsers/_response.py     |  34 ++++
 platypush/plugins/inspect/_parsers/_schema.py |  97 +++++++++++
 12 files changed, 428 insertions(+), 70 deletions(-)
 create mode 100644 platypush/plugins/inspect/_parsers/__init__.py
 create mode 100644 platypush/plugins/inspect/_parsers/_backend.py
 create mode 100644 platypush/plugins/inspect/_parsers/_base.py
 create mode 100644 platypush/plugins/inspect/_parsers/_event.py
 create mode 100644 platypush/plugins/inspect/_parsers/_method.py
 create mode 100644 platypush/plugins/inspect/_parsers/_plugin.py
 create mode 100644 platypush/plugins/inspect/_parsers/_response.py
 create mode 100644 platypush/plugins/inspect/_parsers/_schema.py

diff --git a/platypush/backend/http/webapp/src/components/Nav.vue b/platypush/backend/http/webapp/src/components/Nav.vue
index 51deb02abc..4f2fccad54 100644
--- a/platypush/backend/http/webapp/src/components/Nav.vue
+++ b/platypush/backend/http/webapp/src/components/Nav.vue
@@ -156,6 +156,8 @@ nav {
       display: block;
       color: $nav-fg;
       padding: 1em 0.5em;
+      text-decoration: none;
+
       &:hover {
         color: $nav-fg;
       }
diff --git a/platypush/backend/http/webapp/src/components/panels/Execute/Index.vue b/platypush/backend/http/webapp/src/components/panels/Execute/Index.vue
index 93fc79c915..9322d85836 100644
--- a/platypush/backend/http/webapp/src/components/panels/Execute/Index.vue
+++ b/platypush/backend/http/webapp/src/components/panels/Execute/Index.vue
@@ -1,5 +1,5 @@
 <template>
-  <div class="row plugin execute-container">
+  <div class="row plugin execute-container" @click="onClick">
     <Loading v-if="loading" />
     <div class="section command-container">
       <div class="section-title">Execute Action</div>
@@ -32,7 +32,7 @@
 
           <div class="doc-container" v-if="selectedDoc">
             <div class="title">
-              Action documentation
+              <a :href="currentActionDocURL">Action documentation</a>
             </div>
 
             <div class="doc html">
@@ -217,6 +217,24 @@ export default {
     }
   },
 
+  computed: {
+    currentActionDocURL() {
+      if (!this.action?.name?.length)
+        return undefined
+
+      const plugin = this.action.name.split('.').slice(0, -1).join('.')
+      const actionName = this.action.name.split('.').slice(-1)
+      const actionClass = this.action.name
+        .split('.')
+        .slice(0, -1)
+        .map((token) => token.slice(0, 1).toUpperCase() + token.slice(1))
+        .join('') + 'Plugin'
+
+      return 'https://docs.platypush.tech/platypush/plugins/' +
+        `${plugin}.html#platypush.plugins.${plugin}.${actionClass}.${actionName}`
+    },
+  },
+
   methods: {
     async refresh() {
       this.loading = true
@@ -457,6 +475,15 @@ export default {
       this.request('procedure.' + this.selectedProcedure.name, args)
           .then(this.onResponse).catch(this.onError).finally(this.onDone)
     },
+
+    onClick(event) {
+      // Intercept any clicks from RST rendered links and open them in a new tab
+      if (event.target.tagName.toLowerCase() === 'a') {
+        event.stopPropagation()
+        event.preventDefault()
+        window.open(event.target.getAttribute('href', '_blank'))
+      }
+    },
   },
 
   mounted() {
@@ -857,6 +884,10 @@ $request-headers-btn-width: 7.5em;
         background: $procedure-submit-btn-bg;
       }
     }
+
+    .action-param-value {
+      margin: 0.25em 0;
+    }
   }
 
   pre {
diff --git a/platypush/backend/http/webapp/src/style/components.scss b/platypush/backend/http/webapp/src/style/components.scss
index d261f30df3..0ca65f3b86 100644
--- a/platypush/backend/http/webapp/src/style/components.scss
+++ b/platypush/backend/http/webapp/src/style/components.scss
@@ -42,8 +42,13 @@ ul {
 }
 
 a {
+  color: $default-link-fg;
+  text-decoration: underline dotted;
   cursor: pointer;
-  text-decoration: none;
+
+  &:hover {
+    color: $default-hover-fg;
+  }
 }
 
 ::-webkit-scrollbar {
diff --git a/platypush/plugins/inspect/_model.py b/platypush/plugins/inspect/_model.py
index be73783fcd..31bd55bd48 100644
--- a/platypush/plugins/inspect/_model.py
+++ b/platypush/plugins/inspect/_model.py
@@ -1,7 +1,7 @@
 import inspect
 import json
 import re
-from typing import Optional, Type
+from typing import Callable, Optional, Type
 
 from platypush.backend import Backend
 from platypush.message.event import Event
@@ -9,12 +9,30 @@ from platypush.message.response import Response
 from platypush.plugins import Plugin
 from platypush.utils import get_decorators
 
+from ._parsers import (
+    BackendParser,
+    EventParser,
+    MethodParser,
+    PluginParser,
+    ResponseParser,
+    SchemaParser,
+)
+
 
 class Model:
     """
     Base class for component models.
     """
 
+    _parsers = [
+        BackendParser,
+        EventParser,
+        MethodParser,
+        PluginParser,
+        ResponseParser,
+        SchemaParser,
+    ]
+
     def __init__(
         self,
         obj_type: type,
@@ -32,8 +50,22 @@ class Model:
         self._obj_type = obj_type
         self.package = obj_type.__module__[len(prefix) :]
         self.name = name or self.package
-        self.doc = doc or obj_type.__doc__
         self.last_modified = last_modified
+        self.doc, argsdoc = self._parse_docstring(doc or obj_type.__doc__ or '')
+        self.args = {}
+        self.has_kwargs = False
+
+        for arg in list(inspect.signature(obj_type).parameters.values())[1:]:
+            if arg.kind == arg.VAR_KEYWORD:
+                self.has_kwargs = True
+                continue
+
+            self.args[arg.name] = {
+                'default': arg.default
+                if not issubclass(arg.default.__class__, type)
+                else None,
+                'doc': argsdoc.get(arg.name),
+            }
 
     def __str__(self):
         """
@@ -51,9 +83,62 @@ class Model:
         """
         Iterator for the model public attributes/values pairs.
         """
-        for attr in ['name', 'doc']:
+        for attr in ['name', 'args', 'doc', 'has_kwargs']:
             yield attr, getattr(self, attr)
 
+    @classmethod
+    def _parse_docstring(cls, docstring: str):
+        new_docstring = ''
+        params = {}
+        cur_param = None
+        cur_param_docstring = ''
+
+        if not docstring:
+            return None, {}
+
+        for line in docstring.split('\n'):
+            m = re.match(r'^\s*:param ([^:]+):\s*(.*)', line)
+            if m:
+                if cur_param:
+                    params[cur_param] = cur_param_docstring
+
+                cur_param = m.group(1)
+                cur_param_docstring = m.group(2)
+                continue
+
+            m = re.match(r'^\s*:return:\s+(.*)', line)
+            if m:
+                if cur_param:
+                    params[cur_param] = cur_param_docstring
+
+                new_docstring += '\n\n**Returns:**\n\n' + m.group(1).strip() + ' '
+                cur_param = None
+                continue
+
+            if cur_param:
+                if not line.strip():
+                    params[cur_param] = cur_param_docstring
+                    cur_param = None
+                    cur_param_docstring = ''
+                else:
+                    cur_param_docstring += '\n' + line.strip() + ' '
+            else:
+                new_docstring += line + '\n'
+
+        if cur_param:
+            params[cur_param] = cur_param_docstring
+
+        for param, doc in params.items():
+            params[param] = cls._post_process_docstring(doc)
+
+        return cls._post_process_docstring(new_docstring), params
+
+    @classmethod
+    def _post_process_docstring(cls, docstring: str) -> str:
+        for parsers in cls._parsers:
+            docstring = parsers.parse(docstring)
+        return docstring.strip()
+
 
 # pylint: disable=too-few-public-methods
 class BackendModel(Model):
@@ -90,7 +175,7 @@ class PluginModel(Model):
         Overrides the default implementation of ``__iter__`` to also include
         plugin actions.
         """
-        for attr in ['name', 'actions', 'doc']:
+        for attr in ['name', 'args', 'actions', 'doc', 'has_kwargs']:
             if attr == 'actions':
                 yield attr, {
                     name: dict(action) for name, action in self.actions.items()
@@ -122,66 +207,5 @@ class ActionModel(Model):
     Model for plugin action components.
     """
 
-    def __init__(self, action, **kwargs):
-        doc, argsdoc = self._parse_docstring(action.__doc__)
-        super().__init__(action, name=action.__name__, doc=doc, **kwargs)
-
-        self.args = {}
-        self.has_kwargs = False
-
-        for arg in list(inspect.signature(action).parameters.values())[1:]:
-            if arg.kind == arg.VAR_KEYWORD:
-                self.has_kwargs = True
-                continue
-
-            self.args[arg.name] = {
-                'default': arg.default
-                if not issubclass(arg.default.__class__, type)
-                else None,
-                'doc': argsdoc.get(arg.name),
-            }
-
-    @classmethod
-    def _parse_docstring(cls, docstring: str):
-        new_docstring = ''
-        params = {}
-        cur_param = None
-        cur_param_docstring = ''
-
-        if not docstring:
-            return None, {}
-
-        for line in docstring.split('\n'):
-            if m := re.match(r'^\s*:param ([^:]+):\s*(.*)', line):
-                if cur_param:
-                    params[cur_param] = cur_param_docstring
-
-                cur_param = m.group(1)
-                cur_param_docstring = m.group(2)
-            elif m := re.match(r'^\s*:return:\s+(.*)', line):
-                if cur_param:
-                    params[cur_param] = cur_param_docstring
-
-                new_docstring += '\n\n**Returns:**\n\n' + m.group(1).strip() + ' '
-                cur_param = None
-            elif re.match(r'^\s*:[^:]+:\s*.*', line):
-                continue
-            else:
-                if cur_param:
-                    if not line.strip():
-                        params[cur_param] = cur_param_docstring
-                        cur_param = None
-                        cur_param_docstring = ''
-                    else:
-                        cur_param_docstring += '\n' + line.strip() + ' '
-                else:
-                    new_docstring += line.strip() + ' '
-
-        if cur_param:
-            params[cur_param] = cur_param_docstring
-
-        return new_docstring.strip(), params
-
-    def __iter__(self):
-        for attr in ['name', 'args', 'doc', 'has_kwargs']:
-            yield attr, getattr(self, attr)
+    def __init__(self, obj_type: Type[Callable], *args, **kwargs):
+        super().__init__(obj_type, name=obj_type.__name__, *args, **kwargs)
diff --git a/platypush/plugins/inspect/_parsers/__init__.py b/platypush/plugins/inspect/_parsers/__init__.py
new file mode 100644
index 0000000000..9c74b8bcce
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/__init__.py
@@ -0,0 +1,16 @@
+from ._backend import BackendParser
+from ._event import EventParser
+from ._method import MethodParser
+from ._plugin import PluginParser
+from ._response import ResponseParser
+from ._schema import SchemaParser
+
+
+__all__ = [
+    'BackendParser',
+    'EventParser',
+    'MethodParser',
+    'PluginParser',
+    'ResponseParser',
+    'SchemaParser',
+]
diff --git a/platypush/plugins/inspect/_parsers/_backend.py b/platypush/plugins/inspect/_parsers/_backend.py
new file mode 100644
index 0000000000..9d56aabdbd
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_backend.py
@@ -0,0 +1,34 @@
+import re
+from typing_extensions import override
+
+from ._base import Parser
+
+
+class BackendParser(Parser):
+    """
+    Parse backend references in the docstrings with rendered links to their
+    respective documentation.
+    """
+
+    _backend_regex = re.compile(
+        r'(\s*):class:`(platypush\.backend\.(.+?))`', re.MULTILINE
+    )
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._backend_regex.search(docstring)
+            if not m:
+                break
+
+            class_name = m.group(3).split('.')[-1]
+            package = '.'.join(m.group(3).split('.')[:-1])
+            docstring = cls._backend_regex.sub(
+                f'{m.group(1)}`{class_name} '
+                f'<https://docs.platypush.tech/platypush/backend/{package}.html#{m.group(2)}>`_',
+                docstring,
+                count=1,
+            )
+
+        return docstring
diff --git a/platypush/plugins/inspect/_parsers/_base.py b/platypush/plugins/inspect/_parsers/_base.py
new file mode 100644
index 0000000000..f1616ad04e
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_base.py
@@ -0,0 +1,12 @@
+from abc import ABC, abstractmethod
+
+
+class Parser(ABC):
+    """
+    Base class for parsers.
+    """
+
+    @classmethod
+    @abstractmethod
+    def parse(cls, docstring: str) -> str:
+        raise NotImplementedError()
diff --git a/platypush/plugins/inspect/_parsers/_event.py b/platypush/plugins/inspect/_parsers/_event.py
new file mode 100644
index 0000000000..5f21ab2261
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_event.py
@@ -0,0 +1,34 @@
+import re
+from typing_extensions import override
+
+from ._base import Parser
+
+
+class EventParser(Parser):
+    """
+    Parse event references in the docstrings with rendered links to their
+    respective documentation.
+    """
+
+    _event_regex = re.compile(
+        r'(\s*):class:`(platypush\.message\.event\.(.+?))`', re.MULTILINE
+    )
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._event_regex.search(docstring)
+            if not m:
+                break
+
+            class_name = m.group(3).split('.')[-1]
+            package = '.'.join(m.group(3).split('.')[:-1])
+            docstring = cls._event_regex.sub(
+                f'{m.group(1)}`{class_name} '
+                f'<https://docs.platypush.tech/platypush/events/{package}.html#{m.group(2)}>`_',
+                docstring,
+                count=1,
+            )
+
+        return docstring
diff --git a/platypush/plugins/inspect/_parsers/_method.py b/platypush/plugins/inspect/_parsers/_method.py
new file mode 100644
index 0000000000..686c87b191
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_method.py
@@ -0,0 +1,35 @@
+import re
+from typing_extensions import override
+
+from ._base import Parser
+
+
+class MethodParser(Parser):
+    """
+    Parse method references in the docstrings with rendered links to their
+    respective documentation.
+    """
+
+    _method_regex = re.compile(
+        r'(\s*):meth:`(platypush\.plugins\.(.+?))`', re.MULTILINE
+    )
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._method_regex.search(docstring)
+            if not m:
+                break
+
+            tokens = m.group(3).split('.')
+            method = tokens[-1]
+            package = '.'.join(tokens[:-2])
+            docstring = cls._method_regex.sub(
+                f'{m.group(1)}`{package}.{method} '
+                f'<https://docs.platypush.tech/platypush/plugins/{package}.html#{m.group(2)}>`_',
+                docstring,
+                count=1,
+            )
+
+        return docstring
diff --git a/platypush/plugins/inspect/_parsers/_plugin.py b/platypush/plugins/inspect/_parsers/_plugin.py
new file mode 100644
index 0000000000..aa40380922
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_plugin.py
@@ -0,0 +1,34 @@
+import re
+from typing_extensions import override
+
+from ._base import Parser
+
+
+class PluginParser(Parser):
+    """
+    Parse plugin references in the docstrings with rendered links to their
+    respective documentation.
+    """
+
+    _plugin_regex = re.compile(
+        r'(\s*):class:`(platypush\.plugins\.(.+?))`', re.MULTILINE
+    )
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._plugin_regex.search(docstring)
+            if not m:
+                break
+
+            class_name = m.group(3).split('.')[-1]
+            package = '.'.join(m.group(3).split('.')[:-1])
+            docstring = cls._plugin_regex.sub(
+                f'{m.group(1)}`{class_name} '
+                f'<https://docs.platypush.tech/platypush/plugins/{package}.html#{m.group(2)}>`_',
+                docstring,
+                count=1,
+            )
+
+        return docstring
diff --git a/platypush/plugins/inspect/_parsers/_response.py b/platypush/plugins/inspect/_parsers/_response.py
new file mode 100644
index 0000000000..222595e1d7
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_response.py
@@ -0,0 +1,34 @@
+import re
+from typing_extensions import override
+
+from ._base import Parser
+
+
+class ResponseParser(Parser):
+    """
+    Parse response references in the docstrings with rendered links to their
+    respective documentation.
+    """
+
+    _response_regex = re.compile(
+        r'(\s*):class:`(platypush\.message\.response\.(.+?))`', re.MULTILINE
+    )
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._response_regex.search(docstring)
+            if not m:
+                break
+
+            class_name = m.group(3).split('.')[-1]
+            package = '.'.join(m.group(3).split('.')[:-1])
+            docstring = cls._response_regex.sub(
+                f'{m.group(1)}`{class_name} '
+                f'<https://docs.platypush.tech/platypush/responses/{package}.html#{m.group(2)}>`_',
+                docstring,
+                count=1,
+            )
+
+        return docstring
diff --git a/platypush/plugins/inspect/_parsers/_schema.py b/platypush/plugins/inspect/_parsers/_schema.py
new file mode 100644
index 0000000000..7e66fcd58e
--- /dev/null
+++ b/platypush/plugins/inspect/_parsers/_schema.py
@@ -0,0 +1,97 @@
+import importlib
+import inspect
+import json
+import os
+from random import randint
+import re
+import textwrap
+from typing_extensions import override
+
+from marshmallow import fields
+
+import platypush.schemas
+
+from ._base import Parser
+
+
+class SchemaParser(Parser):
+    """
+    Support for response/message schemas in the docs. Format: ``.. schema:: rel_path.SchemaClass(arg1=value1, ...)``,
+    where ``rel_path`` is the path of the schema relative to ``platypush/schemas``.
+    """
+
+    _schemas_path = os.path.dirname(inspect.getfile(platypush.schemas))
+    _schema_regex = re.compile(
+        r'^(\s*)\.\.\s+schema::\s*([a-zA-Z0-9._]+)\s*(\((.+?)\))?', re.MULTILINE
+    )
+
+    @classmethod
+    def _get_field_value(cls, field):
+        metadata = getattr(field, 'metadata', {})
+        if metadata.get('example'):
+            return metadata['example']
+        if metadata.get('description'):
+            return metadata['description']
+
+        if isinstance(field, fields.Number):
+            return randint(1, 99)
+        if isinstance(field, fields.Boolean):
+            return bool(randint(0, 1))
+        if isinstance(field, fields.URL):
+            return 'https://example.org'
+        if isinstance(field, fields.List):
+            return [cls._get_field_value(field.inner)]
+        if isinstance(field, fields.Dict):
+            return {
+                cls._get_field_value(field.key_field)
+                if field.key_field
+                else 'key': cls._get_field_value(field.value_field)
+                if field.value_field
+                else 'value'
+            }
+        if isinstance(field, fields.Nested):
+            ret = {
+                name: cls._get_field_value(f)
+                for name, f in field.nested().fields.items()
+            }
+
+            return [ret] if field.many else ret
+
+        return str(field.__class__.__name__).lower()
+
+    @override
+    @classmethod
+    def parse(cls, docstring: str) -> str:
+        while True:
+            m = cls._schema_regex.search(docstring)
+            if not m:
+                break
+
+            schema_module_name = '.'.join(
+                ['platypush.schemas', *(m.group(2).split('.')[:-1])]
+            )
+            schema_module = importlib.import_module(schema_module_name)
+            schema_class = getattr(schema_module, m.group(2).split('.')[-1])
+            schema_args = eval(f'dict({m.group(4)})') if m.group(4) else {}
+            schema = schema_class(**schema_args)
+            parsed_schema = {
+                name: cls._get_field_value(field)
+                for name, field in schema.fields.items()
+                if not field.load_only
+            }
+
+            if schema.many:
+                parsed_schema = [parsed_schema]
+
+            padding = m.group(1)
+            docstring = cls._schema_regex.sub(
+                textwrap.indent('\n\n.. code-block:: json\n\n', padding)
+                + textwrap.indent(
+                    json.dumps(parsed_schema, sort_keys=True, indent=2),
+                    padding + '  ',
+                ).replace('\n\n', '\n')
+                + '\n\n',
+                docstring,
+            )
+
+        return docstring