New components cache management.

The cache is no longer generated at runtime when the application runs -
which may take quite a while, especially on fresh installations.

A `components.json.gz` file is instead generated by the CI/CD process on
every commit to the main branch, and distributed with the package.

The application will read this file when the `inspect` plugin is first
initialized.

This also means that we can no longer cache the `install_cmds`, as they
change depending on the target OS. A new
`application.get_install_commands` action has now been made available to
the UI, so it can get the correct installation commands at runtime.

.drone.yml

@@ -62,7 +62,8 @@ steps:
 
     # Backup the original git configuration before changing attributes
     - export GIT_CONF=$PWD/.git/config
-    - cp $GIT_CONF /tmp/git.config.orig
+    - export TMP_GIT_CONF=/tmp/git.config.orig
+    - cp $GIT_CONF $TMP_GIT_CONF
     - git config --global --add safe.directory $PWD
 
     # Install the SSH and PGP keys
@@ -95,7 +96,7 @@ steps:
     - git checkout master
 
     # Restore the original git configuration
-    - mv /tmp/git.config.orig $GIT_CONF
+    - mv $TMP_GIT_CONF $GIT_CONF
 
   when:
     event:
@@ -187,7 +188,8 @@ steps:
 
     # Backup the original git configuration before changing attributes
     - export GIT_CONF=$PWD/.git/config
-    - cp $GIT_CONF /tmp/git.config.orig
+    - export TMP_GIT_CONF=/tmp/git.config.orig
+    - cp $GIT_CONF $TMP_GIT_CONF
 
     - git config --global --add safe.directory $PWD
     - cd platypush/backend/http/webapp
@@ -242,7 +244,93 @@ steps:
     - git push -f origin master
 
     # Restore the original git configuration
-    - mv /tmp/git.config.orig $GIT_CONF
+    - mv $TMP_GIT_CONF $GIT_CONF
+
+###
+### Regenerate the components.json cache
+###
+
+- name: update-components-cache
+  image: alpine
+
+  environment:
+    SSH_PUBKEY:
+      from_secret: ssh_pubkey
+    SSH_PRIVKEY:
+      from_secret: ssh_privkey
+    PGP_KEY:
+      from_secret: pgp_key
+    PGP_KEY_ID:
+      from_secret: pgp_key_id
+
+  when:
+    branch:
+      - master
+    event:
+      - push
+
+  depends_on:
+    - build-ui
+
+  commands:
+    - export SKIPCI="$PWD/.skipci"
+    - export CACHEFILE="$PWD/platypush/components.json.gz"
+    - |
+      [ -f "$SKIPCI" ] && exit 0
+
+    - apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
+    - pip install . --break-system-packages
+    - |
+      python - <<EOF
+      from platypush import get_plugin
+
+      get_plugin('inspect').refresh_cache(force=True)
+      EOF
+
+    - |
+      if [ -z "$(git status "$CACHEFILE" --porcelain)" ]; then
+        echo 'No changes to the components file'
+        exit 0
+      fi
+
+    # Backup the original git configuration before changing attributes
+    - apk add --update --no-cache git
+    - export GIT_CONF=$PWD/.git/config
+    - export TMP_GIT_CONF=/tmp/git.config.orig
+    - cp $GIT_CONF $TMP_GIT_CONF
+    - git config --global --add safe.directory $PWD
+
+    # Create a .skipci file to mark the fact that the next steps should be skipped
+    # (we're going to do another push anyway, so another pipeline will be triggered)
+    - touch "$SKIPCI"
+
+    - apk add --update --no-cache bash gnupg openssh
+    - mkdir -p ~/.ssh
+    - |
+      cat <<EOF | gpg --import --armor
+      $PGP_KEY
+      EOF
+
+    - echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
+    - |
+      cat <<EOF > ~/.ssh/id_rsa
+      $SSH_PRIVKEY
+      EOF
+    - chmod 0600 ~/.ssh/id_rsa
+    - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
+
+    - git config user.name "Platypush CI/CD Automation"
+    - git config user.email "admin@platypush.tech"
+    - git config commit.gpgsign true
+    - git config user.signingkey $PGP_KEY_ID
+    - git add "$CACHEFILE"
+    - git commit dist -S -m "[Automatic] Updated components cache" --no-verify
+    - git remote rm origin
+    - git remote add origin git@git.platypush.tech:platypush/platypush.git
+    - git push -f origin master
+
+    # Restore the original git configuration
+    - mv $TMP_GIT_CONF $GIT_CONF
 
 ###
 ### Update the Arch packages
@@ -264,7 +352,7 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
     - |
@@ -364,7 +452,7 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
     - |
@@ -463,7 +551,7 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
     - |
@@ -774,7 +862,7 @@ steps:
       - push
 
   depends_on:
-    - build-ui
+    - update-components-cache
 
   commands:
     - |

MANIFEST.in

@@ -3,3 +3,4 @@ recursive-include platypush/install *
 include platypush/plugins/http/webpage/mercury-parser.js
 include platypush/config/*.yaml
 global-include manifest.yaml
+global-include components.json.gz

platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue

@@ -7,11 +7,11 @@
 
       <div class="body">
         <div class="container install-cmd-container">
-          <CopyButton :text="installCmd" />
+          <CopyButton :text="installCmd" v-if="installCmd" />
-          <pre><code v-html="highlightedInstallCmd" /></pre>
+          <pre><Loading v-if="loading" /><code v-html="highlightedInstallCmd" v-else /></pre>
         </div>
 
-        <div class="buttons install-btn" v-if="installCmd?.length">
+        <div class="buttons install-btn" v-if="installCmd">
           <RestartButton v-if="installDone" />
           <button type="button"
                   class="btn btn-default"
@@ -73,22 +73,26 @@ export default {
       installRunning: false,
       installDone: false,
       installOutput: null,
+      installCmds: [],
       pendingCommands: 0,
       error: null,
+      loading: false,
     }
   },
 
   computed: {
     installCmd() {
-      const cmd = this.extension.deps.install_cmd.join('\n').trim()
+      if (this.installCmds.length)
-      return cmd?.length ? cmd : null
+        return this.installCmds.join('\n').trim()
+
+      return null
     },
 
     highlightedInstallCmd() {
       return (
         hljs.highlight(
           this.installCmd ?
-          this.extension.deps.install_cmd
+          this.installCmds
           .map((cmd) => `$ ${cmd}`)
           .join('\n')
           .trim() :
@@ -159,12 +163,15 @@ export default {
     },
 
     installExtension() {
+      if (!this.installCmd)
+        return
+
       this.error = null
       this.installRunning = true
       this.installOutput = ''
       this.$emit('install-start', this.extension)
 
-      const cmd = (this.extension.deps.install_cmd || []).join(';\n')
+      const cmd = this.installCmds.join(';\n')
       this.request('shell.exec', {
         cmd: cmd,
         ws: true,
@@ -176,9 +183,25 @@ export default {
         this.$emit('install-end', this.extension)
       })
     },
+
+    async refreshInstallCmds() {
+      this.loading = true
+      try {
+        this.installCmds = await this.request('application.get_install_commands', {
+          extension: this.extension.name,
+        })
+      } finally {
+        this.loading = false
+      }
+    },
   },
 
   mounted() {
+    this.refreshInstallCmds()
+    this.$watch('extension.name', () => {
+      this.refreshInstallCmds()
+    })
+
     this.$watch('installOutput', () => {
       this.$nextTick(() => {
         this.$refs.installOutput.focus()

platypush/plugins/application/__init__.py

@@ -1,7 +1,8 @@
 import inspect
 import pathlib
 import subprocess
-from typing import Optional
+from functools import lru_cache
+from typing import List, Optional
 
 from platypush.commands import CommandStream, RestartCommand, StopCommand
 from platypush.common.db import override_definitions
@@ -48,6 +49,29 @@ class ApplicationPlugin(Plugin):
             will be prefixed by ``backend.`` (e.g. ``backend.http`` or
             ``backend.tcp``).
         """
+        install_cmds = self._get_install_cmds(extension)
+        if not install_cmds:
+            self.logger.info('No extra requirements found for extension %s', extension)
+            return
+
+        for cmd in install_cmds:
+            self.logger.info('> %s', cmd)
+            subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
+
+    @action
+    def get_install_commands(self, extension: str) -> List[str]:
+        """
+        Get the installation commands for an extension.
+
+        :param extension: Extension name. For plugins, it will be the plugin
+            name (e.g. ``light.hue`` or ``music.mpd``); for backend, the name
+            will be prefixed by ``backend.`` (e.g. ``backend.http`` or
+            ``backend.tcp``).
+        """
+        return self._get_install_cmds(extension)
+
+    @lru_cache(maxsize=256)  # noqa
+    def _get_install_cmds(self, extension: str) -> List[str]:
         getter = get_plugin_class_by_name
         if extension.startswith('backend.'):
             extension = extension[len('backend.') :]
@@ -58,14 +82,4 @@ class ApplicationPlugin(Plugin):
 
         assert ext, f'Could not find extension {extension}'
         manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
-        install_cmds = list(
+        return list(Manifest.from_file(manifest_file).install.to_install_commands())
-            Manifest.from_file(manifest_file).install.to_install_commands()
-        )
-
-        if not install_cmds:
-            self.logger.info('No extra requirements found for extension %s', extension)
-            return
-
-        for cmd in install_cmds:
-            self.logger.info('> %s', cmd)
-            subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)

platypush/plugins/inspect/__init__.py

@@ -33,34 +33,45 @@ class InspectPlugin(Plugin):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self._cache_file = os.path.join(Config.get_cachedir(), 'components.json')
         self._cache = Cache()
         self._load_cache()
 
+    @property
+    def cache_file(self) -> str:
+        """
+        :return: The path to the components cache file.
+        """
+        import platypush
+
+        return os.path.join(
+            os.path.dirname(inspect.getfile(platypush)),
+            'components.json.gz',
+        )
+
     def _load_cache(self):
         """
         Loads the components cache from disk.
         """
         with self._cache.lock(), auto_mocks(), override_definitions():
             try:
-                self._cache = Cache.load(self._cache_file)
+                self._cache = Cache.load(self.cache_file)
             except Exception as e:
                 self.logger.warning(
                     'Could not initialize the components cache from %s: %s',
-                    self._cache_file,
+                    self.cache_file,
                     e,
                 )
-                self._cache = Cache()
 
-            self._refresh_cache()
+    def refresh_cache(self, force: bool = False):
-
-    def _refresh_cache(self):
         """
         Refreshes the components cache.
         """
         cache_version_differs = self._cache.version != Cache.cur_version
+        force = force or cache_version_differs
 
-        with ThreadPoolExecutor(self._num_workers) as pool:
+        with self._cache.lock(), auto_mocks(), override_definitions(), ThreadPoolExecutor(
+            self._num_workers
+        ) as pool:
             futures = []
 
             for base_type in [Plugin, Backend]:
@@ -69,7 +80,7 @@ class InspectPlugin(Plugin):
                         self._scan_integrations,
                         base_type,
                         pool=pool,
-                        force_refresh=cache_version_differs,
+                        force_refresh=force,
                         futures=futures,
                     )
                 )
@@ -80,7 +91,7 @@ class InspectPlugin(Plugin):
                         self._scan_modules,
                         base_type,
                         pool=pool,
-                        force_refresh=cache_version_differs,
+                        force_refresh=force,
                         futures=futures,
                     )
                 )
@@ -89,10 +100,12 @@ class InspectPlugin(Plugin):
                 futures.pop().result()
 
         if self._cache.has_changes:
-            self.logger.info('Saving new components cache to %s', self._cache_file)
+            self.logger.info('Saving new components cache to %s', self.cache_file)
-            self._cache.dump(self._cache_file)
+            self._cache.dump(self.cache_file)
             self._cache.loaded_at = self._cache.saved_at
 
+        return self._cache
+
     def _scan_integration(self, manifest: Manifest):
         """
         Scans a single integration from the manifest and adds it to the cache.
@@ -188,9 +201,11 @@ class InspectPlugin(Plugin):
         """
         :return: True if the given file needs to be refreshed in the cache.
         """
-        return os.lstat(os.path.dirname(filename)).st_mtime > (
+        dirname = os.path.dirname(filename)
-            self._cache.saved_at or 0
+        if not os.path.isdir(dirname):
-        )
+            return True
+
+        return os.lstat(dirname).st_mtime > (self._cache.saved_at or 0)
 
     @staticmethod
     def _module_filename(path: str, modname: str) -> str:

platypush/plugins/inspect/_cache.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+import gzip
 import json
 import logging
 from collections import defaultdict
@@ -69,7 +70,7 @@ class Cache:
 
     """
 
-    cur_version = 1
+    cur_version = 1.1
     """
     Cache version, used to detect breaking changes in the cache logic that require a cache refresh.
     """
@@ -79,7 +80,7 @@ class Cache:
         items: Optional[Dict[type, Dict[type, dict]]] = None,
         saved_at: Optional[float] = None,
         loaded_at: Optional[float] = None,
-        version: int = cur_version,
+        version: float = cur_version,
     ):
         self.saved_at = saved_at
         self.loaded_at = loaded_at
@@ -99,9 +100,10 @@ class Cache:
 
         :param cache_file: Cache file path.
         """
-        with open(cache_file, 'r') as f:
+        with gzip.open(cache_file, 'rb') as f:
-            data = json.load(f)
+            data = f.read()
-            return cls.from_dict(data)
+
+        return cls.from_dict(json.loads(data.decode()))
 
     def dump(self, cache_file: str):
         """
@@ -112,18 +114,20 @@ class Cache:
         from platypush.message import Message
 
         self.version = self.cur_version
-
+        self.saved_at = time()
-        with open(cache_file, 'w') as f:
+        compressed_cache = gzip.compress(
-            self.saved_at = time()
+            json.dumps(
-            json.dump(
                 {
                     'saved_at': self.saved_at,
                     'version': self.version,
                     'items': self.to_dict(),
                 },
-                f,
                 cls=Message.Encoder,
-            )
+            ).encode()
+        )
+
+        with open(cache_file, 'wb') as f:
+            f.write(compressed_cache)
 
         self.has_changes = False
 

platypush/utils/manifest.py

@@ -279,7 +279,10 @@ class Dependencies:
             'packages': list(self.packages),
             'pip': self.pip,
             'after': self.after,
-            'install_cmd': list(self.to_install_commands()),
+            'by_pkg_manager': {
+                pkg_manager.value.executable: list(pkgs)
+                for pkg_manager, pkgs in self.by_pkg_manager.items()
+            },
         }
 
     @property

setup.py

@@ -48,6 +48,7 @@ setup(
             'install/scripts/**/*',
             'install/requirements/*',
             'install/docker/*',
+            'components.json.gz',
         ],
     },
     entry_points={