From 6e1ab922982773c6b978bd9b25bc6b369acde7b0 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 6 Jan 2024 22:21:59 +0100 Subject: [PATCH] New components cache management. The cache is no longer generated at runtime when the application runs - which may take quite a while, especially on fresh installations. A `components.json.gz` file is instead generated by the CI/CD process on every commit to the main branch, and distributed with the package. The application will read this file when the `inspect` plugin is first initialized. This also means that we can no longer cache the `install_cmds`, as they change depending on the target OS. A new `application.get_install_commands` action has now been made available to the UI, so it can get the correct installation commands at runtime. --- .drone.yml | 104 ++++++++++++++++-- MANIFEST.in | 1 + .../components/panels/Extensions/Install.vue | 37 +++++-- platypush/components.json.gz | Bin 0 -> 269184 bytes platypush/plugins/application/__init__.py | 38 +++++-- platypush/plugins/inspect/__init__.py | 45 +++++--- platypush/plugins/inspect/_cache.py | 26 +++-- platypush/utils/manifest.py | 5 +- setup.py | 1 + 9 files changed, 203 insertions(+), 54 deletions(-) create mode 100644 platypush/components.json.gz diff --git a/.drone.yml b/.drone.yml index c03c0e52cb..9ba97735a6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -62,7 +62,8 @@ steps: # Backup the original git configuration before changing attributes - export GIT_CONF=$PWD/.git/config - - cp $GIT_CONF /tmp/git.config.orig + - export TMP_GIT_CONF=/tmp/git.config.orig + - cp $GIT_CONF $TMP_GIT_CONF - git config --global --add safe.directory $PWD # Install the SSH and PGP keys @@ -95,7 +96,7 @@ steps: - git checkout master # Restore the original git configuration - - mv /tmp/git.config.orig $GIT_CONF + - mv $TMP_GIT_CONF $GIT_CONF when: event: @@ -187,7 +188,8 @@ steps: # Backup the original git configuration before changing attributes - export GIT_CONF=$PWD/.git/config - - cp $GIT_CONF /tmp/git.config.orig + - export TMP_GIT_CONF=/tmp/git.config.orig + - cp $GIT_CONF $TMP_GIT_CONF - git config --global --add safe.directory $PWD - cd platypush/backend/http/webapp @@ -242,7 +244,93 @@ steps: - git push -f origin master # Restore the original git configuration - - mv /tmp/git.config.orig $GIT_CONF + - mv $TMP_GIT_CONF $GIT_CONF + +### +### Regenerate the components.json cache +### + +- name: update-components-cache + image: alpine + + environment: + SSH_PUBKEY: + from_secret: ssh_pubkey + SSH_PRIVKEY: + from_secret: ssh_privkey + PGP_KEY: + from_secret: pgp_key + PGP_KEY_ID: + from_secret: pgp_key_id + + when: + branch: + - master + event: + - push + + depends_on: + - build-ui + + commands: + - export SKIPCI="$PWD/.skipci" + - export CACHEFILE="$PWD/platypush/components.json.gz" + - | + [ -f "$SKIPCI" ] && exit 0 + + - apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt) + - pip install . --break-system-packages + - | + python - < ~/.ssh/id_rsa.pub + - | + cat < ~/.ssh/id_rsa + $SSH_PRIVKEY + EOF + - chmod 0600 ~/.ssh/id_rsa + - ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null + + - git config user.name "Platypush CI/CD Automation" + - git config user.email "admin@platypush.tech" + - git config commit.gpgsign true + - git config user.signingkey $PGP_KEY_ID + - git add "$CACHEFILE" + - git commit dist -S -m "[Automatic] Updated components cache" --no-verify + - git remote rm origin + - git remote add origin git@git.platypush.tech:platypush/platypush.git + - git push -f origin master + + # Restore the original git configuration + - mv $TMP_GIT_CONF $GIT_CONF ### ### Update the Arch packages @@ -264,7 +352,7 @@ steps: - push depends_on: - - build-ui + - update-components-cache commands: - | @@ -364,7 +452,7 @@ steps: - push depends_on: - - build-ui + - update-components-cache commands: - | @@ -463,7 +551,7 @@ steps: - push depends_on: - - build-ui + - update-components-cache commands: - | @@ -774,7 +862,7 @@ steps: - push depends_on: - - build-ui + - update-components-cache commands: - | diff --git a/MANIFEST.in b/MANIFEST.in index ac702393c3..310df247fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ recursive-include platypush/install * include platypush/plugins/http/webpage/mercury-parser.js include platypush/config/*.yaml global-include manifest.yaml +global-include components.json.gz diff --git a/platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue b/platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue index d12cefc0dd..b118a4f90d 100644 --- a/platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue +++ b/platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue @@ -7,11 +7,11 @@
- -
+ +
-
+