blacklight/platypush
1
0
Fork 0
forked from platypush/platypush
Code Issues Pull requests Projects Releases Packages Wiki Activity

New components cache management.

Browse source 
The cache is no longer generated at runtime when the application runs -
which may take quite a while, especially on fresh installations.

A `components.json.gz` file is instead generated by the CI/CD process on
every commit to the main branch, and distributed with the package.

The application will read this file when the `inspect` plugin is first
initialized.

This also means that we can no longer cache the `install_cmds`, as they
change depending on the target OS. A new
`application.get_install_commands` action has now been made available to
the UI, so it can get the correct installation commands at runtime.
This commit is contained in:
Fabio Manganiello 2024-01-06 22:21:59 +01:00
parent ef885e096f
commit 6e1ab92298
Signed by: blacklight
GPG key ID: D90FBA7F76362774
9 changed files with 203 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
.drone.yml
View file

@ -62,7 +62,8 @@ steps:
# Backup the original git configuration before changing attributes
- export GIT_CONF=$PWD/.git/config
- cp $GIT_CONF /tmp/git.config.orig
- export TMP_GIT_CONF=/tmp/git.config.orig
- cp $GIT_CONF $TMP_GIT_CONF
- git config --global --add safe.directory $PWD
# Install the SSH and PGP keys
@ -95,7 +96,7 @@ steps:
- git checkout master
# Restore the original git configuration
- mv /tmp/git.config.orig $GIT_CONF
- mv $TMP_GIT_CONF $GIT_CONF
when:
event:
@ -187,7 +188,8 @@ steps:
# Backup the original git configuration before changing attributes
- export GIT_CONF=$PWD/.git/config
- cp $GIT_CONF /tmp/git.config.orig
- export TMP_GIT_CONF=/tmp/git.config.orig
- cp $GIT_CONF $TMP_GIT_CONF
- git config --global --add safe.directory $PWD
- cd platypush/backend/http/webapp
@ -242,7 +244,93 @@ steps:
- git push -f origin master
# Restore the original git configuration
- mv /tmp/git.config.orig $GIT_CONF
- mv $TMP_GIT_CONF $GIT_CONF
###
### Regenerate the components.json cache
###
- name: update-components-cache
image: alpine
environment:
SSH_PUBKEY:
from_secret: ssh_pubkey
SSH_PRIVKEY:
from_secret: ssh_privkey
PGP_KEY:
from_secret: pgp_key
PGP_KEY_ID:
from_secret: pgp_key_id
when:
branch:
- master
event:
- push
depends_on:
- build-ui
commands:
- export SKIPCI="$PWD/.skipci"
- export CACHEFILE="$PWD/platypush/components.json.gz"
- |
[ -f "$SKIPCI" ] && exit 0
- apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
- pip install . --break-system-packages
- |
python - <<EOF
from platypush import get_plugin
get_plugin('inspect').refresh_cache(force=True)
EOF
- |
if [ -z "$(git status "$CACHEFILE" --porcelain)" ]; then
echo 'No changes to the components file'
exit 0
fi
# Backup the original git configuration before changing attributes
- apk add --update --no-cache git
- export GIT_CONF=$PWD/.git/config
- export TMP_GIT_CONF=/tmp/git.config.orig
- cp $GIT_CONF $TMP_GIT_CONF
- git config --global --add safe.directory $PWD
# Create a .skipci file to mark the fact that the next steps should be skipped
# (we're going to do another push anyway, so another pipeline will be triggered)
- touch "$SKIPCI"
- apk add --update --no-cache bash gnupg openssh
- mkdir -p ~/.ssh
- |
cat <<EOF | gpg --import --armor
$PGP_KEY
EOF
- echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
- |
cat <<EOF > ~/.ssh/id_rsa
$SSH_PRIVKEY
EOF
- chmod 0600 ~/.ssh/id_rsa
- ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
- git config user.name "Platypush CI/CD Automation"
- git config user.email "admin@platypush.tech"
- git config commit.gpgsign true
- git config user.signingkey $PGP_KEY_ID
- git add "$CACHEFILE"
- git commit dist -S -m "[Automatic] Updated components cache" --no-verify
- git remote rm origin
- git remote add origin git@git.platypush.tech:platypush/platypush.git
- git push -f origin master
# Restore the original git configuration
- mv $TMP_GIT_CONF $GIT_CONF
###
### Update the Arch packages
@ -264,7 +352,7 @@ steps:
- push
depends_on:
- build-ui
- update-components-cache
commands:
- |
@ -364,7 +452,7 @@ steps:
- push
depends_on:
- build-ui
- update-components-cache
commands:
- |
@ -463,7 +551,7 @@ steps:
- push
depends_on:
- build-ui
- update-components-cache
commands:
- |
@ -774,7 +862,7 @@ steps:
- push
depends_on:
- build-ui
- update-components-cache
commands:
- |

1
MANIFEST.in
View file

@ -3,3 +3,4 @@ recursive-include platypush/install *
include platypush/plugins/http/webpage/mercury-parser.js
include platypush/config/*.yaml
global-include manifest.yaml
global-include components.json.gz

37
platypush/backend/http/webapp/src/components/panels/Extensions/Install.vue
View file

@ -7,11 +7,11 @@
<div class="body">
<div class="container install-cmd-container">
<CopyButton :text="installCmd" />
<pre><code v-html="highlightedInstallCmd" /></pre>
<CopyButton :text="installCmd" v-if="installCmd" />
<pre><Loading v-if="loading" /><code v-html="highlightedInstallCmd" v-else /></pre>
</div>
<div class="buttons install-btn" v-if="installCmd?.length">
<div class="buttons install-btn" v-if="installCmd">
<RestartButton v-if="installDone" />
<button type="button"
class="btn btn-default"
@ -73,22 +73,26 @@ export default {
installRunning: false,
installDone: false,
installOutput: null,
installCmds: [],
pendingCommands: 0,
error: null,
loading: false,
}
},
computed: {
installCmd() {
const cmd = this.extension.deps.install_cmd.join('\n').trim()
return cmd?.length ? cmd : null
if (this.installCmds.length)
return this.installCmds.join('\n').trim()
return null
},
highlightedInstallCmd() {
return (
hljs.highlight(
this.installCmd ?
this.extension.deps.install_cmd
this.installCmds
.map((cmd) => `$ ${cmd}`)
.join('\n')
.trim() :
@ -159,12 +163,15 @@ export default {
},
installExtension() {
if (!this.installCmd)
return
this.error = null
this.installRunning = true
this.installOutput = ''
this.$emit('install-start', this.extension)
const cmd = (this.extension.deps.install_cmd || []).join(';\n')
const cmd = this.installCmds.join(';\n')
this.request('shell.exec', {
cmd: cmd,
ws: true,
@ -176,9 +183,25 @@ export default {
this.$emit('install-end', this.extension)
})
},
async refreshInstallCmds() {
this.loading = true
try {
this.installCmds = await this.request('application.get_install_commands', {
extension: this.extension.name,
})
} finally {
this.loading = false
}
},
},
mounted() {
this.refreshInstallCmds()
this.$watch('extension.name', () => {
this.refreshInstallCmds()
})
this.$watch('installOutput', () => {
this.$nextTick(() => {
this.$refs.installOutput.focus()

BIN
platypush/components.json.gz Normal file
View file

Binary file not shown.

38
platypush/plugins/application/__init__.py
View file

@ -1,7 +1,8 @@
import inspect
import pathlib
import subprocess
from typing import Optional
from functools import lru_cache
from typing import List, Optional
from platypush.commands import CommandStream, RestartCommand, StopCommand
from platypush.common.db import override_definitions
@ -48,6 +49,29 @@ class ApplicationPlugin(Plugin):
will be prefixed by ``backend.`` (e.g. ``backend.http`` or
``backend.tcp``).
"""
install_cmds = self._get_install_cmds(extension)
if not install_cmds:
self.logger.info('No extra requirements found for extension %s', extension)
return
for cmd in install_cmds:
self.logger.info('> %s', cmd)
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
@action
def get_install_commands(self, extension: str) -> List[str]:
"""
Get the installation commands for an extension.
:param extension: Extension name. For plugins, it will be the plugin
name (e.g. ``light.hue`` or ``music.mpd``); for backend, the name
will be prefixed by ``backend.`` (e.g. ``backend.http`` or
``backend.tcp``).
"""
return self._get_install_cmds(extension)
@lru_cache(maxsize=256) # noqa
def _get_install_cmds(self, extension: str) -> List[str]:
getter = get_plugin_class_by_name
if extension.startswith('backend.'):
extension = extension[len('backend.') :]
@ -58,14 +82,4 @@ class ApplicationPlugin(Plugin):
assert ext, f'Could not find extension {extension}'
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
install_cmds = list(
Manifest.from_file(manifest_file).install.to_install_commands()
)
if not install_cmds:
self.logger.info('No extra requirements found for extension %s', extension)
return
for cmd in install_cmds:
self.logger.info('> %s', cmd)
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
return list(Manifest.from_file(manifest_file).install.to_install_commands())

45
platypush/plugins/inspect/__init__.py
View file

@ -33,34 +33,45 @@ class InspectPlugin(Plugin):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._cache_file = os.path.join(Config.get_cachedir(), 'components.json')
self._cache = Cache()
self._load_cache()
@property
def cache_file(self) -> str:
"""
:return: The path to the components cache file.
"""
import platypush
return os.path.join(
os.path.dirname(inspect.getfile(platypush)),
'components.json.gz',
)
def _load_cache(self):
"""
Loads the components cache from disk.
"""
with self._cache.lock(), auto_mocks(), override_definitions():
try:
self._cache = Cache.load(self._cache_file)
self._cache = Cache.load(self.cache_file)
except Exception as e:
self.logger.warning(
'Could not initialize the components cache from %s: %s',
self._cache_file,
self.cache_file,
e,
)
self._cache = Cache()
self._refresh_cache()
def _refresh_cache(self):
def refresh_cache(self, force: bool = False):
"""
Refreshes the components cache.
"""
cache_version_differs = self._cache.version != Cache.cur_version
force = force or cache_version_differs
with ThreadPoolExecutor(self._num_workers) as pool:
with self._cache.lock(), auto_mocks(), override_definitions(), ThreadPoolExecutor(
self._num_workers
) as pool:
futures = []
for base_type in [Plugin, Backend]:
@ -69,7 +80,7 @@ class InspectPlugin(Plugin):
self._scan_integrations,
base_type,
pool=pool,
force_refresh=cache_version_differs,
force_refresh=force,
futures=futures,
)
)
@ -80,7 +91,7 @@ class InspectPlugin(Plugin):
self._scan_modules,
base_type,
pool=pool,
force_refresh=cache_version_differs,
force_refresh=force,
futures=futures,
)
)
@ -89,10 +100,12 @@ class InspectPlugin(Plugin):
futures.pop().result()
if self._cache.has_changes:
self.logger.info('Saving new components cache to %s', self._cache_file)
self._cache.dump(self._cache_file)
self.logger.info('Saving new components cache to %s', self.cache_file)
self._cache.dump(self.cache_file)
self._cache.loaded_at = self._cache.saved_at
return self._cache
def _scan_integration(self, manifest: Manifest):
"""
Scans a single integration from the manifest and adds it to the cache.
@ -188,9 +201,11 @@ class InspectPlugin(Plugin):
"""
:return: True if the given file needs to be refreshed in the cache.
"""
return os.lstat(os.path.dirname(filename)).st_mtime > (
self._cache.saved_at or 0
)
dirname = os.path.dirname(filename)
if not os.path.isdir(dirname):
return True
return os.lstat(dirname).st_mtime > (self._cache.saved_at or 0)
@staticmethod
def _module_filename(path: str, modname: str) -> str:

22
platypush/plugins/inspect/_cache.py
View file

@ -1,4 +1,5 @@
from contextlib import contextmanager
import gzip
import json
import logging
from collections import defaultdict
@ -69,7 +70,7 @@ class Cache:
"""
cur_version = 1
cur_version = 1.1
"""
Cache version, used to detect breaking changes in the cache logic that require a cache refresh.
"""
@ -79,7 +80,7 @@ class Cache:
items: Optional[Dict[type, Dict[type, dict]]] = None,
saved_at: Optional[float] = None,
loaded_at: Optional[float] = None,
version: int = cur_version,
version: float = cur_version,
):
self.saved_at = saved_at
self.loaded_at = loaded_at
@ -99,9 +100,10 @@ class Cache:
:param cache_file: Cache file path.
"""
with open(cache_file, 'r') as f:
data = json.load(f)
return cls.from_dict(data)
with gzip.open(cache_file, 'rb') as f:
data = f.read()
return cls.from_dict(json.loads(data.decode()))
def dump(self, cache_file: str):
"""
@ -112,19 +114,21 @@ class Cache:
from platypush.message import Message
self.version = self.cur_version
with open(cache_file, 'w') as f:
self.saved_at = time()
json.dump(
compressed_cache = gzip.compress(
json.dumps(
{
'saved_at': self.saved_at,
'version': self.version,
'items': self.to_dict(),
},
f,
cls=Message.Encoder,
).encode()
)
with open(cache_file, 'wb') as f:
f.write(compressed_cache)
self.has_changes = False
@classmethod

5
platypush/utils/manifest.py
View file

@ -279,7 +279,10 @@ class Dependencies:
'packages': list(self.packages),
'pip': self.pip,
'after': self.after,
'install_cmd': list(self.to_install_commands()),
'by_pkg_manager': {
pkg_manager.value.executable: list(pkgs)
for pkg_manager, pkgs in self.by_pkg_manager.items()
},
}
@property

1
setup.py
View file

@ -48,6 +48,7 @@ setup(
'install/scripts/**/*',
'install/requirements/*',
'install/docker/*',
'components.json.gz',
],
},
entry_points={