New components cache management.
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
The cache is no longer generated at runtime when the application runs - which may take quite a while, especially on fresh installations. A `components.json.gz` file is instead generated by the CI/CD process on every commit to the main branch, and distributed with the package. The application will read this file when the `inspect` plugin is first initialized. This also means that we can no longer cache the `install_cmds`, as they change depending on the target OS. A new `application.get_install_commands` action has now been made available to the UI, so it can get the correct installation commands at runtime.
This commit is contained in:
parent
ef885e096f
commit
6e1ab92298
104
.drone.yml
104
.drone.yml
|
@ -62,7 +62,8 @@ steps:
|
|||
|
||||
# Backup the original git configuration before changing attributes
|
||||
- export GIT_CONF=$PWD/.git/config
|
||||
- cp $GIT_CONF /tmp/git.config.orig
|
||||
- export TMP_GIT_CONF=/tmp/git.config.orig
|
||||
- cp $GIT_CONF $TMP_GIT_CONF
|
||||
- git config --global --add safe.directory $PWD
|
||||
|
||||
# Install the SSH and PGP keys
|
||||
|
@ -95,7 +96,7 @@ steps:
|
|||
- git checkout master
|
||||
|
||||
# Restore the original git configuration
|
||||
- mv /tmp/git.config.orig $GIT_CONF
|
||||
- mv $TMP_GIT_CONF $GIT_CONF
|
||||
|
||||
when:
|
||||
event:
|
||||
|
@ -187,7 +188,8 @@ steps:
|
|||
|
||||
# Backup the original git configuration before changing attributes
|
||||
- export GIT_CONF=$PWD/.git/config
|
||||
- cp $GIT_CONF /tmp/git.config.orig
|
||||
- export TMP_GIT_CONF=/tmp/git.config.orig
|
||||
- cp $GIT_CONF $TMP_GIT_CONF
|
||||
|
||||
- git config --global --add safe.directory $PWD
|
||||
- cd platypush/backend/http/webapp
|
||||
|
@ -242,7 +244,93 @@ steps:
|
|||
- git push -f origin master
|
||||
|
||||
# Restore the original git configuration
|
||||
- mv /tmp/git.config.orig $GIT_CONF
|
||||
- mv $TMP_GIT_CONF $GIT_CONF
|
||||
|
||||
###
|
||||
### Regenerate the components.json cache
|
||||
###
|
||||
|
||||
- name: update-components-cache
|
||||
image: alpine
|
||||
|
||||
environment:
|
||||
SSH_PUBKEY:
|
||||
from_secret: ssh_pubkey
|
||||
SSH_PRIVKEY:
|
||||
from_secret: ssh_privkey
|
||||
PGP_KEY:
|
||||
from_secret: pgp_key
|
||||
PGP_KEY_ID:
|
||||
from_secret: pgp_key_id
|
||||
|
||||
when:
|
||||
branch:
|
||||
- master
|
||||
event:
|
||||
- push
|
||||
|
||||
depends_on:
|
||||
- build-ui
|
||||
|
||||
commands:
|
||||
- export SKIPCI="$PWD/.skipci"
|
||||
- export CACHEFILE="$PWD/platypush/components.json.gz"
|
||||
- |
|
||||
[ -f "$SKIPCI" ] && exit 0
|
||||
|
||||
- apk add --update --no-cache $(cat platypush/install/requirements/alpine.txt)
|
||||
- pip install . --break-system-packages
|
||||
- |
|
||||
python - <<EOF
|
||||
from platypush import get_plugin
|
||||
|
||||
get_plugin('inspect').refresh_cache(force=True)
|
||||
EOF
|
||||
|
||||
- |
|
||||
if [ -z "$(git status "$CACHEFILE" --porcelain)" ]; then
|
||||
echo 'No changes to the components file'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Backup the original git configuration before changing attributes
|
||||
- apk add --update --no-cache git
|
||||
- export GIT_CONF=$PWD/.git/config
|
||||
- export TMP_GIT_CONF=/tmp/git.config.orig
|
||||
- cp $GIT_CONF $TMP_GIT_CONF
|
||||
- git config --global --add safe.directory $PWD
|
||||
|
||||
# Create a .skipci file to mark the fact that the next steps should be skipped
|
||||
# (we're going to do another push anyway, so another pipeline will be triggered)
|
||||
- touch "$SKIPCI"
|
||||
|
||||
- apk add --update --no-cache bash gnupg openssh
|
||||
- mkdir -p ~/.ssh
|
||||
- |
|
||||
cat <<EOF | gpg --import --armor
|
||||
$PGP_KEY
|
||||
EOF
|
||||
|
||||
- echo $SSH_PUBKEY > ~/.ssh/id_rsa.pub
|
||||
- |
|
||||
cat <<EOF > ~/.ssh/id_rsa
|
||||
$SSH_PRIVKEY
|
||||
EOF
|
||||
- chmod 0600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan git.platypush.tech >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
- git config user.name "Platypush CI/CD Automation"
|
||||
- git config user.email "admin@platypush.tech"
|
||||
- git config commit.gpgsign true
|
||||
- git config user.signingkey $PGP_KEY_ID
|
||||
- git add "$CACHEFILE"
|
||||
- git commit dist -S -m "[Automatic] Updated components cache" --no-verify
|
||||
- git remote rm origin
|
||||
- git remote add origin git@git.platypush.tech:platypush/platypush.git
|
||||
- git push -f origin master
|
||||
|
||||
# Restore the original git configuration
|
||||
- mv $TMP_GIT_CONF $GIT_CONF
|
||||
|
||||
###
|
||||
### Update the Arch packages
|
||||
|
@ -264,7 +352,7 @@ steps:
|
|||
- push
|
||||
|
||||
depends_on:
|
||||
- build-ui
|
||||
- update-components-cache
|
||||
|
||||
commands:
|
||||
- |
|
||||
|
@ -364,7 +452,7 @@ steps:
|
|||
- push
|
||||
|
||||
depends_on:
|
||||
- build-ui
|
||||
- update-components-cache
|
||||
|
||||
commands:
|
||||
- |
|
||||
|
@ -463,7 +551,7 @@ steps:
|
|||
- push
|
||||
|
||||
depends_on:
|
||||
- build-ui
|
||||
- update-components-cache
|
||||
|
||||
commands:
|
||||
- |
|
||||
|
@ -774,7 +862,7 @@ steps:
|
|||
- push
|
||||
|
||||
depends_on:
|
||||
- build-ui
|
||||
- update-components-cache
|
||||
|
||||
commands:
|
||||
- |
|
||||
|
|
|
@ -3,3 +3,4 @@ recursive-include platypush/install *
|
|||
include platypush/plugins/http/webpage/mercury-parser.js
|
||||
include platypush/config/*.yaml
|
||||
global-include manifest.yaml
|
||||
global-include components.json.gz
|
||||
|
|
|
@ -7,11 +7,11 @@
|
|||
|
||||
<div class="body">
|
||||
<div class="container install-cmd-container">
|
||||
<CopyButton :text="installCmd" />
|
||||
<pre><code v-html="highlightedInstallCmd" /></pre>
|
||||
<CopyButton :text="installCmd" v-if="installCmd" />
|
||||
<pre><Loading v-if="loading" /><code v-html="highlightedInstallCmd" v-else /></pre>
|
||||
</div>
|
||||
|
||||
<div class="buttons install-btn" v-if="installCmd?.length">
|
||||
<div class="buttons install-btn" v-if="installCmd">
|
||||
<RestartButton v-if="installDone" />
|
||||
<button type="button"
|
||||
class="btn btn-default"
|
||||
|
@ -73,22 +73,26 @@ export default {
|
|||
installRunning: false,
|
||||
installDone: false,
|
||||
installOutput: null,
|
||||
installCmds: [],
|
||||
pendingCommands: 0,
|
||||
error: null,
|
||||
loading: false,
|
||||
}
|
||||
},
|
||||
|
||||
computed: {
|
||||
installCmd() {
|
||||
const cmd = this.extension.deps.install_cmd.join('\n').trim()
|
||||
return cmd?.length ? cmd : null
|
||||
if (this.installCmds.length)
|
||||
return this.installCmds.join('\n').trim()
|
||||
|
||||
return null
|
||||
},
|
||||
|
||||
highlightedInstallCmd() {
|
||||
return (
|
||||
hljs.highlight(
|
||||
this.installCmd ?
|
||||
this.extension.deps.install_cmd
|
||||
this.installCmds
|
||||
.map((cmd) => `$ ${cmd}`)
|
||||
.join('\n')
|
||||
.trim() :
|
||||
|
@ -159,12 +163,15 @@ export default {
|
|||
},
|
||||
|
||||
installExtension() {
|
||||
if (!this.installCmd)
|
||||
return
|
||||
|
||||
this.error = null
|
||||
this.installRunning = true
|
||||
this.installOutput = ''
|
||||
this.$emit('install-start', this.extension)
|
||||
|
||||
const cmd = (this.extension.deps.install_cmd || []).join(';\n')
|
||||
const cmd = this.installCmds.join(';\n')
|
||||
this.request('shell.exec', {
|
||||
cmd: cmd,
|
||||
ws: true,
|
||||
|
@ -176,9 +183,25 @@ export default {
|
|||
this.$emit('install-end', this.extension)
|
||||
})
|
||||
},
|
||||
|
||||
async refreshInstallCmds() {
|
||||
this.loading = true
|
||||
try {
|
||||
this.installCmds = await this.request('application.get_install_commands', {
|
||||
extension: this.extension.name,
|
||||
})
|
||||
} finally {
|
||||
this.loading = false
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
mounted() {
|
||||
this.refreshInstallCmds()
|
||||
this.$watch('extension.name', () => {
|
||||
this.refreshInstallCmds()
|
||||
})
|
||||
|
||||
this.$watch('installOutput', () => {
|
||||
this.$nextTick(() => {
|
||||
this.$refs.installOutput.focus()
|
||||
|
|
BIN
platypush/components.json.gz
Normal file
BIN
platypush/components.json.gz
Normal file
Binary file not shown.
|
@ -1,7 +1,8 @@
|
|||
import inspect
|
||||
import pathlib
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
from functools import lru_cache
|
||||
from typing import List, Optional
|
||||
|
||||
from platypush.commands import CommandStream, RestartCommand, StopCommand
|
||||
from platypush.common.db import override_definitions
|
||||
|
@ -48,6 +49,29 @@ class ApplicationPlugin(Plugin):
|
|||
will be prefixed by ``backend.`` (e.g. ``backend.http`` or
|
||||
``backend.tcp``).
|
||||
"""
|
||||
install_cmds = self._get_install_cmds(extension)
|
||||
if not install_cmds:
|
||||
self.logger.info('No extra requirements found for extension %s', extension)
|
||||
return
|
||||
|
||||
for cmd in install_cmds:
|
||||
self.logger.info('> %s', cmd)
|
||||
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
|
||||
|
||||
@action
|
||||
def get_install_commands(self, extension: str) -> List[str]:
|
||||
"""
|
||||
Get the installation commands for an extension.
|
||||
|
||||
:param extension: Extension name. For plugins, it will be the plugin
|
||||
name (e.g. ``light.hue`` or ``music.mpd``); for backend, the name
|
||||
will be prefixed by ``backend.`` (e.g. ``backend.http`` or
|
||||
``backend.tcp``).
|
||||
"""
|
||||
return self._get_install_cmds(extension)
|
||||
|
||||
@lru_cache(maxsize=256) # noqa
|
||||
def _get_install_cmds(self, extension: str) -> List[str]:
|
||||
getter = get_plugin_class_by_name
|
||||
if extension.startswith('backend.'):
|
||||
extension = extension[len('backend.') :]
|
||||
|
@ -58,14 +82,4 @@ class ApplicationPlugin(Plugin):
|
|||
|
||||
assert ext, f'Could not find extension {extension}'
|
||||
manifest_file = str(pathlib.Path(inspect.getfile(ext)).parent / 'manifest.yaml')
|
||||
install_cmds = list(
|
||||
Manifest.from_file(manifest_file).install.to_install_commands()
|
||||
)
|
||||
|
||||
if not install_cmds:
|
||||
self.logger.info('No extra requirements found for extension %s', extension)
|
||||
return
|
||||
|
||||
for cmd in install_cmds:
|
||||
self.logger.info('> %s', cmd)
|
||||
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
|
||||
return list(Manifest.from_file(manifest_file).install.to_install_commands())
|
||||
|
|
|
@ -33,34 +33,45 @@ class InspectPlugin(Plugin):
|
|||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._cache_file = os.path.join(Config.get_cachedir(), 'components.json')
|
||||
self._cache = Cache()
|
||||
self._load_cache()
|
||||
|
||||
@property
|
||||
def cache_file(self) -> str:
|
||||
"""
|
||||
:return: The path to the components cache file.
|
||||
"""
|
||||
import platypush
|
||||
|
||||
return os.path.join(
|
||||
os.path.dirname(inspect.getfile(platypush)),
|
||||
'components.json.gz',
|
||||
)
|
||||
|
||||
def _load_cache(self):
|
||||
"""
|
||||
Loads the components cache from disk.
|
||||
"""
|
||||
with self._cache.lock(), auto_mocks(), override_definitions():
|
||||
try:
|
||||
self._cache = Cache.load(self._cache_file)
|
||||
self._cache = Cache.load(self.cache_file)
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
'Could not initialize the components cache from %s: %s',
|
||||
self._cache_file,
|
||||
self.cache_file,
|
||||
e,
|
||||
)
|
||||
self._cache = Cache()
|
||||
|
||||
self._refresh_cache()
|
||||
|
||||
def _refresh_cache(self):
|
||||
def refresh_cache(self, force: bool = False):
|
||||
"""
|
||||
Refreshes the components cache.
|
||||
"""
|
||||
cache_version_differs = self._cache.version != Cache.cur_version
|
||||
force = force or cache_version_differs
|
||||
|
||||
with ThreadPoolExecutor(self._num_workers) as pool:
|
||||
with self._cache.lock(), auto_mocks(), override_definitions(), ThreadPoolExecutor(
|
||||
self._num_workers
|
||||
) as pool:
|
||||
futures = []
|
||||
|
||||
for base_type in [Plugin, Backend]:
|
||||
|
@ -69,7 +80,7 @@ class InspectPlugin(Plugin):
|
|||
self._scan_integrations,
|
||||
base_type,
|
||||
pool=pool,
|
||||
force_refresh=cache_version_differs,
|
||||
force_refresh=force,
|
||||
futures=futures,
|
||||
)
|
||||
)
|
||||
|
@ -80,7 +91,7 @@ class InspectPlugin(Plugin):
|
|||
self._scan_modules,
|
||||
base_type,
|
||||
pool=pool,
|
||||
force_refresh=cache_version_differs,
|
||||
force_refresh=force,
|
||||
futures=futures,
|
||||
)
|
||||
)
|
||||
|
@ -89,10 +100,12 @@ class InspectPlugin(Plugin):
|
|||
futures.pop().result()
|
||||
|
||||
if self._cache.has_changes:
|
||||
self.logger.info('Saving new components cache to %s', self._cache_file)
|
||||
self._cache.dump(self._cache_file)
|
||||
self.logger.info('Saving new components cache to %s', self.cache_file)
|
||||
self._cache.dump(self.cache_file)
|
||||
self._cache.loaded_at = self._cache.saved_at
|
||||
|
||||
return self._cache
|
||||
|
||||
def _scan_integration(self, manifest: Manifest):
|
||||
"""
|
||||
Scans a single integration from the manifest and adds it to the cache.
|
||||
|
@ -188,9 +201,11 @@ class InspectPlugin(Plugin):
|
|||
"""
|
||||
:return: True if the given file needs to be refreshed in the cache.
|
||||
"""
|
||||
return os.lstat(os.path.dirname(filename)).st_mtime > (
|
||||
self._cache.saved_at or 0
|
||||
)
|
||||
dirname = os.path.dirname(filename)
|
||||
if not os.path.isdir(dirname):
|
||||
return True
|
||||
|
||||
return os.lstat(dirname).st_mtime > (self._cache.saved_at or 0)
|
||||
|
||||
@staticmethod
|
||||
def _module_filename(path: str, modname: str) -> str:
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
from contextlib import contextmanager
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
@ -69,7 +70,7 @@ class Cache:
|
|||
|
||||
"""
|
||||
|
||||
cur_version = 1
|
||||
cur_version = 1.1
|
||||
"""
|
||||
Cache version, used to detect breaking changes in the cache logic that require a cache refresh.
|
||||
"""
|
||||
|
@ -79,7 +80,7 @@ class Cache:
|
|||
items: Optional[Dict[type, Dict[type, dict]]] = None,
|
||||
saved_at: Optional[float] = None,
|
||||
loaded_at: Optional[float] = None,
|
||||
version: int = cur_version,
|
||||
version: float = cur_version,
|
||||
):
|
||||
self.saved_at = saved_at
|
||||
self.loaded_at = loaded_at
|
||||
|
@ -99,9 +100,10 @@ class Cache:
|
|||
|
||||
:param cache_file: Cache file path.
|
||||
"""
|
||||
with open(cache_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
return cls.from_dict(data)
|
||||
with gzip.open(cache_file, 'rb') as f:
|
||||
data = f.read()
|
||||
|
||||
return cls.from_dict(json.loads(data.decode()))
|
||||
|
||||
def dump(self, cache_file: str):
|
||||
"""
|
||||
|
@ -112,18 +114,20 @@ class Cache:
|
|||
from platypush.message import Message
|
||||
|
||||
self.version = self.cur_version
|
||||
|
||||
with open(cache_file, 'w') as f:
|
||||
self.saved_at = time()
|
||||
json.dump(
|
||||
self.saved_at = time()
|
||||
compressed_cache = gzip.compress(
|
||||
json.dumps(
|
||||
{
|
||||
'saved_at': self.saved_at,
|
||||
'version': self.version,
|
||||
'items': self.to_dict(),
|
||||
},
|
||||
f,
|
||||
cls=Message.Encoder,
|
||||
)
|
||||
).encode()
|
||||
)
|
||||
|
||||
with open(cache_file, 'wb') as f:
|
||||
f.write(compressed_cache)
|
||||
|
||||
self.has_changes = False
|
||||
|
||||
|
|
|
@ -279,7 +279,10 @@ class Dependencies:
|
|||
'packages': list(self.packages),
|
||||
'pip': self.pip,
|
||||
'after': self.after,
|
||||
'install_cmd': list(self.to_install_commands()),
|
||||
'by_pkg_manager': {
|
||||
pkg_manager.value.executable: list(pkgs)
|
||||
for pkg_manager, pkgs in self.by_pkg_manager.items()
|
||||
},
|
||||
}
|
||||
|
||||
@property
|
||||
|
|
Loading…
Reference in a new issue