platypush/platypush
platypush
/
platypush
2
2
Fork 4
Code Issues 44 Pull Requests Packages Projects 4 Releases Wiki Activity

Refactored the interface of Platydock and manifest utils.

This commit is contained in:
Fabio Manganiello 2023-08-19 22:46:37 +02:00
parent a99ffea37c
commit 71c5291190
Signed by: blacklight
GPG Key ID: D90FBA7F76362774
2 changed files with 285 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
platypush/platydock/__init__.py
View File

@ -4,6 +4,7 @@ Dockerfile for Platypush starting from a configuration file.
"""
import argparse
from enum import Enum
import inspect
import os
import pathlib
@ -11,73 +12,108 @@ import sys
from typing import Iterable
from platypush.config import Config
from platypush.utils.manifest import Dependencies
ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m '
ERR_SUFFIX = '\033[0m'
from platypush.utils.manifest import Dependencies, InstallContext, PackageManagers
def generate_dockerfile(cfgfile: str) -> str:
class BaseImage(Enum):
"""
Generate a Dockerfile based on a configuration file.
Supported base images for Dockerfiles.
"""
ALPINE = 'alpine'
UBUNTU = 'ubuntu'
def __str__(self) -> str:
"""
Explicit __str__ override for argparse purposes.
"""
return self.value
class DockerfileGenerator:
"""
Generate a Dockerfile from on a configuration file.
:param cfgfile: Path to the configuration file.
:return: The content of the generated Dockerfile.
:param image: The base image to use.
"""
Config.init(cfgfile)
new_file_lines = []
ports = _get_exposed_ports()
deps = Dependencies.from_config(cfgfile, pkg_manager='apk')
is_after_expose_cmd = False
base_file = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile'
)
with open(base_file, 'r') as f:
file_lines = [line.rstrip() for line in f.readlines()]
for line in file_lines:
if line.startswith('RUN cd /install '):
for new_line in deps.before:
new_file_lines.append('RUN ' + new_line)
for new_line in deps.to_pkg_install_commands(
pkg_manager='apk', skip_sudo=True
):
new_file_lines.append('RUN ' + new_line)
elif line == 'RUN rm -rf /install':
for new_line in deps.to_pip_install_commands():
new_file_lines.append('RUN ' + new_line)
for new_line in deps.after:
new_file_lines.append('RUN' + new_line)
elif line.startswith('EXPOSE ') and ports:
if not is_after_expose_cmd:
new_file_lines.extend([f'EXPOSE {port}' for port in ports])
is_after_expose_cmd = True
continue
new_file_lines.append(line)
return '\n'.join(new_file_lines)
def _get_exposed_ports() -> Iterable[int]:
"""
:return: The listen ports used by the backends enabled in the configuration
file.
"""
backends_config = Config.get_backends()
return {
int(port)
for port in (
backends_config.get('http', {}).get('port'),
backends_config.get('tcp', {}).get('port'),
)
if port
_pkg_manager_by_base_image = {
BaseImage.ALPINE: PackageManagers.APK,
BaseImage.UBUNTU: PackageManagers.APT,
}
def __init__(self, cfgfile: str, image: BaseImage) -> None:
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
self.image = image
def generate(self) -> str:
"""
Generate a Dockerfile based on a configuration file.
:param cfgfile: Path to the configuration file.
:return: The content of the generated Dockerfile.
"""
Config.init(self.cfgfile)
new_file_lines = []
ports = self._get_exposed_ports()
pkg_manager = self._pkg_manager_by_base_image[self.image]
deps = Dependencies.from_config(
self.cfgfile,
pkg_manager=pkg_manager,
install_context=InstallContext.DOCKER,
)
is_after_expose_cmd = False
base_file = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent),
'docker',
f'{self.image}.Dockerfile',
)
with open(base_file, 'r') as f:
file_lines = [line.rstrip() for line in f.readlines()]
for line in file_lines:
if line.startswith('RUN cd /install '):
for new_line in deps.before:
new_file_lines.append('RUN ' + new_line)
for new_line in deps.to_pkg_install_commands():
new_file_lines.append('RUN ' + new_line)
elif line == 'RUN rm -rf /install':
for new_line in deps.to_pip_install_commands():
new_file_lines.append('RUN ' + new_line)
for new_line in deps.after:
new_file_lines.append('RUN' + new_line)
elif line.startswith('EXPOSE ') and ports:
if not is_after_expose_cmd:
new_file_lines.extend([f'EXPOSE {port}' for port in ports])
is_after_expose_cmd = True
continue
new_file_lines.append(line)
return '\n'.join(new_file_lines)
@staticmethod
def _get_exposed_ports() -> Iterable[int]:
"""
:return: The listen ports used by the backends enabled in the configuration
file.
"""
backends_config = Config.get_backends()
return {
int(port)
for port in (
backends_config.get('http', {}).get('port'),
backends_config.get('tcp', {}).get('port'),
)
if port
}
def main():
"""
@ -87,22 +123,44 @@ def main():
prog='platydock',
add_help=False,
description='Create a Platypush Dockerfile from a config.yaml.',
epilog='Use platydock <action> --help to get additional help.',
)
parser.add_argument('-h', '--help', action='store_true', help='Show usage')
parser.add_argument(
'cfgfile', type=str, nargs=1, help='The path to the configuration file.'
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
)
parser.add_argument(
'cfgfile', type=str, nargs='?', help='The path to the configuration file.'
)
parser.add_argument(
'--image',
'-i',
dest='image',
required=False,
type=BaseImage,
choices=list(BaseImage),
default=BaseImage.ALPINE,
help='Base image to use for the Dockerfile.',
)
opts, _ = parser.parse_known_args(sys.argv[1:])
cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0]))
dockerfile = generate_dockerfile(cfgfile)
if opts.show_usage:
parser.print_help()
return 0
if not opts.cfgfile:
print(
f'Please specify a configuration file.\nRun {sys.argv[0]} --help to get the available options.',
file=sys.stderr,
)
return 1
dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate()
print(dockerfile)
return 0
if __name__ == '__main__':
main()
sys.exit(main())
# vim:sw=4:ts=4:et:

250
platypush/utils/manifest.py
View File

@ -1,5 +1,6 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import enum
from enum import Enum
import importlib
import inspect
import json
@ -20,22 +21,102 @@ from typing import (
Type,
Union,
)
from typing_extensions import override
import yaml
from platypush.message.event import Event
supported_package_managers = {
'apk': ['apk', 'add', '--update', '--no-interactive', '--no-cache'],
'apt': ['apt', 'install', '-y'],
'pacman': ['pacman', '-S', '--noconfirm'],
}
_available_package_manager = None
logger = logging.getLogger(__name__)
class ManifestType(enum.Enum):
@dataclass
class PackageManager:
"""
Representation of a package manager.
"""
executable: str
""" The executable name. """
command: Iterable[str] = field(default_factory=tuple)
""" The command to execute, as a sequence of strings. """
class PackageManagers(Enum):
"""
Supported package managers.
"""
APK = PackageManager(
executable='apk',
command=('apk', 'add', '--update', '--no-interactive', '--no-cache'),
)
APT = PackageManager(
executable='apt',
command=('DEBIAN_FRONTEND=noninteractive', 'apt', 'install', '-y'),
)
PACMAN = PackageManager(
executable='pacman',
command=('pacman', '-S', '--noconfirm'),
)
@classmethod
def get_command(cls, name: str) -> Iterable[str]:
"""
:param name: The name of the package manager executable to get the
command for.
:return: The base command to execute, as a sequence of strings.
"""
pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None)
if not pkg_manager:
raise ValueError(f'Unknown package manager: {name}')
return pkg_manager.value.command
@classmethod
def scan(cls) -> Optional["PackageManagers"]:
"""
Get the name of the available package manager on the system, if supported.
"""
# pylint: disable=global-statement
global _available_package_manager
if _available_package_manager:
return _available_package_manager
available_package_managers = [
pkg_manager
for pkg_manager in cls
if shutil.which(pkg_manager.value.executable)
]
if not available_package_managers:
logger.warning(
'\nYour OS does not provide any of the supported package managers.\n'
'You may have to install some optional dependencies manually.\n'
'Supported package managers: %s.\n',
', '.join([pm.value.executable for pm in cls]),
)
return None
_available_package_manager = available_package_managers[0]
return _available_package_manager
class InstallContext(Enum):
"""
Supported installation contexts.
"""
NONE = None
DOCKER = 'docker'
VENV = 'venv'
class ManifestType(Enum):
"""
Manifest types.
"""
@ -58,15 +139,32 @@ class Dependencies:
""" pip dependencies. """
after: List[str] = field(default_factory=list)
""" Commands to execute after the component is installed. """
pkg_manager: Optional[PackageManagers] = None
""" Override the default package manager detected on the system. """
install_context: InstallContext = InstallContext.NONE
@property
def _is_venv(self) -> bool:
"""
:return: True if the dependencies scanning logic is running either in a
virtual environment or in a virtual environment preparation
context.
"""
return (
self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix
)
@classmethod
def from_config(
cls, conf_file: Optional[str] = None, pkg_manager: Optional[str] = None
cls,
conf_file: Optional[str] = None,
pkg_manager: Optional[PackageManagers] = None,
install_context: InstallContext = InstallContext.NONE,
) -> "Dependencies":
"""
Parse the required dependencies from a configuration file.
"""
deps = cls()
deps = cls(pkg_manager=pkg_manager, install_context=install_context)
for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager):
deps.before += manifest.install.before
@ -76,26 +174,19 @@ class Dependencies:
return deps
def to_pkg_install_commands(
self, pkg_manager: Optional[str] = None, skip_sudo: bool = False
) -> Generator[str, None, None]:
def to_pkg_install_commands(self) -> Generator[str, None, None]:
"""
Generates the package manager commands required to install the given
dependencies on the system.
:param pkg_manager: Force package manager to use (default: looks for
the one available on the system).
:param skip_sudo: Skip sudo when installing packages (default: it will
look if the current user is root and use sudo otherwise).
"""
wants_sudo = not skip_sudo and os.getuid() != 0
pkg_manager = pkg_manager or get_available_package_manager()
wants_sudo = self.install_context != InstallContext.DOCKER and os.getuid() != 0
pkg_manager = self.pkg_manager or PackageManagers.scan()
if self.packages and pkg_manager:
yield ' '.join(
[
*(['sudo'] if wants_sudo else []),
*supported_package_managers[pkg_manager],
*sorted(self.packages),
*pkg_manager.value.command,
*sorted(self.packages), # type: ignore
]
)
@ -104,11 +195,14 @@ class Dependencies:
Generates the pip commands required to install the given dependencies on
the system.
"""
# Recent versions want an explicit --break-system-packages option when
# installing packages via pip outside of a virtual environment
wants_break_system_packages = (
sys.version_info > (3, 10)
and sys.prefix == sys.base_prefix # We're not in a venv
wants_break_system_packages = not (
# Docker installations shouldn't require --break-system-packages in pip
self.install_context == InstallContext.DOCKER
# --break-system-packages has been introduced in Python 3.10
or sys.version_info < (3, 11)
# If we're in a virtual environment then we don't need
# --break-system-packages
or self._is_venv
)
if self.pip:
@ -118,24 +212,15 @@ class Dependencies:
+ ' '.join(sorted(self.pip))
)
def to_install_commands(
self, pkg_manager: Optional[str] = None, skip_sudo: bool = False
) -> Generator[str, None, None]:
def to_install_commands(self) -> Generator[str, None, None]:
"""
Generates the commands required to install the given dependencies on
this system.
:param pkg_manager: Force package manager to use (default: looks for
the one available on the system).
:param skip_sudo: Skip sudo when installing packages (default: it will
look if the current user is root and use sudo otherwise).
"""
for cmd in self.before:
yield cmd
for cmd in self.to_pkg_install_commands(
pkg_manager=pkg_manager, skip_sudo=skip_sudo
):
for cmd in self.to_pkg_install_commands():
yield cmd
for cmd in self.to_pip_install_commands():
@ -145,7 +230,7 @@ class Dependencies:
yield cmd
class Manifest:
class Manifest(ABC):
"""
Base class for plugin/backend manifests.
"""
@ -156,10 +241,10 @@ class Manifest:
description: Optional[str] = None,
install: Optional[Dict[str, Iterable[str]]] = None,
events: Optional[Mapping] = None,
pkg_manager: Optional[str] = None,
pkg_manager: Optional[PackageManagers] = None,
**_,
):
self._pkg_manager = pkg_manager or get_available_package_manager()
self._pkg_manager = pkg_manager or PackageManagers.scan()
self.description = description
self.install = self._init_deps(install or {})
self.events = self._init_events(events or {})
@ -167,6 +252,13 @@ class Manifest:
self.component_name = '.'.join(package.split('.')[2:])
self.component = None
@property
@abstractmethod
def manifest_type(self) -> ManifestType:
"""
:return: The type of the manifest.
"""
def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
deps = Dependencies()
for key, items in install.items():
@ -176,7 +268,7 @@ class Manifest:
deps.before += items
elif key == 'after':
deps.after += items
elif key == self._pkg_manager:
elif self._pkg_manager and key == self._pkg_manager.value.executable:
deps.packages.update(items)
return deps
@ -201,7 +293,9 @@ class Manifest:
return ret
@classmethod
def from_file(cls, filename: str, pkg_manager: Optional[str] = None) -> "Manifest":
def from_file(
cls, filename: str, pkg_manager: Optional[PackageManagers] = None
) -> "Manifest":
"""
Parse a manifest filename into a ``Manifest`` class.
"""
@ -210,9 +304,21 @@ class Manifest:
assert 'type' in manifest, f'Manifest file {filename} has no type field'
comp_type = ManifestType(manifest.pop('type'))
manifest_class = _manifest_class_by_type[comp_type]
manifest_class = cls.by_type(comp_type)
return manifest_class(**manifest, pkg_manager=pkg_manager)
@classmethod
def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]:
"""
:return: The manifest class corresponding to the given manifest type.
"""
if manifest_type == ManifestType.PLUGIN:
return PluginManifest
if manifest_type == ManifestType.BACKEND:
return BackendManifest
raise ValueError(f'Unknown manifest type: {manifest_type}')
def __repr__(self):
"""
:return: A JSON serialized representation of the manifest.
@ -225,19 +331,23 @@ class Manifest:
'.'.join([evt_type.__module__, evt_type.__name__]): doc
for evt_type, doc in self.events.items()
},
'type': _manifest_type_by_class[self.__class__].value,
'type': self.manifest_type.value,
'package': self.package,
'component_name': self.component_name,
}
)
# pylint: disable=too-few-public-methods
class PluginManifest(Manifest):
"""
Plugin manifest.
"""
@property
@override
def manifest_type(self) -> ManifestType:
return ManifestType.PLUGIN
# pylint: disable=too-few-public-methods
class BackendManifest(Manifest):
@ -245,6 +355,11 @@ class BackendManifest(Manifest):
Backend manifest.
"""
@property
@override
def manifest_type(self) -> ManifestType:
return ManifestType.BACKEND
class Manifests:
"""
@ -253,7 +368,7 @@ class Manifests:
@staticmethod
def by_base_class(
base_class: Type, pkg_manager: Optional[str] = None
base_class: Type, pkg_manager: Optional[PackageManagers] = None
) -> Generator[Manifest, None, None]:
"""
Get all the manifest files declared under the base path of a given class
@ -267,7 +382,7 @@ class Manifests:
@staticmethod
def by_config(
conf_file: Optional[str] = None,
pkg_manager: Optional[str] = None,
pkg_manager: Optional[PackageManagers] = None,
) -> Generator[Manifest, None, None]:
"""
Get all the manifest objects associated to the extensions declared in a
@ -294,42 +409,3 @@ class Manifests:
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
pkg_manager=pkg_manager,
)
def get_available_package_manager() -> Optional[str]:
"""
Get the name of the available package manager on the system, if supported.
"""
# pylint: disable=global-statement
global _available_package_manager
if _available_package_manager:
return _available_package_manager
available_package_managers = [
pkg_manager
for pkg_manager in supported_package_managers
if shutil.which(pkg_manager)
]
if not available_package_managers:
logger.warning(
'\nYour OS does not provide any of the supported package managers.\n'
'You may have to install some optional dependencies manually.\n'
'Supported package managers: %s.\n',
', '.join(supported_package_managers.keys()),
)
return None
_available_package_manager = available_package_managers[0]
return _available_package_manager
_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = {
ManifestType.PLUGIN: PluginManifest,
ManifestType.BACKEND: BackendManifest,
}
_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = {
cls: t for t, cls in _manifest_class_by_type.items()
}