From 71c529119058875086022d1b2dbac307c318942e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 22:46:37 +0200 Subject: [PATCH] Refactored the interface of Platydock and manifest utils. --- platypush/platydock/__init__.py | 186 ++++++++++++++++-------- platypush/utils/manifest.py | 250 +++++++++++++++++++++----------- 2 files changed, 285 insertions(+), 151 deletions(-) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 1725bf4d5..2f49d3cb2 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -4,6 +4,7 @@ Dockerfile for Platypush starting from a configuration file. """ import argparse +from enum import Enum import inspect import os import pathlib @@ -11,73 +12,108 @@ import sys from typing import Iterable from platypush.config import Config -from platypush.utils.manifest import Dependencies - -ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m ' -ERR_SUFFIX = '\033[0m' +from platypush.utils.manifest import Dependencies, InstallContext, PackageManagers -def generate_dockerfile(cfgfile: str) -> str: +class BaseImage(Enum): """ - Generate a Dockerfile based on a configuration file. + Supported base images for Dockerfiles. + """ + + ALPINE = 'alpine' + UBUNTU = 'ubuntu' + + def __str__(self) -> str: + """ + Explicit __str__ override for argparse purposes. + """ + return self.value + + +class DockerfileGenerator: + """ + Generate a Dockerfile from on a configuration file. :param cfgfile: Path to the configuration file. - :return: The content of the generated Dockerfile. + :param image: The base image to use. """ - Config.init(cfgfile) - new_file_lines = [] - ports = _get_exposed_ports() - deps = Dependencies.from_config(cfgfile, pkg_manager='apk') - is_after_expose_cmd = False - base_file = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile' - ) - with open(base_file, 'r') as f: - file_lines = [line.rstrip() for line in f.readlines()] - - for line in file_lines: - if line.startswith('RUN cd /install '): - for new_line in deps.before: - new_file_lines.append('RUN ' + new_line) - - for new_line in deps.to_pkg_install_commands( - pkg_manager='apk', skip_sudo=True - ): - new_file_lines.append('RUN ' + new_line) - elif line == 'RUN rm -rf /install': - for new_line in deps.to_pip_install_commands(): - new_file_lines.append('RUN ' + new_line) - - for new_line in deps.after: - new_file_lines.append('RUN' + new_line) - elif line.startswith('EXPOSE ') and ports: - if not is_after_expose_cmd: - new_file_lines.extend([f'EXPOSE {port}' for port in ports]) - is_after_expose_cmd = True - - continue - - new_file_lines.append(line) - - return '\n'.join(new_file_lines) - - -def _get_exposed_ports() -> Iterable[int]: - """ - :return: The listen ports used by the backends enabled in the configuration - file. - """ - backends_config = Config.get_backends() - return { - int(port) - for port in ( - backends_config.get('http', {}).get('port'), - backends_config.get('tcp', {}).get('port'), - ) - if port + _pkg_manager_by_base_image = { + BaseImage.ALPINE: PackageManagers.APK, + BaseImage.UBUNTU: PackageManagers.APT, } + def __init__(self, cfgfile: str, image: BaseImage) -> None: + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.image = image + + def generate(self) -> str: + """ + Generate a Dockerfile based on a configuration file. + + :param cfgfile: Path to the configuration file. + :return: The content of the generated Dockerfile. + """ + Config.init(self.cfgfile) + new_file_lines = [] + ports = self._get_exposed_ports() + pkg_manager = self._pkg_manager_by_base_image[self.image] + deps = Dependencies.from_config( + self.cfgfile, + pkg_manager=pkg_manager, + install_context=InstallContext.DOCKER, + ) + + is_after_expose_cmd = False + base_file = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'docker', + f'{self.image}.Dockerfile', + ) + + with open(base_file, 'r') as f: + file_lines = [line.rstrip() for line in f.readlines()] + + for line in file_lines: + if line.startswith('RUN cd /install '): + for new_line in deps.before: + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.to_pkg_install_commands(): + new_file_lines.append('RUN ' + new_line) + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.after: + new_file_lines.append('RUN' + new_line) + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + new_file_lines.extend([f'EXPOSE {port}' for port in ports]) + is_after_expose_cmd = True + + continue + + new_file_lines.append(line) + + return '\n'.join(new_file_lines) + + @staticmethod + def _get_exposed_ports() -> Iterable[int]: + """ + :return: The listen ports used by the backends enabled in the configuration + file. + """ + backends_config = Config.get_backends() + return { + int(port) + for port in ( + backends_config.get('http', {}).get('port'), + backends_config.get('tcp', {}).get('port'), + ) + if port + } + def main(): """ @@ -87,22 +123,44 @@ def main(): prog='platydock', add_help=False, description='Create a Platypush Dockerfile from a config.yaml.', - epilog='Use platydock --help to get additional help.', ) - parser.add_argument('-h', '--help', action='store_true', help='Show usage') parser.add_argument( - 'cfgfile', type=str, nargs=1, help='The path to the configuration file.' + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + parser.add_argument( + 'cfgfile', type=str, nargs='?', help='The path to the configuration file.' + ) + parser.add_argument( + '--image', + '-i', + dest='image', + required=False, + type=BaseImage, + choices=list(BaseImage), + default=BaseImage.ALPINE, + help='Base image to use for the Dockerfile.', ) opts, _ = parser.parse_known_args(sys.argv[1:]) - cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0])) - dockerfile = generate_dockerfile(cfgfile) + if opts.show_usage: + parser.print_help() + return 0 + + if not opts.cfgfile: + print( + f'Please specify a configuration file.\nRun {sys.argv[0]} --help to get the available options.', + file=sys.stderr, + ) + return 1 + + dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate() print(dockerfile) + return 0 if __name__ == '__main__': - main() + sys.exit(main()) # vim:sw=4:ts=4:et: diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 1436b705f..c217c464b 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -1,5 +1,6 @@ +from abc import ABC, abstractmethod from dataclasses import dataclass, field -import enum +from enum import Enum import importlib import inspect import json @@ -20,22 +21,102 @@ from typing import ( Type, Union, ) +from typing_extensions import override import yaml from platypush.message.event import Event -supported_package_managers = { - 'apk': ['apk', 'add', '--update', '--no-interactive', '--no-cache'], - 'apt': ['apt', 'install', '-y'], - 'pacman': ['pacman', '-S', '--noconfirm'], -} - _available_package_manager = None logger = logging.getLogger(__name__) -class ManifestType(enum.Enum): +@dataclass +class PackageManager: + """ + Representation of a package manager. + """ + + executable: str + """ The executable name. """ + command: Iterable[str] = field(default_factory=tuple) + """ The command to execute, as a sequence of strings. """ + + +class PackageManagers(Enum): + """ + Supported package managers. + """ + + APK = PackageManager( + executable='apk', + command=('apk', 'add', '--update', '--no-interactive', '--no-cache'), + ) + + APT = PackageManager( + executable='apt', + command=('DEBIAN_FRONTEND=noninteractive', 'apt', 'install', '-y'), + ) + + PACMAN = PackageManager( + executable='pacman', + command=('pacman', '-S', '--noconfirm'), + ) + + @classmethod + def get_command(cls, name: str) -> Iterable[str]: + """ + :param name: The name of the package manager executable to get the + command for. + :return: The base command to execute, as a sequence of strings. + """ + pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None) + if not pkg_manager: + raise ValueError(f'Unknown package manager: {name}') + + return pkg_manager.value.command + + @classmethod + def scan(cls) -> Optional["PackageManagers"]: + """ + Get the name of the available package manager on the system, if supported. + """ + # pylint: disable=global-statement + global _available_package_manager + if _available_package_manager: + return _available_package_manager + + available_package_managers = [ + pkg_manager + for pkg_manager in cls + if shutil.which(pkg_manager.value.executable) + ] + + if not available_package_managers: + logger.warning( + '\nYour OS does not provide any of the supported package managers.\n' + 'You may have to install some optional dependencies manually.\n' + 'Supported package managers: %s.\n', + ', '.join([pm.value.executable for pm in cls]), + ) + + return None + + _available_package_manager = available_package_managers[0] + return _available_package_manager + + +class InstallContext(Enum): + """ + Supported installation contexts. + """ + + NONE = None + DOCKER = 'docker' + VENV = 'venv' + + +class ManifestType(Enum): """ Manifest types. """ @@ -58,15 +139,32 @@ class Dependencies: """ pip dependencies. """ after: List[str] = field(default_factory=list) """ Commands to execute after the component is installed. """ + pkg_manager: Optional[PackageManagers] = None + """ Override the default package manager detected on the system. """ + install_context: InstallContext = InstallContext.NONE + + @property + def _is_venv(self) -> bool: + """ + :return: True if the dependencies scanning logic is running either in a + virtual environment or in a virtual environment preparation + context. + """ + return ( + self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix + ) @classmethod def from_config( - cls, conf_file: Optional[str] = None, pkg_manager: Optional[str] = None + cls, + conf_file: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, + install_context: InstallContext = InstallContext.NONE, ) -> "Dependencies": """ Parse the required dependencies from a configuration file. """ - deps = cls() + deps = cls(pkg_manager=pkg_manager, install_context=install_context) for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager): deps.before += manifest.install.before @@ -76,26 +174,19 @@ class Dependencies: return deps - def to_pkg_install_commands( - self, pkg_manager: Optional[str] = None, skip_sudo: bool = False - ) -> Generator[str, None, None]: + def to_pkg_install_commands(self) -> Generator[str, None, None]: """ Generates the package manager commands required to install the given dependencies on the system. - - :param pkg_manager: Force package manager to use (default: looks for - the one available on the system). - :param skip_sudo: Skip sudo when installing packages (default: it will - look if the current user is root and use sudo otherwise). """ - wants_sudo = not skip_sudo and os.getuid() != 0 - pkg_manager = pkg_manager or get_available_package_manager() + wants_sudo = self.install_context != InstallContext.DOCKER and os.getuid() != 0 + pkg_manager = self.pkg_manager or PackageManagers.scan() if self.packages and pkg_manager: yield ' '.join( [ *(['sudo'] if wants_sudo else []), - *supported_package_managers[pkg_manager], - *sorted(self.packages), + *pkg_manager.value.command, + *sorted(self.packages), # type: ignore ] ) @@ -104,11 +195,14 @@ class Dependencies: Generates the pip commands required to install the given dependencies on the system. """ - # Recent versions want an explicit --break-system-packages option when - # installing packages via pip outside of a virtual environment - wants_break_system_packages = ( - sys.version_info > (3, 10) - and sys.prefix == sys.base_prefix # We're not in a venv + wants_break_system_packages = not ( + # Docker installations shouldn't require --break-system-packages in pip + self.install_context == InstallContext.DOCKER + # --break-system-packages has been introduced in Python 3.10 + or sys.version_info < (3, 11) + # If we're in a virtual environment then we don't need + # --break-system-packages + or self._is_venv ) if self.pip: @@ -118,24 +212,15 @@ class Dependencies: + ' '.join(sorted(self.pip)) ) - def to_install_commands( - self, pkg_manager: Optional[str] = None, skip_sudo: bool = False - ) -> Generator[str, None, None]: + def to_install_commands(self) -> Generator[str, None, None]: """ Generates the commands required to install the given dependencies on this system. - - :param pkg_manager: Force package manager to use (default: looks for - the one available on the system). - :param skip_sudo: Skip sudo when installing packages (default: it will - look if the current user is root and use sudo otherwise). """ for cmd in self.before: yield cmd - for cmd in self.to_pkg_install_commands( - pkg_manager=pkg_manager, skip_sudo=skip_sudo - ): + for cmd in self.to_pkg_install_commands(): yield cmd for cmd in self.to_pip_install_commands(): @@ -145,7 +230,7 @@ class Dependencies: yield cmd -class Manifest: +class Manifest(ABC): """ Base class for plugin/backend manifests. """ @@ -156,10 +241,10 @@ class Manifest: description: Optional[str] = None, install: Optional[Dict[str, Iterable[str]]] = None, events: Optional[Mapping] = None, - pkg_manager: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, **_, ): - self._pkg_manager = pkg_manager or get_available_package_manager() + self._pkg_manager = pkg_manager or PackageManagers.scan() self.description = description self.install = self._init_deps(install or {}) self.events = self._init_events(events or {}) @@ -167,6 +252,13 @@ class Manifest: self.component_name = '.'.join(package.split('.')[2:]) self.component = None + @property + @abstractmethod + def manifest_type(self) -> ManifestType: + """ + :return: The type of the manifest. + """ + def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies: deps = Dependencies() for key, items in install.items(): @@ -176,7 +268,7 @@ class Manifest: deps.before += items elif key == 'after': deps.after += items - elif key == self._pkg_manager: + elif self._pkg_manager and key == self._pkg_manager.value.executable: deps.packages.update(items) return deps @@ -201,7 +293,9 @@ class Manifest: return ret @classmethod - def from_file(cls, filename: str, pkg_manager: Optional[str] = None) -> "Manifest": + def from_file( + cls, filename: str, pkg_manager: Optional[PackageManagers] = None + ) -> "Manifest": """ Parse a manifest filename into a ``Manifest`` class. """ @@ -210,9 +304,21 @@ class Manifest: assert 'type' in manifest, f'Manifest file {filename} has no type field' comp_type = ManifestType(manifest.pop('type')) - manifest_class = _manifest_class_by_type[comp_type] + manifest_class = cls.by_type(comp_type) return manifest_class(**manifest, pkg_manager=pkg_manager) + @classmethod + def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]: + """ + :return: The manifest class corresponding to the given manifest type. + """ + if manifest_type == ManifestType.PLUGIN: + return PluginManifest + if manifest_type == ManifestType.BACKEND: + return BackendManifest + + raise ValueError(f'Unknown manifest type: {manifest_type}') + def __repr__(self): """ :return: A JSON serialized representation of the manifest. @@ -225,19 +331,23 @@ class Manifest: '.'.join([evt_type.__module__, evt_type.__name__]): doc for evt_type, doc in self.events.items() }, - 'type': _manifest_type_by_class[self.__class__].value, + 'type': self.manifest_type.value, 'package': self.package, 'component_name': self.component_name, } ) -# pylint: disable=too-few-public-methods class PluginManifest(Manifest): """ Plugin manifest. """ + @property + @override + def manifest_type(self) -> ManifestType: + return ManifestType.PLUGIN + # pylint: disable=too-few-public-methods class BackendManifest(Manifest): @@ -245,6 +355,11 @@ class BackendManifest(Manifest): Backend manifest. """ + @property + @override + def manifest_type(self) -> ManifestType: + return ManifestType.BACKEND + class Manifests: """ @@ -253,7 +368,7 @@ class Manifests: @staticmethod def by_base_class( - base_class: Type, pkg_manager: Optional[str] = None + base_class: Type, pkg_manager: Optional[PackageManagers] = None ) -> Generator[Manifest, None, None]: """ Get all the manifest files declared under the base path of a given class @@ -267,7 +382,7 @@ class Manifests: @staticmethod def by_config( conf_file: Optional[str] = None, - pkg_manager: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, ) -> Generator[Manifest, None, None]: """ Get all the manifest objects associated to the extensions declared in a @@ -294,42 +409,3 @@ class Manifests: os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'), pkg_manager=pkg_manager, ) - - -def get_available_package_manager() -> Optional[str]: - """ - Get the name of the available package manager on the system, if supported. - """ - # pylint: disable=global-statement - global _available_package_manager - if _available_package_manager: - return _available_package_manager - - available_package_managers = [ - pkg_manager - for pkg_manager in supported_package_managers - if shutil.which(pkg_manager) - ] - - if not available_package_managers: - logger.warning( - '\nYour OS does not provide any of the supported package managers.\n' - 'You may have to install some optional dependencies manually.\n' - 'Supported package managers: %s.\n', - ', '.join(supported_package_managers.keys()), - ) - - return None - - _available_package_manager = available_package_managers[0] - return _available_package_manager - - -_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = { - ManifestType.PLUGIN: PluginManifest, - ManifestType.BACKEND: BackendManifest, -} - -_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = { - cls: t for t, cls in _manifest_class_by_type.items() -}