forked from platypush/platypush
Refactored the interface of Platydock and manifest utils.
This commit is contained in:
parent
a99ffea37c
commit
71c5291190
GPG key ID:
D90FBA7F76362774
2 changed files with 285 additions and 151 deletions
|
|
@ -4,6 +4,7 @@ Dockerfile for Platypush starting from a configuration file.
|
|||
"""
|
||||
|
||||
import argparse
|
||||
from enum import Enum
|
||||
import inspect
|
||||
import os
|
||||
import pathlib
|
||||
|
|
@ -11,26 +12,63 @@ import sys
|
|||
from typing import Iterable
|
||||
|
||||
from platypush.config import Config
|
||||
from platypush.utils.manifest import Dependencies
|
||||
|
||||
ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m '
|
||||
ERR_SUFFIX = '\033[0m'
|
||||
from platypush.utils.manifest import Dependencies, InstallContext, PackageManagers
|
||||
|
||||
|
||||
def generate_dockerfile(cfgfile: str) -> str:
|
||||
class BaseImage(Enum):
|
||||
"""
|
||||
Supported base images for Dockerfiles.
|
||||
"""
|
||||
|
||||
ALPINE = 'alpine'
|
||||
UBUNTU = 'ubuntu'
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""
|
||||
Explicit __str__ override for argparse purposes.
|
||||
"""
|
||||
return self.value
|
||||
|
||||
|
||||
class DockerfileGenerator:
|
||||
"""
|
||||
Generate a Dockerfile from on a configuration file.
|
||||
|
||||
:param cfgfile: Path to the configuration file.
|
||||
:param image: The base image to use.
|
||||
"""
|
||||
|
||||
_pkg_manager_by_base_image = {
|
||||
BaseImage.ALPINE: PackageManagers.APK,
|
||||
BaseImage.UBUNTU: PackageManagers.APT,
|
||||
}
|
||||
|
||||
def __init__(self, cfgfile: str, image: BaseImage) -> None:
|
||||
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
|
||||
self.image = image
|
||||
|
||||
def generate(self) -> str:
|
||||
"""
|
||||
Generate a Dockerfile based on a configuration file.
|
||||
|
||||
:param cfgfile: Path to the configuration file.
|
||||
:return: The content of the generated Dockerfile.
|
||||
"""
|
||||
Config.init(cfgfile)
|
||||
Config.init(self.cfgfile)
|
||||
new_file_lines = []
|
||||
ports = _get_exposed_ports()
|
||||
deps = Dependencies.from_config(cfgfile, pkg_manager='apk')
|
||||
ports = self._get_exposed_ports()
|
||||
pkg_manager = self._pkg_manager_by_base_image[self.image]
|
||||
deps = Dependencies.from_config(
|
||||
self.cfgfile,
|
||||
pkg_manager=pkg_manager,
|
||||
install_context=InstallContext.DOCKER,
|
||||
)
|
||||
|
||||
is_after_expose_cmd = False
|
||||
base_file = os.path.join(
|
||||
str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile'
|
||||
str(pathlib.Path(inspect.getfile(Config)).parent),
|
||||
'docker',
|
||||
f'{self.image}.Dockerfile',
|
||||
)
|
||||
|
||||
with open(base_file, 'r') as f:
|
||||
|
|
@ -41,9 +79,7 @@ def generate_dockerfile(cfgfile: str) -> str:
|
|||
for new_line in deps.before:
|
||||
new_file_lines.append('RUN ' + new_line)
|
||||
|
||||
for new_line in deps.to_pkg_install_commands(
|
||||
pkg_manager='apk', skip_sudo=True
|
||||
):
|
||||
for new_line in deps.to_pkg_install_commands():
|
||||
new_file_lines.append('RUN ' + new_line)
|
||||
elif line == 'RUN rm -rf /install':
|
||||
for new_line in deps.to_pip_install_commands():
|
||||
|
|
@ -62,8 +98,8 @@ def generate_dockerfile(cfgfile: str) -> str:
|
|||
|
||||
return '\n'.join(new_file_lines)
|
||||
|
||||
|
||||
def _get_exposed_ports() -> Iterable[int]:
|
||||
@staticmethod
|
||||
def _get_exposed_ports() -> Iterable[int]:
|
||||
"""
|
||||
:return: The listen ports used by the backends enabled in the configuration
|
||||
file.
|
||||
|
|
@ -87,22 +123,44 @@ def main():
|
|||
prog='platydock',
|
||||
add_help=False,
|
||||
description='Create a Platypush Dockerfile from a config.yaml.',
|
||||
epilog='Use platydock <action> --help to get additional help.',
|
||||
)
|
||||
|
||||
parser.add_argument('-h', '--help', action='store_true', help='Show usage')
|
||||
parser.add_argument(
|
||||
'cfgfile', type=str, nargs=1, help='The path to the configuration file.'
|
||||
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
|
||||
)
|
||||
parser.add_argument(
|
||||
'cfgfile', type=str, nargs='?', help='The path to the configuration file.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--image',
|
||||
'-i',
|
||||
dest='image',
|
||||
required=False,
|
||||
type=BaseImage,
|
||||
choices=list(BaseImage),
|
||||
default=BaseImage.ALPINE,
|
||||
help='Base image to use for the Dockerfile.',
|
||||
)
|
||||
|
||||
opts, _ = parser.parse_known_args(sys.argv[1:])
|
||||
cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0]))
|
||||
dockerfile = generate_dockerfile(cfgfile)
|
||||
if opts.show_usage:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
if not opts.cfgfile:
|
||||
print(
|
||||
f'Please specify a configuration file.\nRun {sys.argv[0]} --help to get the available options.',
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate()
|
||||
print(dockerfile)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
# vim:sw=4:ts=4:et:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
import enum
|
||||
from enum import Enum
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -20,22 +21,102 @@ from typing import (
|
|||
Type,
|
||||
Union,
|
||||
)
|
||||
from typing_extensions import override
|
||||
|
||||
import yaml
|
||||
|
||||
from platypush.message.event import Event
|
||||
|
||||
supported_package_managers = {
|
||||
'apk': ['apk', 'add', '--update', '--no-interactive', '--no-cache'],
|
||||
'apt': ['apt', 'install', '-y'],
|
||||
'pacman': ['pacman', '-S', '--noconfirm'],
|
||||
}
|
||||
|
||||
_available_package_manager = None
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManifestType(enum.Enum):
|
||||
@dataclass
|
||||
class PackageManager:
|
||||
"""
|
||||
Representation of a package manager.
|
||||
"""
|
||||
|
||||
executable: str
|
||||
""" The executable name. """
|
||||
command: Iterable[str] = field(default_factory=tuple)
|
||||
""" The command to execute, as a sequence of strings. """
|
||||
|
||||
|
||||
class PackageManagers(Enum):
|
||||
"""
|
||||
Supported package managers.
|
||||
"""
|
||||
|
||||
APK = PackageManager(
|
||||
executable='apk',
|
||||
command=('apk', 'add', '--update', '--no-interactive', '--no-cache'),
|
||||
)
|
||||
|
||||
APT = PackageManager(
|
||||
executable='apt',
|
||||
command=('DEBIAN_FRONTEND=noninteractive', 'apt', 'install', '-y'),
|
||||
)
|
||||
|
||||
PACMAN = PackageManager(
|
||||
executable='pacman',
|
||||
command=('pacman', '-S', '--noconfirm'),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_command(cls, name: str) -> Iterable[str]:
|
||||
"""
|
||||
:param name: The name of the package manager executable to get the
|
||||
command for.
|
||||
:return: The base command to execute, as a sequence of strings.
|
||||
"""
|
||||
pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None)
|
||||
if not pkg_manager:
|
||||
raise ValueError(f'Unknown package manager: {name}')
|
||||
|
||||
return pkg_manager.value.command
|
||||
|
||||
@classmethod
|
||||
def scan(cls) -> Optional["PackageManagers"]:
|
||||
"""
|
||||
Get the name of the available package manager on the system, if supported.
|
||||
"""
|
||||
# pylint: disable=global-statement
|
||||
global _available_package_manager
|
||||
if _available_package_manager:
|
||||
return _available_package_manager
|
||||
|
||||
available_package_managers = [
|
||||
pkg_manager
|
||||
for pkg_manager in cls
|
||||
if shutil.which(pkg_manager.value.executable)
|
||||
]
|
||||
|
||||
if not available_package_managers:
|
||||
logger.warning(
|
||||
'\nYour OS does not provide any of the supported package managers.\n'
|
||||
'You may have to install some optional dependencies manually.\n'
|
||||
'Supported package managers: %s.\n',
|
||||
', '.join([pm.value.executable for pm in cls]),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
_available_package_manager = available_package_managers[0]
|
||||
return _available_package_manager
|
||||
|
||||
|
||||
class InstallContext(Enum):
|
||||
"""
|
||||
Supported installation contexts.
|
||||
"""
|
||||
|
||||
NONE = None
|
||||
DOCKER = 'docker'
|
||||
VENV = 'venv'
|
||||
|
||||
|
||||
class ManifestType(Enum):
|
||||
"""
|
||||
Manifest types.
|
||||
"""
|
||||
|
|
@ -58,15 +139,32 @@ class Dependencies:
|
|||
""" pip dependencies. """
|
||||
after: List[str] = field(default_factory=list)
|
||||
""" Commands to execute after the component is installed. """
|
||||
pkg_manager: Optional[PackageManagers] = None
|
||||
""" Override the default package manager detected on the system. """
|
||||
install_context: InstallContext = InstallContext.NONE
|
||||
|
||||
@property
|
||||
def _is_venv(self) -> bool:
|
||||
"""
|
||||
:return: True if the dependencies scanning logic is running either in a
|
||||
virtual environment or in a virtual environment preparation
|
||||
context.
|
||||
"""
|
||||
return (
|
||||
self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls, conf_file: Optional[str] = None, pkg_manager: Optional[str] = None
|
||||
cls,
|
||||
conf_file: Optional[str] = None,
|
||||
pkg_manager: Optional[PackageManagers] = None,
|
||||
install_context: InstallContext = InstallContext.NONE,
|
||||
) -> "Dependencies":
|
||||
"""
|
||||
Parse the required dependencies from a configuration file.
|
||||
"""
|
||||
deps = cls()
|
||||
deps = cls(pkg_manager=pkg_manager, install_context=install_context)
|
||||
|
||||
for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager):
|
||||
deps.before += manifest.install.before
|
||||
|
|
@ -76,26 +174,19 @@ class Dependencies:
|
|||
|
||||
return deps
|
||||
|
||||
def to_pkg_install_commands(
|
||||
self, pkg_manager: Optional[str] = None, skip_sudo: bool = False
|
||||
) -> Generator[str, None, None]:
|
||||
def to_pkg_install_commands(self) -> Generator[str, None, None]:
|
||||
"""
|
||||
Generates the package manager commands required to install the given
|
||||
dependencies on the system.
|
||||
|
||||
:param pkg_manager: Force package manager to use (default: looks for
|
||||
the one available on the system).
|
||||
:param skip_sudo: Skip sudo when installing packages (default: it will
|
||||
look if the current user is root and use sudo otherwise).
|
||||
"""
|
||||
wants_sudo = not skip_sudo and os.getuid() != 0
|
||||
pkg_manager = pkg_manager or get_available_package_manager()
|
||||
wants_sudo = self.install_context != InstallContext.DOCKER and os.getuid() != 0
|
||||
pkg_manager = self.pkg_manager or PackageManagers.scan()
|
||||
if self.packages and pkg_manager:
|
||||
yield ' '.join(
|
||||
[
|
||||
*(['sudo'] if wants_sudo else []),
|
||||
*supported_package_managers[pkg_manager],
|
||||
*sorted(self.packages),
|
||||
*pkg_manager.value.command,
|
||||
*sorted(self.packages), # type: ignore
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -104,11 +195,14 @@ class Dependencies:
|
|||
Generates the pip commands required to install the given dependencies on
|
||||
the system.
|
||||
"""
|
||||
# Recent versions want an explicit --break-system-packages option when
|
||||
# installing packages via pip outside of a virtual environment
|
||||
wants_break_system_packages = (
|
||||
sys.version_info > (3, 10)
|
||||
and sys.prefix == sys.base_prefix # We're not in a venv
|
||||
wants_break_system_packages = not (
|
||||
# Docker installations shouldn't require --break-system-packages in pip
|
||||
self.install_context == InstallContext.DOCKER
|
||||
# --break-system-packages has been introduced in Python 3.10
|
||||
or sys.version_info < (3, 11)
|
||||
# If we're in a virtual environment then we don't need
|
||||
# --break-system-packages
|
||||
or self._is_venv
|
||||
)
|
||||
|
||||
if self.pip:
|
||||
|
|
@ -118,24 +212,15 @@ class Dependencies:
|
|||
+ ' '.join(sorted(self.pip))
|
||||
)
|
||||
|
||||
def to_install_commands(
|
||||
self, pkg_manager: Optional[str] = None, skip_sudo: bool = False
|
||||
) -> Generator[str, None, None]:
|
||||
def to_install_commands(self) -> Generator[str, None, None]:
|
||||
"""
|
||||
Generates the commands required to install the given dependencies on
|
||||
this system.
|
||||
|
||||
:param pkg_manager: Force package manager to use (default: looks for
|
||||
the one available on the system).
|
||||
:param skip_sudo: Skip sudo when installing packages (default: it will
|
||||
look if the current user is root and use sudo otherwise).
|
||||
"""
|
||||
for cmd in self.before:
|
||||
yield cmd
|
||||
|
||||
for cmd in self.to_pkg_install_commands(
|
||||
pkg_manager=pkg_manager, skip_sudo=skip_sudo
|
||||
):
|
||||
for cmd in self.to_pkg_install_commands():
|
||||
yield cmd
|
||||
|
||||
for cmd in self.to_pip_install_commands():
|
||||
|
|
@ -145,7 +230,7 @@ class Dependencies:
|
|||
yield cmd
|
||||
|
||||
|
||||
class Manifest:
|
||||
class Manifest(ABC):
|
||||
"""
|
||||
Base class for plugin/backend manifests.
|
||||
"""
|
||||
|
|
@ -156,10 +241,10 @@ class Manifest:
|
|||
description: Optional[str] = None,
|
||||
install: Optional[Dict[str, Iterable[str]]] = None,
|
||||
events: Optional[Mapping] = None,
|
||||
pkg_manager: Optional[str] = None,
|
||||
pkg_manager: Optional[PackageManagers] = None,
|
||||
**_,
|
||||
):
|
||||
self._pkg_manager = pkg_manager or get_available_package_manager()
|
||||
self._pkg_manager = pkg_manager or PackageManagers.scan()
|
||||
self.description = description
|
||||
self.install = self._init_deps(install or {})
|
||||
self.events = self._init_events(events or {})
|
||||
|
|
@ -167,6 +252,13 @@ class Manifest:
|
|||
self.component_name = '.'.join(package.split('.')[2:])
|
||||
self.component = None
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def manifest_type(self) -> ManifestType:
|
||||
"""
|
||||
:return: The type of the manifest.
|
||||
"""
|
||||
|
||||
def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies:
|
||||
deps = Dependencies()
|
||||
for key, items in install.items():
|
||||
|
|
@ -176,7 +268,7 @@ class Manifest:
|
|||
deps.before += items
|
||||
elif key == 'after':
|
||||
deps.after += items
|
||||
elif key == self._pkg_manager:
|
||||
elif self._pkg_manager and key == self._pkg_manager.value.executable:
|
||||
deps.packages.update(items)
|
||||
|
||||
return deps
|
||||
|
|
@ -201,7 +293,9 @@ class Manifest:
|
|||
return ret
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, filename: str, pkg_manager: Optional[str] = None) -> "Manifest":
|
||||
def from_file(
|
||||
cls, filename: str, pkg_manager: Optional[PackageManagers] = None
|
||||
) -> "Manifest":
|
||||
"""
|
||||
Parse a manifest filename into a ``Manifest`` class.
|
||||
"""
|
||||
|
|
@ -210,9 +304,21 @@ class Manifest:
|
|||
|
||||
assert 'type' in manifest, f'Manifest file {filename} has no type field'
|
||||
comp_type = ManifestType(manifest.pop('type'))
|
||||
manifest_class = _manifest_class_by_type[comp_type]
|
||||
manifest_class = cls.by_type(comp_type)
|
||||
return manifest_class(**manifest, pkg_manager=pkg_manager)
|
||||
|
||||
@classmethod
|
||||
def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]:
|
||||
"""
|
||||
:return: The manifest class corresponding to the given manifest type.
|
||||
"""
|
||||
if manifest_type == ManifestType.PLUGIN:
|
||||
return PluginManifest
|
||||
if manifest_type == ManifestType.BACKEND:
|
||||
return BackendManifest
|
||||
|
||||
raise ValueError(f'Unknown manifest type: {manifest_type}')
|
||||
|
||||
def __repr__(self):
|
||||
"""
|
||||
:return: A JSON serialized representation of the manifest.
|
||||
|
|
@ -225,19 +331,23 @@ class Manifest:
|
|||
'.'.join([evt_type.__module__, evt_type.__name__]): doc
|
||||
for evt_type, doc in self.events.items()
|
||||
},
|
||||
'type': _manifest_type_by_class[self.__class__].value,
|
||||
'type': self.manifest_type.value,
|
||||
'package': self.package,
|
||||
'component_name': self.component_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class PluginManifest(Manifest):
|
||||
"""
|
||||
Plugin manifest.
|
||||
"""
|
||||
|
||||
@property
|
||||
@override
|
||||
def manifest_type(self) -> ManifestType:
|
||||
return ManifestType.PLUGIN
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class BackendManifest(Manifest):
|
||||
|
|
@ -245,6 +355,11 @@ class BackendManifest(Manifest):
|
|||
Backend manifest.
|
||||
"""
|
||||
|
||||
@property
|
||||
@override
|
||||
def manifest_type(self) -> ManifestType:
|
||||
return ManifestType.BACKEND
|
||||
|
||||
|
||||
class Manifests:
|
||||
"""
|
||||
|
|
@ -253,7 +368,7 @@ class Manifests:
|
|||
|
||||
@staticmethod
|
||||
def by_base_class(
|
||||
base_class: Type, pkg_manager: Optional[str] = None
|
||||
base_class: Type, pkg_manager: Optional[PackageManagers] = None
|
||||
) -> Generator[Manifest, None, None]:
|
||||
"""
|
||||
Get all the manifest files declared under the base path of a given class
|
||||
|
|
@ -267,7 +382,7 @@ class Manifests:
|
|||
@staticmethod
|
||||
def by_config(
|
||||
conf_file: Optional[str] = None,
|
||||
pkg_manager: Optional[str] = None,
|
||||
pkg_manager: Optional[PackageManagers] = None,
|
||||
) -> Generator[Manifest, None, None]:
|
||||
"""
|
||||
Get all the manifest objects associated to the extensions declared in a
|
||||
|
|
@ -294,42 +409,3 @@ class Manifests:
|
|||
os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'),
|
||||
pkg_manager=pkg_manager,
|
||||
)
|
||||
|
||||
|
||||
def get_available_package_manager() -> Optional[str]:
|
||||
"""
|
||||
Get the name of the available package manager on the system, if supported.
|
||||
"""
|
||||
# pylint: disable=global-statement
|
||||
global _available_package_manager
|
||||
if _available_package_manager:
|
||||
return _available_package_manager
|
||||
|
||||
available_package_managers = [
|
||||
pkg_manager
|
||||
for pkg_manager in supported_package_managers
|
||||
if shutil.which(pkg_manager)
|
||||
]
|
||||
|
||||
if not available_package_managers:
|
||||
logger.warning(
|
||||
'\nYour OS does not provide any of the supported package managers.\n'
|
||||
'You may have to install some optional dependencies manually.\n'
|
||||
'Supported package managers: %s.\n',
|
||||
', '.join(supported_package_managers.keys()),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
_available_package_manager = available_package_managers[0]
|
||||
return _available_package_manager
|
||||
|
||||
|
||||
_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = {
|
||||
ManifestType.PLUGIN: PluginManifest,
|
||||
ManifestType.BACKEND: BackendManifest,
|
||||
}
|
||||
|
||||
_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = {
|
||||
cls: t for t, cls in _manifest_class_by_type.items()
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue