Refactored Platydock and Platyvenv.

The two scripts now share the same command interface, behaviour and base
class.

Also, Platydock now builds a Docker image instead of just printing a
Dockerfile, unless the `--print` option is passed.
This commit is contained in:
Fabio Manganiello 2023-08-31 01:16:56 +02:00
parent 429658e7c8
commit 4dd713ffd2
Signed by untrusted user: blacklight
GPG key ID: D90FBA7F76362774
5 changed files with 473 additions and 233 deletions

View file

@ -0,0 +1,3 @@
from ._base import BaseBuilder
__all__ = ["BaseBuilder"]

198
platypush/builder/_base.py Normal file
View file

@ -0,0 +1,198 @@
from abc import ABC, abstractmethod
import argparse
import inspect
import logging
import os
import pathlib
import sys
from typing import Final, Optional, Sequence
from platypush.config import Config
from platypush.utils.manifest import (
Dependencies,
InstallContext,
)
logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger()
class BaseBuilder(ABC):
"""
Base interface and utility methods for Platypush builders.
A Platypush builder is a script/piece of logic that can build a Platypush
installation, with all its base and required extra dependencies, given a
configuration file.
This class is currently implemented by the :module:`platypush.platyvenv`
and :module:`platypush.platydock` modules/scripts.
"""
REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git'
"""
We use the Github URL here rather than the self-hosted Gitea URL to prevent
too many requests to the Gitea server.
"""
def __init__(
self,
cfgfile: str,
gitref: str,
output: str,
install_context: InstallContext,
*_,
verbose: bool = False,
device_id: Optional[str] = None,
**__,
) -> None:
"""
:param cfgfile: The path to the configuration file.
:param gitref: The git reference to use. It can be a branch name, a tag
name or a commit hash.
:param output: The path to the output file or directory.
:param install_context: The installation context for this builder.
:param verbose: Whether to log debug traces.
:param device_id: A device name that will be used to uniquely identify
this installation.
"""
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
self.output = os.path.abspath(os.path.expanduser(output))
self.gitref = gitref
self.install_context = install_context
self.device_id = device_id
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
@classmethod
@abstractmethod
def get_name(cls) -> str:
"""
:return: The name of the builder.
"""
@classmethod
@abstractmethod
def get_description(cls) -> str:
"""
:return: The description of the builder.
"""
@property
def deps(self) -> Dependencies:
"""
:return: The dependencies for this builder, given the configuration
file and the installation context.
"""
return Dependencies.from_config(
self.cfgfile,
install_context=self.install_context,
)
def _print_instructions(self, s: str) -> None:
GREEN = '\033[92m'
NORM = '\033[0m'
helper_lines = s.split('\n')
wrapper_line = '=' * max(len(t) for t in helper_lines)
helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n'
print(GREEN + helper + NORM)
@abstractmethod
def build(self):
"""
Builds the application. To be implemented by the subclasses.
"""
@classmethod
def _get_arg_parser(cls) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog=cls.get_name(),
add_help=False,
description=cls.get_description(),
)
parser.add_argument(
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
)
parser.add_argument(
'-v',
'--verbose',
dest='verbose',
action='store_true',
help='Enable debug traces',
)
parser.add_argument(
'-c',
'--config',
type=str,
dest='cfgfile',
required=False,
default=None,
help='The path to the configuration file. If not specified, a minimal '
'installation including only the base dependencies will be generated.',
)
parser.add_argument(
'-o',
'--output',
dest='output',
type=str,
required=False,
default='.',
help='Target directory (default: current directory). For Platydock, '
'this is the directory where the Dockerfile will be generated. For '
'Platyvenv, this is the base directory of the new virtual '
'environment.',
)
parser.add_argument(
'-d',
'--device-id',
dest='device_id',
type=str,
required=False,
default=None,
help='A name that will be used to uniquely identify this device. '
'Default: a random name for Docker containers, and the '
'hostname of the machine for virtual environments.',
)
parser.add_argument(
'-r',
'--ref',
dest='gitref',
required=False,
type=str,
default='master',
help='If the script is not run from a Platypush installation directory, '
'it will clone the sources via git. You can specify through this '
'option which branch, tag or commit hash to use. Defaults to master.',
)
return parser
@classmethod
def from_cmdline(cls, args: Sequence[str]):
"""
Create a builder instance from command line arguments.
:param args: Command line arguments.
:return: A builder instance.
"""
parser = cls._get_arg_parser()
opts, _ = parser.parse_known_args(args)
if opts.show_usage:
parser.print_help()
sys.exit(0)
if not opts.cfgfile:
opts.cfgfile = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent),
'config.auto.yaml',
)
logger.info('No configuration file specified. Using %s.', opts.cfgfile)
return cls(**vars(opts))

View file

@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace:
"""
Parse command-line arguments from a list of strings.
"""
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(
'platypush',
description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.',
)
parser.add_argument(
'--config',

View file

@ -4,14 +4,19 @@ Dockerfile for Platypush starting from a configuration file.
"""
import argparse
from contextlib import contextmanager
import inspect
import logging
import os
import pathlib
import re
import subprocess
import sys
import textwrap
from typing import Iterable, Sequence
from typing import IO, Generator, Iterable
from typing_extensions import override
from platypush.builder import BaseBuilder
from platypush.config import Config
from platypush.utils.manifest import (
BaseImage,
@ -20,14 +25,12 @@ from platypush.utils.manifest import (
PackageManagers,
)
logger = logging.getLogger()
# pylint: disable=too-few-public-methods
class DockerfileGenerator:
class DockerBuilder(BaseBuilder):
"""
Generate a Dockerfile from on a configuration file.
:param cfgfile: Path to the configuration file.
:param image: The base image to use.
Creates a Platypush Docker image from a configuration file.
"""
_pkg_manager_by_base_image = {
@ -49,7 +52,7 @@ class DockerfileGenerator:
# docker run --rm --name platypush \\
# -v /path/to/your/config/dir:/etc/platypush \\
# -v /path/to/your/workdir:/var/lib/platypush \\
# -p 8080:8080 \\
# -p 8008:8008 \\
# platypush\n
"""
)
@ -61,31 +64,63 @@ class DockerfileGenerator:
"""
)
def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None:
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
def __init__(
self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs
):
kwargs['install_context'] = InstallContext.DOCKER
super().__init__(*args, **kwargs)
self.image = image
self.gitref = gitref
self.tag = tag
self.print_only = print_only # TODO
def generate(self) -> str:
@override
@classmethod
def get_name(cls):
return "platydock"
@override
@classmethod
def get_description(cls):
return "Build a Platypush Docker image from a configuration file."
@property
def dockerfile_dir(self) -> str:
"""
Generate a Dockerfile based on a configuration file.
Proxy property for the output Dockerfile directory.
"""
output = self.output
parent = os.path.dirname(output)
:return: The content of the generated Dockerfile.
if os.path.isfile(output):
return parent
if os.path.isdir(output):
return output
logger.info('%s directory does not exist, creating it', output)
pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True)
return output
@property
def dockerfile(self) -> str:
"""
Proxy property for the output Dockerfile.
"""
return os.path.join(self.dockerfile_dir, 'Dockerfile')
@property
def pkg_manager(self) -> PackageManagers:
"""
Proxy property for the package manager to use.
"""
return self._pkg_manager_by_base_image[self.image]
def _read_base_dockerfile_lines(self) -> Generator[str, None, None]:
"""
:return: The lines of the base Dockerfile.
"""
import platypush
Config.init(self.cfgfile)
new_file_lines = []
ports = self._get_exposed_ports()
pkg_manager = self._pkg_manager_by_base_image[self.image]
deps = Dependencies.from_config(
self.cfgfile,
pkg_manager=pkg_manager,
install_context=InstallContext.DOCKER,
base_image=self.image,
)
is_after_expose_cmd = False
base_file = os.path.join(
str(pathlib.Path(inspect.getfile(platypush)).parent),
'install',
@ -94,40 +129,145 @@ class DockerfileGenerator:
)
with open(base_file, 'r') as f:
file_lines = [line.rstrip() for line in f.readlines()]
for line in f:
yield line.rstrip()
new_file_lines.extend(self._header.split('\n'))
@property
@override
def deps(self) -> Dependencies:
return Dependencies.from_config(
self.cfgfile,
pkg_manager=self.pkg_manager,
install_context=InstallContext.DOCKER,
base_image=self.image,
)
for line in file_lines:
if re.match(
r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh',
line.strip(),
):
new_file_lines.append(self._generate_git_clone_command())
elif line.startswith('RUN cd /install '):
for new_line in deps.before:
new_file_lines.append('RUN ' + new_line)
def _create_dockerfile_parser(self):
"""
Closure for a context-aware parser for the default Dockerfile.
"""
is_after_expose_cmd = False
deps = self.deps
ports = self._get_exposed_ports()
for new_line in deps.to_pkg_install_commands():
new_file_lines.append('RUN ' + new_line)
elif line == 'RUN rm -rf /install':
for new_line in deps.to_pip_install_commands():
new_file_lines.append('RUN ' + new_line)
def parser():
nonlocal is_after_expose_cmd
for new_line in deps.after:
new_file_lines.append('RUN' + new_line)
elif line.startswith('EXPOSE ') and ports:
if not is_after_expose_cmd:
new_file_lines.extend([f'EXPOSE {port}' for port in ports])
is_after_expose_cmd = True
for line in self._read_base_dockerfile_lines():
if re.match(
r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh',
line.strip(),
):
yield self._generate_git_clone_command()
elif line.startswith('RUN cd /install '):
for new_line in deps.before:
yield 'RUN ' + new_line
continue
elif line.startswith('CMD'):
new_file_lines.extend(self._footer.split('\n'))
for new_line in deps.to_pkg_install_commands():
yield 'RUN ' + new_line
elif line == 'RUN rm -rf /install':
for new_line in deps.to_pip_install_commands():
yield 'RUN ' + new_line
new_file_lines.append(line)
for new_line in deps.after:
yield 'RUN' + new_line
elif line.startswith('EXPOSE ') and ports:
if not is_after_expose_cmd:
yield from [f'EXPOSE {port}' for port in ports]
is_after_expose_cmd = True
return '\n'.join(new_file_lines)
continue
elif line.startswith('CMD'):
yield from self._footer.split('\n')
yield line
if line.startswith('CMD') and self.device_id:
yield f'\t--device-id {self.device_id} \\'
return parser
@override
def build(self):
"""
Build a Dockerfile based on a configuration file.
:return: The content of the generated Dockerfile.
"""
# Set the DOCKER_CTX environment variable so any downstream logic knows
# that we're running in a Docker build context.
os.environ['DOCKER_CTX'] = '1'
self._generate_dockerfile()
if self.print_only:
return
self._build_image()
self._print_instructions(
textwrap.dedent(
f"""
A Docker image has been built from the configuration file {self.cfgfile}.
The Dockerfile is available under {self.dockerfile}.
You can run the Docker image with the following command:
docker run \\
--rm --name platypush \\
-v {os.path.dirname(self.cfgfile)}:/etc/platypush \\
-v /path/to/your/workdir:/var/lib/platypush \\
-p 8008:8008 \\
platypush
"""
)
)
def _build_image(self):
"""
Build a Platypush Docker image from the generated Dockerfile.
"""
logger.info('Building Docker image...')
cmd = [
'docker',
'build',
'-f',
self.dockerfile,
'-t',
self.tag,
'.',
]
subprocess.run(cmd, check=True)
def _generate_dockerfile(self):
"""
Parses the configuration file and generates a Dockerfile based on it.
"""
@contextmanager
def open_writer() -> Generator[IO, None, None]:
# flake8: noqa
f = sys.stdout if self.print_only else open(self.dockerfile, 'w')
try:
yield f
finally:
if f is not sys.stdout:
f.close()
if not self.print_only:
logger.info('Parsing configuration file %s...', self.cfgfile)
Config.init(self.cfgfile)
if not self.print_only:
logger.info('Generating Dockerfile %s...', self.dockerfile)
parser = self._create_dockerfile_parser()
with open_writer() as f:
f.write(self._header + '\n')
for line in parser():
f.write(line + '\n')
def _generate_git_clone_command(self) -> str:
"""
@ -135,16 +275,15 @@ class DockerfileGenerator:
and the right git reference, if the application sources aren't already
available under /install.
"""
pkg_manager = self._pkg_manager_by_base_image[self.image]
install_cmd = ' '.join(pkg_manager.value.install)
uninstall_cmd = ' '.join(pkg_manager.value.uninstall)
install_cmd = ' '.join(self.pkg_manager.value.install)
uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall)
return textwrap.dedent(
f"""
RUN if [ ! -f "/install/setup.py" ]; then \\
echo "Platypush source not found under the current directory, downloading it" && \\
{install_cmd} git && \\
rm -rf /install && \\
git clone https://github.com/BlackLight/platypush.git /install && \\
git clone --recursive https://github.com/BlackLight/platypush.git /install && \\
cd /install && \\
git checkout {self.gitref} && \\
{uninstall_cmd} git; \\
@ -153,71 +292,41 @@ class DockerfileGenerator:
)
@classmethod
def from_cmdline(cls, args: Sequence[str]) -> 'DockerfileGenerator':
"""
Create a DockerfileGenerator instance from command line arguments.
:param args: Command line arguments.
:return: A DockerfileGenerator instance.
"""
parser = argparse.ArgumentParser(
prog='platydock',
add_help=False,
description='Create a Platypush Dockerfile from a config.yaml.',
)
@override
def _get_arg_parser(cls) -> argparse.ArgumentParser:
parser = super()._get_arg_parser()
parser.add_argument(
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
)
parser.add_argument(
'cfgfile',
type=str,
nargs='?',
help='The path to the configuration file. If not specified a minimal '
'Dockerfile with no extra dependencies will be generated.',
)
parser.add_argument(
'--image',
'-i',
'--image',
dest='image',
required=False,
type=BaseImage,
choices=list(BaseImage),
default=BaseImage.ALPINE,
help='Base image to use for the Dockerfile.',
help='Base image to use for the Dockerfile (default: alpine).',
)
parser.add_argument(
'--ref',
'-r',
dest='gitref',
'-t',
'--tag',
dest='tag',
required=False,
type=str,
default='master',
help='If platydock is not run from a Platypush installation directory, '
'it will clone the source via git. You can specify through this '
'option which branch, tag or commit hash to use. Defaults to master.',
default='platypush:latest',
help='Tag name to be used for the built image '
'(default: "platypush:latest").',
)
opts, _ = parser.parse_known_args(args)
if opts.show_usage:
parser.print_help()
sys.exit(0)
parser.add_argument(
'--print',
dest='print_only',
action='store_true',
help='Use this flag if you only want to print the Dockerfile to '
'stdout instead of generating an image.',
)
if not opts.cfgfile:
opts.cfgfile = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent),
'config.auto.yaml',
)
print(
f'No configuration file specified. Using {opts.cfgfile}.',
file=sys.stderr,
)
return cls(opts.cfgfile, image=opts.image, gitref=opts.gitref)
return parser
@staticmethod
def _get_exposed_ports() -> Iterable[int]:
@ -240,7 +349,7 @@ def main():
"""
Generates a Dockerfile based on the configuration file.
"""
print(DockerfileGenerator.from_cmdline(sys.argv[1:]).generate())
DockerBuilder.from_cmdline(sys.argv[1:]).build()
return 0

View file

@ -3,11 +3,9 @@ Platyvenv is a helper script that allows you to automatically create a
virtual environment for Platypush starting from a configuration file.
"""
import argparse
from contextlib import contextmanager
import inspect
import logging
import os
import pathlib
import re
import shutil
import subprocess
@ -17,26 +15,36 @@ import textwrap
from typing import Generator, Sequence
import venv
from typing_extensions import override
from platypush.builder import BaseBuilder
from platypush.config import Config
from platypush.utils.manifest import (
Dependencies,
InstallContext,
)
logger = logging.getLogger()
# pylint: disable=too-few-public-methods
class VenvBuilder:
class VenvBuilder(BaseBuilder):
"""
Build a virtual environment from on a configuration file.
:param cfgfile: Path to the configuration file.
:param image: The base image to use.
"""
def __init__(self, cfgfile: str, gitref: str, output_dir: str) -> None:
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
self.output_dir = os.path.abspath(os.path.expanduser(output_dir))
self.gitref = gitref
def __init__(self, *args, **kwargs) -> None:
kwargs['install_context'] = InstallContext.DOCKER
super().__init__(*args, **kwargs)
@override
@classmethod
def get_name(cls):
return "platyvenv"
@override
@classmethod
def get_description(cls):
return "Build a Platypush virtual environment from a configuration file."
@property
def _pip_cmd(self) -> Sequence[str]:
@ -44,7 +52,7 @@ class VenvBuilder:
:return: The pip install command to use for the selected environment.
"""
return (
os.path.join(self.output_dir, 'bin', 'python'),
os.path.join(self.output, 'bin', 'python'),
'-m',
'pip',
'install',
@ -58,7 +66,7 @@ class VenvBuilder:
Install the required system packages.
"""
for cmd in deps.to_pkg_install_commands():
print(f'Installing system packages: {cmd}')
logger.info('Installing system packages: %s', cmd)
subprocess.call(re.split(r'\s+', cmd.strip()))
@contextmanager
@ -74,11 +82,11 @@ class VenvBuilder:
"""
setup_py_path = os.path.join(os.getcwd(), 'setup.py')
if os.path.isfile(setup_py_path):
print('Using local checkout of the Platypush source code')
logger.info('Using local checkout of the Platypush source code')
yield os.getcwd()
else:
checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git')
print(f'Cloning Platypush source code from git into {checkout_dir}')
logger.info('Cloning Platypush source code from git into %s', checkout_dir)
subprocess.call(
[
'git',
@ -95,72 +103,42 @@ class VenvBuilder:
yield checkout_dir
os.chdir(pwd)
print(f'Cleaning up {checkout_dir}')
logger.info('Cleaning up %s', checkout_dir)
shutil.rmtree(checkout_dir, ignore_errors=True)
def _prepare_venv(self) -> None:
"""
Installs the virtual environment under the configured output_dir.
Install the virtual environment under the configured output.
"""
print(f'Creating virtual environment under {self.output_dir}...')
logger.info('Creating virtual environment under %s...', self.output)
venv.create(
self.output_dir,
self.output,
symlinks=True,
with_pip=True,
upgrade_deps=True,
)
print(
f'Installing base Python dependencies under {self.output_dir}...',
)
logger.info('Installing base Python dependencies under %s...', self.output)
subprocess.call([*self._pip_cmd, 'pip', '.'])
def _install_extra_pip_packages(self, deps: Dependencies):
"""
Install the extra pip dependencies parsed through the
Install the extra pip dependencies inferred from the configured
extensions.
"""
pip_deps = list(deps.to_pip_install_commands(full_command=False))
if not pip_deps:
return
print(
f'Installing extra pip dependencies under {self.output_dir}: '
+ ' '.join(pip_deps)
logger.info(
'Installing extra pip dependencies under %s: %s',
self.output,
' '.join(pip_deps),
)
subprocess.call([*self._pip_cmd, *pip_deps])
def _generate_run_sh(self) -> str:
"""
Generate a ``run.sh`` script to run the application from a newly built
virtual environment.
:return: The location of the generated ``run.sh`` script.
"""
run_sh_path = os.path.join(self.output_dir, 'bin', 'run.sh')
with open(run_sh_path, 'w') as run_sh:
run_sh.write(
textwrap.dedent(
f"""
#!/bin/bash
cd {self.output_dir}
# Activate the virtual environment
source bin/activate
# Run the application with the configuration file passed
# during build
platypush -c {self.cfgfile} $*
"""
)
)
os.chmod(run_sh_path, 0o750)
return run_sh_path
def build(self):
"""
Build a Dockerfile based on a configuration file.
@ -178,78 +156,26 @@ class VenvBuilder:
self._prepare_venv()
self._install_extra_pip_packages(deps)
run_sh_path = self._generate_run_sh()
print(
f'\nVirtual environment created under {self.output_dir}.\n'
f'You can run the application through the {run_sh_path} script.\n'
)
self._print_instructions(
textwrap.dedent(
f"""
Virtual environment created under {self.output}.
To run the application:
@classmethod
def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder':
"""
Create a DockerfileGenerator instance from command line arguments.
source {self.output}/bin/activate
platypush -c {self.cfgfile} {
"--device_id " + self.device_id if self.device_id else ""
}
:param args: Command line arguments.
:return: A DockerfileGenerator instance.
"""
parser = argparse.ArgumentParser(
prog='platyvenv',
add_help=False,
description='Create a Platypush virtual environment from a config.yaml.',
)
Platypush requires a Redis instance. If you don't want to use a
stand-alone server, you can pass the --start-redis option to
the executable (optionally with --redis-port).
parser.add_argument(
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
)
parser.add_argument(
'cfgfile',
type=str,
nargs='?',
help='The path to the configuration file. If not specified a minimal '
'virtual environment only with the base dependencies will be '
'generated.',
)
parser.add_argument(
'-o',
'--output',
dest='output_dir',
type=str,
required=False,
default='venv',
help='Target directory for the virtual environment (default: ./venv)',
)
parser.add_argument(
'--ref',
'-r',
dest='gitref',
required=False,
type=str,
default='master',
help='If platyvenv is not run from a Platypush installation directory, '
'it will clone the sources via git. You can specify through this '
'option which branch, tag or commit hash to use. Defaults to master.',
)
opts, _ = parser.parse_known_args(args)
if opts.show_usage:
parser.print_help()
sys.exit(0)
if not opts.cfgfile:
opts.cfgfile = os.path.join(
str(pathlib.Path(inspect.getfile(Config)).parent),
'config.auto.yaml',
Platypush will then start its own local instance and it will
terminate it once the application is stopped.
"""
)
print(
f'No configuration file specified. Using {opts.cfgfile}.',
file=sys.stderr,
)
return cls(opts.cfgfile, gitref=opts.gitref, output_dir=opts.output_dir)
)
def main():
@ -257,6 +183,7 @@ def main():
Generates a virtual environment based on the configuration file.
"""
VenvBuilder.from_cmdline(sys.argv[1:]).build()
return 0
if __name__ == '__main__':