diff --git a/platypush/builder/__init__.py b/platypush/builder/__init__.py new file mode 100644 index 0000000000..6be2232a0a --- /dev/null +++ b/platypush/builder/__init__.py @@ -0,0 +1,3 @@ +from ._base import BaseBuilder + +__all__ = ["BaseBuilder"] diff --git a/platypush/builder/_base.py b/platypush/builder/_base.py new file mode 100644 index 0000000000..fe228fb258 --- /dev/null +++ b/platypush/builder/_base.py @@ -0,0 +1,198 @@ +from abc import ABC, abstractmethod +import argparse +import inspect +import logging +import os +import pathlib +import sys +from typing import Final, Optional, Sequence + +from platypush.config import Config +from platypush.utils.manifest import ( + Dependencies, + InstallContext, +) + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger() + + +class BaseBuilder(ABC): + """ + Base interface and utility methods for Platypush builders. + + A Platypush builder is a script/piece of logic that can build a Platypush + installation, with all its base and required extra dependencies, given a + configuration file. + + This class is currently implemented by the :module:`platypush.platyvenv` + and :module:`platypush.platydock` modules/scripts. + """ + + REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git' + """ + We use the Github URL here rather than the self-hosted Gitea URL to prevent + too many requests to the Gitea server. + """ + + def __init__( + self, + cfgfile: str, + gitref: str, + output: str, + install_context: InstallContext, + *_, + verbose: bool = False, + device_id: Optional[str] = None, + **__, + ) -> None: + """ + :param cfgfile: The path to the configuration file. + :param gitref: The git reference to use. It can be a branch name, a tag + name or a commit hash. + :param output: The path to the output file or directory. + :param install_context: The installation context for this builder. + :param verbose: Whether to log debug traces. + :param device_id: A device name that will be used to uniquely identify + this installation. + """ + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.output = os.path.abspath(os.path.expanduser(output)) + self.gitref = gitref + self.install_context = install_context + self.device_id = device_id + logger.setLevel(logging.DEBUG if verbose else logging.INFO) + + @classmethod + @abstractmethod + def get_name(cls) -> str: + """ + :return: The name of the builder. + """ + + @classmethod + @abstractmethod + def get_description(cls) -> str: + """ + :return: The description of the builder. + """ + + @property + def deps(self) -> Dependencies: + """ + :return: The dependencies for this builder, given the configuration + file and the installation context. + """ + return Dependencies.from_config( + self.cfgfile, + install_context=self.install_context, + ) + + def _print_instructions(self, s: str) -> None: + GREEN = '\033[92m' + NORM = '\033[0m' + + helper_lines = s.split('\n') + wrapper_line = '=' * max(len(t) for t in helper_lines) + helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n' + print(GREEN + helper + NORM) + + @abstractmethod + def build(self): + """ + Builds the application. To be implemented by the subclasses. + """ + + @classmethod + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=cls.get_name(), + add_help=False, + description=cls.get_description(), + ) + + parser.add_argument( + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + + parser.add_argument( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='Enable debug traces', + ) + + parser.add_argument( + '-c', + '--config', + type=str, + dest='cfgfile', + required=False, + default=None, + help='The path to the configuration file. If not specified, a minimal ' + 'installation including only the base dependencies will be generated.', + ) + + parser.add_argument( + '-o', + '--output', + dest='output', + type=str, + required=False, + default='.', + help='Target directory (default: current directory). For Platydock, ' + 'this is the directory where the Dockerfile will be generated. For ' + 'Platyvenv, this is the base directory of the new virtual ' + 'environment.', + ) + + parser.add_argument( + '-d', + '--device-id', + dest='device_id', + type=str, + required=False, + default=None, + help='A name that will be used to uniquely identify this device. ' + 'Default: a random name for Docker containers, and the ' + 'hostname of the machine for virtual environments.', + ) + + parser.add_argument( + '-r', + '--ref', + dest='gitref', + required=False, + type=str, + default='master', + help='If the script is not run from a Platypush installation directory, ' + 'it will clone the sources via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + + return parser + + @classmethod + def from_cmdline(cls, args: Sequence[str]): + """ + Create a builder instance from command line arguments. + + :param args: Command line arguments. + :return: A builder instance. + """ + parser = cls._get_arg_parser() + opts, _ = parser.parse_known_args(args) + if opts.show_usage: + parser.print_help() + sys.exit(0) + + if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.auto.yaml', + ) + + logger.info('No configuration file specified. Using %s.', opts.cfgfile) + + return cls(**vars(opts)) diff --git a/platypush/cli.py b/platypush/cli.py index f3768b199c..b9bec5564c 100644 --- a/platypush/cli.py +++ b/platypush/cli.py @@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace: """ Parse command-line arguments from a list of strings. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + 'platypush', + description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.', + ) parser.add_argument( '--config', diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 3f1a425aec..7ad3eb74c0 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -4,14 +4,19 @@ Dockerfile for Platypush starting from a configuration file. """ import argparse +from contextlib import contextmanager import inspect +import logging import os import pathlib import re +import subprocess import sys import textwrap -from typing import Iterable, Sequence +from typing import IO, Generator, Iterable +from typing_extensions import override +from platypush.builder import BaseBuilder from platypush.config import Config from platypush.utils.manifest import ( BaseImage, @@ -20,14 +25,12 @@ from platypush.utils.manifest import ( PackageManagers, ) +logger = logging.getLogger() -# pylint: disable=too-few-public-methods -class DockerfileGenerator: + +class DockerBuilder(BaseBuilder): """ - Generate a Dockerfile from on a configuration file. - - :param cfgfile: Path to the configuration file. - :param image: The base image to use. + Creates a Platypush Docker image from a configuration file. """ _pkg_manager_by_base_image = { @@ -49,7 +52,7 @@ class DockerfileGenerator: # docker run --rm --name platypush \\ # -v /path/to/your/config/dir:/etc/platypush \\ # -v /path/to/your/workdir:/var/lib/platypush \\ - # -p 8080:8080 \\ + # -p 8008:8008 \\ # platypush\n """ ) @@ -61,31 +64,63 @@ class DockerfileGenerator: """ ) - def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None: - self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + def __init__( + self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs + ): + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) self.image = image - self.gitref = gitref + self.tag = tag + self.print_only = print_only # TODO - def generate(self) -> str: + @override + @classmethod + def get_name(cls): + return "platydock" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush Docker image from a configuration file." + + @property + def dockerfile_dir(self) -> str: """ - Generate a Dockerfile based on a configuration file. + Proxy property for the output Dockerfile directory. + """ + output = self.output + parent = os.path.dirname(output) - :return: The content of the generated Dockerfile. + if os.path.isfile(output): + return parent + + if os.path.isdir(output): + return output + + logger.info('%s directory does not exist, creating it', output) + pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True) + return output + + @property + def dockerfile(self) -> str: + """ + Proxy property for the output Dockerfile. + """ + return os.path.join(self.dockerfile_dir, 'Dockerfile') + + @property + def pkg_manager(self) -> PackageManagers: + """ + Proxy property for the package manager to use. + """ + return self._pkg_manager_by_base_image[self.image] + + def _read_base_dockerfile_lines(self) -> Generator[str, None, None]: + """ + :return: The lines of the base Dockerfile. """ import platypush - Config.init(self.cfgfile) - new_file_lines = [] - ports = self._get_exposed_ports() - pkg_manager = self._pkg_manager_by_base_image[self.image] - deps = Dependencies.from_config( - self.cfgfile, - pkg_manager=pkg_manager, - install_context=InstallContext.DOCKER, - base_image=self.image, - ) - - is_after_expose_cmd = False base_file = os.path.join( str(pathlib.Path(inspect.getfile(platypush)).parent), 'install', @@ -94,40 +129,145 @@ class DockerfileGenerator: ) with open(base_file, 'r') as f: - file_lines = [line.rstrip() for line in f.readlines()] + for line in f: + yield line.rstrip() - new_file_lines.extend(self._header.split('\n')) + @property + @override + def deps(self) -> Dependencies: + return Dependencies.from_config( + self.cfgfile, + pkg_manager=self.pkg_manager, + install_context=InstallContext.DOCKER, + base_image=self.image, + ) - for line in file_lines: - if re.match( - r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', - line.strip(), - ): - new_file_lines.append(self._generate_git_clone_command()) - elif line.startswith('RUN cd /install '): - for new_line in deps.before: - new_file_lines.append('RUN ' + new_line) + def _create_dockerfile_parser(self): + """ + Closure for a context-aware parser for the default Dockerfile. + """ + is_after_expose_cmd = False + deps = self.deps + ports = self._get_exposed_ports() - for new_line in deps.to_pkg_install_commands(): - new_file_lines.append('RUN ' + new_line) - elif line == 'RUN rm -rf /install': - for new_line in deps.to_pip_install_commands(): - new_file_lines.append('RUN ' + new_line) + def parser(): + nonlocal is_after_expose_cmd - for new_line in deps.after: - new_file_lines.append('RUN' + new_line) - elif line.startswith('EXPOSE ') and ports: - if not is_after_expose_cmd: - new_file_lines.extend([f'EXPOSE {port}' for port in ports]) - is_after_expose_cmd = True + for line in self._read_base_dockerfile_lines(): + if re.match( + r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', + line.strip(), + ): + yield self._generate_git_clone_command() + elif line.startswith('RUN cd /install '): + for new_line in deps.before: + yield 'RUN ' + new_line - continue - elif line.startswith('CMD'): - new_file_lines.extend(self._footer.split('\n')) + for new_line in deps.to_pkg_install_commands(): + yield 'RUN ' + new_line + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + yield 'RUN ' + new_line - new_file_lines.append(line) + for new_line in deps.after: + yield 'RUN' + new_line + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + yield from [f'EXPOSE {port}' for port in ports] + is_after_expose_cmd = True - return '\n'.join(new_file_lines) + continue + elif line.startswith('CMD'): + yield from self._footer.split('\n') + + yield line + + if line.startswith('CMD') and self.device_id: + yield f'\t--device-id {self.device_id} \\' + + return parser + + @override + def build(self): + """ + Build a Dockerfile based on a configuration file. + + :return: The content of the generated Dockerfile. + """ + + # Set the DOCKER_CTX environment variable so any downstream logic knows + # that we're running in a Docker build context. + os.environ['DOCKER_CTX'] = '1' + + self._generate_dockerfile() + if self.print_only: + return + + self._build_image() + self._print_instructions( + textwrap.dedent( + f""" + A Docker image has been built from the configuration file {self.cfgfile}. + The Dockerfile is available under {self.dockerfile}. + You can run the Docker image with the following command: + + docker run \\ + --rm --name platypush \\ + -v {os.path.dirname(self.cfgfile)}:/etc/platypush \\ + -v /path/to/your/workdir:/var/lib/platypush \\ + -p 8008:8008 \\ + platypush + """ + ) + ) + + def _build_image(self): + """ + Build a Platypush Docker image from the generated Dockerfile. + """ + logger.info('Building Docker image...') + cmd = [ + 'docker', + 'build', + '-f', + self.dockerfile, + '-t', + self.tag, + '.', + ] + + subprocess.run(cmd, check=True) + + def _generate_dockerfile(self): + """ + Parses the configuration file and generates a Dockerfile based on it. + """ + + @contextmanager + def open_writer() -> Generator[IO, None, None]: + # flake8: noqa + f = sys.stdout if self.print_only else open(self.dockerfile, 'w') + + try: + yield f + finally: + if f is not sys.stdout: + f.close() + + if not self.print_only: + logger.info('Parsing configuration file %s...', self.cfgfile) + + Config.init(self.cfgfile) + + if not self.print_only: + logger.info('Generating Dockerfile %s...', self.dockerfile) + + parser = self._create_dockerfile_parser() + + with open_writer() as f: + f.write(self._header + '\n') + for line in parser(): + f.write(line + '\n') def _generate_git_clone_command(self) -> str: """ @@ -135,16 +275,15 @@ class DockerfileGenerator: and the right git reference, if the application sources aren't already available under /install. """ - pkg_manager = self._pkg_manager_by_base_image[self.image] - install_cmd = ' '.join(pkg_manager.value.install) - uninstall_cmd = ' '.join(pkg_manager.value.uninstall) + install_cmd = ' '.join(self.pkg_manager.value.install) + uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall) return textwrap.dedent( f""" RUN if [ ! -f "/install/setup.py" ]; then \\ echo "Platypush source not found under the current directory, downloading it" && \\ {install_cmd} git && \\ rm -rf /install && \\ - git clone https://github.com/BlackLight/platypush.git /install && \\ + git clone --recursive https://github.com/BlackLight/platypush.git /install && \\ cd /install && \\ git checkout {self.gitref} && \\ {uninstall_cmd} git; \\ @@ -153,71 +292,41 @@ class DockerfileGenerator: ) @classmethod - def from_cmdline(cls, args: Sequence[str]) -> 'DockerfileGenerator': - """ - Create a DockerfileGenerator instance from command line arguments. - - :param args: Command line arguments. - :return: A DockerfileGenerator instance. - """ - parser = argparse.ArgumentParser( - prog='platydock', - add_help=False, - description='Create a Platypush Dockerfile from a config.yaml.', - ) + @override + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = super()._get_arg_parser() parser.add_argument( - '-h', '--help', dest='show_usage', action='store_true', help='Show usage' - ) - - parser.add_argument( - 'cfgfile', - type=str, - nargs='?', - help='The path to the configuration file. If not specified a minimal ' - 'Dockerfile with no extra dependencies will be generated.', - ) - - parser.add_argument( - '--image', '-i', + '--image', dest='image', required=False, type=BaseImage, choices=list(BaseImage), default=BaseImage.ALPINE, - help='Base image to use for the Dockerfile.', + help='Base image to use for the Dockerfile (default: alpine).', ) parser.add_argument( - '--ref', - '-r', - dest='gitref', + '-t', + '--tag', + dest='tag', required=False, type=str, - default='master', - help='If platydock is not run from a Platypush installation directory, ' - 'it will clone the source via git. You can specify through this ' - 'option which branch, tag or commit hash to use. Defaults to master.', + default='platypush:latest', + help='Tag name to be used for the built image ' + '(default: "platypush:latest").', ) - opts, _ = parser.parse_known_args(args) - if opts.show_usage: - parser.print_help() - sys.exit(0) + parser.add_argument( + '--print', + dest='print_only', + action='store_true', + help='Use this flag if you only want to print the Dockerfile to ' + 'stdout instead of generating an image.', + ) - if not opts.cfgfile: - opts.cfgfile = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', - ) - - print( - f'No configuration file specified. Using {opts.cfgfile}.', - file=sys.stderr, - ) - - return cls(opts.cfgfile, image=opts.image, gitref=opts.gitref) + return parser @staticmethod def _get_exposed_ports() -> Iterable[int]: @@ -240,7 +349,7 @@ def main(): """ Generates a Dockerfile based on the configuration file. """ - print(DockerfileGenerator.from_cmdline(sys.argv[1:]).generate()) + DockerBuilder.from_cmdline(sys.argv[1:]).build() return 0 diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py index ea161cf01b..0564b719f1 100755 --- a/platypush/platyvenv/__init__.py +++ b/platypush/platyvenv/__init__.py @@ -3,11 +3,9 @@ Platyvenv is a helper script that allows you to automatically create a virtual environment for Platypush starting from a configuration file. """ -import argparse from contextlib import contextmanager -import inspect +import logging import os -import pathlib import re import shutil import subprocess @@ -17,26 +15,36 @@ import textwrap from typing import Generator, Sequence import venv +from typing_extensions import override + +from platypush.builder import BaseBuilder from platypush.config import Config from platypush.utils.manifest import ( Dependencies, InstallContext, ) +logger = logging.getLogger() -# pylint: disable=too-few-public-methods -class VenvBuilder: + +class VenvBuilder(BaseBuilder): """ Build a virtual environment from on a configuration file. - - :param cfgfile: Path to the configuration file. - :param image: The base image to use. """ - def __init__(self, cfgfile: str, gitref: str, output_dir: str) -> None: - self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) - self.output_dir = os.path.abspath(os.path.expanduser(output_dir)) - self.gitref = gitref + def __init__(self, *args, **kwargs) -> None: + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) + + @override + @classmethod + def get_name(cls): + return "platyvenv" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush virtual environment from a configuration file." @property def _pip_cmd(self) -> Sequence[str]: @@ -44,7 +52,7 @@ class VenvBuilder: :return: The pip install command to use for the selected environment. """ return ( - os.path.join(self.output_dir, 'bin', 'python'), + os.path.join(self.output, 'bin', 'python'), '-m', 'pip', 'install', @@ -58,7 +66,7 @@ class VenvBuilder: Install the required system packages. """ for cmd in deps.to_pkg_install_commands(): - print(f'Installing system packages: {cmd}') + logger.info('Installing system packages: %s', cmd) subprocess.call(re.split(r'\s+', cmd.strip())) @contextmanager @@ -74,11 +82,11 @@ class VenvBuilder: """ setup_py_path = os.path.join(os.getcwd(), 'setup.py') if os.path.isfile(setup_py_path): - print('Using local checkout of the Platypush source code') + logger.info('Using local checkout of the Platypush source code') yield os.getcwd() else: checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git') - print(f'Cloning Platypush source code from git into {checkout_dir}') + logger.info('Cloning Platypush source code from git into %s', checkout_dir) subprocess.call( [ 'git', @@ -95,72 +103,42 @@ class VenvBuilder: yield checkout_dir os.chdir(pwd) - print(f'Cleaning up {checkout_dir}') + logger.info('Cleaning up %s', checkout_dir) shutil.rmtree(checkout_dir, ignore_errors=True) def _prepare_venv(self) -> None: """ - Installs the virtual environment under the configured output_dir. + Install the virtual environment under the configured output. """ - print(f'Creating virtual environment under {self.output_dir}...') + logger.info('Creating virtual environment under %s...', self.output) venv.create( - self.output_dir, + self.output, symlinks=True, with_pip=True, upgrade_deps=True, ) - print( - f'Installing base Python dependencies under {self.output_dir}...', - ) - + logger.info('Installing base Python dependencies under %s...', self.output) subprocess.call([*self._pip_cmd, 'pip', '.']) def _install_extra_pip_packages(self, deps: Dependencies): """ - Install the extra pip dependencies parsed through the + Install the extra pip dependencies inferred from the configured + extensions. """ pip_deps = list(deps.to_pip_install_commands(full_command=False)) if not pip_deps: return - print( - f'Installing extra pip dependencies under {self.output_dir}: ' - + ' '.join(pip_deps) + logger.info( + 'Installing extra pip dependencies under %s: %s', + self.output, + ' '.join(pip_deps), ) subprocess.call([*self._pip_cmd, *pip_deps]) - def _generate_run_sh(self) -> str: - """ - Generate a ``run.sh`` script to run the application from a newly built - virtual environment. - - :return: The location of the generated ``run.sh`` script. - """ - run_sh_path = os.path.join(self.output_dir, 'bin', 'run.sh') - with open(run_sh_path, 'w') as run_sh: - run_sh.write( - textwrap.dedent( - f""" - #!/bin/bash - - cd {self.output_dir} - - # Activate the virtual environment - source bin/activate - - # Run the application with the configuration file passed - # during build - platypush -c {self.cfgfile} $* - """ - ) - ) - - os.chmod(run_sh_path, 0o750) - return run_sh_path - def build(self): """ Build a Dockerfile based on a configuration file. @@ -178,78 +156,26 @@ class VenvBuilder: self._prepare_venv() self._install_extra_pip_packages(deps) - run_sh_path = self._generate_run_sh() - print( - f'\nVirtual environment created under {self.output_dir}.\n' - f'You can run the application through the {run_sh_path} script.\n' - ) + self._print_instructions( + textwrap.dedent( + f""" + Virtual environment created under {self.output}. + To run the application: - @classmethod - def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder': - """ - Create a DockerfileGenerator instance from command line arguments. + source {self.output}/bin/activate + platypush -c {self.cfgfile} { + "--device_id " + self.device_id if self.device_id else "" + } - :param args: Command line arguments. - :return: A DockerfileGenerator instance. - """ - parser = argparse.ArgumentParser( - prog='platyvenv', - add_help=False, - description='Create a Platypush virtual environment from a config.yaml.', - ) + Platypush requires a Redis instance. If you don't want to use a + stand-alone server, you can pass the --start-redis option to + the executable (optionally with --redis-port). - parser.add_argument( - '-h', '--help', dest='show_usage', action='store_true', help='Show usage' - ) - - parser.add_argument( - 'cfgfile', - type=str, - nargs='?', - help='The path to the configuration file. If not specified a minimal ' - 'virtual environment only with the base dependencies will be ' - 'generated.', - ) - - parser.add_argument( - '-o', - '--output', - dest='output_dir', - type=str, - required=False, - default='venv', - help='Target directory for the virtual environment (default: ./venv)', - ) - - parser.add_argument( - '--ref', - '-r', - dest='gitref', - required=False, - type=str, - default='master', - help='If platyvenv is not run from a Platypush installation directory, ' - 'it will clone the sources via git. You can specify through this ' - 'option which branch, tag or commit hash to use. Defaults to master.', - ) - - opts, _ = parser.parse_known_args(args) - if opts.show_usage: - parser.print_help() - sys.exit(0) - - if not opts.cfgfile: - opts.cfgfile = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', + Platypush will then start its own local instance and it will + terminate it once the application is stopped. + """ ) - - print( - f'No configuration file specified. Using {opts.cfgfile}.', - file=sys.stderr, - ) - - return cls(opts.cfgfile, gitref=opts.gitref, output_dir=opts.output_dir) + ) def main(): @@ -257,6 +183,7 @@ def main(): Generates a virtual environment based on the configuration file. """ VenvBuilder.from_cmdline(sys.argv[1:]).build() + return 0 if __name__ == '__main__':