forked from platypush/platypush
Refactored Platydock and Platyvenv.
The two scripts now share the same command interface, behaviour and base class. Also, Platydock now builds a Docker image instead of just printing a Dockerfile, unless the `--print` option is passed.
This commit is contained in:
parent
429658e7c8
commit
4dd713ffd2
5 changed files with 473 additions and 233 deletions
3
platypush/builder/__init__.py
Normal file
3
platypush/builder/__init__.py
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
from ._base import BaseBuilder
|
||||||
|
|
||||||
|
__all__ = ["BaseBuilder"]
|
198
platypush/builder/_base.py
Normal file
198
platypush/builder/_base.py
Normal file
|
@ -0,0 +1,198 @@
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
import argparse
|
||||||
|
import inspect
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
from typing import Final, Optional, Sequence
|
||||||
|
|
||||||
|
from platypush.config import Config
|
||||||
|
from platypush.utils.manifest import (
|
||||||
|
Dependencies,
|
||||||
|
InstallContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
logging.basicConfig(stream=sys.stdout)
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
|
class BaseBuilder(ABC):
|
||||||
|
"""
|
||||||
|
Base interface and utility methods for Platypush builders.
|
||||||
|
|
||||||
|
A Platypush builder is a script/piece of logic that can build a Platypush
|
||||||
|
installation, with all its base and required extra dependencies, given a
|
||||||
|
configuration file.
|
||||||
|
|
||||||
|
This class is currently implemented by the :module:`platypush.platyvenv`
|
||||||
|
and :module:`platypush.platydock` modules/scripts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git'
|
||||||
|
"""
|
||||||
|
We use the Github URL here rather than the self-hosted Gitea URL to prevent
|
||||||
|
too many requests to the Gitea server.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
cfgfile: str,
|
||||||
|
gitref: str,
|
||||||
|
output: str,
|
||||||
|
install_context: InstallContext,
|
||||||
|
*_,
|
||||||
|
verbose: bool = False,
|
||||||
|
device_id: Optional[str] = None,
|
||||||
|
**__,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
:param cfgfile: The path to the configuration file.
|
||||||
|
:param gitref: The git reference to use. It can be a branch name, a tag
|
||||||
|
name or a commit hash.
|
||||||
|
:param output: The path to the output file or directory.
|
||||||
|
:param install_context: The installation context for this builder.
|
||||||
|
:param verbose: Whether to log debug traces.
|
||||||
|
:param device_id: A device name that will be used to uniquely identify
|
||||||
|
this installation.
|
||||||
|
"""
|
||||||
|
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
|
||||||
|
self.output = os.path.abspath(os.path.expanduser(output))
|
||||||
|
self.gitref = gitref
|
||||||
|
self.install_context = install_context
|
||||||
|
self.device_id = device_id
|
||||||
|
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@abstractmethod
|
||||||
|
def get_name(cls) -> str:
|
||||||
|
"""
|
||||||
|
:return: The name of the builder.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@abstractmethod
|
||||||
|
def get_description(cls) -> str:
|
||||||
|
"""
|
||||||
|
:return: The description of the builder.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def deps(self) -> Dependencies:
|
||||||
|
"""
|
||||||
|
:return: The dependencies for this builder, given the configuration
|
||||||
|
file and the installation context.
|
||||||
|
"""
|
||||||
|
return Dependencies.from_config(
|
||||||
|
self.cfgfile,
|
||||||
|
install_context=self.install_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _print_instructions(self, s: str) -> None:
|
||||||
|
GREEN = '\033[92m'
|
||||||
|
NORM = '\033[0m'
|
||||||
|
|
||||||
|
helper_lines = s.split('\n')
|
||||||
|
wrapper_line = '=' * max(len(t) for t in helper_lines)
|
||||||
|
helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n'
|
||||||
|
print(GREEN + helper + NORM)
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def build(self):
|
||||||
|
"""
|
||||||
|
Builds the application. To be implemented by the subclasses.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_arg_parser(cls) -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog=cls.get_name(),
|
||||||
|
add_help=False,
|
||||||
|
description=cls.get_description(),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-v',
|
||||||
|
'--verbose',
|
||||||
|
dest='verbose',
|
||||||
|
action='store_true',
|
||||||
|
help='Enable debug traces',
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-c',
|
||||||
|
'--config',
|
||||||
|
type=str,
|
||||||
|
dest='cfgfile',
|
||||||
|
required=False,
|
||||||
|
default=None,
|
||||||
|
help='The path to the configuration file. If not specified, a minimal '
|
||||||
|
'installation including only the base dependencies will be generated.',
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-o',
|
||||||
|
'--output',
|
||||||
|
dest='output',
|
||||||
|
type=str,
|
||||||
|
required=False,
|
||||||
|
default='.',
|
||||||
|
help='Target directory (default: current directory). For Platydock, '
|
||||||
|
'this is the directory where the Dockerfile will be generated. For '
|
||||||
|
'Platyvenv, this is the base directory of the new virtual '
|
||||||
|
'environment.',
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-d',
|
||||||
|
'--device-id',
|
||||||
|
dest='device_id',
|
||||||
|
type=str,
|
||||||
|
required=False,
|
||||||
|
default=None,
|
||||||
|
help='A name that will be used to uniquely identify this device. '
|
||||||
|
'Default: a random name for Docker containers, and the '
|
||||||
|
'hostname of the machine for virtual environments.',
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-r',
|
||||||
|
'--ref',
|
||||||
|
dest='gitref',
|
||||||
|
required=False,
|
||||||
|
type=str,
|
||||||
|
default='master',
|
||||||
|
help='If the script is not run from a Platypush installation directory, '
|
||||||
|
'it will clone the sources via git. You can specify through this '
|
||||||
|
'option which branch, tag or commit hash to use. Defaults to master.',
|
||||||
|
)
|
||||||
|
|
||||||
|
return parser
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_cmdline(cls, args: Sequence[str]):
|
||||||
|
"""
|
||||||
|
Create a builder instance from command line arguments.
|
||||||
|
|
||||||
|
:param args: Command line arguments.
|
||||||
|
:return: A builder instance.
|
||||||
|
"""
|
||||||
|
parser = cls._get_arg_parser()
|
||||||
|
opts, _ = parser.parse_known_args(args)
|
||||||
|
if opts.show_usage:
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
if not opts.cfgfile:
|
||||||
|
opts.cfgfile = os.path.join(
|
||||||
|
str(pathlib.Path(inspect.getfile(Config)).parent),
|
||||||
|
'config.auto.yaml',
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info('No configuration file specified. Using %s.', opts.cfgfile)
|
||||||
|
|
||||||
|
return cls(**vars(opts))
|
|
@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace:
|
||||||
"""
|
"""
|
||||||
Parse command-line arguments from a list of strings.
|
Parse command-line arguments from a list of strings.
|
||||||
"""
|
"""
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser(
|
||||||
|
'platypush',
|
||||||
|
description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.',
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--config',
|
'--config',
|
||||||
|
|
|
@ -4,14 +4,19 @@ Dockerfile for Platypush starting from a configuration file.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
import inspect
|
import inspect
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import textwrap
|
import textwrap
|
||||||
from typing import Iterable, Sequence
|
from typing import IO, Generator, Iterable
|
||||||
|
from typing_extensions import override
|
||||||
|
|
||||||
|
from platypush.builder import BaseBuilder
|
||||||
from platypush.config import Config
|
from platypush.config import Config
|
||||||
from platypush.utils.manifest import (
|
from platypush.utils.manifest import (
|
||||||
BaseImage,
|
BaseImage,
|
||||||
|
@ -20,14 +25,12 @@ from platypush.utils.manifest import (
|
||||||
PackageManagers,
|
PackageManagers,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
class DockerfileGenerator:
|
class DockerBuilder(BaseBuilder):
|
||||||
"""
|
"""
|
||||||
Generate a Dockerfile from on a configuration file.
|
Creates a Platypush Docker image from a configuration file.
|
||||||
|
|
||||||
:param cfgfile: Path to the configuration file.
|
|
||||||
:param image: The base image to use.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_pkg_manager_by_base_image = {
|
_pkg_manager_by_base_image = {
|
||||||
|
@ -49,7 +52,7 @@ class DockerfileGenerator:
|
||||||
# docker run --rm --name platypush \\
|
# docker run --rm --name platypush \\
|
||||||
# -v /path/to/your/config/dir:/etc/platypush \\
|
# -v /path/to/your/config/dir:/etc/platypush \\
|
||||||
# -v /path/to/your/workdir:/var/lib/platypush \\
|
# -v /path/to/your/workdir:/var/lib/platypush \\
|
||||||
# -p 8080:8080 \\
|
# -p 8008:8008 \\
|
||||||
# platypush\n
|
# platypush\n
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
@ -61,31 +64,63 @@ class DockerfileGenerator:
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None:
|
def __init__(
|
||||||
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
|
self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs
|
||||||
|
):
|
||||||
|
kwargs['install_context'] = InstallContext.DOCKER
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
self.image = image
|
self.image = image
|
||||||
self.gitref = gitref
|
self.tag = tag
|
||||||
|
self.print_only = print_only # TODO
|
||||||
|
|
||||||
def generate(self) -> str:
|
@override
|
||||||
|
@classmethod
|
||||||
|
def get_name(cls):
|
||||||
|
return "platydock"
|
||||||
|
|
||||||
|
@override
|
||||||
|
@classmethod
|
||||||
|
def get_description(cls):
|
||||||
|
return "Build a Platypush Docker image from a configuration file."
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dockerfile_dir(self) -> str:
|
||||||
"""
|
"""
|
||||||
Generate a Dockerfile based on a configuration file.
|
Proxy property for the output Dockerfile directory.
|
||||||
|
"""
|
||||||
|
output = self.output
|
||||||
|
parent = os.path.dirname(output)
|
||||||
|
|
||||||
:return: The content of the generated Dockerfile.
|
if os.path.isfile(output):
|
||||||
|
return parent
|
||||||
|
|
||||||
|
if os.path.isdir(output):
|
||||||
|
return output
|
||||||
|
|
||||||
|
logger.info('%s directory does not exist, creating it', output)
|
||||||
|
pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
return output
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dockerfile(self) -> str:
|
||||||
|
"""
|
||||||
|
Proxy property for the output Dockerfile.
|
||||||
|
"""
|
||||||
|
return os.path.join(self.dockerfile_dir, 'Dockerfile')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pkg_manager(self) -> PackageManagers:
|
||||||
|
"""
|
||||||
|
Proxy property for the package manager to use.
|
||||||
|
"""
|
||||||
|
return self._pkg_manager_by_base_image[self.image]
|
||||||
|
|
||||||
|
def _read_base_dockerfile_lines(self) -> Generator[str, None, None]:
|
||||||
|
"""
|
||||||
|
:return: The lines of the base Dockerfile.
|
||||||
"""
|
"""
|
||||||
import platypush
|
import platypush
|
||||||
|
|
||||||
Config.init(self.cfgfile)
|
|
||||||
new_file_lines = []
|
|
||||||
ports = self._get_exposed_ports()
|
|
||||||
pkg_manager = self._pkg_manager_by_base_image[self.image]
|
|
||||||
deps = Dependencies.from_config(
|
|
||||||
self.cfgfile,
|
|
||||||
pkg_manager=pkg_manager,
|
|
||||||
install_context=InstallContext.DOCKER,
|
|
||||||
base_image=self.image,
|
|
||||||
)
|
|
||||||
|
|
||||||
is_after_expose_cmd = False
|
|
||||||
base_file = os.path.join(
|
base_file = os.path.join(
|
||||||
str(pathlib.Path(inspect.getfile(platypush)).parent),
|
str(pathlib.Path(inspect.getfile(platypush)).parent),
|
||||||
'install',
|
'install',
|
||||||
|
@ -94,40 +129,145 @@ class DockerfileGenerator:
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(base_file, 'r') as f:
|
with open(base_file, 'r') as f:
|
||||||
file_lines = [line.rstrip() for line in f.readlines()]
|
for line in f:
|
||||||
|
yield line.rstrip()
|
||||||
|
|
||||||
new_file_lines.extend(self._header.split('\n'))
|
@property
|
||||||
|
@override
|
||||||
|
def deps(self) -> Dependencies:
|
||||||
|
return Dependencies.from_config(
|
||||||
|
self.cfgfile,
|
||||||
|
pkg_manager=self.pkg_manager,
|
||||||
|
install_context=InstallContext.DOCKER,
|
||||||
|
base_image=self.image,
|
||||||
|
)
|
||||||
|
|
||||||
for line in file_lines:
|
def _create_dockerfile_parser(self):
|
||||||
if re.match(
|
"""
|
||||||
r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh',
|
Closure for a context-aware parser for the default Dockerfile.
|
||||||
line.strip(),
|
"""
|
||||||
):
|
is_after_expose_cmd = False
|
||||||
new_file_lines.append(self._generate_git_clone_command())
|
deps = self.deps
|
||||||
elif line.startswith('RUN cd /install '):
|
ports = self._get_exposed_ports()
|
||||||
for new_line in deps.before:
|
|
||||||
new_file_lines.append('RUN ' + new_line)
|
|
||||||
|
|
||||||
for new_line in deps.to_pkg_install_commands():
|
def parser():
|
||||||
new_file_lines.append('RUN ' + new_line)
|
nonlocal is_after_expose_cmd
|
||||||
elif line == 'RUN rm -rf /install':
|
|
||||||
for new_line in deps.to_pip_install_commands():
|
|
||||||
new_file_lines.append('RUN ' + new_line)
|
|
||||||
|
|
||||||
for new_line in deps.after:
|
for line in self._read_base_dockerfile_lines():
|
||||||
new_file_lines.append('RUN' + new_line)
|
if re.match(
|
||||||
elif line.startswith('EXPOSE ') and ports:
|
r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh',
|
||||||
if not is_after_expose_cmd:
|
line.strip(),
|
||||||
new_file_lines.extend([f'EXPOSE {port}' for port in ports])
|
):
|
||||||
is_after_expose_cmd = True
|
yield self._generate_git_clone_command()
|
||||||
|
elif line.startswith('RUN cd /install '):
|
||||||
|
for new_line in deps.before:
|
||||||
|
yield 'RUN ' + new_line
|
||||||
|
|
||||||
continue
|
for new_line in deps.to_pkg_install_commands():
|
||||||
elif line.startswith('CMD'):
|
yield 'RUN ' + new_line
|
||||||
new_file_lines.extend(self._footer.split('\n'))
|
elif line == 'RUN rm -rf /install':
|
||||||
|
for new_line in deps.to_pip_install_commands():
|
||||||
|
yield 'RUN ' + new_line
|
||||||
|
|
||||||
new_file_lines.append(line)
|
for new_line in deps.after:
|
||||||
|
yield 'RUN' + new_line
|
||||||
|
elif line.startswith('EXPOSE ') and ports:
|
||||||
|
if not is_after_expose_cmd:
|
||||||
|
yield from [f'EXPOSE {port}' for port in ports]
|
||||||
|
is_after_expose_cmd = True
|
||||||
|
|
||||||
return '\n'.join(new_file_lines)
|
continue
|
||||||
|
elif line.startswith('CMD'):
|
||||||
|
yield from self._footer.split('\n')
|
||||||
|
|
||||||
|
yield line
|
||||||
|
|
||||||
|
if line.startswith('CMD') and self.device_id:
|
||||||
|
yield f'\t--device-id {self.device_id} \\'
|
||||||
|
|
||||||
|
return parser
|
||||||
|
|
||||||
|
@override
|
||||||
|
def build(self):
|
||||||
|
"""
|
||||||
|
Build a Dockerfile based on a configuration file.
|
||||||
|
|
||||||
|
:return: The content of the generated Dockerfile.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Set the DOCKER_CTX environment variable so any downstream logic knows
|
||||||
|
# that we're running in a Docker build context.
|
||||||
|
os.environ['DOCKER_CTX'] = '1'
|
||||||
|
|
||||||
|
self._generate_dockerfile()
|
||||||
|
if self.print_only:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._build_image()
|
||||||
|
self._print_instructions(
|
||||||
|
textwrap.dedent(
|
||||||
|
f"""
|
||||||
|
A Docker image has been built from the configuration file {self.cfgfile}.
|
||||||
|
The Dockerfile is available under {self.dockerfile}.
|
||||||
|
You can run the Docker image with the following command:
|
||||||
|
|
||||||
|
docker run \\
|
||||||
|
--rm --name platypush \\
|
||||||
|
-v {os.path.dirname(self.cfgfile)}:/etc/platypush \\
|
||||||
|
-v /path/to/your/workdir:/var/lib/platypush \\
|
||||||
|
-p 8008:8008 \\
|
||||||
|
platypush
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _build_image(self):
|
||||||
|
"""
|
||||||
|
Build a Platypush Docker image from the generated Dockerfile.
|
||||||
|
"""
|
||||||
|
logger.info('Building Docker image...')
|
||||||
|
cmd = [
|
||||||
|
'docker',
|
||||||
|
'build',
|
||||||
|
'-f',
|
||||||
|
self.dockerfile,
|
||||||
|
'-t',
|
||||||
|
self.tag,
|
||||||
|
'.',
|
||||||
|
]
|
||||||
|
|
||||||
|
subprocess.run(cmd, check=True)
|
||||||
|
|
||||||
|
def _generate_dockerfile(self):
|
||||||
|
"""
|
||||||
|
Parses the configuration file and generates a Dockerfile based on it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def open_writer() -> Generator[IO, None, None]:
|
||||||
|
# flake8: noqa
|
||||||
|
f = sys.stdout if self.print_only else open(self.dockerfile, 'w')
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield f
|
||||||
|
finally:
|
||||||
|
if f is not sys.stdout:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
if not self.print_only:
|
||||||
|
logger.info('Parsing configuration file %s...', self.cfgfile)
|
||||||
|
|
||||||
|
Config.init(self.cfgfile)
|
||||||
|
|
||||||
|
if not self.print_only:
|
||||||
|
logger.info('Generating Dockerfile %s...', self.dockerfile)
|
||||||
|
|
||||||
|
parser = self._create_dockerfile_parser()
|
||||||
|
|
||||||
|
with open_writer() as f:
|
||||||
|
f.write(self._header + '\n')
|
||||||
|
for line in parser():
|
||||||
|
f.write(line + '\n')
|
||||||
|
|
||||||
def _generate_git_clone_command(self) -> str:
|
def _generate_git_clone_command(self) -> str:
|
||||||
"""
|
"""
|
||||||
|
@ -135,16 +275,15 @@ class DockerfileGenerator:
|
||||||
and the right git reference, if the application sources aren't already
|
and the right git reference, if the application sources aren't already
|
||||||
available under /install.
|
available under /install.
|
||||||
"""
|
"""
|
||||||
pkg_manager = self._pkg_manager_by_base_image[self.image]
|
install_cmd = ' '.join(self.pkg_manager.value.install)
|
||||||
install_cmd = ' '.join(pkg_manager.value.install)
|
uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall)
|
||||||
uninstall_cmd = ' '.join(pkg_manager.value.uninstall)
|
|
||||||
return textwrap.dedent(
|
return textwrap.dedent(
|
||||||
f"""
|
f"""
|
||||||
RUN if [ ! -f "/install/setup.py" ]; then \\
|
RUN if [ ! -f "/install/setup.py" ]; then \\
|
||||||
echo "Platypush source not found under the current directory, downloading it" && \\
|
echo "Platypush source not found under the current directory, downloading it" && \\
|
||||||
{install_cmd} git && \\
|
{install_cmd} git && \\
|
||||||
rm -rf /install && \\
|
rm -rf /install && \\
|
||||||
git clone https://github.com/BlackLight/platypush.git /install && \\
|
git clone --recursive https://github.com/BlackLight/platypush.git /install && \\
|
||||||
cd /install && \\
|
cd /install && \\
|
||||||
git checkout {self.gitref} && \\
|
git checkout {self.gitref} && \\
|
||||||
{uninstall_cmd} git; \\
|
{uninstall_cmd} git; \\
|
||||||
|
@ -153,71 +292,41 @@ class DockerfileGenerator:
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_cmdline(cls, args: Sequence[str]) -> 'DockerfileGenerator':
|
@override
|
||||||
"""
|
def _get_arg_parser(cls) -> argparse.ArgumentParser:
|
||||||
Create a DockerfileGenerator instance from command line arguments.
|
parser = super()._get_arg_parser()
|
||||||
|
|
||||||
:param args: Command line arguments.
|
|
||||||
:return: A DockerfileGenerator instance.
|
|
||||||
"""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock',
|
|
||||||
add_help=False,
|
|
||||||
description='Create a Platypush Dockerfile from a config.yaml.',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'cfgfile',
|
|
||||||
type=str,
|
|
||||||
nargs='?',
|
|
||||||
help='The path to the configuration file. If not specified a minimal '
|
|
||||||
'Dockerfile with no extra dependencies will be generated.',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'--image',
|
|
||||||
'-i',
|
'-i',
|
||||||
|
'--image',
|
||||||
dest='image',
|
dest='image',
|
||||||
required=False,
|
required=False,
|
||||||
type=BaseImage,
|
type=BaseImage,
|
||||||
choices=list(BaseImage),
|
choices=list(BaseImage),
|
||||||
default=BaseImage.ALPINE,
|
default=BaseImage.ALPINE,
|
||||||
help='Base image to use for the Dockerfile.',
|
help='Base image to use for the Dockerfile (default: alpine).',
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--ref',
|
'-t',
|
||||||
'-r',
|
'--tag',
|
||||||
dest='gitref',
|
dest='tag',
|
||||||
required=False,
|
required=False,
|
||||||
type=str,
|
type=str,
|
||||||
default='master',
|
default='platypush:latest',
|
||||||
help='If platydock is not run from a Platypush installation directory, '
|
help='Tag name to be used for the built image '
|
||||||
'it will clone the source via git. You can specify through this '
|
'(default: "platypush:latest").',
|
||||||
'option which branch, tag or commit hash to use. Defaults to master.',
|
|
||||||
)
|
)
|
||||||
|
|
||||||
opts, _ = parser.parse_known_args(args)
|
parser.add_argument(
|
||||||
if opts.show_usage:
|
'--print',
|
||||||
parser.print_help()
|
dest='print_only',
|
||||||
sys.exit(0)
|
action='store_true',
|
||||||
|
help='Use this flag if you only want to print the Dockerfile to '
|
||||||
|
'stdout instead of generating an image.',
|
||||||
|
)
|
||||||
|
|
||||||
if not opts.cfgfile:
|
return parser
|
||||||
opts.cfgfile = os.path.join(
|
|
||||||
str(pathlib.Path(inspect.getfile(Config)).parent),
|
|
||||||
'config.auto.yaml',
|
|
||||||
)
|
|
||||||
|
|
||||||
print(
|
|
||||||
f'No configuration file specified. Using {opts.cfgfile}.',
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
return cls(opts.cfgfile, image=opts.image, gitref=opts.gitref)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_exposed_ports() -> Iterable[int]:
|
def _get_exposed_ports() -> Iterable[int]:
|
||||||
|
@ -240,7 +349,7 @@ def main():
|
||||||
"""
|
"""
|
||||||
Generates a Dockerfile based on the configuration file.
|
Generates a Dockerfile based on the configuration file.
|
||||||
"""
|
"""
|
||||||
print(DockerfileGenerator.from_cmdline(sys.argv[1:]).generate())
|
DockerBuilder.from_cmdline(sys.argv[1:]).build()
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -3,11 +3,9 @@ Platyvenv is a helper script that allows you to automatically create a
|
||||||
virtual environment for Platypush starting from a configuration file.
|
virtual environment for Platypush starting from a configuration file.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
import inspect
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
@ -17,26 +15,36 @@ import textwrap
|
||||||
from typing import Generator, Sequence
|
from typing import Generator, Sequence
|
||||||
import venv
|
import venv
|
||||||
|
|
||||||
|
from typing_extensions import override
|
||||||
|
|
||||||
|
from platypush.builder import BaseBuilder
|
||||||
from platypush.config import Config
|
from platypush.config import Config
|
||||||
from platypush.utils.manifest import (
|
from platypush.utils.manifest import (
|
||||||
Dependencies,
|
Dependencies,
|
||||||
InstallContext,
|
InstallContext,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
class VenvBuilder:
|
class VenvBuilder(BaseBuilder):
|
||||||
"""
|
"""
|
||||||
Build a virtual environment from on a configuration file.
|
Build a virtual environment from on a configuration file.
|
||||||
|
|
||||||
:param cfgfile: Path to the configuration file.
|
|
||||||
:param image: The base image to use.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cfgfile: str, gitref: str, output_dir: str) -> None:
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile))
|
kwargs['install_context'] = InstallContext.DOCKER
|
||||||
self.output_dir = os.path.abspath(os.path.expanduser(output_dir))
|
super().__init__(*args, **kwargs)
|
||||||
self.gitref = gitref
|
|
||||||
|
@override
|
||||||
|
@classmethod
|
||||||
|
def get_name(cls):
|
||||||
|
return "platyvenv"
|
||||||
|
|
||||||
|
@override
|
||||||
|
@classmethod
|
||||||
|
def get_description(cls):
|
||||||
|
return "Build a Platypush virtual environment from a configuration file."
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _pip_cmd(self) -> Sequence[str]:
|
def _pip_cmd(self) -> Sequence[str]:
|
||||||
|
@ -44,7 +52,7 @@ class VenvBuilder:
|
||||||
:return: The pip install command to use for the selected environment.
|
:return: The pip install command to use for the selected environment.
|
||||||
"""
|
"""
|
||||||
return (
|
return (
|
||||||
os.path.join(self.output_dir, 'bin', 'python'),
|
os.path.join(self.output, 'bin', 'python'),
|
||||||
'-m',
|
'-m',
|
||||||
'pip',
|
'pip',
|
||||||
'install',
|
'install',
|
||||||
|
@ -58,7 +66,7 @@ class VenvBuilder:
|
||||||
Install the required system packages.
|
Install the required system packages.
|
||||||
"""
|
"""
|
||||||
for cmd in deps.to_pkg_install_commands():
|
for cmd in deps.to_pkg_install_commands():
|
||||||
print(f'Installing system packages: {cmd}')
|
logger.info('Installing system packages: %s', cmd)
|
||||||
subprocess.call(re.split(r'\s+', cmd.strip()))
|
subprocess.call(re.split(r'\s+', cmd.strip()))
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
|
@ -74,11 +82,11 @@ class VenvBuilder:
|
||||||
"""
|
"""
|
||||||
setup_py_path = os.path.join(os.getcwd(), 'setup.py')
|
setup_py_path = os.path.join(os.getcwd(), 'setup.py')
|
||||||
if os.path.isfile(setup_py_path):
|
if os.path.isfile(setup_py_path):
|
||||||
print('Using local checkout of the Platypush source code')
|
logger.info('Using local checkout of the Platypush source code')
|
||||||
yield os.getcwd()
|
yield os.getcwd()
|
||||||
else:
|
else:
|
||||||
checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git')
|
checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git')
|
||||||
print(f'Cloning Platypush source code from git into {checkout_dir}')
|
logger.info('Cloning Platypush source code from git into %s', checkout_dir)
|
||||||
subprocess.call(
|
subprocess.call(
|
||||||
[
|
[
|
||||||
'git',
|
'git',
|
||||||
|
@ -95,72 +103,42 @@ class VenvBuilder:
|
||||||
yield checkout_dir
|
yield checkout_dir
|
||||||
|
|
||||||
os.chdir(pwd)
|
os.chdir(pwd)
|
||||||
print(f'Cleaning up {checkout_dir}')
|
logger.info('Cleaning up %s', checkout_dir)
|
||||||
shutil.rmtree(checkout_dir, ignore_errors=True)
|
shutil.rmtree(checkout_dir, ignore_errors=True)
|
||||||
|
|
||||||
def _prepare_venv(self) -> None:
|
def _prepare_venv(self) -> None:
|
||||||
"""
|
"""
|
||||||
Installs the virtual environment under the configured output_dir.
|
Install the virtual environment under the configured output.
|
||||||
"""
|
"""
|
||||||
print(f'Creating virtual environment under {self.output_dir}...')
|
logger.info('Creating virtual environment under %s...', self.output)
|
||||||
|
|
||||||
venv.create(
|
venv.create(
|
||||||
self.output_dir,
|
self.output,
|
||||||
symlinks=True,
|
symlinks=True,
|
||||||
with_pip=True,
|
with_pip=True,
|
||||||
upgrade_deps=True,
|
upgrade_deps=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
print(
|
logger.info('Installing base Python dependencies under %s...', self.output)
|
||||||
f'Installing base Python dependencies under {self.output_dir}...',
|
|
||||||
)
|
|
||||||
|
|
||||||
subprocess.call([*self._pip_cmd, 'pip', '.'])
|
subprocess.call([*self._pip_cmd, 'pip', '.'])
|
||||||
|
|
||||||
def _install_extra_pip_packages(self, deps: Dependencies):
|
def _install_extra_pip_packages(self, deps: Dependencies):
|
||||||
"""
|
"""
|
||||||
Install the extra pip dependencies parsed through the
|
Install the extra pip dependencies inferred from the configured
|
||||||
|
extensions.
|
||||||
"""
|
"""
|
||||||
pip_deps = list(deps.to_pip_install_commands(full_command=False))
|
pip_deps = list(deps.to_pip_install_commands(full_command=False))
|
||||||
if not pip_deps:
|
if not pip_deps:
|
||||||
return
|
return
|
||||||
|
|
||||||
print(
|
logger.info(
|
||||||
f'Installing extra pip dependencies under {self.output_dir}: '
|
'Installing extra pip dependencies under %s: %s',
|
||||||
+ ' '.join(pip_deps)
|
self.output,
|
||||||
|
' '.join(pip_deps),
|
||||||
)
|
)
|
||||||
|
|
||||||
subprocess.call([*self._pip_cmd, *pip_deps])
|
subprocess.call([*self._pip_cmd, *pip_deps])
|
||||||
|
|
||||||
def _generate_run_sh(self) -> str:
|
|
||||||
"""
|
|
||||||
Generate a ``run.sh`` script to run the application from a newly built
|
|
||||||
virtual environment.
|
|
||||||
|
|
||||||
:return: The location of the generated ``run.sh`` script.
|
|
||||||
"""
|
|
||||||
run_sh_path = os.path.join(self.output_dir, 'bin', 'run.sh')
|
|
||||||
with open(run_sh_path, 'w') as run_sh:
|
|
||||||
run_sh.write(
|
|
||||||
textwrap.dedent(
|
|
||||||
f"""
|
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
cd {self.output_dir}
|
|
||||||
|
|
||||||
# Activate the virtual environment
|
|
||||||
source bin/activate
|
|
||||||
|
|
||||||
# Run the application with the configuration file passed
|
|
||||||
# during build
|
|
||||||
platypush -c {self.cfgfile} $*
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
os.chmod(run_sh_path, 0o750)
|
|
||||||
return run_sh_path
|
|
||||||
|
|
||||||
def build(self):
|
def build(self):
|
||||||
"""
|
"""
|
||||||
Build a Dockerfile based on a configuration file.
|
Build a Dockerfile based on a configuration file.
|
||||||
|
@ -178,78 +156,26 @@ class VenvBuilder:
|
||||||
self._prepare_venv()
|
self._prepare_venv()
|
||||||
|
|
||||||
self._install_extra_pip_packages(deps)
|
self._install_extra_pip_packages(deps)
|
||||||
run_sh_path = self._generate_run_sh()
|
self._print_instructions(
|
||||||
print(
|
textwrap.dedent(
|
||||||
f'\nVirtual environment created under {self.output_dir}.\n'
|
f"""
|
||||||
f'You can run the application through the {run_sh_path} script.\n'
|
Virtual environment created under {self.output}.
|
||||||
)
|
To run the application:
|
||||||
|
|
||||||
@classmethod
|
source {self.output}/bin/activate
|
||||||
def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder':
|
platypush -c {self.cfgfile} {
|
||||||
"""
|
"--device_id " + self.device_id if self.device_id else ""
|
||||||
Create a DockerfileGenerator instance from command line arguments.
|
}
|
||||||
|
|
||||||
:param args: Command line arguments.
|
Platypush requires a Redis instance. If you don't want to use a
|
||||||
:return: A DockerfileGenerator instance.
|
stand-alone server, you can pass the --start-redis option to
|
||||||
"""
|
the executable (optionally with --redis-port).
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platyvenv',
|
|
||||||
add_help=False,
|
|
||||||
description='Create a Platypush virtual environment from a config.yaml.',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
Platypush will then start its own local instance and it will
|
||||||
'-h', '--help', dest='show_usage', action='store_true', help='Show usage'
|
terminate it once the application is stopped.
|
||||||
)
|
"""
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'cfgfile',
|
|
||||||
type=str,
|
|
||||||
nargs='?',
|
|
||||||
help='The path to the configuration file. If not specified a minimal '
|
|
||||||
'virtual environment only with the base dependencies will be '
|
|
||||||
'generated.',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-o',
|
|
||||||
'--output',
|
|
||||||
dest='output_dir',
|
|
||||||
type=str,
|
|
||||||
required=False,
|
|
||||||
default='venv',
|
|
||||||
help='Target directory for the virtual environment (default: ./venv)',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'--ref',
|
|
||||||
'-r',
|
|
||||||
dest='gitref',
|
|
||||||
required=False,
|
|
||||||
type=str,
|
|
||||||
default='master',
|
|
||||||
help='If platyvenv is not run from a Platypush installation directory, '
|
|
||||||
'it will clone the sources via git. You can specify through this '
|
|
||||||
'option which branch, tag or commit hash to use. Defaults to master.',
|
|
||||||
)
|
|
||||||
|
|
||||||
opts, _ = parser.parse_known_args(args)
|
|
||||||
if opts.show_usage:
|
|
||||||
parser.print_help()
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
if not opts.cfgfile:
|
|
||||||
opts.cfgfile = os.path.join(
|
|
||||||
str(pathlib.Path(inspect.getfile(Config)).parent),
|
|
||||||
'config.auto.yaml',
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
print(
|
|
||||||
f'No configuration file specified. Using {opts.cfgfile}.',
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
return cls(opts.cfgfile, gitref=opts.gitref, output_dir=opts.output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
@ -257,6 +183,7 @@ def main():
|
||||||
Generates a virtual environment based on the configuration file.
|
Generates a virtual environment based on the configuration file.
|
||||||
"""
|
"""
|
||||||
VenvBuilder.from_cmdline(sys.argv[1:]).build()
|
VenvBuilder.from_cmdline(sys.argv[1:]).build()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
Loading…
Reference in a new issue