forked from platypush/platypush
Rewritten platydock utility.
Platydock now will only print out a Dockerfile given a configuration file. No more maintaining the state of containers, storing separate workdirs and configuration directories etc. - that introduced way too much overhead over Docker.
This commit is contained in:
parent
9002f3034a
commit
980af16984
GPG key ID:
D90FBA7F76362774
1 changed files with 76 additions and 430 deletions
|
|
@ -1,462 +1,108 @@
|
||||||
"""
|
"""
|
||||||
Platydock
|
Platydock is a helper script that allows you to automatically create a
|
||||||
|
Dockerfile for Platypush starting from a configuration file.
|
||||||
Platydock is a helper that allows you to easily manage (create, destroy, start,
|
|
||||||
stop and list) Platypush instances as Docker images.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import enum
|
import inspect
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
import sys
|
||||||
import textwrap
|
from typing import Iterable
|
||||||
import traceback as tb
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
from platypush.config import Config
|
from platypush.config import Config
|
||||||
from platypush.utils import manifest
|
from platypush.utils.manifest import Dependencies
|
||||||
|
|
||||||
workdir = os.path.join(
|
ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m '
|
||||||
os.path.expanduser('~'), '.local', 'share', 'platypush', 'platydock'
|
ERR_SUFFIX = '\033[0m'
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Action(enum.Enum):
|
def generate_dockerfile(cfgfile: str) -> str:
|
||||||
build = 'build'
|
"""
|
||||||
start = 'start'
|
Generate a Dockerfile based on a configuration file.
|
||||||
stop = 'stop'
|
|
||||||
rm = 'rm'
|
|
||||||
ls = 'ls'
|
|
||||||
|
|
||||||
def __str__(self):
|
:param cfgfile: Path to the configuration file.
|
||||||
return self.value
|
:return: The content of the generated Dockerfile.
|
||||||
|
"""
|
||||||
|
Config.init(cfgfile)
|
||||||
def _parse_deps(cls):
|
new_file_lines = []
|
||||||
deps = []
|
ports = _get_exposed_ports()
|
||||||
|
deps = Dependencies.from_config(cfgfile, pkg_manager='apk')
|
||||||
for line in cls.__doc__.split('\n'):
|
is_after_expose_cmd = False
|
||||||
m = re.search(r'\(``pip install (.+)``\)', line)
|
base_file = os.path.join(
|
||||||
if m:
|
str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile'
|
||||||
deps.append(m.group(1))
|
|
||||||
|
|
||||||
return deps
|
|
||||||
|
|
||||||
|
|
||||||
def generate_dockerfile(deps, ports, cfgfile, device_dir, python_version):
|
|
||||||
device_id = Config.get('device_id')
|
|
||||||
if not device_id:
|
|
||||||
raise RuntimeError(
|
|
||||||
(
|
|
||||||
'You need to specify a device_id in {} - Docker '
|
|
||||||
+ 'containers cannot rely on hostname'
|
|
||||||
).format(cfgfile)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
os.makedirs(device_dir, exist_ok=True)
|
with open(base_file, 'r') as f:
|
||||||
content = textwrap.dedent(
|
file_lines = [line.rstrip() for line in f.readlines()]
|
||||||
'''
|
|
||||||
FROM python:{python_version}-slim-bookworm
|
|
||||||
|
|
||||||
RUN mkdir -p /app
|
for line in file_lines:
|
||||||
RUN mkdir -p /etc/platypush
|
if line.startswith('RUN cd /install '):
|
||||||
RUN mkdir -p /usr/local/share/platypush\n
|
for new_line in deps.before:
|
||||||
'''.format(
|
new_file_lines.append('RUN ' + new_line)
|
||||||
python_version=python_version
|
|
||||||
)
|
|
||||||
).lstrip()
|
|
||||||
|
|
||||||
srcdir = os.path.dirname(cfgfile)
|
for new_line in deps.to_pkg_install_commands(
|
||||||
cfgfile_copy = os.path.join(device_dir, 'config.yaml')
|
pkg_manager='apk', skip_sudo=True
|
||||||
shutil.copy(cfgfile, cfgfile_copy, follow_symlinks=True)
|
|
||||||
content += 'COPY config.yaml /etc/platypush/\n'
|
|
||||||
backend_config = Config.get_backends()
|
|
||||||
|
|
||||||
# Redis configuration for Docker
|
|
||||||
if 'redis' not in backend_config:
|
|
||||||
backend_config['redis'] = {
|
|
||||||
'redis_args': {
|
|
||||||
'host': 'redis',
|
|
||||||
'port': 6379,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
with open(cfgfile_copy, 'a') as f:
|
|
||||||
f.write(
|
|
||||||
'\n# Automatically added by platydock, do not remove\n'
|
|
||||||
+ yaml.dump(
|
|
||||||
{
|
|
||||||
'backend.redis': backend_config['redis'],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
+ '\n'
|
|
||||||
)
|
|
||||||
|
|
||||||
# Main database configuration
|
|
||||||
has_main_db = False
|
|
||||||
with open(cfgfile_copy, 'r') as f:
|
|
||||||
for line in f.readlines():
|
|
||||||
if re.match(r'^(main.)?db.*', line):
|
|
||||||
has_main_db = True
|
|
||||||
break
|
|
||||||
|
|
||||||
if not has_main_db:
|
|
||||||
with open(cfgfile_copy, 'a') as f:
|
|
||||||
f.write(
|
|
||||||
'\n# Automatically added by platydock, do not remove\n'
|
|
||||||
+ yaml.dump(
|
|
||||||
{
|
|
||||||
'main.db': {
|
|
||||||
'engine': 'sqlite:////platypush.db',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
+ '\n'
|
|
||||||
)
|
|
||||||
|
|
||||||
# Copy included files
|
|
||||||
# noinspection PyProtectedMember
|
|
||||||
for include in Config._included_files:
|
|
||||||
incdir = os.path.relpath(os.path.dirname(include), srcdir)
|
|
||||||
destdir = os.path.join(device_dir, incdir)
|
|
||||||
pathlib.Path(destdir).mkdir(parents=True, exist_ok=True)
|
|
||||||
shutil.copy(include, destdir, follow_symlinks=True)
|
|
||||||
content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n'
|
|
||||||
content += (
|
|
||||||
'COPY '
|
|
||||||
+ os.path.relpath(include, srcdir)
|
|
||||||
+ ' /etc/platypush/'
|
|
||||||
+ incdir
|
|
||||||
+ '\n'
|
|
||||||
)
|
|
||||||
|
|
||||||
# Copy script files
|
|
||||||
scripts_dir = os.path.join(os.path.dirname(cfgfile), 'scripts')
|
|
||||||
if os.path.isdir(scripts_dir):
|
|
||||||
local_scripts_dir = os.path.join(device_dir, 'scripts')
|
|
||||||
remote_scripts_dir = '/etc/platypush/scripts'
|
|
||||||
shutil.copytree(
|
|
||||||
scripts_dir, local_scripts_dir, symlinks=True, dirs_exist_ok=True
|
|
||||||
)
|
|
||||||
content += f'RUN mkdir -p {remote_scripts_dir}\n'
|
|
||||||
content += f'COPY scripts/ {remote_scripts_dir}\n'
|
|
||||||
|
|
||||||
packages = deps.pop('packages', None)
|
|
||||||
pip = deps.pop('pip', None)
|
|
||||||
exec_cmds = deps.pop('exec', None)
|
|
||||||
pkg_cmd = (
|
|
||||||
f'\n\t&& apt-get install --no-install-recommends -y {" ".join(packages)} \\'
|
|
||||||
if packages
|
|
||||||
else ''
|
|
||||||
)
|
|
||||||
pip_cmd = f'\n\t&& pip install {" ".join(pip)} \\' if pip else ''
|
|
||||||
content += f'''
|
|
||||||
RUN dpkg --configure -a \\
|
|
||||||
&& apt-get -f install \\
|
|
||||||
&& apt-get --fix-missing install \\
|
|
||||||
&& apt-get clean \\
|
|
||||||
&& apt-get update \\
|
|
||||||
&& apt-get -y upgrade \\
|
|
||||||
&& apt-get -y dist-upgrade \\
|
|
||||||
&& apt-get install --no-install-recommends -y apt-utils \\
|
|
||||||
&& apt-get install --no-install-recommends -y build-essential \\
|
|
||||||
&& apt-get install --no-install-recommends -y git \\
|
|
||||||
&& apt-get install --no-install-recommends -y sudo \\
|
|
||||||
&& apt-get install --no-install-recommends -y libffi-dev \\
|
|
||||||
&& apt-get install --no-install-recommends -y libcap-dev \\
|
|
||||||
&& apt-get install --no-install-recommends -y libjpeg-dev \\{pkg_cmd}{pip_cmd}'''
|
|
||||||
|
|
||||||
for exec_cmd in exec_cmds:
|
|
||||||
content += f'\n\t&& {exec_cmd} \\'
|
|
||||||
content += '''
|
|
||||||
&& apt-get install --no-install-recommends -y zlib1g-dev
|
|
||||||
|
|
||||||
RUN git clone --recursive https://git.platypush.tech/platypush/platypush.git /app \\
|
|
||||||
&& cd /app \\
|
|
||||||
&& pip install -r requirements.txt
|
|
||||||
|
|
||||||
RUN apt-get remove -y git \\
|
|
||||||
&& apt-get remove -y build-essential \\
|
|
||||||
&& apt-get remove -y libffi-dev \\
|
|
||||||
&& apt-get remove -y libjpeg-dev \\
|
|
||||||
&& apt-get remove -y libcap-dev \\
|
|
||||||
&& apt-get remove -y zlib1g-dev \\
|
|
||||||
&& apt-get remove -y apt-utils \\
|
|
||||||
&& apt-get clean \\
|
|
||||||
&& apt-get autoremove -y \\
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
'''
|
|
||||||
|
|
||||||
for port in ports:
|
|
||||||
content += 'EXPOSE {}\n'.format(port)
|
|
||||||
|
|
||||||
content += textwrap.dedent(
|
|
||||||
'''
|
|
||||||
|
|
||||||
ENV PYTHONPATH /app:$PYTHONPATH
|
|
||||||
CMD ["python", "-m", "platypush"]
|
|
||||||
'''
|
|
||||||
)
|
|
||||||
|
|
||||||
dockerfile = os.path.join(device_dir, 'Dockerfile')
|
|
||||||
print('Generating Dockerfile {}'.format(dockerfile))
|
|
||||||
|
|
||||||
with open(dockerfile, 'w') as f:
|
|
||||||
f.write(content)
|
|
||||||
|
|
||||||
|
|
||||||
def build(args):
|
|
||||||
global workdir
|
|
||||||
|
|
||||||
ports = set()
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock build', description='Build a Platypush image from a config.yaml'
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-c',
|
|
||||||
'--config',
|
|
||||||
type=str,
|
|
||||||
required=True,
|
|
||||||
help='Path to the platypush configuration file',
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'-p',
|
|
||||||
'--python-version',
|
|
||||||
type=str,
|
|
||||||
default='3.9',
|
|
||||||
help='Python version to be used',
|
|
||||||
)
|
|
||||||
|
|
||||||
opts, args = parser.parse_known_args(args)
|
|
||||||
|
|
||||||
cfgfile = os.path.abspath(os.path.expanduser(opts.config))
|
|
||||||
manifest._available_package_manager = (
|
|
||||||
'apt' # Force apt for Debian-based Docker images
|
|
||||||
)
|
|
||||||
install_cmds = manifest.get_dependencies_from_conf(cfgfile)
|
|
||||||
python_version = opts.python_version
|
|
||||||
backend_config = Config.get_backends()
|
|
||||||
|
|
||||||
# Container exposed ports
|
|
||||||
if backend_config.get('http'):
|
|
||||||
from platypush.backend.http import HttpBackend
|
|
||||||
|
|
||||||
# noinspection PyProtectedMember
|
|
||||||
ports.add(backend_config['http'].get('port', HttpBackend._DEFAULT_HTTP_PORT))
|
|
||||||
|
|
||||||
if backend_config.get('tcp'):
|
|
||||||
ports.add(backend_config['tcp']['port'])
|
|
||||||
|
|
||||||
dev_dir = os.path.join(workdir, Config.get('device_id'))
|
|
||||||
generate_dockerfile(
|
|
||||||
deps=dict(install_cmds),
|
|
||||||
ports=ports,
|
|
||||||
cfgfile=cfgfile,
|
|
||||||
device_dir=dev_dir,
|
|
||||||
python_version=python_version,
|
|
||||||
)
|
|
||||||
|
|
||||||
subprocess.call(
|
|
||||||
[
|
|
||||||
'docker',
|
|
||||||
'build',
|
|
||||||
'-t',
|
|
||||||
'platypush-{}'.format(Config.get('device_id')),
|
|
||||||
dev_dir,
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def start(args):
|
|
||||||
global workdir
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock start',
|
|
||||||
description='Start a Platypush container',
|
|
||||||
epilog=textwrap.dedent(
|
|
||||||
'''
|
|
||||||
You can append additional options that
|
|
||||||
will be passed to the docker container.
|
|
||||||
Example:
|
|
||||||
|
|
||||||
--add-host='myhost:192.168.1.1'
|
|
||||||
'''
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument('image', type=str, help='Platypush image to start')
|
|
||||||
parser.add_argument(
|
|
||||||
'-p',
|
|
||||||
'--publish',
|
|
||||||
action='append',
|
|
||||||
nargs='*',
|
|
||||||
default=[],
|
|
||||||
help=textwrap.dedent(
|
|
||||||
'''
|
|
||||||
Container's ports to expose to the host.
|
|
||||||
Note that the default exposed ports from
|
|
||||||
the container service will be exposed unless
|
|
||||||
these mappings override them (e.g. port 8008
|
|
||||||
on the container will be mapped to 8008 on
|
|
||||||
the host).
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
-p 18008:8008
|
|
||||||
'''
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-a',
|
|
||||||
'--attach',
|
|
||||||
action='store_true',
|
|
||||||
default=False,
|
|
||||||
help=textwrap.dedent(
|
|
||||||
'''
|
|
||||||
If set, then attach to the container after starting it up (default: false).
|
|
||||||
'''
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
opts, args = parser.parse_known_args(args)
|
|
||||||
ports = {}
|
|
||||||
dockerfile = os.path.join(workdir, opts.image, 'Dockerfile')
|
|
||||||
|
|
||||||
with open(dockerfile) as f:
|
|
||||||
for line in f:
|
|
||||||
m = re.match(r'expose (\d+)', line.strip().lower())
|
|
||||||
if m:
|
|
||||||
ports[m.group(1)] = m.group(1)
|
|
||||||
|
|
||||||
for mapping in opts.publish:
|
|
||||||
host_port, container_port = mapping[0].split(':')
|
|
||||||
ports[container_port] = host_port
|
|
||||||
|
|
||||||
print('Preparing Redis support container')
|
|
||||||
subprocess.call(['docker', 'pull', 'redis'])
|
|
||||||
subprocess.call(
|
|
||||||
['docker', 'run', '--rm', '--name', 'redis-' + opts.image, '-d', 'redis']
|
|
||||||
)
|
|
||||||
|
|
||||||
docker_cmd = [
|
|
||||||
'docker',
|
|
||||||
'run',
|
|
||||||
'--rm',
|
|
||||||
'--name',
|
|
||||||
opts.image,
|
|
||||||
'-it',
|
|
||||||
'--link',
|
|
||||||
'redis-' + opts.image + ':redis',
|
|
||||||
]
|
|
||||||
|
|
||||||
for container_port, host_port in ports.items():
|
|
||||||
docker_cmd += ['-p', host_port + ':' + container_port]
|
|
||||||
|
|
||||||
docker_cmd += args
|
|
||||||
docker_cmd += ['-d', 'platypush-' + opts.image]
|
|
||||||
|
|
||||||
print('Starting Platypush container {}'.format(opts.image))
|
|
||||||
subprocess.call(docker_cmd)
|
|
||||||
|
|
||||||
if opts.attach:
|
|
||||||
subprocess.call(['docker', 'attach', opts.image])
|
|
||||||
|
|
||||||
|
|
||||||
def stop(args):
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock stop', description='Stop a Platypush container'
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument('container', type=str, help='Platypush container to stop')
|
|
||||||
opts, args = parser.parse_known_args(args)
|
|
||||||
|
|
||||||
print('Stopping Platypush container {}'.format(opts.container))
|
|
||||||
subprocess.call(['docker', 'kill', '{}'.format(opts.container)])
|
|
||||||
|
|
||||||
print('Stopping Redis support container')
|
|
||||||
subprocess.call(['docker', 'stop', 'redis-{}'.format(opts.container)])
|
|
||||||
|
|
||||||
|
|
||||||
def rm(args):
|
|
||||||
global workdir
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock rm',
|
|
||||||
description='Remove a Platypush image. '
|
|
||||||
+ 'NOTE: make sure that no container is '
|
|
||||||
+ 'running nor linked to the image before '
|
|
||||||
+ 'removing it',
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument('image', type=str, help='Platypush image to remove')
|
|
||||||
opts, args = parser.parse_known_args(args)
|
|
||||||
|
|
||||||
subprocess.call(['docker', 'rmi', 'platypush-{}'.format(opts.image)])
|
|
||||||
shutil.rmtree(os.path.join(workdir, opts.image), ignore_errors=True)
|
|
||||||
|
|
||||||
|
|
||||||
def ls(args):
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
prog='platydock ls', description='List available Platypush containers'
|
|
||||||
)
|
|
||||||
parser.add_argument('filter', type=str, nargs='?', help='Image name filter')
|
|
||||||
|
|
||||||
opts, args = parser.parse_known_args(args)
|
|
||||||
|
|
||||||
p = subprocess.Popen(['docker', 'images'], stdout=subprocess.PIPE)
|
|
||||||
output = p.communicate()[0].decode().split('\n')
|
|
||||||
header = output.pop(0)
|
|
||||||
images = []
|
|
||||||
|
|
||||||
for line in output:
|
|
||||||
if re.match(r'^platypush-(.+?)\s.*', line) and (
|
|
||||||
not opts.filter or (opts.filter and opts.filter in line)
|
|
||||||
):
|
):
|
||||||
images.append(line)
|
new_file_lines.append('RUN ' + new_line)
|
||||||
|
elif line == 'RUN rm -rf /install':
|
||||||
|
for new_line in deps.to_pip_install_commands():
|
||||||
|
new_file_lines.append('RUN ' + new_line)
|
||||||
|
|
||||||
if images:
|
for new_line in deps.after:
|
||||||
print(header)
|
new_file_lines.append('RUN' + new_line)
|
||||||
|
elif line.startswith('EXPOSE ') and ports:
|
||||||
|
if not is_after_expose_cmd:
|
||||||
|
new_file_lines.extend([f'EXPOSE {port}' for port in ports])
|
||||||
|
is_after_expose_cmd = True
|
||||||
|
|
||||||
for image in images:
|
continue
|
||||||
print(image)
|
|
||||||
|
new_file_lines.append(line)
|
||||||
|
|
||||||
|
return '\n'.join(new_file_lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_exposed_ports() -> Iterable[int]:
|
||||||
|
"""
|
||||||
|
:return: The listen ports used by the backends enabled in the configuration
|
||||||
|
file.
|
||||||
|
"""
|
||||||
|
backends_config = Config.get_backends()
|
||||||
|
return {
|
||||||
|
int(port)
|
||||||
|
for port in (
|
||||||
|
backends_config.get('http', {}).get('port'),
|
||||||
|
backends_config.get('tcp', {}).get('port'),
|
||||||
|
)
|
||||||
|
if port
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
"""
|
||||||
|
Generates a Dockerfile based on the configuration file.
|
||||||
|
"""
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
prog='platydock',
|
prog='platydock',
|
||||||
add_help=False,
|
add_help=False,
|
||||||
description='Manage Platypush docker containers',
|
description='Create a Platypush Dockerfile from a config.yaml.',
|
||||||
epilog='Use platydock <action> --help to ' + 'get additional help',
|
epilog='Use platydock <action> --help to get additional help.',
|
||||||
)
|
)
|
||||||
|
|
||||||
# noinspection PyTypeChecker
|
|
||||||
parser.add_argument(
|
|
||||||
'action', nargs='?', type=Action, choices=list(Action), help='Action to execute'
|
|
||||||
)
|
|
||||||
parser.add_argument('-h', '--help', action='store_true', help='Show usage')
|
parser.add_argument('-h', '--help', action='store_true', help='Show usage')
|
||||||
opts, args = parser.parse_known_args(sys.argv[1:])
|
parser.add_argument(
|
||||||
|
'cfgfile', type=str, nargs=1, help='The path to the configuration file.'
|
||||||
|
)
|
||||||
|
|
||||||
if (opts.help and not opts.action) or (not opts.help and not opts.action):
|
opts, _ = parser.parse_known_args(sys.argv[1:])
|
||||||
parser.print_help()
|
cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0]))
|
||||||
return 1
|
dockerfile = generate_dockerfile(cfgfile)
|
||||||
|
print(dockerfile)
|
||||||
globals()[str(opts.action)](sys.argv[2:])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m '
|
|
||||||
ERR_SUFFIX = '\033[0m'
|
|
||||||
|
|
||||||
try:
|
|
||||||
main()
|
main()
|
||||||
except Exception as e:
|
|
||||||
tb.print_exc(file=sys.stdout)
|
|
||||||
print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr)
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue