platypush/platypush
platypush
/
platypush
2
2
Fork 4
Code Issues 47 Pull Requests Packages Projects 4 Releases Wiki Activity

Refactored platydock and platyvenv

This commit is contained in:
Fabio Manganiello 2019-12-01 22:27:54 +01:00
parent f27e1efdd6
commit f8d3ea5197
3 changed files with 95 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
Dockerfile
View File

@ -1,18 +1,22 @@
FROM python:alpine3.7 # Sample Dockerfile. Use platydock -c /path/to/custom/config.yamlr
# to generate your custom Dockerfile.
RUN mkdir /app
COPY . /app
WORKDIR /app FROM python:alpine3.7
RUN apk add --update --no-cache --virtual build-base \
&& apk add --update --no-cache libffi-dev \ RUN mkdir /app
&& apk add --update --no-cache zlib-dev \ COPY . /app
&& apk add --update --no-cache libjpeg-turbo-dev \ WORKDIR /app
&& pip install -r requirements.txt \ RUN apk add --update --no-cache --virtual build-base \
&& apk del build-base \ && apk add --update --no-cache libffi-dev \
&& apk del libffi-dev \ && apk add --update --no-cache zlib-dev \
&& apk del libjpeg-turbo-dev \ && apk add --update --no-cache libjpeg-turbo-dev \
&& apk del zlib-dev && pip install -r requirements.txt \
&& apk del build-base \
RUN cd /app && python setup.py build install && apk del libffi-dev \
CMD python -m platypush && apk del libjpeg-turbo-dev \
&& apk del zlib-dev
RUN cd /app && python setup.py build install
CMD python -m platypush

88
bin/platyvenv
View File

@ -17,7 +17,7 @@ function build {
cfgfile= cfgfile=
while getopts ':c:' opt; do while getopts ':c:' opt; do
case $opt in case ${opt} in
c) c)
cfgfile=$OPTARG;; cfgfile=$OPTARG;;
\?) \?)
@ -29,7 +29,7 @@ function build {
esac esac
done done
if [ -z "$cfgfile" ]; then if [[ -z "$cfgfile" ]]; then
echo "Usage: $0 build -c <path-to-platypush-config-file>" >&2 echo "Usage: $0 build -c <path-to-platypush-config-file>" >&2
exit 1 exit 1
fi fi
@ -39,18 +39,18 @@ function build {
includes=() includes=()
while read -r line; do while read -r line; do
echo $line | egrep '``pip install .+?``' > /dev/null 2>&1 echo ${line} | egrep '``pip install .+?``' > /dev/null 2>&1
if (( $? != 0 )); then if (( $? != 0 )); then
continue continue
fi fi
dep=$(echo $line | sed -r -e 's/.*``pip install (.+?)``.*/\1/') dep=$(echo ${line} | sed -r -e 's/.*``pip install (.+?)``.*/\1/')
deps+=("$dep") deps+=("$dep")
done <<< $(python <<EOF done <<< $(python <<EOF
from platypush.config import Config from platypush.config import Config
from platypush.context import get_plugin, get_backend, register_backends from platypush.context import get_plugin, get_backend, register_backends
Config.init('`realpath $cfgfile`') Config.init('`realpath ${cfgfile}`')
register_backends() register_backends()
backend_config = Config.get_backends() backend_config = Config.get_backends()
@ -68,12 +68,12 @@ EOF
) )
while read -r include; do while read -r include; do
includes+=($include) includes+=(${include})
done <<< $(python <<EOF done <<< $(python <<EOF
from platypush.config import Config from platypush.config import Config
from platypush.context import get_plugin, get_backend, register_backends from platypush.context import get_plugin, get_backend, register_backends
Config.init('`realpath $cfgfile`') Config.init('`realpath ${cfgfile}`')
for include in Config._included_files: for include in Config._included_files:
print(include) print(include)
@ -83,13 +83,13 @@ EOF
device_id=$(python <<EOF device_id=$(python <<EOF
from platypush.config import Config from platypush.config import Config
Config.init('`realpath $cfgfile`') Config.init('`realpath ${cfgfile}`')
print(Config.get('device_id')) print(Config.get('device_id'))
EOF EOF
) )
envdir=$workdir/$device_id envdir=${workdir}/${device_id}
etcdir=$envdir/etc/platypush etcdir=${envdir}/etc/platypush
echo "Preparing virtual environment for device $device_id" echo "Preparing virtual environment for device $device_id"
mkdir -p "$envdir" mkdir -p "$envdir"
@ -105,18 +105,18 @@ EOF
done done
cp "$cfgfile" "$etcdir/config.yaml" cp "$cfgfile" "$etcdir/config.yaml"
cfgfile=$etcdir/config.yaml cfgfile=${etcdir}/config.yaml
python3 -m venv $envdir python3 -m venv ${envdir}
cd $envdir cd ${envdir}
source $envdir/bin/activate source ${envdir}/bin/activate
echo "Installing required dependencies" echo "Installing required dependencies"
for ((i=0; $i < ${#deps[@]}; i++)); do for ((i=0; $i < ${#deps[@]}; i++)); do
echo ${deps[$i]} echo ${deps[$i]}
done | sort -u | while read dep; do done | sort -u | while read dep; do
pip install $dep pip install ${dep}
done done
pip install --upgrade git+https://github.com/BlackLight/platypush.git pip install --upgrade git+https://github.com/BlackLight/platypush.git
@ -124,29 +124,29 @@ EOF
} }
function start { function start {
if [ -z "$1" ]; then if [[ -z "$1" ]]; then
echo "Usage: $0 start <env-name>" >&2 echo "Usage: $0 start <env-name>" >&2
exit 1 exit 1
fi fi
env=$1 env=$1
envdir=$workdir/$env envdir=${workdir}/${env}
logsdir=$envdir/var/log/platypush logsdir=${envdir}/var/log/platypush
rundir=$envdir/var/run rundir=${envdir}/var/run
pidfile=$rundir/platypush.pid pidfile=${rundir}/platypush.pid
cfgfile=$envdir/etc/platypush/config.yaml cfgfile=${envdir}/etc/platypush/config.yaml
if [ ! -d "$envdir" ]; then if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2 echo "No such directory: $envdir" >&2
exit 1 exit 1
fi fi
mkdir -p $logsdir mkdir -p ${logsdir}
mkdir -p $rundir mkdir -p ${rundir}
if [[ -f "$pidfile" ]]; then if [[ -f "$pidfile" ]]; then
pid=`cat "$pidfile"` pid=`cat "$pidfile"`
if ps -p $pid | grep platypush; then if ps -p ${pid} | grep platypush; then
echo "Another instance (PID $pid) is running, please stop that instance first" echo "Another instance (PID $pid) is running, please stop that instance first"
exit 1 exit 1
fi fi
@ -155,10 +155,10 @@ function start {
rm -f "$pidfile" rm -f "$pidfile"
fi fi
python3 -m venv $envdir python3 -m venv ${envdir}
cd $envdir cd ${envdir}
source bin/activate source bin/activate
bin/platypush -c "$cfgfile" -P "$pidfile" > $logsdir/stdout.log 2> $logsdir/stderr.log & bin/platypush -c "$cfgfile" -P "$pidfile" > ${logsdir}/stdout.log 2> ${logsdir}/stderr.log &
start_time=`date +'%s'` start_time=`date +'%s'`
timeout=30 timeout=30
@ -166,7 +166,7 @@ function start {
[[ -f "$pidfile" ]] && break [[ -f "$pidfile" ]] && break
now=`date +'%s'` now=`date +'%s'`
let elapsed=$now-$start_time let elapsed=$now-$start_time
if (( $elapsed >= $timeout )); then if (( ${elapsed} >= ${timeout} )); then
echo "Platypush instance '$env' didn't start within $timeout seconds" >&2 echo "Platypush instance '$env' didn't start within $timeout seconds" >&2
exit 1 exit 1
fi fi
@ -181,44 +181,44 @@ function start {
} }
function stop { function stop {
if [ -z "$1" ]; then if [[ -z "$1" ]]; then
echo "Usage: $0 stop <env-name>" >&2 echo "Usage: $0 stop <env-name>" >&2
exit 1 exit 1
fi fi
env=$1 env=$1
envdir=$workdir/$env envdir=${workdir}/${env}
rundir=$envdir/var/run rundir=${envdir}/var/run
pidfile=$rundir/platypush.pid pidfile=${rundir}/platypush.pid
if [ ! -d "$envdir" ]; then if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2 echo "No such directory: $envdir" >&2
exit 1 exit 1
fi fi
if [[ ! -f "$pidfile" ]]; then if [[ ! -f "$pidfile" ]]; then
echo "No pidfile found for instance "$env"" echo "No pidfile found for instance "${env}""
exit 1 exit 1
fi fi
pid=`cat "$pidfile"` pid=`cat "$pidfile"`
pids="$pid `ps --no-headers -o pid= --ppid $pid`" pids="$pid `ps --no-headers -o pid= --ppid $pid`"
kill -9 $pids kill -9 ${pids}
rm -f "$pidfile" rm -f "$pidfile"
echo "Instance '$env' with PID $pid stopped" echo "Instance '$env' with PID $pid stopped"
} }
function rme { function rme {
if [ -z "$1" ]; then if [[ -z "$1" ]]; then
echo "Usage: $0 rm <env-name>" >&2 echo "Usage: $0 rm <env-name>" >&2
exit 1 exit 1
fi fi
envdir=$workdir/$1 envdir=${workdir}/$1
rundir=$envdir/var/run rundir=${envdir}/var/run
pidfile=$rundir/platypush.pid pidfile=${rundir}/platypush.pid
if [ ! -d "$envdir" ]; then if [[ ! -d "$envdir" ]]; then
echo "No such directory: $envdir" >&2 echo "No such directory: $envdir" >&2
exit 1 exit 1
fi fi
@ -247,10 +247,10 @@ fi
action=$1 action=$1
shift shift
mkdir -p $workdir mkdir -p ${workdir}
case $action in case ${action} in
'build') build $*;; 'build') build;;
'start') start $*;; 'start') start $*;;
'stop') stop $*;; 'stop') stop $*;;
'rm') rme $*;; 'rm') rme $*;;

59
platypush/platydock/__init__.py
View File

@ -17,21 +17,19 @@ import sys
import textwrap import textwrap
import traceback as tb import traceback as tb
import platypush
from platypush.config import Config from platypush.config import Config
from platypush.context import register_backends, get_plugin, get_backend from platypush.context import register_backends, get_plugin, get_backend
workdir = os.path.join(os.path.expanduser('~'), '.local', 'share', workdir = os.path.join(os.path.expanduser('~'), '.local', 'share',
'platypush', 'platydock') 'platypush', 'platydock')
class Action(enum.Enum): class Action(enum.Enum):
build = 'build' build = 'build'
start = 'start' start = 'start'
stop = 'stop' stop = 'stop'
rm = 'rm' rm = 'rm'
ls = 'ls' ls = 'ls'
def __str__(self): def __str__(self):
return self.value return self.value
@ -47,30 +45,28 @@ def _parse_deps(cls):
return deps return deps
def generate_dockerfile(deps, ports, cfgfile, devdir):
def generate_dockerfile(deps, ports, cfgfile, devdir, python_version):
device_id = Config.get('device_id') device_id = Config.get('device_id')
if not device_id: if not device_id:
raise RuntimeError(('You need to specify a device_id in {} - Docker ' + raise RuntimeError(('You need to specify a device_id in {} - Docker ' +
'containers cannot rely on hostname').format(cfgfile)) 'containers cannot rely on hostname').format(cfgfile))
try:
os.makedirs(devdir)
except FileExistsError:
pass
os.makedirs(devdir, exist_ok=True)
content = textwrap.dedent( content = textwrap.dedent(
''' '''
FROM python:alpine3.7 FROM python:alpine{python_version}
RUN mkdir -p /etc/platypush RUN mkdir -p /etc/platypush
RUN mkdir -p /usr/local/share/platypush\n RUN mkdir -p /usr/local/share/platypush\n
''').lstrip() '''.format(python_version=python_version)).lstrip()
srcdir = os.path.dirname(cfgfile) srcdir = os.path.dirname(cfgfile)
cfgfile_copy = os.path.join(devdir, 'config.yaml') cfgfile_copy = os.path.join(devdir, 'config.yaml')
subprocess.call(['cp', cfgfile, cfgfile_copy]) subprocess.call(['cp', cfgfile, cfgfile_copy])
content += 'COPY config.yaml /etc/platypush/\n' content += 'COPY config.yaml /etc/platypush/\n'
# noinspection PyProtectedMember
for include in Config._included_files: for include in Config._included_files:
incdir = os.path.relpath(os.path.dirname(include), srcdir) incdir = os.path.relpath(os.path.dirname(include), srcdir)
destdir = os.path.join(devdir, incdir) destdir = os.path.join(devdir, incdir)
@ -83,8 +79,7 @@ def generate_dockerfile(deps, ports, cfgfile, devdir):
subprocess.call(['cp', include, destdir]) subprocess.call(['cp', include, destdir])
content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n' content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n'
content += 'COPY ' + os.path.relpath(include, srcdir) + \ content += 'COPY ' + os.path.relpath(include, srcdir) + \
' /etc/platypush/' + incdir + '\n' ' /etc/platypush/' + incdir + '\n'
content += textwrap.dedent( content += textwrap.dedent(
''' '''
@ -96,7 +91,7 @@ def generate_dockerfile(deps, ports, cfgfile, devdir):
content += '\t&& pip install {} \\\n'.format(dep) content += '\t&& pip install {} \\\n'.format(dep)
content += '\t&& pip install ' + \ content += '\t&& pip install ' + \
'git+https://github.com/BlackLight/platypush.git \\\n' 'git+https://github.com/BlackLight/platypush.git \\\n'
content += '\t&& apk del git \\\n' content += '\t&& apk del git \\\n'
content += '\t&& apk del build-base\n\n' content += '\t&& apk del build-base\n\n'
@ -120,14 +115,17 @@ def build(args):
parser = argparse.ArgumentParser(prog='platydock build', parser = argparse.ArgumentParser(prog='platydock build',
description='Build a Platypush image ' + description='Build a Platypush image ' +
'from a config.yaml') 'from a config.yaml')
parser.add_argument('-c', '--config', type=str, required=True, parser.add_argument('-c', '--config', type=str, required=True,
help='Path to the platypush configuration file') help='Path to the platypush configuration file')
parser.add_argument('-p', '--python-version', type=str, default='3.8',
help='Python version to be used')
opts, args = parser.parse_known_args(args) opts, args = parser.parse_known_args(args)
cfgfile = os.path.abspath(os.path.expanduser(opts.config)) cfgfile = os.path.abspath(os.path.expanduser(opts.config))
python_version = opts.python_version
Config.init(cfgfile) Config.init(cfgfile)
register_backends() register_backends()
backend_config = Config.get_backends() backend_config = Config.get_backends()
@ -150,11 +148,11 @@ def build(args):
for name in Config.get_plugins().keys(): for name in Config.get_plugins().keys():
try: try:
deps.update(_parse_deps(get_plugin(name))) deps.update(_parse_deps(get_plugin(name)))
except: except Exception as ex:
pass print('Dependencies parsing error for {}: {}'.format(name, str(ex)))
devdir = os.path.join(workdir, Config.get('device_id')) devdir = os.path.join(workdir, Config.get('device_id'))
generate_dockerfile(deps=deps, ports=ports, cfgfile=cfgfile, devdir=devdir) generate_dockerfile(deps=deps, ports=ports, cfgfile=cfgfile, devdir=devdir, python_version=python_version)
subprocess.call(['docker', 'build', '-t', 'platypush-{}'.format( subprocess.call(['docker', 'build', '-t', 'platypush-{}'.format(
Config.get('device_id')), devdir]) Config.get('device_id')), devdir])
@ -164,8 +162,8 @@ def start(args):
global workdir global workdir
parser = argparse.ArgumentParser(prog='platydock start', parser = argparse.ArgumentParser(prog='platydock start',
description='Start a Platypush container', description='Start a Platypush container',
epilog=textwrap.dedent(''' epilog=textwrap.dedent('''
You can append additional options that You can append additional options that
will be passed to the docker container. will be passed to the docker container.
Example: Example:
@ -239,9 +237,9 @@ def rm(args):
parser = argparse.ArgumentParser(prog='platydock rm', parser = argparse.ArgumentParser(prog='platydock rm',
description='Remove a Platypush image. ' + description='Remove a Platypush image. ' +
'NOTE: make sure that no container is ' + 'NOTE: make sure that no container is ' +
'running nor linked to the image before ' + 'running nor linked to the image before ' +
'removing it') 'removing it')
parser.add_argument('image', type=str, help='Platypush image to remove') parser.add_argument('image', type=str, help='Platypush image to remove')
opts, args = parser.parse_known_args(args) opts, args = parser.parse_known_args(args)
@ -274,12 +272,14 @@ def ls(args):
for image in images: for image in images:
print(image) print(image)
def main(): def main():
parser = argparse.ArgumentParser(prog='platydock', add_help=False, parser = argparse.ArgumentParser(prog='platydock', add_help=False,
description='Manage Platypush docker containers', description='Manage Platypush docker containers',
epilog='Use platydock <action> --help to ' + epilog='Use platydock <action> --help to ' +
'get additional help') 'get additional help')
# noinspection PyTypeChecker
parser.add_argument('action', nargs='?', type=Action, choices=list(Action), parser.add_argument('action', nargs='?', type=Action, choices=list(Action),
help='Action to execute') help='Action to execute')
parser.add_argument('-h', '--help', action='store_true', help='Show usage') parser.add_argument('-h', '--help', action='store_true', help='Show usage')
@ -302,5 +302,4 @@ if __name__ == '__main__':
tb.print_exc(file=sys.stdout) tb.print_exc(file=sys.stdout)
print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr) print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr)
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et: