From f07b774e75f4878f6ea0139ff9b4c60e5594c245 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 03:04:34 +0200 Subject: [PATCH 01/64] A better Dockerfile. --- .dockerignore | 6 ++++++ examples/docker/Dockerfile | 43 +++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..376b7571f6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +**/.git +**/node_modules +**/__pycache__ +**/venv +**/.mypy_cache +**/build diff --git a/examples/docker/Dockerfile b/examples/docker/Dockerfile index 374ffbdc5d..137ed3ec54 100644 --- a/examples/docker/Dockerfile +++ b/examples/docker/Dockerfile @@ -1,19 +1,38 @@ -# Sample Dockerfile. Use platydock -c /path/to/custom/config.yaml -# to generate your custom Dockerfile. +FROM alpine +ADD . /install +RUN apk add --update --no-interactive --no-cache \ + python3 \ + py3-pip \ + py3-alembic \ + py3-bcrypt \ + py3-dateutil \ + py3-docutils \ + py3-flask \ + py3-frozendict \ + py3-greenlet \ + py3-magic \ + py3-mypy-extensions \ + py3-psutil \ + py3-redis \ + py3-requests \ + py3-rsa \ + py3-sqlalchemy \ + py3-tornado \ + py3-typing-extensions \ + py3-tz \ + py3-websocket-client \ + py3-websockets \ + py3-wheel \ + py3-yaml \ + py3-zeroconf \ + redis -FROM python:3.11-alpine - -RUN mkdir -p /install /app -COPY . /install -RUN apk add --update --no-cache redis -RUN apk add --update --no-cache --virtual build-base g++ rust linux-headers -RUN pip install -U pip -RUN cd /install && pip install . -RUN apk del build-base g++ rust linux-headers +RUN cd /install && pip install --no-cache-dir . +RUN cd / && rm -rf /install EXPOSE 8008 VOLUME /app/config VOLUME /app/workdir -CMD python -m platypush --start-redis --config-file /app/config/config.yaml --workdir /app/workdir +CMD platypush --start-redis --config-file /app/config/config.yaml --workdir /app/workdir From 24b04d9103283d1722dd3d9ae84ab2dcd0edd551 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 10:35:40 +0200 Subject: [PATCH 02/64] s/--config-file/--config/ option in Dockerfile. --- examples/docker/Dockerfile | 2 +- platypush/app/__main__.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/docker/Dockerfile b/examples/docker/Dockerfile index 137ed3ec54..7bcd97a42c 100644 --- a/examples/docker/Dockerfile +++ b/examples/docker/Dockerfile @@ -35,4 +35,4 @@ EXPOSE 8008 VOLUME /app/config VOLUME /app/workdir -CMD platypush --start-redis --config-file /app/config/config.yaml --workdir /app/workdir +CMD platypush --start-redis --config /app/config/config.yaml --workdir /app/workdir diff --git a/platypush/app/__main__.py b/platypush/app/__main__.py index ed2e42208f..54dea00ef3 100644 --- a/platypush/app/__main__.py +++ b/platypush/app/__main__.py @@ -2,6 +2,4 @@ import sys from ._app import main - -if __name__ == '__main__': - sys.exit(main(*sys.argv[1:])) +sys.exit(main(*sys.argv[1:])) From afa4de56739b4b750bb9528e560aa93c21789136 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 10:38:21 +0200 Subject: [PATCH 03/64] Dockerfile moved to application root --- examples/docker/Dockerfile => Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/docker/Dockerfile => Dockerfile (100%) diff --git a/examples/docker/Dockerfile b/Dockerfile similarity index 100% rename from examples/docker/Dockerfile rename to Dockerfile From 657b2cc87d98d01d800692907df995ba18617337 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 11:25:49 +0200 Subject: [PATCH 04/64] Create the default configuration file even if --config is supplied but the file doesn't exist. --- platypush/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 4e6e1ee53d..82c52eac3d 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -104,7 +104,7 @@ class Config: if cfgfile is None: cfgfile = self._get_default_cfgfile() - if cfgfile is None: + if cfgfile is None or not os.path.exists(cfgfile): cfgfile = self._create_default_config() self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) From ac83b43f98499ad3274544e8aa459ea652eed4f9 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 21:59:28 +0200 Subject: [PATCH 05/64] Support for custom key-value overrides on `Config.init`. --- platypush/config/__init__.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 82c52eac3d..9ad77f2f44 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -463,13 +463,31 @@ class Config: return None @classmethod - def init(cls, cfgfile: Optional[str] = None): + def init( + cls, + cfgfile: Optional[str] = None, + device_id: Optional[str] = None, + workdir: Optional[str] = None, + ctrl_sock: Optional[str] = None, + **_, + ): """ Initializes the config object singleton - Params: - cfgfile -- path to the config file - default: _cfgfile_locations + + :param cfgfile: Path to the config file (default: _cfgfile_locations) + :param device_id: Override the configured device_id. + :param workdir: Override the configured working directory. + :param ctrl_sock: Override the configured control socket. """ - return cls._get_instance(cfgfile, force_reload=True) + cfg = cls._get_instance(cfgfile, force_reload=True) + if device_id: + cfg.set('device_id', device_id) + if workdir: + cfg.set('workdir', workdir) + if ctrl_sock: + cfg.set('ctrl_sock', ctrl_sock) + + return cfg @classmethod @property From ec64b0ef8b83fa7a13501b59c238a39133584166 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 23:16:24 +0200 Subject: [PATCH 06/64] Added `--device_id` command line option. --- platypush/app/_app.py | 19 ++++++++++++++----- platypush/cli.py | 11 +++++++++++ platypush/platydock/__main__.py | 1 - 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/platypush/app/_app.py b/platypush/app/_app.py index e95d0db3f6..c11e1af0e9 100644 --- a/platypush/app/_app.py +++ b/platypush/app/_app.py @@ -42,6 +42,7 @@ class Application: config_file: Optional[str] = None, workdir: Optional[str] = None, logsdir: Optional[str] = None, + device_id: Optional[str] = None, pidfile: Optional[str] = None, requests_to_process: Optional[int] = None, no_capture_stdout: bool = False, @@ -61,6 +62,10 @@ class Application: ``filename`` setting under the ``logging`` section of the configuration file is used. If not set, logging will be sent to stdout and stderr. + :param device_id: Override the device ID used to identify this + instance. If not passed here, it is inferred from the configuration + (device_id field). If not present there either, it is inferred from + the hostname. :param pidfile: File where platypush will store its PID upon launch, useful if you're planning to integrate the application within a service or a launcher script (default: None). @@ -97,11 +102,14 @@ class Application: os.path.abspath(os.path.expanduser(logsdir)) if logsdir else None ) - Config.init(self.config_file) - Config.set('ctrl_sock', ctrl_sock) - - if workdir: - Config.set('workdir', os.path.abspath(os.path.expanduser(workdir))) + Config.init( + self.config_file, + device_id=device_id, + workdir=os.path.abspath(os.path.expanduser(workdir)) if workdir else None, + ctrl_sock=os.path.abspath(os.path.expanduser(ctrl_sock)) + if ctrl_sock + else None, + ) self.no_capture_stdout = no_capture_stdout self.no_capture_stderr = no_capture_stderr @@ -199,6 +207,7 @@ class Application: config_file=opts.config, workdir=opts.workdir, logsdir=opts.logsdir, + device_id=opts.device_id, pidfile=opts.pidfile, no_capture_stdout=opts.no_capture_stdout, no_capture_stderr=opts.no_capture_stderr, diff --git a/platypush/cli.py b/platypush/cli.py index 9864c16be0..f3768b199c 100644 --- a/platypush/cli.py +++ b/platypush/cli.py @@ -29,6 +29,17 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace: help='Custom working directory to be used for the application', ) + parser.add_argument( + '--device-id', + '-d', + dest='device_id', + required=False, + default=None, + help='Override the device ID used to identify this instance. If not ' + 'passed here, it is inferred from the configuration (device_id field).' + 'If not present there either, it is inferred from the hostname.', + ) + parser.add_argument( '--logsdir', '-l', diff --git a/platypush/platydock/__main__.py b/platypush/platydock/__main__.py index f216837d4e..bc0461ab7f 100644 --- a/platypush/platydock/__main__.py +++ b/platypush/platydock/__main__.py @@ -3,4 +3,3 @@ from platypush.platydock import main main() # vim:sw=4:ts=4:et: - From e463a52435ae79967384804de0f111e19da6649e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 12:40:57 +0200 Subject: [PATCH 07/64] Use `sys.executable` rather than `'python'` to launch the application. --- platypush/runner/_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platypush/runner/_app.py b/platypush/runner/_app.py index 6ff9207be5..718780e76b 100644 --- a/platypush/runner/_app.py +++ b/platypush/runner/_app.py @@ -32,7 +32,7 @@ class ApplicationProcess(ControllableProcess): self.logger.info('Starting application...') with subprocess.Popen( - ['python', '-m', 'platypush.app', *self.args], + [sys.executable, '-m', 'platypush.app', *self.args], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, From c2b3ec8ce3c8db01474a3c0a8e2e4269b4d9e00a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 12:54:33 +0200 Subject: [PATCH 08/64] Fixed manifest files with outdated formats. --- platypush/backend/music/spotify/manifest.yaml | 2 +- platypush/plugins/gpio/manifest.yaml | 3 +-- platypush/plugins/http/webpage/manifest.yaml | 2 +- platypush/plugins/music/tidal/manifest.yaml | 3 +-- platypush/plugins/sensor/distance/vl53l1x/manifest.yaml | 2 +- platypush/plugins/serial/manifest.yaml | 2 +- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/platypush/backend/music/spotify/manifest.yaml b/platypush/backend/music/spotify/manifest.yaml index 8c165cec21..83e16714ff 100644 --- a/platypush/backend/music/spotify/manifest.yaml +++ b/platypush/backend/music/spotify/manifest.yaml @@ -18,7 +18,7 @@ manifest: pacman: - sudo - cargo - exec: + after: - sudo cargo install librespot package: platypush.backend.music.spotify type: backend diff --git a/platypush/plugins/gpio/manifest.yaml b/platypush/plugins/gpio/manifest.yaml index dfe358cc9f..dc52185a2b 100644 --- a/platypush/plugins/gpio/manifest.yaml +++ b/platypush/plugins/gpio/manifest.yaml @@ -1,7 +1,6 @@ manifest: events: - - platypush.message.event.gpio.GPIOEvent: - When the value of a monitored PIN changes. + - platypush.message.event.gpio.GPIOEvent install: pip: - RPi.GPIO diff --git a/platypush/plugins/http/webpage/manifest.yaml b/platypush/plugins/http/webpage/manifest.yaml index 04560d806a..c266b4aceb 100644 --- a/platypush/plugins/http/webpage/manifest.yaml +++ b/platypush/plugins/http/webpage/manifest.yaml @@ -15,7 +15,7 @@ manifest: - npm pip: - weasyprint - exec: + after: - sudo npm install -g @postlight/mercury-parser package: platypush.plugins.http.webpage type: plugin diff --git a/platypush/plugins/music/tidal/manifest.yaml b/platypush/plugins/music/tidal/manifest.yaml index e047ad5ddf..ba06b25759 100644 --- a/platypush/plugins/music/tidal/manifest.yaml +++ b/platypush/plugins/music/tidal/manifest.yaml @@ -1,7 +1,6 @@ manifest: events: - - platypush.message.event.music.TidalPlaylistUpdatedEvent: when a user playlist - is updated. + - platypush.message.event.music.tidal.TidalPlaylistUpdatedEvent install: pip: - tidalapi >= 0.7.0 diff --git a/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml b/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml index f041778e20..4aa00b1abf 100644 --- a/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml +++ b/platypush/plugins/sensor/distance/vl53l1x/manifest.yaml @@ -1,6 +1,6 @@ manifest: events: - platypush.message.event.sensor import SensorDataChangeEvent: + - platypush.message.event.sensor.SensorDataChangeEvent install: pip: diff --git a/platypush/plugins/serial/manifest.yaml b/platypush/plugins/serial/manifest.yaml index 1af0e50f7f..2756fc8347 100644 --- a/platypush/plugins/serial/manifest.yaml +++ b/platypush/plugins/serial/manifest.yaml @@ -1,6 +1,6 @@ manifest: events: - - platypush.message.event.sensor.SensorDataChangeEvent: + - platypush.message.event.sensor.SensorDataChangeEvent install: apk: - py3-pyserial From 181da63c89184376381db2ddf2c4c666073d326f Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:02:05 +0200 Subject: [PATCH 09/64] Pass the database engine to the Alembic process as an extra argument. If the path of the default database engine is overridden via `--workdir` option then it won't be visible to the new `python` subprocess spawned for Alembic. --- platypush/entities/_base.py | 27 ++++++++++++++++++++++----- platypush/migrations/alembic/env.py | 13 +++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/platypush/entities/_base.py b/platypush/entities/_base.py index c6d5c18b3c..c3089dde56 100644 --- a/platypush/entities/_base.py +++ b/platypush/entities/_base.py @@ -24,6 +24,7 @@ from sqlalchemy import ( UniqueConstraint, inspect as schema_inspect, ) +from sqlalchemy.engine import Engine from sqlalchemy.orm import ColumnProperty, backref, relationship from sqlalchemy.orm.exc import ObjectDeletedError @@ -303,6 +304,24 @@ def _discover_entity_types(): entities_registry[obj] = {} # type: ignore +def _get_db(): + """ + Utility method to get the db plugin. + """ + from platypush.context import get_plugin + + db = get_plugin('db') + assert db + return db + + +def _get_db_engine() -> Engine: + """ + Utility method to get the db engine. + """ + return _get_db().get_engine() + + def get_entities_registry() -> EntityRegistryType: """ :returns: A copy of the entities registry. @@ -314,13 +333,9 @@ def init_entities_db(): """ Initializes the entities database. """ - from platypush.context import get_plugin - run_db_migrations() _discover_entity_types() - db = get_plugin('db') - assert db - db.create_all(db.get_engine(), Base) + _get_db().create_all(_get_db_engine(), Base) def run_db_migrations(): @@ -339,6 +354,8 @@ def run_db_migrations(): 'alembic', '-c', alembic_ini, + '-x', + f'DBNAME={_get_db_engine().url}', 'upgrade', 'head', ], diff --git a/platypush/migrations/alembic/env.py b/platypush/migrations/alembic/env.py index a9de8c4f26..406b1f6e2a 100644 --- a/platypush/migrations/alembic/env.py +++ b/platypush/migrations/alembic/env.py @@ -74,14 +74,15 @@ def run_migrations_online() -> None: def set_db_engine(): - db_conf = Config.get('db') - assert db_conf, 'Could not retrieve the database configuration' - engine = db_conf['engine'] - assert engine, 'No database engine configured' + engine_url = context.get_x_argument(as_dictionary=True).get('DBNAME') + if not engine_url: + db_conf = Config.get('db') + assert db_conf, 'Could not retrieve the database configuration' + engine_url = db_conf['engine'] + assert engine_url, 'No database engine configured' - config = context.config section = config.config_ini_section - config.set_section_option(section, 'DB_ENGINE', engine) + config.set_section_option(section, 'DB_ENGINE', engine_url) set_db_engine() From 5bc82dfe640a77cf25e5aca0750c772d0751c8ee Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:13:36 +0200 Subject: [PATCH 10/64] s/Config._cfgfile/Config.config_file/g --- platypush/config/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 9ad77f2f44..eb7e274fc1 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -83,10 +83,10 @@ class Config: self.dashboards = {} self._plugin_manifests = {} self._backend_manifests = {} - self._cfgfile = '' + self.config_file = '' self._init_cfgfile(cfgfile) - self._config = self._read_config_file(self._cfgfile) + self._config = self._read_config_file(self.config_file) self._init_secrets() self._init_dirs() @@ -107,7 +107,7 @@ class Config: if cfgfile is None or not os.path.exists(cfgfile): cfgfile = self._create_default_config() - self._cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.config_file = os.path.abspath(os.path.expanduser(cfgfile)) def _init_logging(self): logging_config = { @@ -171,13 +171,13 @@ class Config: if 'scripts_dir' not in self._config: self._config['scripts_dir'] = os.path.join( - os.path.dirname(self._cfgfile), 'scripts' + os.path.dirname(self.config_file), 'scripts' ) os.makedirs(self._config['scripts_dir'], mode=0o755, exist_ok=True) if 'dashboards_dir' not in self._config: self._config['dashboards_dir'] = os.path.join( - os.path.dirname(self._cfgfile), 'dashboards' + os.path.dirname(self.config_file), 'dashboards' ) os.makedirs(self._config['dashboards_dir'], mode=0o755, exist_ok=True) @@ -216,7 +216,6 @@ class Config: def _read_config_file(self, cfgfile): cfgfile_dir = os.path.dirname(os.path.abspath(os.path.expanduser(cfgfile))) - config = {} try: From a8836f95f511969696bac64605725514276dd51d Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:15:29 +0200 Subject: [PATCH 11/64] Support explicit `workdir` parameter override in `Config` constructor. --- platypush/config/__init__.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index eb7e274fc1..67541f1761 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -65,13 +65,14 @@ class Config: _included_files: Set[str] = set() - def __init__(self, cfgfile: Optional[str] = None): + def __init__(self, cfgfile: Optional[str] = None, workdir: Optional[str] = None): """ Constructor. Always use the class as a singleton (i.e. through Config.init), you won't probably need to call the constructor directly :param cfgfile: Config file path (default: retrieve the first available location in _cfgfile_locations). + :param workdir: Overrides the default working directory. """ self.backends = {} @@ -89,7 +90,7 @@ class Config: self._config = self._read_config_file(self.config_file) self._init_secrets() - self._init_dirs() + self._init_dirs(workdir=workdir) self._init_db() self._init_logging() self._init_device_id() @@ -163,11 +164,16 @@ class Config: for k, v in self._config['environment'].items(): os.environ[k] = str(v) - def _init_dirs(self): - if 'workdir' not in self._config: + def _init_dirs(self, workdir: Optional[str] = None): + if workdir: + self._config['workdir'] = workdir + if not self._config.get('workdir'): self._config['workdir'] = self._workdir_location - self._config['workdir'] = os.path.expanduser(self._config['workdir']) - os.makedirs(self._config['workdir'], exist_ok=True) + + self._config['workdir'] = os.path.expanduser( + os.path.expanduser(self._config['workdir']) + ) + pathlib.Path(self._config['workdir']).mkdir(parents=True, exist_ok=True) if 'scripts_dir' not in self._config: self._config['scripts_dir'] = os.path.join( @@ -396,14 +402,17 @@ class Config: @classmethod def _get_instance( - cls, cfgfile: Optional[str] = None, force_reload: bool = False + cls, + cfgfile: Optional[str] = None, + workdir: Optional[str] = None, + force_reload: bool = False, ) -> "Config": """ Lazy getter/setter for the default configuration instance. """ if force_reload or cls._instance is None: cfg_args = [cfgfile] if cfgfile else [] - cls._instance = Config(*cfg_args) + cls._instance = Config(*cfg_args, workdir=workdir) return cls._instance @classmethod @@ -478,7 +487,7 @@ class Config: :param workdir: Override the configured working directory. :param ctrl_sock: Override the configured control socket. """ - cfg = cls._get_instance(cfgfile, force_reload=True) + cfg = cls._get_instance(cfgfile, workdir=workdir, force_reload=True) if device_id: cfg.set('device_id', device_id) if workdir: From 1825b492b32cdc247aadf2b00db170d52878e760 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:21:24 +0200 Subject: [PATCH 12/64] Replaced `Config.workdir` with `Config.get_workdir()`. Again, Python < 3.9 doesn't like class properties. --- platypush/config/__init__.py | 3 +-- platypush/plugins/xmpp/__init__.py | 2 +- platypush/utils/__init__.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 67541f1761..d2de1fdef1 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -498,8 +498,7 @@ class Config: return cfg @classmethod - @property - def workdir(cls) -> str: + def get_workdir(cls) -> str: """ :return: The path of the configured working directory. """ diff --git a/platypush/plugins/xmpp/__init__.py b/platypush/plugins/xmpp/__init__.py index 2d8ecc9de0..a7de007667 100644 --- a/platypush/plugins/xmpp/__init__.py +++ b/platypush/plugins/xmpp/__init__.py @@ -117,7 +117,7 @@ class XmppPlugin(AsyncRunnablePlugin, XmppBasePlugin): auto_accept_invites=auto_accept_invites, restore_state=restore_state, state_file=os.path.expanduser( - state_file or os.path.join(Config.workdir, 'xmpp', 'state.json') + state_file or os.path.join(Config.get_workdir(), 'xmpp', 'state.json') ), ) self._loaded_state = SerializedState() diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py index 20763fc907..7f16cce826 100644 --- a/platypush/utils/__init__.py +++ b/platypush/utils/__init__.py @@ -523,7 +523,7 @@ def get_or_generate_jwt_rsa_key_pair(): """ from platypush.config import Config - key_dir = os.path.join(Config.workdir, 'jwt') + key_dir = os.path.join(Config.get_workdir(), 'jwt') priv_key_file = os.path.join(key_dir, 'id_rsa') pub_key_file = priv_key_file + '.pub' From a8255f3621972df2e70945a833192e8c135d44ae Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:23:20 +0200 Subject: [PATCH 13/64] Pass the configuration file used by the application to the Alembic process. The database settings could also be overridden in the configuration file besides the command line. We should therefore pass the path to the runtime configuration file, so the Alembic process can initialize its configuration from the same file and use the same settings. --- platypush/config/__init__.py | 7 +++++++ platypush/entities/_base.py | 3 +++ platypush/migrations/alembic/env.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index d2de1fdef1..28c6beaaa0 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -530,5 +530,12 @@ class Config: # pylint: disable=protected-access cls._get_instance()._config[key] = value + @classmethod + def get_file(cls) -> str: + """ + :return: The main configuration file path. + """ + return cls._get_instance().config_file + # vim:sw=4:ts=4:et: diff --git a/platypush/entities/_base.py b/platypush/entities/_base.py index c3089dde56..2a739124a6 100644 --- a/platypush/entities/_base.py +++ b/platypush/entities/_base.py @@ -29,6 +29,7 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship from sqlalchemy.orm.exc import ObjectDeletedError import platypush +from platypush.config import Config from platypush.common.db import Base from platypush.message import JSONAble, Message @@ -355,6 +356,8 @@ def run_db_migrations(): '-c', alembic_ini, '-x', + f'CFGFILE={Config.get_file()}', + '-x', f'DBNAME={_get_db_engine().url}', 'upgrade', 'head', diff --git a/platypush/migrations/alembic/env.py b/platypush/migrations/alembic/env.py index 406b1f6e2a..d4aba7078b 100644 --- a/platypush/migrations/alembic/env.py +++ b/platypush/migrations/alembic/env.py @@ -74,6 +74,10 @@ def run_migrations_online() -> None: def set_db_engine(): + app_conf_file = context.get_x_argument(as_dictionary=True).get('CFGFILE') + if app_conf_file: + Config.init(app_conf_file) + engine_url = context.get_x_argument(as_dictionary=True).get('DBNAME') if not engine_url: db_conf = Config.get('db') From dd3a701a2e9bef25eacbcdf4ad323d98f56e7711 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:28:40 +0200 Subject: [PATCH 14/64] Full rewrite of `platypush.utils.manifest`. The new version encapsulates all the utility functions into three classes - `Manifest`, `Manifests` and `Dependencies`. --- platypush/utils/manifest.py | 425 ++++++++++++++++++++++++------------ setup.cfg | 1 + 2 files changed, 281 insertions(+), 145 deletions(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index b26573e7eb..3bfea8fb83 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -1,99 +1,328 @@ +from dataclasses import dataclass, field import enum +import importlib import inspect import json import logging import os import pathlib import shutil +import sys + +from typing import ( + Dict, + Generator, + List, + Optional, + Iterable, + Mapping, + Set, + Type, + Union, +) + import yaml -from abc import ABC, abstractmethod -from typing import Optional, Iterable, Mapping, Callable, Type +from platypush.message.event import Event supported_package_managers = { - 'pacman': 'pacman -S', - 'apt': 'apt-get install', + 'apk': ['apk', 'add', '--no-cache', '--no-progress'], + 'apt': ['apt', 'install', '-y', '-q'], + 'pacman': ['pacman', '-S', '--noconfirm', '--noprogressbar'], } _available_package_manager = None +logger = logging.getLogger(__name__) class ManifestType(enum.Enum): + """ + Manifest types. + """ + PLUGIN = 'plugin' BACKEND = 'backend' -class Manifest(ABC): +@dataclass +class Dependencies: + """ + Dependencies for a plugin/backend. + """ + + before: List[str] = field(default_factory=list) + """ Commands to execute before the component is installed. """ + packages: Set[str] = field(default_factory=set) + """ System packages required by the component. """ + pip: Set[str] = field(default_factory=set) + """ pip dependencies. """ + after: List[str] = field(default_factory=list) + """ Commands to execute after the component is installed. """ + + @classmethod + def from_config( + cls, conf_file: Optional[str] = None, pkg_manager: Optional[str] = None + ) -> "Dependencies": + """ + Parse the required dependencies from a configuration file. + """ + deps = cls() + + for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager): + deps.before += manifest.install.before + deps.pip.update(manifest.install.pip) + deps.packages.update(manifest.install.packages) + deps.after += manifest.install.after + + return deps + + def to_pkg_install_commands( + self, pkg_manager: Optional[str] = None, skip_sudo: bool = False + ) -> Generator[str, None, None]: + """ + Generates the package manager commands required to install the given + dependencies on the system. + + :param pkg_manager: Force package manager to use (default: looks for + the one available on the system). + :param skip_sudo: Skip sudo when installing packages (default: it will + look if the current user is root and use sudo otherwise). + """ + wants_sudo = not skip_sudo and os.getuid() != 0 + pkg_manager = pkg_manager or get_available_package_manager() + if self.packages and pkg_manager: + yield ' '.join( + [ + *(['sudo'] if wants_sudo else []), + *supported_package_managers[pkg_manager], + *sorted(self.packages), + ] + ) + + def to_pip_install_commands(self) -> Generator[str, None, None]: + """ + Generates the pip commands required to install the given dependencies on + the system. + """ + # Recent versions want an explicit --break-system-packages option when + # installing packages via pip outside of a virtual environment + wants_break_system_packages = ( + sys.version_info > (3, 10) + and sys.prefix == sys.base_prefix # We're not in a venv + ) + + if self.pip: + yield ( + 'pip install -U --no-input --no-cache-dir ' + + ('--break-system-packages ' if wants_break_system_packages else '') + + ' '.join(sorted(self.pip)) + ) + + def to_install_commands( + self, pkg_manager: Optional[str] = None, skip_sudo: bool = False + ) -> Generator[str, None, None]: + """ + Generates the commands required to install the given dependencies on + this system. + + :param pkg_manager: Force package manager to use (default: looks for + the one available on the system). + :param skip_sudo: Skip sudo when installing packages (default: it will + look if the current user is root and use sudo otherwise). + """ + for cmd in self.before: + yield cmd + + for cmd in self.to_pkg_install_commands( + pkg_manager=pkg_manager, skip_sudo=skip_sudo + ): + yield cmd + + for cmd in self.to_pip_install_commands(): + yield cmd + + for cmd in self.after: + yield cmd + + +class Manifest: """ Base class for plugin/backend manifests. """ - def __init__(self, package: str, description: Optional[str] = None, - install: Optional[Iterable[str]] = None, events: Optional[Mapping] = None, **_): + + def __init__( + self, + package: str, + description: Optional[str] = None, + install: Optional[Dict[str, Iterable[str]]] = None, + events: Optional[Mapping] = None, + pkg_manager: Optional[str] = None, + **_, + ): + self._pkg_manager = pkg_manager or get_available_package_manager() self.description = description - self.install = install or {} - self.events = events or {} - self.logger = logging.getLogger(__name__) + self.install = self._init_deps(install or {}) + self.events = self._init_events(events or {}) self.package = package self.component_name = '.'.join(package.split('.')[2:]) self.component = None - @classmethod - @property - @abstractmethod - def component_getter(self) -> Callable[[str], object]: - raise NotImplementedError + def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies: + deps = Dependencies() + for key, items in install.items(): + if key == 'pip': + deps.pip.update(items) + elif key == 'before': + deps.before += items + elif key == 'after': + deps.after += items + elif key == self._pkg_manager: + deps.packages.update(items) + + return deps + + @staticmethod + def _init_events( + events: Union[Iterable[str], Mapping[str, Optional[str]]] + ) -> Dict[Type[Event], str]: + evt_dict = events if isinstance(events, Mapping) else {e: None for e in events} + ret = {} + + for evt_name, doc in evt_dict.items(): + evt_module_name, evt_class_name = evt_name.rsplit('.', 1) + try: + evt_module = importlib.import_module(evt_module_name) + evt_class = getattr(evt_module, evt_class_name) + except Exception as e: + raise AssertionError(f'Could not load event {evt_name}: {e}') from e + + ret[evt_class] = doc or evt_class.__doc__ + + return ret @classmethod - def from_file(cls, filename: str) -> "Manifest": + def from_file(cls, filename: str, pkg_manager: Optional[str] = None) -> "Manifest": + """ + Parse a manifest filename into a ``Manifest`` class. + """ with open(str(filename), 'r') as f: manifest = yaml.safe_load(f).get('manifest', {}) assert 'type' in manifest, f'Manifest file {filename} has no type field' comp_type = ManifestType(manifest.pop('type')) manifest_class = _manifest_class_by_type[comp_type] - return manifest_class(**manifest) - - @classmethod - def from_class(cls, clazz) -> "Manifest": - return cls.from_file(os.path.dirname(inspect.getfile(clazz))) - - @classmethod - def from_component(cls, comp) -> "Manifest": - return cls.from_class(comp.__class__) - - def get_component(self): - try: - self.component = self.component_getter(self.component_name) - except Exception as e: - self.logger.warning(f'Could not load {self.component_name}: {e}') - - return self.component + return manifest_class(**manifest, pkg_manager=pkg_manager) def __repr__(self): - return json.dumps({ - 'description': self.description, - 'install': self.install, - 'events': self.events, - 'type': _manifest_type_by_class[self.__class__].value, - 'package': self.package, - 'component_name': self.component_name, - }) + """ + :return: A JSON serialized representation of the manifest. + """ + return json.dumps( + { + 'description': self.description, + 'install': self.install, + 'events': { + '.'.join([evt_type.__module__, evt_type.__name__]): doc + for evt_type, doc in self.events.items() + }, + 'type': _manifest_type_by_class[self.__class__].value, + 'package': self.package, + 'component_name': self.component_name, + } + ) +# pylint: disable=too-few-public-methods class PluginManifest(Manifest): - @classmethod - @property - def component_getter(self): - from platypush.context import get_plugin - return get_plugin + """ + Plugin manifest. + """ +# pylint: disable=too-few-public-methods class BackendManifest(Manifest): - @classmethod - @property - def component_getter(self): - from platypush.context import get_backend - return get_backend + """ + Backend manifest. + """ + + +class Manifests: + """ + General-purpose manifests utilities. + """ + + @staticmethod + def by_base_class( + base_class: Type, pkg_manager: Optional[str] = None + ) -> Generator[Manifest, None, None]: + """ + Get all the manifest files declared under the base path of a given class + and parse them into :class:`Manifest` objects. + """ + for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob( + 'manifest.yaml' + ): + yield Manifest.from_file(str(mf), pkg_manager=pkg_manager) + + @staticmethod + def by_config( + conf_file: Optional[str] = None, + pkg_manager: Optional[str] = None, + ) -> Generator[Manifest, None, None]: + """ + Get all the manifest objects associated to the extensions declared in a + given configuration file. + """ + import platypush + from platypush.config import Config + + conf_args = [] + if conf_file: + conf_args.append(conf_file) + + Config.init(*conf_args) + app_dir = os.path.dirname(inspect.getfile(platypush)) + + for name in Config.get_backends().keys(): + yield Manifest.from_file( + os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml'), + pkg_manager=pkg_manager, + ) + + for name in Config.get_plugins().keys(): + yield Manifest.from_file( + os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'), + pkg_manager=pkg_manager, + ) + + +def get_available_package_manager() -> Optional[str]: + """ + Get the name of the available package manager on the system, if supported. + """ + # pylint: disable=global-statement + global _available_package_manager + if _available_package_manager: + return _available_package_manager + + available_package_managers = [ + pkg_manager + for pkg_manager in supported_package_managers + if shutil.which(pkg_manager) + ] + + if not available_package_managers: + logger.warning( + '\nYour OS does not provide any of the supported package managers.\n' + 'You may have to install some optional dependencies manually.\n' + 'Supported package managers: %s.\n', + ', '.join(supported_package_managers.keys()), + ) + + return None + + _available_package_manager = available_package_managers[0] + return _available_package_manager _manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = { @@ -104,97 +333,3 @@ _manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = { _manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = { cls: t for t, cls in _manifest_class_by_type.items() } - - -def scan_manifests(base_class: Type) -> Iterable[str]: - for mf in pathlib.Path(os.path.dirname(inspect.getfile(base_class))).rglob('manifest.yaml'): - yield str(mf) - - -def get_manifests(base_class: Type) -> Iterable[Manifest]: - return [ - Manifest.from_file(mf) - for mf in scan_manifests(base_class) - ] - - -def get_components(base_class: Type) -> Iterable: - manifests = get_manifests(base_class) - components = {mf.get_component() for mf in manifests} - return {comp for comp in components if comp is not None} - - -def get_manifests_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Manifest]: - import platypush - from platypush.config import Config - - conf_args = [] - if conf_file: - conf_args.append(conf_file) - - Config.init(*conf_args) - app_dir = os.path.dirname(inspect.getfile(platypush)) - manifest_files = set() - - for name in Config.get_backends().keys(): - manifest_files.add(os.path.join(app_dir, 'backend', *name.split('.'), 'manifest.yaml')) - - for name in Config.get_plugins().keys(): - manifest_files.add(os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml')) - - return { - manifest_file: Manifest.from_file(manifest_file) - for manifest_file in manifest_files - } - - -def get_dependencies_from_conf(conf_file: Optional[str] = None) -> Mapping[str, Iterable[str]]: - manifests = get_manifests_from_conf(conf_file) - deps = { - 'pip': set(), - 'packages': set(), - 'exec': set(), - } - - for manifest in manifests.values(): - deps['pip'].update(manifest.install.get('pip', set())) - deps['exec'].update(manifest.install.get('exec', set())) - has_requires_packages = len([ - section for section in manifest.install.keys() - if section in supported_package_managers - ]) > 0 - - if has_requires_packages: - pkg_manager = get_available_package_manager() - deps['packages'].update(manifest.install.get(pkg_manager, set())) - - return deps - - -def get_install_commands_from_conf(conf_file: Optional[str] = None) -> Mapping[str, str]: - deps = get_dependencies_from_conf(conf_file) - return { - 'pip': f'pip install {" ".join(deps["pip"])}', - 'exec': deps["exec"], - 'packages': f'{supported_package_managers[_available_package_manager]} {" ".join(deps["packages"])}' - if deps['packages'] else None, - } - - -def get_available_package_manager() -> str: - global _available_package_manager - if _available_package_manager: - return _available_package_manager - - available_package_managers = [ - pkg_manager for pkg_manager in supported_package_managers.keys() - if shutil.which(pkg_manager) - ] - - assert available_package_managers, ( - 'Your OS does not provide any of the supported package managers. ' - f'Supported package managers: {supported_package_managers.keys()}' - ) - - _available_package_manager = available_package_managers[0] - return _available_package_manager diff --git a/setup.cfg b/setup.cfg index e80481961c..a6014566ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,5 +11,6 @@ max-line-length = 120 extend-ignore = E203 W503 + SIM104 SIM105 From 043f303761a229257b3e44a865b7bec2bd22d18f Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:30:51 +0200 Subject: [PATCH 15/64] `s/HttpBackend._DEFAULT_HTTP_PORT/HttpBackend.DEFAULT_HTTP_PORT/g` --- platypush/backend/http/__init__.py | 4 ++-- platypush/backend/http/app/utils/routes.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/platypush/backend/http/__init__.py b/platypush/backend/http/__init__.py index 6bda90657e..a682262755 100644 --- a/platypush/backend/http/__init__.py +++ b/platypush/backend/http/__init__.py @@ -192,7 +192,7 @@ class HttpBackend(Backend): """ - _DEFAULT_HTTP_PORT = 8008 + DEFAULT_HTTP_PORT = 8008 """The default listen port for the webserver.""" _STOP_TIMEOUT = 5 @@ -200,7 +200,7 @@ class HttpBackend(Backend): def __init__( self, - port: int = _DEFAULT_HTTP_PORT, + port: int = DEFAULT_HTTP_PORT, bind_address: str = '0.0.0.0', resource_dirs: Optional[Mapping[str, str]] = None, secret_key_file: Optional[str] = None, diff --git a/platypush/backend/http/app/utils/routes.py b/platypush/backend/http/app/utils/routes.py index f7de249da5..68a3fd9e18 100644 --- a/platypush/backend/http/app/utils/routes.py +++ b/platypush/backend/http/app/utils/routes.py @@ -14,7 +14,7 @@ def get_http_port(): from platypush.backend.http import HttpBackend http_conf = Config.get('backend.http') or {} - return http_conf.get('port', HttpBackend._DEFAULT_HTTP_PORT) + return http_conf.get('port', HttpBackend.DEFAULT_HTTP_PORT) def get_routes(): From 1cb686bdab9ccda97928b489329284a4cc7e9ff8 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:31:48 +0200 Subject: [PATCH 16/64] Updated the `inspect` plugin to the new manifest utils interface. --- platypush/plugins/inspect/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/platypush/plugins/inspect/__init__.py b/platypush/plugins/inspect/__init__.py index 01bb6cfdb2..ad0f1492a5 100644 --- a/platypush/plugins/inspect/__init__.py +++ b/platypush/plugins/inspect/__init__.py @@ -20,7 +20,7 @@ from platypush.utils import ( get_plugin_class_by_name, get_plugin_name_by_class, ) -from platypush.utils.manifest import Manifest, scan_manifests +from platypush.utils.manifest import Manifests from ._context import ComponentContext from ._model import ( @@ -116,8 +116,7 @@ class InspectPlugin(Plugin): A generator that scans the manifest files given a ``base_type`` (``Plugin`` or ``Backend``) and yields the parsed submodules. """ - for mf_file in scan_manifests(base_type): - manifest = Manifest.from_file(mf_file) + for manifest in Manifests.by_base_class(base_type): try: yield importlib.import_module(manifest.package) except Exception as e: From 69706eaabe6e60d4a4c2c6208818b09417fc54f2 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:32:19 +0200 Subject: [PATCH 17/64] `s/logger/_logger/` in the `plugins` module. The `logger` name may clash with the context of an action, where `logger` may have been set to something else. --- platypush/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platypush/plugins/__init__.py b/platypush/plugins/__init__.py index a66a7e34eb..b0090f60b4 100644 --- a/platypush/plugins/__init__.py +++ b/platypush/plugins/__init__.py @@ -15,7 +15,7 @@ from platypush.message.response import Response from platypush.utils import get_decorators, get_plugin_name_by_class PLUGIN_STOP_TIMEOUT = 5 # Plugin stop timeout in seconds -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def action(f: Callable[..., Any]) -> Callable[..., Response]: @@ -33,7 +33,7 @@ def action(f: Callable[..., Any]) -> Callable[..., Response]: try: result = f(*args, **kwargs) except TypeError as e: - logger.exception(e) + _logger.exception(e) result = Response(errors=[str(e)]) if result and isinstance(result, Response): From 9002f3034a79676be6ede5762eb7add93af3081b Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:46:08 +0200 Subject: [PATCH 18/64] Tweaked package managers install command arguments. --- platypush/utils/manifest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 3bfea8fb83..1436b705f3 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -26,9 +26,9 @@ import yaml from platypush.message.event import Event supported_package_managers = { - 'apk': ['apk', 'add', '--no-cache', '--no-progress'], - 'apt': ['apt', 'install', '-y', '-q'], - 'pacman': ['pacman', '-S', '--noconfirm', '--noprogressbar'], + 'apk': ['apk', 'add', '--update', '--no-interactive', '--no-cache'], + 'apt': ['apt', 'install', '-y'], + 'pacman': ['pacman', '-S', '--noconfirm'], } _available_package_manager = None From 980af169840271a927b0f09f183dffe11463884d Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:47:43 +0200 Subject: [PATCH 19/64] Rewritten platydock utility. Platydock now will only print out a Dockerfile given a configuration file. No more maintaining the state of containers, storing separate workdirs and configuration directories etc. - that introduced way too much overhead over Docker. --- platypush/platydock/__init__.py | 506 +++++--------------------------- 1 file changed, 76 insertions(+), 430 deletions(-) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 77e9778c9d..1725bf4d5c 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -1,462 +1,108 @@ """ -Platydock - -Platydock is a helper that allows you to easily manage (create, destroy, start, -stop and list) Platypush instances as Docker images. +Platydock is a helper script that allows you to automatically create a +Dockerfile for Platypush starting from a configuration file. """ import argparse -import enum +import inspect import os import pathlib -import re -import shutil -import subprocess import sys -import textwrap -import traceback as tb -import yaml +from typing import Iterable from platypush.config import Config -from platypush.utils import manifest +from platypush.utils.manifest import Dependencies -workdir = os.path.join( - os.path.expanduser('~'), '.local', 'share', 'platypush', 'platydock' -) +ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m ' +ERR_SUFFIX = '\033[0m' -class Action(enum.Enum): - build = 'build' - start = 'start' - stop = 'stop' - rm = 'rm' - ls = 'ls' +def generate_dockerfile(cfgfile: str) -> str: + """ + Generate a Dockerfile based on a configuration file. - def __str__(self): - return self.value + :param cfgfile: Path to the configuration file. + :return: The content of the generated Dockerfile. + """ + Config.init(cfgfile) + new_file_lines = [] + ports = _get_exposed_ports() + deps = Dependencies.from_config(cfgfile, pkg_manager='apk') + is_after_expose_cmd = False + base_file = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile' + ) + + with open(base_file, 'r') as f: + file_lines = [line.rstrip() for line in f.readlines()] + + for line in file_lines: + if line.startswith('RUN cd /install '): + for new_line in deps.before: + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.to_pkg_install_commands( + pkg_manager='apk', skip_sudo=True + ): + new_file_lines.append('RUN ' + new_line) + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.after: + new_file_lines.append('RUN' + new_line) + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + new_file_lines.extend([f'EXPOSE {port}' for port in ports]) + is_after_expose_cmd = True + + continue + + new_file_lines.append(line) + + return '\n'.join(new_file_lines) -def _parse_deps(cls): - deps = [] - - for line in cls.__doc__.split('\n'): - m = re.search(r'\(``pip install (.+)``\)', line) - if m: - deps.append(m.group(1)) - - return deps - - -def generate_dockerfile(deps, ports, cfgfile, device_dir, python_version): - device_id = Config.get('device_id') - if not device_id: - raise RuntimeError( - ( - 'You need to specify a device_id in {} - Docker ' - + 'containers cannot rely on hostname' - ).format(cfgfile) +def _get_exposed_ports() -> Iterable[int]: + """ + :return: The listen ports used by the backends enabled in the configuration + file. + """ + backends_config = Config.get_backends() + return { + int(port) + for port in ( + backends_config.get('http', {}).get('port'), + backends_config.get('tcp', {}).get('port'), ) - - os.makedirs(device_dir, exist_ok=True) - content = textwrap.dedent( - ''' - FROM python:{python_version}-slim-bookworm - - RUN mkdir -p /app - RUN mkdir -p /etc/platypush - RUN mkdir -p /usr/local/share/platypush\n - '''.format( - python_version=python_version - ) - ).lstrip() - - srcdir = os.path.dirname(cfgfile) - cfgfile_copy = os.path.join(device_dir, 'config.yaml') - shutil.copy(cfgfile, cfgfile_copy, follow_symlinks=True) - content += 'COPY config.yaml /etc/platypush/\n' - backend_config = Config.get_backends() - - # Redis configuration for Docker - if 'redis' not in backend_config: - backend_config['redis'] = { - 'redis_args': { - 'host': 'redis', - 'port': 6379, - } - } - - with open(cfgfile_copy, 'a') as f: - f.write( - '\n# Automatically added by platydock, do not remove\n' - + yaml.dump( - { - 'backend.redis': backend_config['redis'], - } - ) - + '\n' - ) - - # Main database configuration - has_main_db = False - with open(cfgfile_copy, 'r') as f: - for line in f.readlines(): - if re.match(r'^(main.)?db.*', line): - has_main_db = True - break - - if not has_main_db: - with open(cfgfile_copy, 'a') as f: - f.write( - '\n# Automatically added by platydock, do not remove\n' - + yaml.dump( - { - 'main.db': { - 'engine': 'sqlite:////platypush.db', - } - } - ) - + '\n' - ) - - # Copy included files - # noinspection PyProtectedMember - for include in Config._included_files: - incdir = os.path.relpath(os.path.dirname(include), srcdir) - destdir = os.path.join(device_dir, incdir) - pathlib.Path(destdir).mkdir(parents=True, exist_ok=True) - shutil.copy(include, destdir, follow_symlinks=True) - content += 'RUN mkdir -p /etc/platypush/' + incdir + '\n' - content += ( - 'COPY ' - + os.path.relpath(include, srcdir) - + ' /etc/platypush/' - + incdir - + '\n' - ) - - # Copy script files - scripts_dir = os.path.join(os.path.dirname(cfgfile), 'scripts') - if os.path.isdir(scripts_dir): - local_scripts_dir = os.path.join(device_dir, 'scripts') - remote_scripts_dir = '/etc/platypush/scripts' - shutil.copytree( - scripts_dir, local_scripts_dir, symlinks=True, dirs_exist_ok=True - ) - content += f'RUN mkdir -p {remote_scripts_dir}\n' - content += f'COPY scripts/ {remote_scripts_dir}\n' - - packages = deps.pop('packages', None) - pip = deps.pop('pip', None) - exec_cmds = deps.pop('exec', None) - pkg_cmd = ( - f'\n\t&& apt-get install --no-install-recommends -y {" ".join(packages)} \\' - if packages - else '' - ) - pip_cmd = f'\n\t&& pip install {" ".join(pip)} \\' if pip else '' - content += f''' -RUN dpkg --configure -a \\ - && apt-get -f install \\ - && apt-get --fix-missing install \\ - && apt-get clean \\ - && apt-get update \\ - && apt-get -y upgrade \\ - && apt-get -y dist-upgrade \\ - && apt-get install --no-install-recommends -y apt-utils \\ - && apt-get install --no-install-recommends -y build-essential \\ - && apt-get install --no-install-recommends -y git \\ - && apt-get install --no-install-recommends -y sudo \\ - && apt-get install --no-install-recommends -y libffi-dev \\ - && apt-get install --no-install-recommends -y libcap-dev \\ - && apt-get install --no-install-recommends -y libjpeg-dev \\{pkg_cmd}{pip_cmd}''' - - for exec_cmd in exec_cmds: - content += f'\n\t&& {exec_cmd} \\' - content += ''' - && apt-get install --no-install-recommends -y zlib1g-dev - -RUN git clone --recursive https://git.platypush.tech/platypush/platypush.git /app \\ - && cd /app \\ - && pip install -r requirements.txt - -RUN apt-get remove -y git \\ - && apt-get remove -y build-essential \\ - && apt-get remove -y libffi-dev \\ - && apt-get remove -y libjpeg-dev \\ - && apt-get remove -y libcap-dev \\ - && apt-get remove -y zlib1g-dev \\ - && apt-get remove -y apt-utils \\ - && apt-get clean \\ - && apt-get autoremove -y \\ - && rm -rf /var/lib/apt/lists/* -''' - - for port in ports: - content += 'EXPOSE {}\n'.format(port) - - content += textwrap.dedent( - ''' - - ENV PYTHONPATH /app:$PYTHONPATH - CMD ["python", "-m", "platypush"] - ''' - ) - - dockerfile = os.path.join(device_dir, 'Dockerfile') - print('Generating Dockerfile {}'.format(dockerfile)) - - with open(dockerfile, 'w') as f: - f.write(content) - - -def build(args): - global workdir - - ports = set() - parser = argparse.ArgumentParser( - prog='platydock build', description='Build a Platypush image from a config.yaml' - ) - - parser.add_argument( - '-c', - '--config', - type=str, - required=True, - help='Path to the platypush configuration file', - ) - parser.add_argument( - '-p', - '--python-version', - type=str, - default='3.9', - help='Python version to be used', - ) - - opts, args = parser.parse_known_args(args) - - cfgfile = os.path.abspath(os.path.expanduser(opts.config)) - manifest._available_package_manager = ( - 'apt' # Force apt for Debian-based Docker images - ) - install_cmds = manifest.get_dependencies_from_conf(cfgfile) - python_version = opts.python_version - backend_config = Config.get_backends() - - # Container exposed ports - if backend_config.get('http'): - from platypush.backend.http import HttpBackend - - # noinspection PyProtectedMember - ports.add(backend_config['http'].get('port', HttpBackend._DEFAULT_HTTP_PORT)) - - if backend_config.get('tcp'): - ports.add(backend_config['tcp']['port']) - - dev_dir = os.path.join(workdir, Config.get('device_id')) - generate_dockerfile( - deps=dict(install_cmds), - ports=ports, - cfgfile=cfgfile, - device_dir=dev_dir, - python_version=python_version, - ) - - subprocess.call( - [ - 'docker', - 'build', - '-t', - 'platypush-{}'.format(Config.get('device_id')), - dev_dir, - ] - ) - - -def start(args): - global workdir - - parser = argparse.ArgumentParser( - prog='platydock start', - description='Start a Platypush container', - epilog=textwrap.dedent( - ''' - You can append additional options that - will be passed to the docker container. - Example: - - --add-host='myhost:192.168.1.1' - ''' - ), - ) - - parser.add_argument('image', type=str, help='Platypush image to start') - parser.add_argument( - '-p', - '--publish', - action='append', - nargs='*', - default=[], - help=textwrap.dedent( - ''' - Container's ports to expose to the host. - Note that the default exposed ports from - the container service will be exposed unless - these mappings override them (e.g. port 8008 - on the container will be mapped to 8008 on - the host). - - Example: - - -p 18008:8008 - ''' - ), - ) - - parser.add_argument( - '-a', - '--attach', - action='store_true', - default=False, - help=textwrap.dedent( - ''' - If set, then attach to the container after starting it up (default: false). - ''' - ), - ) - - opts, args = parser.parse_known_args(args) - ports = {} - dockerfile = os.path.join(workdir, opts.image, 'Dockerfile') - - with open(dockerfile) as f: - for line in f: - m = re.match(r'expose (\d+)', line.strip().lower()) - if m: - ports[m.group(1)] = m.group(1) - - for mapping in opts.publish: - host_port, container_port = mapping[0].split(':') - ports[container_port] = host_port - - print('Preparing Redis support container') - subprocess.call(['docker', 'pull', 'redis']) - subprocess.call( - ['docker', 'run', '--rm', '--name', 'redis-' + opts.image, '-d', 'redis'] - ) - - docker_cmd = [ - 'docker', - 'run', - '--rm', - '--name', - opts.image, - '-it', - '--link', - 'redis-' + opts.image + ':redis', - ] - - for container_port, host_port in ports.items(): - docker_cmd += ['-p', host_port + ':' + container_port] - - docker_cmd += args - docker_cmd += ['-d', 'platypush-' + opts.image] - - print('Starting Platypush container {}'.format(opts.image)) - subprocess.call(docker_cmd) - - if opts.attach: - subprocess.call(['docker', 'attach', opts.image]) - - -def stop(args): - parser = argparse.ArgumentParser( - prog='platydock stop', description='Stop a Platypush container' - ) - - parser.add_argument('container', type=str, help='Platypush container to stop') - opts, args = parser.parse_known_args(args) - - print('Stopping Platypush container {}'.format(opts.container)) - subprocess.call(['docker', 'kill', '{}'.format(opts.container)]) - - print('Stopping Redis support container') - subprocess.call(['docker', 'stop', 'redis-{}'.format(opts.container)]) - - -def rm(args): - global workdir - - parser = argparse.ArgumentParser( - prog='platydock rm', - description='Remove a Platypush image. ' - + 'NOTE: make sure that no container is ' - + 'running nor linked to the image before ' - + 'removing it', - ) - - parser.add_argument('image', type=str, help='Platypush image to remove') - opts, args = parser.parse_known_args(args) - - subprocess.call(['docker', 'rmi', 'platypush-{}'.format(opts.image)]) - shutil.rmtree(os.path.join(workdir, opts.image), ignore_errors=True) - - -def ls(args): - parser = argparse.ArgumentParser( - prog='platydock ls', description='List available Platypush containers' - ) - parser.add_argument('filter', type=str, nargs='?', help='Image name filter') - - opts, args = parser.parse_known_args(args) - - p = subprocess.Popen(['docker', 'images'], stdout=subprocess.PIPE) - output = p.communicate()[0].decode().split('\n') - header = output.pop(0) - images = [] - - for line in output: - if re.match(r'^platypush-(.+?)\s.*', line) and ( - not opts.filter or (opts.filter and opts.filter in line) - ): - images.append(line) - - if images: - print(header) - - for image in images: - print(image) + if port + } def main(): + """ + Generates a Dockerfile based on the configuration file. + """ parser = argparse.ArgumentParser( prog='platydock', add_help=False, - description='Manage Platypush docker containers', - epilog='Use platydock --help to ' + 'get additional help', + description='Create a Platypush Dockerfile from a config.yaml.', + epilog='Use platydock --help to get additional help.', ) - # noinspection PyTypeChecker - parser.add_argument( - 'action', nargs='?', type=Action, choices=list(Action), help='Action to execute' - ) parser.add_argument('-h', '--help', action='store_true', help='Show usage') - opts, args = parser.parse_known_args(sys.argv[1:]) + parser.add_argument( + 'cfgfile', type=str, nargs=1, help='The path to the configuration file.' + ) - if (opts.help and not opts.action) or (not opts.help and not opts.action): - parser.print_help() - return 1 - - globals()[str(opts.action)](sys.argv[2:]) + opts, _ = parser.parse_known_args(sys.argv[1:]) + cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0])) + dockerfile = generate_dockerfile(cfgfile) + print(dockerfile) if __name__ == '__main__': - ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m ' - ERR_SUFFIX = '\033[0m' + main() - try: - main() - except Exception as e: - tb.print_exc(file=sys.stdout) - print(ERR_PREFIX + str(e) + ERR_SUFFIX, file=sys.stderr) # vim:sw=4:ts=4:et: From 7889b2f1db0ae525b0b3e02ec0e6149cd0f374c0 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 13:49:31 +0200 Subject: [PATCH 20/64] Updated `generate_missing_docs` to use the new manifest API. --- generate_missing_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generate_missing_docs.py b/generate_missing_docs.py index d34eec7729..3d9f828510 100644 --- a/generate_missing_docs.py +++ b/generate_missing_docs.py @@ -4,7 +4,7 @@ from typing import Iterable, Optional from platypush.backend import Backend from platypush.context import get_plugin from platypush.plugins import Plugin -from platypush.utils.manifest import get_manifests +from platypush.utils.manifest import Manifests def _get_inspect_plugin(): @@ -14,11 +14,11 @@ def _get_inspect_plugin(): def get_all_plugins(): - return sorted([mf.component_name for mf in get_manifests(Plugin)]) + return sorted([mf.component_name for mf in Manifests.by_base_class(Plugin)]) def get_all_backends(): - return sorted([mf.component_name for mf in get_manifests(Backend)]) + return sorted([mf.component_name for mf in Manifests.by_base_class(Backend)]) def get_all_events(): From a99ffea37c885caec9b7e4458dfc4f6ac182f1e7 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 21:46:08 +0200 Subject: [PATCH 21/64] Fixed apt dependencies for `mpd` plugin. --- platypush/plugins/music/mpd/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platypush/plugins/music/mpd/manifest.yaml b/platypush/plugins/music/mpd/manifest.yaml index f96099e6cd..b9d837d80c 100644 --- a/platypush/plugins/music/mpd/manifest.yaml +++ b/platypush/plugins/music/mpd/manifest.yaml @@ -2,7 +2,7 @@ manifest: events: {} install: apt: - - python-mpd + - python3-mpd pacman: - python-mpd2 pip: From 71c529119058875086022d1b2dbac307c318942e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 19 Aug 2023 22:46:37 +0200 Subject: [PATCH 22/64] Refactored the interface of Platydock and manifest utils. --- platypush/platydock/__init__.py | 186 ++++++++++++++++-------- platypush/utils/manifest.py | 250 +++++++++++++++++++++----------- 2 files changed, 285 insertions(+), 151 deletions(-) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 1725bf4d5c..2f49d3cb2f 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -4,6 +4,7 @@ Dockerfile for Platypush starting from a configuration file. """ import argparse +from enum import Enum import inspect import os import pathlib @@ -11,73 +12,108 @@ import sys from typing import Iterable from platypush.config import Config -from platypush.utils.manifest import Dependencies - -ERR_PREFIX = '\n\033[6;31;47mERROR\033[0;91m ' -ERR_SUFFIX = '\033[0m' +from platypush.utils.manifest import Dependencies, InstallContext, PackageManagers -def generate_dockerfile(cfgfile: str) -> str: +class BaseImage(Enum): """ - Generate a Dockerfile based on a configuration file. + Supported base images for Dockerfiles. + """ + + ALPINE = 'alpine' + UBUNTU = 'ubuntu' + + def __str__(self) -> str: + """ + Explicit __str__ override for argparse purposes. + """ + return self.value + + +class DockerfileGenerator: + """ + Generate a Dockerfile from on a configuration file. :param cfgfile: Path to the configuration file. - :return: The content of the generated Dockerfile. + :param image: The base image to use. """ - Config.init(cfgfile) - new_file_lines = [] - ports = _get_exposed_ports() - deps = Dependencies.from_config(cfgfile, pkg_manager='apk') - is_after_expose_cmd = False - base_file = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), 'docker', 'base.Dockerfile' - ) - with open(base_file, 'r') as f: - file_lines = [line.rstrip() for line in f.readlines()] - - for line in file_lines: - if line.startswith('RUN cd /install '): - for new_line in deps.before: - new_file_lines.append('RUN ' + new_line) - - for new_line in deps.to_pkg_install_commands( - pkg_manager='apk', skip_sudo=True - ): - new_file_lines.append('RUN ' + new_line) - elif line == 'RUN rm -rf /install': - for new_line in deps.to_pip_install_commands(): - new_file_lines.append('RUN ' + new_line) - - for new_line in deps.after: - new_file_lines.append('RUN' + new_line) - elif line.startswith('EXPOSE ') and ports: - if not is_after_expose_cmd: - new_file_lines.extend([f'EXPOSE {port}' for port in ports]) - is_after_expose_cmd = True - - continue - - new_file_lines.append(line) - - return '\n'.join(new_file_lines) - - -def _get_exposed_ports() -> Iterable[int]: - """ - :return: The listen ports used by the backends enabled in the configuration - file. - """ - backends_config = Config.get_backends() - return { - int(port) - for port in ( - backends_config.get('http', {}).get('port'), - backends_config.get('tcp', {}).get('port'), - ) - if port + _pkg_manager_by_base_image = { + BaseImage.ALPINE: PackageManagers.APK, + BaseImage.UBUNTU: PackageManagers.APT, } + def __init__(self, cfgfile: str, image: BaseImage) -> None: + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.image = image + + def generate(self) -> str: + """ + Generate a Dockerfile based on a configuration file. + + :param cfgfile: Path to the configuration file. + :return: The content of the generated Dockerfile. + """ + Config.init(self.cfgfile) + new_file_lines = [] + ports = self._get_exposed_ports() + pkg_manager = self._pkg_manager_by_base_image[self.image] + deps = Dependencies.from_config( + self.cfgfile, + pkg_manager=pkg_manager, + install_context=InstallContext.DOCKER, + ) + + is_after_expose_cmd = False + base_file = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'docker', + f'{self.image}.Dockerfile', + ) + + with open(base_file, 'r') as f: + file_lines = [line.rstrip() for line in f.readlines()] + + for line in file_lines: + if line.startswith('RUN cd /install '): + for new_line in deps.before: + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.to_pkg_install_commands(): + new_file_lines.append('RUN ' + new_line) + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + new_file_lines.append('RUN ' + new_line) + + for new_line in deps.after: + new_file_lines.append('RUN' + new_line) + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + new_file_lines.extend([f'EXPOSE {port}' for port in ports]) + is_after_expose_cmd = True + + continue + + new_file_lines.append(line) + + return '\n'.join(new_file_lines) + + @staticmethod + def _get_exposed_ports() -> Iterable[int]: + """ + :return: The listen ports used by the backends enabled in the configuration + file. + """ + backends_config = Config.get_backends() + return { + int(port) + for port in ( + backends_config.get('http', {}).get('port'), + backends_config.get('tcp', {}).get('port'), + ) + if port + } + def main(): """ @@ -87,22 +123,44 @@ def main(): prog='platydock', add_help=False, description='Create a Platypush Dockerfile from a config.yaml.', - epilog='Use platydock --help to get additional help.', ) - parser.add_argument('-h', '--help', action='store_true', help='Show usage') parser.add_argument( - 'cfgfile', type=str, nargs=1, help='The path to the configuration file.' + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + parser.add_argument( + 'cfgfile', type=str, nargs='?', help='The path to the configuration file.' + ) + parser.add_argument( + '--image', + '-i', + dest='image', + required=False, + type=BaseImage, + choices=list(BaseImage), + default=BaseImage.ALPINE, + help='Base image to use for the Dockerfile.', ) opts, _ = parser.parse_known_args(sys.argv[1:]) - cfgfile = os.path.abspath(os.path.expanduser(opts.cfgfile[0])) - dockerfile = generate_dockerfile(cfgfile) + if opts.show_usage: + parser.print_help() + return 0 + + if not opts.cfgfile: + print( + f'Please specify a configuration file.\nRun {sys.argv[0]} --help to get the available options.', + file=sys.stderr, + ) + return 1 + + dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate() print(dockerfile) + return 0 if __name__ == '__main__': - main() + sys.exit(main()) # vim:sw=4:ts=4:et: diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 1436b705f3..c217c464b2 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -1,5 +1,6 @@ +from abc import ABC, abstractmethod from dataclasses import dataclass, field -import enum +from enum import Enum import importlib import inspect import json @@ -20,22 +21,102 @@ from typing import ( Type, Union, ) +from typing_extensions import override import yaml from platypush.message.event import Event -supported_package_managers = { - 'apk': ['apk', 'add', '--update', '--no-interactive', '--no-cache'], - 'apt': ['apt', 'install', '-y'], - 'pacman': ['pacman', '-S', '--noconfirm'], -} - _available_package_manager = None logger = logging.getLogger(__name__) -class ManifestType(enum.Enum): +@dataclass +class PackageManager: + """ + Representation of a package manager. + """ + + executable: str + """ The executable name. """ + command: Iterable[str] = field(default_factory=tuple) + """ The command to execute, as a sequence of strings. """ + + +class PackageManagers(Enum): + """ + Supported package managers. + """ + + APK = PackageManager( + executable='apk', + command=('apk', 'add', '--update', '--no-interactive', '--no-cache'), + ) + + APT = PackageManager( + executable='apt', + command=('DEBIAN_FRONTEND=noninteractive', 'apt', 'install', '-y'), + ) + + PACMAN = PackageManager( + executable='pacman', + command=('pacman', '-S', '--noconfirm'), + ) + + @classmethod + def get_command(cls, name: str) -> Iterable[str]: + """ + :param name: The name of the package manager executable to get the + command for. + :return: The base command to execute, as a sequence of strings. + """ + pkg_manager = next(iter(pm for pm in cls if pm.value.executable == name), None) + if not pkg_manager: + raise ValueError(f'Unknown package manager: {name}') + + return pkg_manager.value.command + + @classmethod + def scan(cls) -> Optional["PackageManagers"]: + """ + Get the name of the available package manager on the system, if supported. + """ + # pylint: disable=global-statement + global _available_package_manager + if _available_package_manager: + return _available_package_manager + + available_package_managers = [ + pkg_manager + for pkg_manager in cls + if shutil.which(pkg_manager.value.executable) + ] + + if not available_package_managers: + logger.warning( + '\nYour OS does not provide any of the supported package managers.\n' + 'You may have to install some optional dependencies manually.\n' + 'Supported package managers: %s.\n', + ', '.join([pm.value.executable for pm in cls]), + ) + + return None + + _available_package_manager = available_package_managers[0] + return _available_package_manager + + +class InstallContext(Enum): + """ + Supported installation contexts. + """ + + NONE = None + DOCKER = 'docker' + VENV = 'venv' + + +class ManifestType(Enum): """ Manifest types. """ @@ -58,15 +139,32 @@ class Dependencies: """ pip dependencies. """ after: List[str] = field(default_factory=list) """ Commands to execute after the component is installed. """ + pkg_manager: Optional[PackageManagers] = None + """ Override the default package manager detected on the system. """ + install_context: InstallContext = InstallContext.NONE + + @property + def _is_venv(self) -> bool: + """ + :return: True if the dependencies scanning logic is running either in a + virtual environment or in a virtual environment preparation + context. + """ + return ( + self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix + ) @classmethod def from_config( - cls, conf_file: Optional[str] = None, pkg_manager: Optional[str] = None + cls, + conf_file: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, + install_context: InstallContext = InstallContext.NONE, ) -> "Dependencies": """ Parse the required dependencies from a configuration file. """ - deps = cls() + deps = cls(pkg_manager=pkg_manager, install_context=install_context) for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager): deps.before += manifest.install.before @@ -76,26 +174,19 @@ class Dependencies: return deps - def to_pkg_install_commands( - self, pkg_manager: Optional[str] = None, skip_sudo: bool = False - ) -> Generator[str, None, None]: + def to_pkg_install_commands(self) -> Generator[str, None, None]: """ Generates the package manager commands required to install the given dependencies on the system. - - :param pkg_manager: Force package manager to use (default: looks for - the one available on the system). - :param skip_sudo: Skip sudo when installing packages (default: it will - look if the current user is root and use sudo otherwise). """ - wants_sudo = not skip_sudo and os.getuid() != 0 - pkg_manager = pkg_manager or get_available_package_manager() + wants_sudo = self.install_context != InstallContext.DOCKER and os.getuid() != 0 + pkg_manager = self.pkg_manager or PackageManagers.scan() if self.packages and pkg_manager: yield ' '.join( [ *(['sudo'] if wants_sudo else []), - *supported_package_managers[pkg_manager], - *sorted(self.packages), + *pkg_manager.value.command, + *sorted(self.packages), # type: ignore ] ) @@ -104,11 +195,14 @@ class Dependencies: Generates the pip commands required to install the given dependencies on the system. """ - # Recent versions want an explicit --break-system-packages option when - # installing packages via pip outside of a virtual environment - wants_break_system_packages = ( - sys.version_info > (3, 10) - and sys.prefix == sys.base_prefix # We're not in a venv + wants_break_system_packages = not ( + # Docker installations shouldn't require --break-system-packages in pip + self.install_context == InstallContext.DOCKER + # --break-system-packages has been introduced in Python 3.10 + or sys.version_info < (3, 11) + # If we're in a virtual environment then we don't need + # --break-system-packages + or self._is_venv ) if self.pip: @@ -118,24 +212,15 @@ class Dependencies: + ' '.join(sorted(self.pip)) ) - def to_install_commands( - self, pkg_manager: Optional[str] = None, skip_sudo: bool = False - ) -> Generator[str, None, None]: + def to_install_commands(self) -> Generator[str, None, None]: """ Generates the commands required to install the given dependencies on this system. - - :param pkg_manager: Force package manager to use (default: looks for - the one available on the system). - :param skip_sudo: Skip sudo when installing packages (default: it will - look if the current user is root and use sudo otherwise). """ for cmd in self.before: yield cmd - for cmd in self.to_pkg_install_commands( - pkg_manager=pkg_manager, skip_sudo=skip_sudo - ): + for cmd in self.to_pkg_install_commands(): yield cmd for cmd in self.to_pip_install_commands(): @@ -145,7 +230,7 @@ class Dependencies: yield cmd -class Manifest: +class Manifest(ABC): """ Base class for plugin/backend manifests. """ @@ -156,10 +241,10 @@ class Manifest: description: Optional[str] = None, install: Optional[Dict[str, Iterable[str]]] = None, events: Optional[Mapping] = None, - pkg_manager: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, **_, ): - self._pkg_manager = pkg_manager or get_available_package_manager() + self._pkg_manager = pkg_manager or PackageManagers.scan() self.description = description self.install = self._init_deps(install or {}) self.events = self._init_events(events or {}) @@ -167,6 +252,13 @@ class Manifest: self.component_name = '.'.join(package.split('.')[2:]) self.component = None + @property + @abstractmethod + def manifest_type(self) -> ManifestType: + """ + :return: The type of the manifest. + """ + def _init_deps(self, install: Mapping[str, Iterable[str]]) -> Dependencies: deps = Dependencies() for key, items in install.items(): @@ -176,7 +268,7 @@ class Manifest: deps.before += items elif key == 'after': deps.after += items - elif key == self._pkg_manager: + elif self._pkg_manager and key == self._pkg_manager.value.executable: deps.packages.update(items) return deps @@ -201,7 +293,9 @@ class Manifest: return ret @classmethod - def from_file(cls, filename: str, pkg_manager: Optional[str] = None) -> "Manifest": + def from_file( + cls, filename: str, pkg_manager: Optional[PackageManagers] = None + ) -> "Manifest": """ Parse a manifest filename into a ``Manifest`` class. """ @@ -210,9 +304,21 @@ class Manifest: assert 'type' in manifest, f'Manifest file {filename} has no type field' comp_type = ManifestType(manifest.pop('type')) - manifest_class = _manifest_class_by_type[comp_type] + manifest_class = cls.by_type(comp_type) return manifest_class(**manifest, pkg_manager=pkg_manager) + @classmethod + def by_type(cls, manifest_type: ManifestType) -> Type["Manifest"]: + """ + :return: The manifest class corresponding to the given manifest type. + """ + if manifest_type == ManifestType.PLUGIN: + return PluginManifest + if manifest_type == ManifestType.BACKEND: + return BackendManifest + + raise ValueError(f'Unknown manifest type: {manifest_type}') + def __repr__(self): """ :return: A JSON serialized representation of the manifest. @@ -225,19 +331,23 @@ class Manifest: '.'.join([evt_type.__module__, evt_type.__name__]): doc for evt_type, doc in self.events.items() }, - 'type': _manifest_type_by_class[self.__class__].value, + 'type': self.manifest_type.value, 'package': self.package, 'component_name': self.component_name, } ) -# pylint: disable=too-few-public-methods class PluginManifest(Manifest): """ Plugin manifest. """ + @property + @override + def manifest_type(self) -> ManifestType: + return ManifestType.PLUGIN + # pylint: disable=too-few-public-methods class BackendManifest(Manifest): @@ -245,6 +355,11 @@ class BackendManifest(Manifest): Backend manifest. """ + @property + @override + def manifest_type(self) -> ManifestType: + return ManifestType.BACKEND + class Manifests: """ @@ -253,7 +368,7 @@ class Manifests: @staticmethod def by_base_class( - base_class: Type, pkg_manager: Optional[str] = None + base_class: Type, pkg_manager: Optional[PackageManagers] = None ) -> Generator[Manifest, None, None]: """ Get all the manifest files declared under the base path of a given class @@ -267,7 +382,7 @@ class Manifests: @staticmethod def by_config( conf_file: Optional[str] = None, - pkg_manager: Optional[str] = None, + pkg_manager: Optional[PackageManagers] = None, ) -> Generator[Manifest, None, None]: """ Get all the manifest objects associated to the extensions declared in a @@ -294,42 +409,3 @@ class Manifests: os.path.join(app_dir, 'plugins', *name.split('.'), 'manifest.yaml'), pkg_manager=pkg_manager, ) - - -def get_available_package_manager() -> Optional[str]: - """ - Get the name of the available package manager on the system, if supported. - """ - # pylint: disable=global-statement - global _available_package_manager - if _available_package_manager: - return _available_package_manager - - available_package_managers = [ - pkg_manager - for pkg_manager in supported_package_managers - if shutil.which(pkg_manager) - ] - - if not available_package_managers: - logger.warning( - '\nYour OS does not provide any of the supported package managers.\n' - 'You may have to install some optional dependencies manually.\n' - 'Supported package managers: %s.\n', - ', '.join(supported_package_managers.keys()), - ) - - return None - - _available_package_manager = available_package_managers[0] - return _available_package_manager - - -_manifest_class_by_type: Mapping[ManifestType, Type[Manifest]] = { - ManifestType.PLUGIN: PluginManifest, - ManifestType.BACKEND: BackendManifest, -} - -_manifest_type_by_class: Mapping[Type[Manifest], ManifestType] = { - cls: t for t, cls in _manifest_class_by_type.items() -} From 199ac5f0f7167f37b7ba196c3df249bae6b6a931 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 01:54:55 +0200 Subject: [PATCH 23/64] New way of managing installation scripts and dependencies. Created `platypush/install` folder that contains: - Dockerfiles for the supported distros - Lists of required base dependencies for the supported distros - Install and run scripts - Added Debian to supported base images --- MANIFEST.in | 1 + platypush/install/docker/alpine.Dockerfile | 14 +++++++++ platypush/install/docker/debian.Dockerfile | 18 +++++++++++ platypush/install/docker/ubuntu.Dockerfile | 18 +++++++++++ platypush/install/requirements/alpine.txt | 26 ++++++++++++++++ platypush/install/requirements/arch.txt | 24 +++++++++++++++ platypush/install/requirements/debian.txt | 28 +++++++++++++++++ platypush/install/requirements/ubuntu.txt | 1 + platypush/install/scripts/alpine/PKGCMD | 1 + platypush/install/scripts/alpine/install.sh | 1 + platypush/install/scripts/arch/PKGCMD | 1 + platypush/install/scripts/arch/install.sh | 1 + platypush/install/scripts/debian/PKGCMD | 1 + platypush/install/scripts/debian/install.sh | 1 + platypush/install/scripts/docker-run.sh | 8 +++++ platypush/install/scripts/install.sh | 34 +++++++++++++++++++++ platypush/install/scripts/ubuntu | 1 + platypush/platydock/__init__.py | 8 ++++- 18 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 platypush/install/docker/alpine.Dockerfile create mode 100644 platypush/install/docker/debian.Dockerfile create mode 100644 platypush/install/docker/ubuntu.Dockerfile create mode 100644 platypush/install/requirements/alpine.txt create mode 100644 platypush/install/requirements/arch.txt create mode 100644 platypush/install/requirements/debian.txt create mode 120000 platypush/install/requirements/ubuntu.txt create mode 100644 platypush/install/scripts/alpine/PKGCMD create mode 120000 platypush/install/scripts/alpine/install.sh create mode 100644 platypush/install/scripts/arch/PKGCMD create mode 120000 platypush/install/scripts/arch/install.sh create mode 100644 platypush/install/scripts/debian/PKGCMD create mode 120000 platypush/install/scripts/debian/install.sh create mode 100755 platypush/install/scripts/docker-run.sh create mode 100755 platypush/install/scripts/install.sh create mode 120000 platypush/install/scripts/ubuntu diff --git a/MANIFEST.in b/MANIFEST.in index 9163c3d772..ac702393c3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ recursive-include platypush/backend/http/webapp/dist * +recursive-include platypush/install * include platypush/plugins/http/webpage/mercury-parser.js include platypush/config/*.yaml global-include manifest.yaml diff --git a/platypush/install/docker/alpine.Dockerfile b/platypush/install/docker/alpine.Dockerfile new file mode 100644 index 0000000000..4f2d0ed097 --- /dev/null +++ b/platypush/install/docker/alpine.Dockerfile @@ -0,0 +1,14 @@ +FROM alpine +ADD . /install +WORKDIR /var/lib/platypush + +RUN DOCKER_CTX=1 /install/platypush/install/scripts/alpine/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD /run.sh diff --git a/platypush/install/docker/debian.Dockerfile b/platypush/install/docker/debian.Dockerfile new file mode 100644 index 0000000000..42e8e3242f --- /dev/null +++ b/platypush/install/docker/debian.Dockerfile @@ -0,0 +1,18 @@ +FROM debian +ADD . /install +WORKDIR /var/lib/platypush + +RUN apt update +RUN DOCKER_CTX=1 /install/platypush/install/scripts/debian/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install +RUN apt autoclean -y +RUN apt autoremove -y +RUN apt clean + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD /run.sh diff --git a/platypush/install/docker/ubuntu.Dockerfile b/platypush/install/docker/ubuntu.Dockerfile new file mode 100644 index 0000000000..79cbca6e57 --- /dev/null +++ b/platypush/install/docker/ubuntu.Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu +ADD . /install +WORKDIR /var/lib/platypush + +RUN apt update +RUN DOCKER_CTX=1 /install/platypush/install/scripts/debian/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install +RUN apt autoclean -y +RUN apt autoremove -y +RUN apt clean + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + +CMD /run.sh diff --git a/platypush/install/requirements/alpine.txt b/platypush/install/requirements/alpine.txt new file mode 100644 index 0000000000..e9f2b4773b --- /dev/null +++ b/platypush/install/requirements/alpine.txt @@ -0,0 +1,26 @@ +python3 +py3-pip +py3-alembic +py3-bcrypt +py3-dateutil +py3-docutils +py3-flask +py3-frozendict +py3-greenlet +py3-magic +py3-mypy-extensions +py3-psutil +py3-redis +py3-requests +py3-rsa +py3-sqlalchemy +py3-tornado +py3-typing-extensions +py3-tz +py3-websocket-client +py3-websockets +py3-wheel +py3-yaml +py3-zeroconf +redis +sudo diff --git a/platypush/install/requirements/arch.txt b/platypush/install/requirements/arch.txt new file mode 100644 index 0000000000..cc1a45ea6e --- /dev/null +++ b/platypush/install/requirements/arch.txt @@ -0,0 +1,24 @@ +python +python-alembic +python-bcrypt +python-dateutil +python-docutils +python-flask +python-frozendict +python-magic +python-marshmallow +python-pip +python-psutil +python-pytz +python-redis +python-requests +python-rsa +python-sqlalchemy +python-tornado +python-websocket-client +python-websockets +python-wheel +python-yaml +python-zeroconf +redis +sudo diff --git a/platypush/install/requirements/debian.txt b/platypush/install/requirements/debian.txt new file mode 100644 index 0000000000..d7fdd1aa79 --- /dev/null +++ b/platypush/install/requirements/debian.txt @@ -0,0 +1,28 @@ +python3 +python3-pip +python3-alembic +python3-bcrypt +python3-dateutil +python3-docutils +python3-flask +python3-frozendict +python3-greenlet +python3-magic +python3-marshmallow +python3-mypy-extensions +python3-psutil +python3-redis +python3-requests +python3-rsa +python3-sqlalchemy +python3-tornado +python3-typing-extensions +python3-typing-inspect +python3-tz +python3-websocket +python3-websockets +python3-wheel +python3-yaml +python3-zeroconf +redis +sudo diff --git a/platypush/install/requirements/ubuntu.txt b/platypush/install/requirements/ubuntu.txt new file mode 120000 index 0000000000..22a908cb66 --- /dev/null +++ b/platypush/install/requirements/ubuntu.txt @@ -0,0 +1 @@ +debian.txt \ No newline at end of file diff --git a/platypush/install/scripts/alpine/PKGCMD b/platypush/install/scripts/alpine/PKGCMD new file mode 100644 index 0000000000..eba0c3d562 --- /dev/null +++ b/platypush/install/scripts/alpine/PKGCMD @@ -0,0 +1 @@ +apk add --update --no-interactive --no-cache diff --git a/platypush/install/scripts/alpine/install.sh b/platypush/install/scripts/alpine/install.sh new file mode 120000 index 0000000000..3f44f994d2 --- /dev/null +++ b/platypush/install/scripts/alpine/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/arch/PKGCMD b/platypush/install/scripts/arch/PKGCMD new file mode 100644 index 0000000000..6d32b41ebd --- /dev/null +++ b/platypush/install/scripts/arch/PKGCMD @@ -0,0 +1 @@ +pacman -S --noconfirm diff --git a/platypush/install/scripts/arch/install.sh b/platypush/install/scripts/arch/install.sh new file mode 120000 index 0000000000..3f44f994d2 --- /dev/null +++ b/platypush/install/scripts/arch/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/debian/PKGCMD b/platypush/install/scripts/debian/PKGCMD new file mode 100644 index 0000000000..0b313dee51 --- /dev/null +++ b/platypush/install/scripts/debian/PKGCMD @@ -0,0 +1 @@ +DEBIAN_FRONTEND=noninteractive apt install -y diff --git a/platypush/install/scripts/debian/install.sh b/platypush/install/scripts/debian/install.sh new file mode 120000 index 0000000000..3f44f994d2 --- /dev/null +++ b/platypush/install/scripts/debian/install.sh @@ -0,0 +1 @@ +../install.sh \ No newline at end of file diff --git a/platypush/install/scripts/docker-run.sh b/platypush/install/scripts/docker-run.sh new file mode 100755 index 0000000000..9a0a81b440 --- /dev/null +++ b/platypush/install/scripts/docker-run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# This script is used as a default entry point for Docker containers + +DOCKER_CTX=1 platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/scripts/install.sh b/platypush/install/scripts/install.sh new file mode 100755 index 0000000000..106745a5cd --- /dev/null +++ b/platypush/install/scripts/install.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +# This script parses the system requirements for a specific OS and it runs the +# appropriate package manager command to install them. + +# This script is usually symlinked in the folders of the individual operating +# systems, and it's not supposed to be invoked directly. +# Instead, it will be called either by the root install.sh script or by a +# Dockerfile. + +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +OS="$(basename "$SCRIPT_PATH")" +CMD="$(cat "${SCRIPT_PATH}/PKGCMD")" +REQUIREMENTS="$(cat "${SCRIPT_PATH}/../../requirements/${OS}.txt" | tr '\n' ' ')" +SUDO= + +# If we are running in a Docker context then we want to copy the docker-run.sh +# script where we can easily find it. +if [ -n "$DOCKER_CTX" ]; then + cp -v /install/platypush/install/scripts/docker-run.sh /run.sh +fi + +# If we aren't running in a Docker context, or the user is not root, we should +# use sudo to install system packages. +if [[ "$(id -u)" != "0" ]] || [ -z "$DOCKER_CTX" ]; then + if ! type sudo >/dev/null; then + echo "sudo executable not found, I can't install system packages" >&2 + exit 1 + fi + + SUDO="sudo" +fi + +${SUDO_ARGS} ${CMD} ${REQUIREMENTS} diff --git a/platypush/install/scripts/ubuntu b/platypush/install/scripts/ubuntu new file mode 120000 index 0000000000..b2f7fd3e91 --- /dev/null +++ b/platypush/install/scripts/ubuntu @@ -0,0 +1 @@ +debian \ No newline at end of file diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 2f49d3cb2f..084b9956c0 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -21,6 +21,7 @@ class BaseImage(Enum): """ ALPINE = 'alpine' + DEBIAN = 'debian' UBUNTU = 'ubuntu' def __str__(self) -> str: @@ -30,6 +31,7 @@ class BaseImage(Enum): return self.value +# pylint: disable=too-few-public-methods class DockerfileGenerator: """ Generate a Dockerfile from on a configuration file. @@ -40,6 +42,7 @@ class DockerfileGenerator: _pkg_manager_by_base_image = { BaseImage.ALPINE: PackageManagers.APK, + BaseImage.DEBIAN: PackageManagers.APT, BaseImage.UBUNTU: PackageManagers.APT, } @@ -54,6 +57,8 @@ class DockerfileGenerator: :param cfgfile: Path to the configuration file. :return: The content of the generated Dockerfile. """ + import platypush + Config.init(self.cfgfile) new_file_lines = [] ports = self._get_exposed_ports() @@ -66,7 +71,8 @@ class DockerfileGenerator: is_after_expose_cmd = False base_file = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), + str(pathlib.Path(inspect.getfile(platypush)).parent), + 'install', 'docker', f'{self.image}.Dockerfile', ) From 28ba04281015a6fb312c5a9120155f97747d5498 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 02:35:25 +0200 Subject: [PATCH 24/64] If no configuration file is passed to platydock generate a minimal Dockerfile. --- platypush/install/docker/alpine.Dockerfile | 11 ++++- platypush/install/docker/debian.Dockerfile | 15 +++++-- platypush/install/docker/ubuntu.Dockerfile | 13 +++++- platypush/install/scripts/debian/PKGCMD | 2 +- platypush/install/scripts/docker-run.sh | 8 ---- platypush/install/scripts/install.sh | 8 +--- platypush/platydock/__init__.py | 39 +++++++++--------- platypush/utils/manifest.py | 47 +++++++++++++++++++--- 8 files changed, 94 insertions(+), 49 deletions(-) delete mode 100755 platypush/install/scripts/docker-run.sh diff --git a/platypush/install/docker/alpine.Dockerfile b/platypush/install/docker/alpine.Dockerfile index 4f2d0ed097..742614f191 100644 --- a/platypush/install/docker/alpine.Dockerfile +++ b/platypush/install/docker/alpine.Dockerfile @@ -1,8 +1,12 @@ FROM alpine + ADD . /install WORKDIR /var/lib/platypush -RUN DOCKER_CTX=1 /install/platypush/install/scripts/alpine/install.sh +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + +RUN /install/platypush/install/scripts/alpine/install.sh RUN cd /install && pip install -U --no-input --no-cache-dir . RUN rm -rf /install @@ -11,4 +15,7 @@ EXPOSE 8008 VOLUME /etc/platypush VOLUME /var/lib/platypush -CMD /run.sh +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/docker/debian.Dockerfile b/platypush/install/docker/debian.Dockerfile index 42e8e3242f..49e59035cd 100644 --- a/platypush/install/docker/debian.Dockerfile +++ b/platypush/install/docker/debian.Dockerfile @@ -1,10 +1,16 @@ FROM debian + ADD . /install WORKDIR /var/lib/platypush +ARG DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + RUN apt update -RUN DOCKER_CTX=1 /install/platypush/install/scripts/debian/install.sh -RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN /install/platypush/install/scripts/debian/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . --break-system-packages RUN rm -rf /install RUN apt autoclean -y RUN apt autoremove -y @@ -15,4 +21,7 @@ EXPOSE 8008 VOLUME /etc/platypush VOLUME /var/lib/platypush -CMD /run.sh +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/docker/ubuntu.Dockerfile b/platypush/install/docker/ubuntu.Dockerfile index 79cbca6e57..14b4008024 100644 --- a/platypush/install/docker/ubuntu.Dockerfile +++ b/platypush/install/docker/ubuntu.Dockerfile @@ -1,9 +1,15 @@ FROM ubuntu + ADD . /install WORKDIR /var/lib/platypush +ARG DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + RUN apt update -RUN DOCKER_CTX=1 /install/platypush/install/scripts/debian/install.sh +RUN /install/platypush/install/scripts/debian/install.sh RUN cd /install && pip install -U --no-input --no-cache-dir . RUN rm -rf /install RUN apt autoclean -y @@ -15,4 +21,7 @@ EXPOSE 8008 VOLUME /etc/platypush VOLUME /var/lib/platypush -CMD /run.sh +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush diff --git a/platypush/install/scripts/debian/PKGCMD b/platypush/install/scripts/debian/PKGCMD index 0b313dee51..3330267457 100644 --- a/platypush/install/scripts/debian/PKGCMD +++ b/platypush/install/scripts/debian/PKGCMD @@ -1 +1 @@ -DEBIAN_FRONTEND=noninteractive apt install -y +apt install -y diff --git a/platypush/install/scripts/docker-run.sh b/platypush/install/scripts/docker-run.sh deleted file mode 100755 index 9a0a81b440..0000000000 --- a/platypush/install/scripts/docker-run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# This script is used as a default entry point for Docker containers - -DOCKER_CTX=1 platypush \ - --start-redis \ - --config /etc/platypush/config.yaml \ - --workdir /var/lib/platypush diff --git a/platypush/install/scripts/install.sh b/platypush/install/scripts/install.sh index 106745a5cd..5eb6882699 100755 --- a/platypush/install/scripts/install.sh +++ b/platypush/install/scripts/install.sh @@ -14,15 +14,9 @@ CMD="$(cat "${SCRIPT_PATH}/PKGCMD")" REQUIREMENTS="$(cat "${SCRIPT_PATH}/../../requirements/${OS}.txt" | tr '\n' ' ')" SUDO= -# If we are running in a Docker context then we want to copy the docker-run.sh -# script where we can easily find it. -if [ -n "$DOCKER_CTX" ]; then - cp -v /install/platypush/install/scripts/docker-run.sh /run.sh -fi - # If we aren't running in a Docker context, or the user is not root, we should # use sudo to install system packages. -if [[ "$(id -u)" != "0" ]] || [ -z "$DOCKER_CTX" ]; then +if [ $(id -u) -ne 0 ] || [ -z "$DOCKER_CTX" ]; then if ! type sudo >/dev/null; then echo "sudo executable not found, I can't install system packages" >&2 exit 1 diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 084b9956c0..86557e4454 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -4,7 +4,6 @@ Dockerfile for Platypush starting from a configuration file. """ import argparse -from enum import Enum import inspect import os import pathlib @@ -12,23 +11,12 @@ import sys from typing import Iterable from platypush.config import Config -from platypush.utils.manifest import Dependencies, InstallContext, PackageManagers - - -class BaseImage(Enum): - """ - Supported base images for Dockerfiles. - """ - - ALPINE = 'alpine' - DEBIAN = 'debian' - UBUNTU = 'ubuntu' - - def __str__(self) -> str: - """ - Explicit __str__ override for argparse purposes. - """ - return self.value +from platypush.utils.manifest import ( + BaseImage, + Dependencies, + InstallContext, + PackageManagers, +) # pylint: disable=too-few-public-methods @@ -67,6 +55,7 @@ class DockerfileGenerator: self.cfgfile, pkg_manager=pkg_manager, install_context=InstallContext.DOCKER, + base_image=self.image, ) is_after_expose_cmd = False @@ -135,7 +124,11 @@ def main(): '-h', '--help', dest='show_usage', action='store_true', help='Show usage' ) parser.add_argument( - 'cfgfile', type=str, nargs='?', help='The path to the configuration file.' + 'cfgfile', + type=str, + nargs='?', + help='The path to the configuration file. If not specified a minimal ' + 'Dockerfile with no extra dependencies will be generated.', ) parser.add_argument( '--image', @@ -154,11 +147,15 @@ def main(): return 0 if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.auto.yaml', + ) + print( - f'Please specify a configuration file.\nRun {sys.argv[0]} --help to get the available options.', + f'No configuration file specified. Using {opts.cfgfile}.', file=sys.stderr, ) - return 1 dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate() print(dockerfile) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index c217c464b2..0be8314113 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -31,6 +31,22 @@ _available_package_manager = None logger = logging.getLogger(__name__) +class BaseImage(Enum): + """ + Supported base images for Dockerfiles. + """ + + ALPINE = 'alpine' + DEBIAN = 'debian' + UBUNTU = 'ubuntu' + + def __str__(self) -> str: + """ + Explicit __str__ override for argparse purposes. + """ + return self.value + + @dataclass class PackageManager: """ @@ -55,7 +71,7 @@ class PackageManagers(Enum): APT = PackageManager( executable='apt', - command=('DEBIAN_FRONTEND=noninteractive', 'apt', 'install', '-y'), + command=('apt', 'install', '-y'), ) PACMAN = PackageManager( @@ -142,6 +158,9 @@ class Dependencies: pkg_manager: Optional[PackageManagers] = None """ Override the default package manager detected on the system. """ install_context: InstallContext = InstallContext.NONE + """ The installation context - Docker, virtual environment or bare metal. """ + base_image: Optional[BaseImage] = None + """ Base image used in case of Docker installations. """ @property def _is_venv(self) -> bool: @@ -154,17 +173,34 @@ class Dependencies: self.install_context == InstallContext.VENV or sys.prefix != sys.base_prefix ) + @property + def _is_docker(self) -> bool: + """ + :return: True if the dependencies scanning logic is running either in a + Docker environment. + """ + return ( + self.install_context == InstallContext.DOCKER + or 'DOCKER_CTX' in os.environ + or os.path.isfile('/.dockerenv') + ) + @classmethod def from_config( cls, conf_file: Optional[str] = None, pkg_manager: Optional[PackageManagers] = None, install_context: InstallContext = InstallContext.NONE, + base_image: Optional[BaseImage] = None, ) -> "Dependencies": """ Parse the required dependencies from a configuration file. """ - deps = cls(pkg_manager=pkg_manager, install_context=install_context) + deps = cls( + pkg_manager=pkg_manager, + install_context=install_context, + base_image=base_image, + ) for manifest in Manifests.by_config(conf_file, pkg_manager=pkg_manager): deps.before += manifest.install.before @@ -179,7 +215,7 @@ class Dependencies: Generates the package manager commands required to install the given dependencies on the system. """ - wants_sudo = self.install_context != InstallContext.DOCKER and os.getuid() != 0 + wants_sudo = not (self._is_docker or os.getuid() == 0) pkg_manager = self.pkg_manager or PackageManagers.scan() if self.packages and pkg_manager: yield ' '.join( @@ -196,8 +232,9 @@ class Dependencies: the system. """ wants_break_system_packages = not ( - # Docker installations shouldn't require --break-system-packages in pip - self.install_context == InstallContext.DOCKER + # Docker installations shouldn't require --break-system-packages in + # pip, except for Debian + (self._is_docker and self.base_image != BaseImage.DEBIAN) # --break-system-packages has been introduced in Python 3.10 or sys.version_info < (3, 11) # If we're in a virtual environment then we don't need From 5efcae64c1c2b268444e9a587bb3154cc5fdeaeb Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 13:31:13 +0200 Subject: [PATCH 25/64] Better Dockerfile logic to retrieve sources. If the /install folder on the container doesn't contain a copy of the source files, then the git repository will be cloned under that folder. The user can specify via `-r/--ref` option which tag/branch/commit they want to install. --- platypush/platydock/__init__.py | 49 +++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 86557e4454..e2acabff6d 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -7,7 +7,9 @@ import argparse import inspect import os import pathlib +import re import sys +import textwrap from typing import Iterable from platypush.config import Config @@ -34,9 +36,10 @@ class DockerfileGenerator: BaseImage.UBUNTU: PackageManagers.APT, } - def __init__(self, cfgfile: str, image: BaseImage) -> None: + def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None: self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) self.image = image + self.gitref = gitref def generate(self) -> str: """ @@ -70,7 +73,12 @@ class DockerfileGenerator: file_lines = [line.rstrip() for line in f.readlines()] for line in file_lines: - if line.startswith('RUN cd /install '): + if re.match( + r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', + line.strip(), + ): + new_file_lines.append(self._generate_git_clone_command()) + elif line.startswith('RUN cd /install '): for new_line in deps.before: new_file_lines.append('RUN ' + new_line) @@ -93,6 +101,24 @@ class DockerfileGenerator: return '\n'.join(new_file_lines) + def _generate_git_clone_command(self) -> str: + pkg_manager = self._pkg_manager_by_base_image[self.image] + install_cmd = ' '.join(pkg_manager.value.install) + uninstall_cmd = ' '.join(pkg_manager.value.uninstall) + return textwrap.dedent( + f""" + RUN if [ ! -f "/install/setup.py" ]; then \\ + echo "Platypush source not found under the current directory, downloading it" && \\ + {install_cmd} git && \\ + rm -rf /install && \\ + git clone https://github.com/BlackLight/platypush.git /install && \\ + cd /install && \\ + git checkout {self.gitref} && \\ + {uninstall_cmd} git; \\ + fi + """ + ) + @staticmethod def _get_exposed_ports() -> Iterable[int]: """ @@ -123,6 +149,7 @@ def main(): parser.add_argument( '-h', '--help', dest='show_usage', action='store_true', help='Show usage' ) + parser.add_argument( 'cfgfile', type=str, @@ -130,6 +157,7 @@ def main(): help='The path to the configuration file. If not specified a minimal ' 'Dockerfile with no extra dependencies will be generated.', ) + parser.add_argument( '--image', '-i', @@ -141,6 +169,18 @@ def main(): help='Base image to use for the Dockerfile.', ) + parser.add_argument( + '--ref', + '-r', + dest='gitref', + required=False, + type=str, + default='master', + help='If platydock is not run from a Platypush installation directory, ' + 'it will clone the source via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + opts, _ = parser.parse_known_args(sys.argv[1:]) if opts.show_usage: parser.print_help() @@ -157,7 +197,10 @@ def main(): file=sys.stderr, ) - dockerfile = DockerfileGenerator(opts.cfgfile, image=opts.image).generate() + dockerfile = DockerfileGenerator( + opts.cfgfile, image=opts.image, gitref=opts.gitref + ).generate() + print(dockerfile) return 0 From a6f8021150eff91d2bf041ff28cc3a932f8c6597 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 13:33:49 +0200 Subject: [PATCH 26/64] `PackageManager` has both `install` and `uninstall`. --- platypush/install/docker/alpine.Dockerfile | 1 + platypush/utils/manifest.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/platypush/install/docker/alpine.Dockerfile b/platypush/install/docker/alpine.Dockerfile index 742614f191..e1e6016b72 100644 --- a/platypush/install/docker/alpine.Dockerfile +++ b/platypush/install/docker/alpine.Dockerfile @@ -6,6 +6,7 @@ WORKDIR /var/lib/platypush ARG DOCKER_CTX=1 ENV DOCKER_CTX=1 +RUN apk update RUN /install/platypush/install/scripts/alpine/install.sh RUN cd /install && pip install -U --no-input --no-cache-dir . RUN rm -rf /install diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 0be8314113..0ab2ed8001 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -55,8 +55,10 @@ class PackageManager: executable: str """ The executable name. """ - command: Iterable[str] = field(default_factory=tuple) - """ The command to execute, as a sequence of strings. """ + install: Iterable[str] = field(default_factory=tuple) + """ The install command, as a sequence of strings. """ + uninstall: Iterable[str] = field(default_factory=tuple) + """ The uninstall command, as a sequence of strings. """ class PackageManagers(Enum): @@ -66,17 +68,20 @@ class PackageManagers(Enum): APK = PackageManager( executable='apk', - command=('apk', 'add', '--update', '--no-interactive', '--no-cache'), + install=('apk', 'add', '--update', '--no-interactive', '--no-cache'), + uninstall=('apk', 'del', '--no-interactive'), ) APT = PackageManager( executable='apt', - command=('apt', 'install', '-y'), + install=('apt', 'install', '-y'), + uninstall=('apt', 'remove', '-y'), ) PACMAN = PackageManager( executable='pacman', - command=('pacman', '-S', '--noconfirm'), + install=('pacman', '-S', '--noconfirm'), + uninstall=('pacman', '-R', '--noconfirm'), ) @classmethod @@ -90,7 +95,7 @@ class PackageManagers(Enum): if not pkg_manager: raise ValueError(f'Unknown package manager: {name}') - return pkg_manager.value.command + return pkg_manager.value.install @classmethod def scan(cls) -> Optional["PackageManagers"]: @@ -221,7 +226,7 @@ class Dependencies: yield ' '.join( [ *(['sudo'] if wants_sudo else []), - *pkg_manager.value.command, + *pkg_manager.value.install, *sorted(self.packages), # type: ignore ] ) From f66c4aa0715c705b8dea689ff7b3ff1c9b969e8a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 13:48:31 +0200 Subject: [PATCH 27/64] Ignore the Dockerfile in the root folder --- .gitignore | 1 + Dockerfile | 38 -------------------------------------- 2 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 Dockerfile diff --git a/.gitignore b/.gitignore index 8befed6126..c446c8336d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ coverage.xml Session.vim /jsconfig.json /package.json +/Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7bcd97a42c..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM alpine -ADD . /install - -RUN apk add --update --no-interactive --no-cache \ - python3 \ - py3-pip \ - py3-alembic \ - py3-bcrypt \ - py3-dateutil \ - py3-docutils \ - py3-flask \ - py3-frozendict \ - py3-greenlet \ - py3-magic \ - py3-mypy-extensions \ - py3-psutil \ - py3-redis \ - py3-requests \ - py3-rsa \ - py3-sqlalchemy \ - py3-tornado \ - py3-typing-extensions \ - py3-tz \ - py3-websocket-client \ - py3-websockets \ - py3-wheel \ - py3-yaml \ - py3-zeroconf \ - redis - -RUN cd /install && pip install --no-cache-dir . -RUN cd / && rm -rf /install - -EXPOSE 8008 -VOLUME /app/config -VOLUME /app/workdir - -CMD platypush --start-redis --config /app/config/config.yaml --workdir /app/workdir From 700b8e1d166640c7d70415101f4767660eaa6963 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 14:05:22 +0200 Subject: [PATCH 28/64] Added header and footer to generated Dockerfile. --- platypush/platydock/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index e2acabff6d..8f4591edaa 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -36,6 +36,31 @@ class DockerfileGenerator: BaseImage.UBUNTU: PackageManagers.APT, } + _header = textwrap.dedent( + """ + # This Dockerfile was automatically generated by Platydock. + # + # You can build a Platypush image from it by running + # `docker build -t platypush .` in the same folder as this file, + # or copy it to the root a Platypush source folder to install the + # checked out version instead of downloading it first. + # + # You can then run your new image through: + # docker run --rm --name platypush \\ + # -v /path/to/your/config/dir:/etc/platypush \\ + # -v /path/to/your/workdir:/var/lib/platypush \\ + # -p 8080:8080 \\ + # platypush\n + """ + ) + + _footer = textwrap.dedent( + """ + # You can customize the name of your installation by passing + # --device-id=... to the launched command. + """ + ) + def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None: self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) self.image = image @@ -72,6 +97,8 @@ class DockerfileGenerator: with open(base_file, 'r') as f: file_lines = [line.rstrip() for line in f.readlines()] + new_file_lines.extend(self._header.split('\n')) + for line in file_lines: if re.match( r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', @@ -96,6 +123,8 @@ class DockerfileGenerator: is_after_expose_cmd = True continue + elif line.startswith('CMD'): + new_file_lines.extend(self._footer.split('\n')) new_file_lines.append(line) From a6752ed034ad36fcc46cada850216548d32b23d0 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 14:08:22 +0200 Subject: [PATCH 29/64] Fixed wrong event path in a manifest file. --- platypush/backend/assistant/google/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platypush/backend/assistant/google/manifest.yaml b/platypush/backend/assistant/google/manifest.yaml index c97ca42ef0..6de6807f12 100644 --- a/platypush/backend/assistant/google/manifest.yaml +++ b/platypush/backend/assistant/google/manifest.yaml @@ -10,7 +10,7 @@ manifest: times out platypush.message.event.assistant.MicMutedEvent: when the microphone is muted. platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted. - platypush.message.event.assistant.NoResponse: when a conversation returned no + platypush.message.event.assistant.NoResponseEvent: when a conversation returned no response platypush.message.event.assistant.ResponseEvent: when the assistant is speaking a response From a28dcb7a8d33b4ace028fcad4acd3786c529d66d Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 14:19:04 +0200 Subject: [PATCH 30/64] Remove /var/cache/apk from container image after installation. --- platypush/install/docker/alpine.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/platypush/install/docker/alpine.Dockerfile b/platypush/install/docker/alpine.Dockerfile index e1e6016b72..5092f782af 100644 --- a/platypush/install/docker/alpine.Dockerfile +++ b/platypush/install/docker/alpine.Dockerfile @@ -10,6 +10,7 @@ RUN apk update RUN /install/platypush/install/scripts/alpine/install.sh RUN cd /install && pip install -U --no-input --no-cache-dir . RUN rm -rf /install +RUN rm -rf /var/cache/apk EXPOSE 8008 From 2c46b6fe1467c0e9a70769f9a2e7b26c94848782 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 21:19:15 +0200 Subject: [PATCH 31/64] Added git a required manifest dependency when needed. It is needed for packages that install pip packages via git. --- platypush/backend/pushbullet/manifest.yaml | 6 ++++++ platypush/plugins/bluetooth/manifest.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/platypush/backend/pushbullet/manifest.yaml b/platypush/backend/pushbullet/manifest.yaml index 27f5a0f255..e49e855aa6 100644 --- a/platypush/backend/pushbullet/manifest.yaml +++ b/platypush/backend/pushbullet/manifest.yaml @@ -1,6 +1,12 @@ manifest: events: platypush.message.event.pushbullet.PushbulletEvent: if a new push is received + apk: + - git + apt: + - git + pacman: + - git install: pip: - git+https://github.com/rbrcsk/pushbullet.py diff --git a/platypush/plugins/bluetooth/manifest.yaml b/platypush/plugins/bluetooth/manifest.yaml index c2e98a3488..893d4e074f 100644 --- a/platypush/plugins/bluetooth/manifest.yaml +++ b/platypush/plugins/bluetooth/manifest.yaml @@ -15,12 +15,15 @@ manifest: install: apk: - py3-pydbus + - git apt: - libbluetooth-dev - python3-pydbus + - git pacman: - python-pydbus - python-bleak + - git pip: - bleak - bluetooth-numbers From 10c0e5fcad9509a11467c746c4e785537732935e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 21:21:37 +0200 Subject: [PATCH 32/64] Added default_os field to PackageManagers enum elements. This is useful to determine which is the default set of scripts that should be used by the installer depending on the detected installed package manager. --- platypush/install/scripts/arch/PKGCMD | 2 +- platypush/utils/manifest.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/platypush/install/scripts/arch/PKGCMD b/platypush/install/scripts/arch/PKGCMD index 6d32b41ebd..eb711b6209 100644 --- a/platypush/install/scripts/arch/PKGCMD +++ b/platypush/install/scripts/arch/PKGCMD @@ -1 +1 @@ -pacman -S --noconfirm +pacman -S --noconfirm --needed diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 0ab2ed8001..f8aef8a37f 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -55,6 +55,11 @@ class PackageManager: executable: str """ The executable name. """ + default_os: str + """ + The default distro whose configuration we should use if this package + manager is detected. + """ install: Iterable[str] = field(default_factory=tuple) """ The install command, as a sequence of strings. """ uninstall: Iterable[str] = field(default_factory=tuple) @@ -70,18 +75,21 @@ class PackageManagers(Enum): executable='apk', install=('apk', 'add', '--update', '--no-interactive', '--no-cache'), uninstall=('apk', 'del', '--no-interactive'), + default_os='alpine', ) APT = PackageManager( executable='apt', install=('apt', 'install', '-y'), uninstall=('apt', 'remove', '-y'), + default_os='debian', ) PACMAN = PackageManager( executable='pacman', - install=('pacman', '-S', '--noconfirm'), + install=('pacman', '-S', '--noconfirm', '--needed'), uninstall=('pacman', '-R', '--noconfirm'), + default_os='arch', ) @classmethod From ce68250b4dcc4c3880546f128db1d4a64f33289a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Tue, 22 Aug 2023 02:49:05 +0200 Subject: [PATCH 33/64] Refactor/documentation round for platydock. --- platypush/platydock/__init__.py | 138 +++++++++++++++++--------------- 1 file changed, 74 insertions(+), 64 deletions(-) diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 8f4591edaa..3f1a425aec 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -10,7 +10,7 @@ import pathlib import re import sys import textwrap -from typing import Iterable +from typing import Iterable, Sequence from platypush.config import Config from platypush.utils.manifest import ( @@ -70,7 +70,6 @@ class DockerfileGenerator: """ Generate a Dockerfile based on a configuration file. - :param cfgfile: Path to the configuration file. :return: The content of the generated Dockerfile. """ import platypush @@ -131,6 +130,11 @@ class DockerfileGenerator: return '\n'.join(new_file_lines) def _generate_git_clone_command(self) -> str: + """ + Generates a git clone command in Dockerfile that checks out the repo + and the right git reference, if the application sources aren't already + available under /install. + """ pkg_manager = self._pkg_manager_by_base_image[self.image] install_cmd = ' '.join(pkg_manager.value.install) uninstall_cmd = ' '.join(pkg_manager.value.uninstall) @@ -148,6 +152,73 @@ class DockerfileGenerator: """ ) + @classmethod + def from_cmdline(cls, args: Sequence[str]) -> 'DockerfileGenerator': + """ + Create a DockerfileGenerator instance from command line arguments. + + :param args: Command line arguments. + :return: A DockerfileGenerator instance. + """ + parser = argparse.ArgumentParser( + prog='platydock', + add_help=False, + description='Create a Platypush Dockerfile from a config.yaml.', + ) + + parser.add_argument( + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + + parser.add_argument( + 'cfgfile', + type=str, + nargs='?', + help='The path to the configuration file. If not specified a minimal ' + 'Dockerfile with no extra dependencies will be generated.', + ) + + parser.add_argument( + '--image', + '-i', + dest='image', + required=False, + type=BaseImage, + choices=list(BaseImage), + default=BaseImage.ALPINE, + help='Base image to use for the Dockerfile.', + ) + + parser.add_argument( + '--ref', + '-r', + dest='gitref', + required=False, + type=str, + default='master', + help='If platydock is not run from a Platypush installation directory, ' + 'it will clone the source via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + + opts, _ = parser.parse_known_args(args) + if opts.show_usage: + parser.print_help() + sys.exit(0) + + if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.auto.yaml', + ) + + print( + f'No configuration file specified. Using {opts.cfgfile}.', + file=sys.stderr, + ) + + return cls(opts.cfgfile, image=opts.image, gitref=opts.gitref) + @staticmethod def _get_exposed_ports() -> Iterable[int]: """ @@ -169,68 +240,7 @@ def main(): """ Generates a Dockerfile based on the configuration file. """ - parser = argparse.ArgumentParser( - prog='platydock', - add_help=False, - description='Create a Platypush Dockerfile from a config.yaml.', - ) - - parser.add_argument( - '-h', '--help', dest='show_usage', action='store_true', help='Show usage' - ) - - parser.add_argument( - 'cfgfile', - type=str, - nargs='?', - help='The path to the configuration file. If not specified a minimal ' - 'Dockerfile with no extra dependencies will be generated.', - ) - - parser.add_argument( - '--image', - '-i', - dest='image', - required=False, - type=BaseImage, - choices=list(BaseImage), - default=BaseImage.ALPINE, - help='Base image to use for the Dockerfile.', - ) - - parser.add_argument( - '--ref', - '-r', - dest='gitref', - required=False, - type=str, - default='master', - help='If platydock is not run from a Platypush installation directory, ' - 'it will clone the source via git. You can specify through this ' - 'option which branch, tag or commit hash to use. Defaults to master.', - ) - - opts, _ = parser.parse_known_args(sys.argv[1:]) - if opts.show_usage: - parser.print_help() - return 0 - - if not opts.cfgfile: - opts.cfgfile = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', - ) - - print( - f'No configuration file specified. Using {opts.cfgfile}.', - file=sys.stderr, - ) - - dockerfile = DockerfileGenerator( - opts.cfgfile, image=opts.image, gitref=opts.gitref - ).generate() - - print(dockerfile) + print(DockerfileGenerator.from_cmdline(sys.argv[1:]).generate()) return 0 From 782ddf5097a02ce597cbf851f5f4f02980783a43 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Tue, 22 Aug 2023 23:49:37 +0200 Subject: [PATCH 34/64] Removed legacy platyvenv bash script. The script is being migrated to a Python implementation. --- bin/platyvenv | 249 -------------------------------------------------- setup.py | 2 +- 2 files changed, 1 insertion(+), 250 deletions(-) delete mode 100755 bin/platyvenv diff --git a/bin/platyvenv b/bin/platyvenv deleted file mode 100755 index 1b0d27de7f..0000000000 --- a/bin/platyvenv +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash - -############################################################################## -# This script allows you to easily manage Platypush instances through Python # -# virtual environment. You can build environments from a config.yaml file # -# and automatically managed the required dependencies, as well as start, # -# stop and remove them # -# # -# @author: Fabio Manganiello # -# @licence: MIT # -############################################################################## - - -workdir="$HOME/.local/share/platypush/venv" - -function build { - cfgfile= - - while getopts ':c:' opt; do - case ${opt} in - c) - cfgfile=$OPTARG;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1;; - :) - echo "Option -$OPTARG requires the path to a Platypush configuration file" >&2 - exit 1;; - esac - done - - if [[ -z "$cfgfile" ]]; then - echo "Usage: $0 build -c " >&2 - exit 1 - fi - - echo "Parsing configuration file" - pip_cmd= - pkg_cmd= - includes=() - cmd_exec=() - - while read -r line; do - if echo "$line" | grep -E "^pip:\s*"; then - pip_cmd="$(echo "$line" | sed -r -e 's/^pip:\s*(.*)'/\\1/)" - elif echo "$line" | grep -E "^packages:\s*"; then - pkg_cmd="$(echo "$line" | sed -r -e 's/^packages:\s*(.*)'/\\1/)" - elif echo "$line" | grep -E "^exec:\s*"; then - cmd_exec+=("$(echo "$line" | sed -r -e 's/^exec:\s*(.*)'/\\1/)") - elif echo "$line" | grep -E "^include:\s*"; then - includes+=("$(echo "$line" | sed -r -e 's/^include:\s*(.*)'/\\1/)") - elif echo "$line" | grep -E "^device_id:\s*"; then - device_id="$(echo "$line" | sed -r -e 's/^device_id:\s*(.*)'/\\1/)" - fi - done <<< "$(python <" >&2 - exit 1 - fi - - env=$1 - envdir="${workdir}/${env}" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - cfgfile="${envdir}/etc/platypush/config.yaml" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - mkdir -p "${rundir}" - - if [[ -f "$pidfile" ]]; then - if pgrep -F "${pidfile}"; then - echo "Another instance (PID $(cat "${pidfile}")) is running, please stop that instance first" - exit 1 - fi - - echo "A PID file was found but the process does not seem to be running, starting anyway" - rm -f "$pidfile" - fi - - python3 -m venv "${envdir}" - cd "${envdir}" || exit 1 - source bin/activate - bin/platypush -c "$cfgfile" -P "$pidfile" & - start_time=$(date +'%s') - timeout=30 - - while :; do - [[ -f "$pidfile" ]] && break - now=$(date +'%s') - elapsed=$(( now-start_time )) - if (( elapsed >= timeout )); then - echo "Platypush instance '$env' did not start within $timeout seconds" >&2 - exit 1 - fi - - echo -n '.' - sleep 1 - done - - pid=$(cat "$pidfile") - echo - echo "Platypush environment $env started with PID $pid" - wait "${pid}" - echo "Platypush environment $env terminated" -} - -function stop { - if [[ -z "$1" ]]; then - echo "Usage: $0 stop " >&2 - exit 1 - fi - - env=$1 - envdir="${workdir}/${env}" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - if [[ ! -f "$pidfile" ]]; then - echo "No pidfile found for instance \"${env}\"" - exit 1 - fi - - pid=$(cat "$pidfile") - pids="$pid $(ps --no-headers -o pid= --ppid "$pid")" - # shellcheck disable=SC2086 - kill -9 ${pids} - rm -f "$pidfile" - echo "Instance '$env' with PID $pid stopped" -} - -function rme { - if [[ -z "$1" ]]; then - echo "Usage: $0 rm " >&2 - exit 1 - fi - - envdir="${workdir}/$1" - rundir="${envdir}/var/run" - pidfile="${rundir}/platypush.pid" - - if [[ ! -d "$envdir" ]]; then - echo "No such directory: $envdir" >&2 - exit 1 - fi - - if [[ -f "$pidfile" ]]; then - if pgrep -F "${pidfile}"; then - echo "Another instance (PID $(cat "$pidfile")) is running, please stop that instance first" - exit 1 - fi - - echo "A PID file was found but the process does not seem to be running, removing anyway" - fi - - echo "WARNING: This operation will permanently remove the Platypush environment $1" - echo -n "Are you sure you want to continue? (y/N) " - IFS= read -r answer - echo "$answer" | grep -E '^[yY]' >/dev/null || exit 0 - rm -rf "$envdir" - echo "$envdir removed" -} - -function usage { - echo "Usage: $0 [options]" >&2 - exit 1 -} - -if (( $# < 1 )); then - usage -fi - -action=$1 -shift -mkdir -p "${workdir}" - -# shellcheck disable=SC2048,SC2086 -case ${action} in - 'build') build $*;; - 'start') start $*;; - 'stop') stop $*;; - 'rm') rme $*;; - *) usage;; -esac diff --git a/setup.py b/setup.py index 1f2bf8d998..b9ea662eb0 100755 --- a/setup.py +++ b/setup.py @@ -49,9 +49,9 @@ setup( 'console_scripts': [ 'platypush=platypush:main', 'platydock=platypush.platydock:main', + 'platyvenv=platypush.platyvenv:main', ], }, - scripts=['bin/platyvenv'], long_description=readfile('README.md'), long_description_content_type='text/markdown', classifiers=[ From 8f39231d311b1116a634359a2673aa5aebe8c29f Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 02:15:50 +0200 Subject: [PATCH 35/64] Added new utility methods to the Dependencies class. --- platypush/utils/manifest.py | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index f8aef8a37f..5f9f78305a 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -26,6 +26,7 @@ from typing_extensions import override import yaml from platypush.message.event import Event +from platypush.utils import get_src_root _available_package_manager = None logger = logging.getLogger(__name__) @@ -198,6 +199,81 @@ class Dependencies: or os.path.isfile('/.dockerenv') ) + @property + def _wants_sudo(self) -> bool: + """ + :return: True if the system dependencies should be installed with sudo. + """ + return not (self._is_docker or os.getuid() == 0) + + @staticmethod + def _get_requirements_dir() -> str: + """ + :return: The root folder for the base installation requirements. + """ + return os.path.join(get_src_root(), 'install', 'requirements') + + @classmethod + def _parse_requirements_file( + cls, + requirements_file: str, + install_context: InstallContext = InstallContext.NONE, + ) -> Iterable[str]: + """ + :param requirements_file: The path to the requirements file. + :return: The list of requirements to install. + """ + with open(requirements_file, 'r') as f: + return { + line.strip() + for line in f + if not ( + not line.strip() + or line.strip().startswith('#') + # Virtual environments will install all the Python + # dependencies via pip, so we should skip them here + or ( + install_context == InstallContext.VENV + and cls._is_python_pkg(line.strip()) + ) + ) + } + + @classmethod + def _get_base_system_dependencies( + cls, + pkg_manager: Optional[PackageManagers] = None, + install_context: InstallContext = InstallContext.NONE, + ) -> Iterable[str]: + """ + :return: The base system dependencies that should be installed on the + system. + """ + + # Docker images will install the base packages through their own + # dedicated shell script, so don't report their base system + # requirements here. + if not (pkg_manager and install_context != InstallContext.DOCKER): + return set() + + return cls._parse_requirements_file( + os.path.join( + cls._get_requirements_dir(), pkg_manager.value.default_os + '.txt' + ), + install_context, + ) + + @staticmethod + def _is_python_pkg(pkg: str) -> bool: + """ + Utility function that returns True if a given package is a Python + system package. These should be skipped during a virtual + environment installation, as the virtual environment will be + installed via pip. + """ + tokens = pkg.split('-') + return len(tokens) > 1 and tokens[0] in {'py3', 'python3', 'python'} + @classmethod def from_config( cls, From cddf318fa723e2896afc127dd222b16ea50a9b0a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 02:16:33 +0200 Subject: [PATCH 36/64] Dependencies.from_config should include the base system deps. --- platypush/utils/manifest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 5f9f78305a..7eaf67261d 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -285,7 +285,15 @@ class Dependencies: """ Parse the required dependencies from a configuration file. """ + if not pkg_manager: + pkg_manager = PackageManagers.scan() + + base_system_deps = cls._get_base_system_dependencies( + pkg_manager=pkg_manager, install_context=install_context + ) + deps = cls( + packages=set(base_system_deps), pkg_manager=pkg_manager, install_context=install_context, base_image=base_image, From 2bff4c9cf1208eafc70e61f659e4f6caf90b11b0 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 02:17:19 +0200 Subject: [PATCH 37/64] Exclude python-* system packages when installing in a venv. --- platypush/utils/manifest.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 7eaf67261d..9de25ed2ba 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -312,6 +312,7 @@ class Dependencies: Generates the package manager commands required to install the given dependencies on the system. """ + wants_sudo = not (self._is_docker or os.getuid() == 0) pkg_manager = self.pkg_manager or PackageManagers.scan() if self.packages and pkg_manager: @@ -319,7 +320,14 @@ class Dependencies: [ *(['sudo'] if wants_sudo else []), *pkg_manager.value.install, - *sorted(self.packages), # type: ignore + *sorted( + pkg + for pkg in self.packages # type: ignore + if not ( + self.install_context == InstallContext.VENV + and self._is_python_pkg(pkg) + ) + ), ] ) From 1ef0d804dbd1eed2d61f34a01090f4b7ac5965fe Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 02:19:54 +0200 Subject: [PATCH 38/64] Added `full_command` argument to `to_pip_install_commands`. This is useful if we just want to get the list of pip dependencies and create our own pip command. --- platypush/utils/manifest.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 9de25ed2ba..ef2350ee79 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -331,10 +331,14 @@ class Dependencies: ] ) - def to_pip_install_commands(self) -> Generator[str, None, None]: + def to_pip_install_commands(self, full_command=True) -> Generator[str, None, None]: """ Generates the pip commands required to install the given dependencies on the system. + + :param full_command: Whether to return the full pip command to execute + (as a single string) or the list of packages that will be installed + through another script. """ wants_break_system_packages = not ( # Docker installations shouldn't require --break-system-packages in @@ -348,11 +352,20 @@ class Dependencies: ) if self.pip: - yield ( - 'pip install -U --no-input --no-cache-dir ' - + ('--break-system-packages ' if wants_break_system_packages else '') - + ' '.join(sorted(self.pip)) - ) + deps = sorted(self.pip) + if full_command: + yield ( + 'pip install -U --no-input --no-cache-dir ' + + ( + '--break-system-packages ' + if wants_break_system_packages + else '' + ) + + ' '.join(deps) + ) + else: + for dep in deps: + yield dep def to_install_commands(self) -> Generator[str, None, None]: """ @@ -530,7 +543,6 @@ class Manifests: Get all the manifest objects associated to the extensions declared in a given configuration file. """ - import platypush from platypush.config import Config conf_args = [] @@ -538,7 +550,7 @@ class Manifests: conf_args.append(conf_file) Config.init(*conf_args) - app_dir = os.path.dirname(inspect.getfile(platypush)) + app_dir = get_src_root() for name in Config.get_backends().keys(): yield Manifest.from_file( From b10ccdb31326590a292afc6c1080d2d0303f0776 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 02:53:31 +0200 Subject: [PATCH 39/64] Added get_src_root utility function. --- platypush/utils/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py index 7f16cce826..3ec6934e09 100644 --- a/platypush/utils/__init__.py +++ b/platypush/utils/__init__.py @@ -646,4 +646,13 @@ def get_remaining_timeout( return cls(max(0, timeout - (time.time() - start))) +def get_src_root() -> str: + """ + :return: The root source folder of the application. + """ + import platypush + + return os.path.dirname(inspect.getfile(platypush)) + + # vim:sw=4:ts=4:et: From 449821673c29ed25e02535c49c578f3a0a5a3a19 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 10:49:58 +0200 Subject: [PATCH 40/64] Added `PackageManager.get_installed`. --- platypush/utils/manifest.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index ef2350ee79..1c14316542 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -7,10 +7,13 @@ import json import logging import os import pathlib +import re import shutil +import subprocess import sys from typing import ( + Callable, Dict, Generator, List, @@ -65,6 +68,8 @@ class PackageManager: """ The install command, as a sequence of strings. """ uninstall: Iterable[str] = field(default_factory=tuple) """ The uninstall command, as a sequence of strings. """ + get_installed: Callable[[], Iterable[str]] = lambda: [] + """ A function that returns the list of installed packages. """ class PackageManagers(Enum): @@ -77,6 +82,19 @@ class PackageManagers(Enum): install=('apk', 'add', '--update', '--no-interactive', '--no-cache'), uninstall=('apk', 'del', '--no-interactive'), default_os='alpine', + get_installed=lambda: { + re.sub(r'.*\s*\{(.+?)\}\s*.*', r'\1', line) + for line in ( + line.strip() + for line in subprocess.Popen( # pylint: disable=consider-using-with + ['apk', 'list', '--installed'], stdout=subprocess.PIPE + ) + .communicate()[0] + .decode() + .split('\n') + ) + if line.strip() + }, ) APT = PackageManager( @@ -84,6 +102,16 @@ class PackageManagers(Enum): install=('apt', 'install', '-y'), uninstall=('apt', 'remove', '-y'), default_os='debian', + get_installed=lambda: { + line.strip().split('/')[0] + for line in subprocess.Popen( # pylint: disable=consider-using-with + ['apt', 'list', '--installed'], stdout=subprocess.PIPE + ) + .communicate()[0] + .decode() + .split('\n') + if line.strip() + }, ) PACMAN = PackageManager( @@ -91,6 +119,16 @@ class PackageManagers(Enum): install=('pacman', '-S', '--noconfirm', '--needed'), uninstall=('pacman', '-R', '--noconfirm'), default_os='arch', + get_installed=lambda: { + line.strip().split(' ')[0] + for line in subprocess.Popen( # pylint: disable=consider-using-with + ['pacman', '-Q'], stdout=subprocess.PIPE + ) + .communicate()[0] + .decode() + .split('\n') + if line.strip() + }, ) @classmethod From f230fa79bbb636ee25921d7fa22c366232acb08b Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 11:51:53 +0200 Subject: [PATCH 41/64] `to_pkg_install_commands` should skip already installed sys packages. --- platypush/utils/manifest.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 1c14316542..b6c5a7f7fb 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -353,22 +353,28 @@ class Dependencies: wants_sudo = not (self._is_docker or os.getuid() == 0) pkg_manager = self.pkg_manager or PackageManagers.scan() + if self.packages and pkg_manager: - yield ' '.join( - [ - *(['sudo'] if wants_sudo else []), - *pkg_manager.value.install, - *sorted( - pkg - for pkg in self.packages # type: ignore - if not ( - self.install_context == InstallContext.VENV - and self._is_python_pkg(pkg) - ) - ), - ] + installed_packages = pkg_manager.value.get_installed() + to_install = sorted( + pkg + for pkg in self.packages # type: ignore + if pkg not in installed_packages + and not ( + self.install_context == InstallContext.VENV + and self._is_python_pkg(pkg) + ) ) + if to_install: + yield ' '.join( + [ + *(['sudo'] if wants_sudo else []), + *pkg_manager.value.install, + *to_install, + ] + ) + def to_pip_install_commands(self, full_command=True) -> Generator[str, None, None]: """ Generates the pip commands required to install the given dependencies on From dafd65dc21a9d251f384009aedad98a8a9c5eba6 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 11:53:14 +0200 Subject: [PATCH 42/64] Added new platyvenv Python script. --- platypush/platyvenv/__init__.py | 196 ++++++++++++++++++++++++++++++++ platypush/platyvenv/__main__.py | 5 + 2 files changed, 201 insertions(+) create mode 100755 platypush/platyvenv/__init__.py create mode 100644 platypush/platyvenv/__main__.py diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py new file mode 100755 index 0000000000..0e370f40c8 --- /dev/null +++ b/platypush/platyvenv/__init__.py @@ -0,0 +1,196 @@ +""" +Platyvenv is a helper script that allows you to automatically create a +virtual environment for Platypush starting from a configuration file. +""" + +import argparse +import inspect +import os +import pathlib +import re +import subprocess +import sys +from typing import Sequence +import venv + +from platypush.config import Config +from platypush.utils import get_src_root +from platypush.utils.manifest import ( + Dependencies, + InstallContext, +) + + +# pylint: disable=too-few-public-methods +class VenvBuilder: + """ + Build a virtual environment from on a configuration file. + + :param cfgfile: Path to the configuration file. + :param image: The base image to use. + """ + + def __init__(self, cfgfile: str, gitref: str, output_dir: str) -> None: + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.output_dir = os.path.abspath(os.path.expanduser(output_dir)) + self.gitref = gitref + + @property + def _pip_cmd(self) -> Sequence[str]: + """ + :return: The pip install command to use for the selected environment. + """ + return ( + os.path.join(self.output_dir, 'bin', 'python'), + '-m', + 'pip', + 'install', + '-U', + '--no-cache-dir', + '--no-input', + ) + + def _install_system_packages(self, deps: Dependencies): + """ + Install the required system packages. + """ + for cmd in deps.to_pkg_install_commands(): + print(f'Installing system packages: {cmd}') + subprocess.call(re.split(r'\s+', cmd.strip())) + + def _prepare_venv(self) -> None: + """ + Installs the virtual environment under the configured output_dir. + """ + print(f'Creating virtual environment under {self.output_dir}...') + + venv.create( + self.output_dir, + symlinks=True, + with_pip=True, + upgrade_deps=True, + ) + + print( + f'Installing base Python dependencies under {self.output_dir}...', + ) + + subprocess.call([*self._pip_cmd, 'pip']) + pwd = os.getcwd() + + try: + os.chdir(os.path.dirname(get_src_root())) + subprocess.call([*self._pip_cmd, '.']) + finally: + os.chdir(pwd) + + def _install_extra_pip_packages(self, deps: Dependencies): + """ + Install the extra pip dependencies parsed through the + """ + pip_deps = list(deps.to_pip_install_commands(full_command=False)) + if not pip_deps: + return + + print( + f'Installing extra pip dependencies under {self.output_dir}: ' + + ' '.join(pip_deps) + ) + + subprocess.call([*self._pip_cmd, *pip_deps]) + + def build(self): + """ + Build a Dockerfile based on a configuration file. + """ + Config.init(self.cfgfile) + + deps = Dependencies.from_config( + self.cfgfile, + install_context=InstallContext.VENV, + ) + + self._install_system_packages(deps) + self._prepare_venv() + self._install_extra_pip_packages(deps) + + @classmethod + def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder': + """ + Create a DockerfileGenerator instance from command line arguments. + + :param args: Command line arguments. + :return: A DockerfileGenerator instance. + """ + parser = argparse.ArgumentParser( + prog='platyvenv', + add_help=False, + description='Create a Platypush virtual environment from a config.yaml.', + ) + + parser.add_argument( + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + + parser.add_argument( + 'cfgfile', + type=str, + nargs='?', + help='The path to the configuration file. If not specified a minimal ' + 'virtual environment only with the base dependencies will be ' + 'generated.', + ) + + parser.add_argument( + '-o', + '--output', + dest='output_dir', + type=str, + required=False, + default='venv', + help='Target directory for the virtual environment (default: ./venv)', + ) + + parser.add_argument( + '--ref', + '-r', + dest='gitref', + required=False, + type=str, + default='master', + help='If platyvenv is not run from a Platypush installation directory, ' + 'it will clone the sources via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + + opts, _ = parser.parse_known_args(args) + if opts.show_usage: + parser.print_help() + sys.exit(0) + + if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.auto.yaml', + ) + + print( + f'No configuration file specified. Using {opts.cfgfile}.', + file=sys.stderr, + ) + + return cls(opts.cfgfile, gitref=opts.gitref, output_dir=opts.output_dir) + + +def main(): + """ + Generates a virtual environment based on the configuration file. + """ + VenvBuilder.from_cmdline(sys.argv[1:]).build() + + +if __name__ == '__main__': + sys.exit(main()) + + +# vim:sw=4:ts=4:et: diff --git a/platypush/platyvenv/__main__.py b/platypush/platyvenv/__main__.py new file mode 100644 index 0000000000..b6726678b5 --- /dev/null +++ b/platypush/platyvenv/__main__.py @@ -0,0 +1,5 @@ +from platypush.platyvenv import main + +main() + +# vim:sw=4:ts=4:et: From 9e6430a9ac523d5ef2c55984bb08df298af5bd0e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 23 Aug 2023 20:02:04 +0200 Subject: [PATCH 43/64] Clone git repo if platyvenv is not running from a srcdir --- platypush/platyvenv/__init__.py | 61 +++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py index 0e370f40c8..723dc8c633 100755 --- a/platypush/platyvenv/__init__.py +++ b/platypush/platyvenv/__init__.py @@ -4,17 +4,19 @@ virtual environment for Platypush starting from a configuration file. """ import argparse +from contextlib import contextmanager import inspect import os import pathlib import re +import shutil import subprocess import sys -from typing import Sequence +import tempfile +from typing import Generator, Sequence import venv from platypush.config import Config -from platypush.utils import get_src_root from platypush.utils.manifest import ( Dependencies, InstallContext, @@ -58,6 +60,43 @@ class VenvBuilder: print(f'Installing system packages: {cmd}') subprocess.call(re.split(r'\s+', cmd.strip())) + @contextmanager + def _prepare_src_dir(self) -> Generator[str, None, None]: + """ + Prepare the source directory used to install the virtual enviornment. + + If platyvenv is launched from a local checkout of the Platypush source + code, then that checkout will be used. + + Otherwise, the source directory will be cloned from git into a + temporary folder. + """ + setup_py_path = os.path.join(os.getcwd(), 'setup.py') + if os.path.isfile(setup_py_path): + print('Using local checkout of the Platypush source code') + yield os.getcwd() + else: + checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git') + print(f'Cloning Platypush source code from git into {checkout_dir}') + subprocess.call( + [ + 'git', + 'clone', + '--recursive', + 'https://github.com/BlackLight/platypush.git', + checkout_dir, + ] + ) + + pwd = os.getcwd() + os.chdir(checkout_dir) + subprocess.call(['git', 'checkout', self.gitref]) + yield checkout_dir + + os.chdir(pwd) + print(f'Cleaning up {checkout_dir}') + shutil.rmtree(checkout_dir, ignore_errors=True) + def _prepare_venv(self) -> None: """ Installs the virtual environment under the configured output_dir. @@ -75,14 +114,7 @@ class VenvBuilder: f'Installing base Python dependencies under {self.output_dir}...', ) - subprocess.call([*self._pip_cmd, 'pip']) - pwd = os.getcwd() - - try: - os.chdir(os.path.dirname(get_src_root())) - subprocess.call([*self._pip_cmd, '.']) - finally: - os.chdir(pwd) + subprocess.call([*self._pip_cmd, 'pip', '.']) def _install_extra_pip_packages(self, deps: Dependencies): """ @@ -111,8 +143,15 @@ class VenvBuilder: ) self._install_system_packages(deps) - self._prepare_venv() + + with self._prepare_src_dir(): + self._prepare_venv() + self._install_extra_pip_packages(deps) + print( + f'\nVirtual environment created at {self.output_dir}.\n' + f'Run source {os.path.join(self.output_dir, "bin", "activate")} to activate it.' + ) @classmethod def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder': From 86e5f74645cbe4cc4d3120ddf238599f473ff759 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 27 Aug 2023 23:20:03 +0200 Subject: [PATCH 44/64] platyvenv should generate (and document) a run.sh helper script. --- platypush/platyvenv/__init__.py | 35 +++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py index 723dc8c633..ea161cf01b 100755 --- a/platypush/platyvenv/__init__.py +++ b/platypush/platyvenv/__init__.py @@ -13,6 +13,7 @@ import shutil import subprocess import sys import tempfile +import textwrap from typing import Generator, Sequence import venv @@ -131,6 +132,35 @@ class VenvBuilder: subprocess.call([*self._pip_cmd, *pip_deps]) + def _generate_run_sh(self) -> str: + """ + Generate a ``run.sh`` script to run the application from a newly built + virtual environment. + + :return: The location of the generated ``run.sh`` script. + """ + run_sh_path = os.path.join(self.output_dir, 'bin', 'run.sh') + with open(run_sh_path, 'w') as run_sh: + run_sh.write( + textwrap.dedent( + f""" + #!/bin/bash + + cd {self.output_dir} + + # Activate the virtual environment + source bin/activate + + # Run the application with the configuration file passed + # during build + platypush -c {self.cfgfile} $* + """ + ) + ) + + os.chmod(run_sh_path, 0o750) + return run_sh_path + def build(self): """ Build a Dockerfile based on a configuration file. @@ -148,9 +178,10 @@ class VenvBuilder: self._prepare_venv() self._install_extra_pip_packages(deps) + run_sh_path = self._generate_run_sh() print( - f'\nVirtual environment created at {self.output_dir}.\n' - f'Run source {os.path.join(self.output_dir, "bin", "activate")} to activate it.' + f'\nVirtual environment created under {self.output_dir}.\n' + f'You can run the application through the {run_sh_path} script.\n' ) @classmethod From 429658e7c88cc97b4f054bbd4c5a6606cc9f8ab6 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 28 Aug 2023 01:26:19 +0200 Subject: [PATCH 45/64] Refactored `PackageManager` classes. Instead of having a custom `get_installed` callable field, with replicated code for each package manager, the field has now been promoted to a class method containing the common logic, and the instances now expect a `list` field (base command to list the installed packages using the specified package manager) and a `parse_list_line` callback field (to extract the base package name given a raw line from the command above). Also, we shouldn't run the list command if we're running within a Docker context - the host and container environments will be different. --- platypush/utils/manifest.py | 84 ++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index b6c5a7f7fb..48a8a0ab28 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -20,6 +20,7 @@ from typing import ( Optional, Iterable, Mapping, + Sequence, Set, Type, Union, @@ -64,12 +65,46 @@ class PackageManager: The default distro whose configuration we should use if this package manager is detected. """ - install: Iterable[str] = field(default_factory=tuple) + install: Sequence[str] = field(default_factory=tuple) """ The install command, as a sequence of strings. """ - uninstall: Iterable[str] = field(default_factory=tuple) + uninstall: Sequence[str] = field(default_factory=tuple) """ The uninstall command, as a sequence of strings. """ - get_installed: Callable[[], Iterable[str]] = lambda: [] - """ A function that returns the list of installed packages. """ + list: Sequence[str] = field(default_factory=tuple) + """ The command to list the installed packages. """ + parse_list_line: Callable[[str], str] = field(default_factory=lambda: lambda s: s) + """ + Internal package-manager dependent function that parses the base package + name from a line returned by the list command. + """ + + def _get_installed(self) -> Sequence[str]: + """ + :return: The install context-aware list of installed packages. + It should only used within the context of :meth:`.get_installed`. + """ + + if os.environ.get('DOCKER_CTX'): + # If we're running in a Docker build context, don't run the package + # manager to retrieve the list of installed packages, as the host + # and guest systems have different environments. + return () + + return tuple( + line.strip() + for line in subprocess.Popen( # pylint: disable=consider-using-with + self.list, stdout=subprocess.PIPE + ) + .communicate()[0] + .decode() + .split('\n') + if line.strip() + ) + + def get_installed(self) -> Sequence[str]: + """ + :return: The list of installed packages. + """ + return tuple(self.parse_list_line(line) for line in self._get_installed()) class PackageManagers(Enum): @@ -81,54 +116,27 @@ class PackageManagers(Enum): executable='apk', install=('apk', 'add', '--update', '--no-interactive', '--no-cache'), uninstall=('apk', 'del', '--no-interactive'), + list=('apk', 'list', '--installed'), default_os='alpine', - get_installed=lambda: { - re.sub(r'.*\s*\{(.+?)\}\s*.*', r'\1', line) - for line in ( - line.strip() - for line in subprocess.Popen( # pylint: disable=consider-using-with - ['apk', 'list', '--installed'], stdout=subprocess.PIPE - ) - .communicate()[0] - .decode() - .split('\n') - ) - if line.strip() - }, + parse_list_line=lambda line: re.sub(r'.*\s*\{(.+?)\}\s*.*', r'\1', line), ) APT = PackageManager( executable='apt', install=('apt', 'install', '-y'), uninstall=('apt', 'remove', '-y'), + list=('apt', 'list', '--installed'), default_os='debian', - get_installed=lambda: { - line.strip().split('/')[0] - for line in subprocess.Popen( # pylint: disable=consider-using-with - ['apt', 'list', '--installed'], stdout=subprocess.PIPE - ) - .communicate()[0] - .decode() - .split('\n') - if line.strip() - }, + parse_list_line=lambda line: line.split('/')[0], ) PACMAN = PackageManager( executable='pacman', install=('pacman', '-S', '--noconfirm', '--needed'), uninstall=('pacman', '-R', '--noconfirm'), + list=('pacman', '-Q'), default_os='arch', - get_installed=lambda: { - line.strip().split(' ')[0] - for line in subprocess.Popen( # pylint: disable=consider-using-with - ['pacman', '-Q'], stdout=subprocess.PIPE - ) - .communicate()[0] - .decode() - .split('\n') - if line.strip() - }, + parse_list_line=lambda line: line.split(' ')[0], ) @classmethod @@ -233,7 +241,7 @@ class Dependencies: """ return ( self.install_context == InstallContext.DOCKER - or 'DOCKER_CTX' in os.environ + or bool(os.environ.get('DOCKER_CTX')) or os.path.isfile('/.dockerenv') ) From 4dd713ffd26fd2fbe3ab9c0aa12068d226e2a0fe Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 01:16:56 +0200 Subject: [PATCH 46/64] Refactored Platydock and Platyvenv. The two scripts now share the same command interface, behaviour and base class. Also, Platydock now builds a Docker image instead of just printing a Dockerfile, unless the `--print` option is passed. --- platypush/builder/__init__.py | 3 + platypush/builder/_base.py | 198 ++++++++++++++++++++ platypush/cli.py | 5 +- platypush/platydock/__init__.py | 323 +++++++++++++++++++++----------- platypush/platyvenv/__init__.py | 177 +++++------------ 5 files changed, 473 insertions(+), 233 deletions(-) create mode 100644 platypush/builder/__init__.py create mode 100644 platypush/builder/_base.py diff --git a/platypush/builder/__init__.py b/platypush/builder/__init__.py new file mode 100644 index 0000000000..6be2232a0a --- /dev/null +++ b/platypush/builder/__init__.py @@ -0,0 +1,3 @@ +from ._base import BaseBuilder + +__all__ = ["BaseBuilder"] diff --git a/platypush/builder/_base.py b/platypush/builder/_base.py new file mode 100644 index 0000000000..fe228fb258 --- /dev/null +++ b/platypush/builder/_base.py @@ -0,0 +1,198 @@ +from abc import ABC, abstractmethod +import argparse +import inspect +import logging +import os +import pathlib +import sys +from typing import Final, Optional, Sequence + +from platypush.config import Config +from platypush.utils.manifest import ( + Dependencies, + InstallContext, +) + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger() + + +class BaseBuilder(ABC): + """ + Base interface and utility methods for Platypush builders. + + A Platypush builder is a script/piece of logic that can build a Platypush + installation, with all its base and required extra dependencies, given a + configuration file. + + This class is currently implemented by the :module:`platypush.platyvenv` + and :module:`platypush.platydock` modules/scripts. + """ + + REPO_URL: Final[str] = 'https://github.com/BlackLight/platypush.git' + """ + We use the Github URL here rather than the self-hosted Gitea URL to prevent + too many requests to the Gitea server. + """ + + def __init__( + self, + cfgfile: str, + gitref: str, + output: str, + install_context: InstallContext, + *_, + verbose: bool = False, + device_id: Optional[str] = None, + **__, + ) -> None: + """ + :param cfgfile: The path to the configuration file. + :param gitref: The git reference to use. It can be a branch name, a tag + name or a commit hash. + :param output: The path to the output file or directory. + :param install_context: The installation context for this builder. + :param verbose: Whether to log debug traces. + :param device_id: A device name that will be used to uniquely identify + this installation. + """ + self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + self.output = os.path.abspath(os.path.expanduser(output)) + self.gitref = gitref + self.install_context = install_context + self.device_id = device_id + logger.setLevel(logging.DEBUG if verbose else logging.INFO) + + @classmethod + @abstractmethod + def get_name(cls) -> str: + """ + :return: The name of the builder. + """ + + @classmethod + @abstractmethod + def get_description(cls) -> str: + """ + :return: The description of the builder. + """ + + @property + def deps(self) -> Dependencies: + """ + :return: The dependencies for this builder, given the configuration + file and the installation context. + """ + return Dependencies.from_config( + self.cfgfile, + install_context=self.install_context, + ) + + def _print_instructions(self, s: str) -> None: + GREEN = '\033[92m' + NORM = '\033[0m' + + helper_lines = s.split('\n') + wrapper_line = '=' * max(len(t) for t in helper_lines) + helper = '\n' + '\n'.join([wrapper_line, *helper_lines, wrapper_line]) + '\n' + print(GREEN + helper + NORM) + + @abstractmethod + def build(self): + """ + Builds the application. To be implemented by the subclasses. + """ + + @classmethod + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=cls.get_name(), + add_help=False, + description=cls.get_description(), + ) + + parser.add_argument( + '-h', '--help', dest='show_usage', action='store_true', help='Show usage' + ) + + parser.add_argument( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='Enable debug traces', + ) + + parser.add_argument( + '-c', + '--config', + type=str, + dest='cfgfile', + required=False, + default=None, + help='The path to the configuration file. If not specified, a minimal ' + 'installation including only the base dependencies will be generated.', + ) + + parser.add_argument( + '-o', + '--output', + dest='output', + type=str, + required=False, + default='.', + help='Target directory (default: current directory). For Platydock, ' + 'this is the directory where the Dockerfile will be generated. For ' + 'Platyvenv, this is the base directory of the new virtual ' + 'environment.', + ) + + parser.add_argument( + '-d', + '--device-id', + dest='device_id', + type=str, + required=False, + default=None, + help='A name that will be used to uniquely identify this device. ' + 'Default: a random name for Docker containers, and the ' + 'hostname of the machine for virtual environments.', + ) + + parser.add_argument( + '-r', + '--ref', + dest='gitref', + required=False, + type=str, + default='master', + help='If the script is not run from a Platypush installation directory, ' + 'it will clone the sources via git. You can specify through this ' + 'option which branch, tag or commit hash to use. Defaults to master.', + ) + + return parser + + @classmethod + def from_cmdline(cls, args: Sequence[str]): + """ + Create a builder instance from command line arguments. + + :param args: Command line arguments. + :return: A builder instance. + """ + parser = cls._get_arg_parser() + opts, _ = parser.parse_known_args(args) + if opts.show_usage: + parser.print_help() + sys.exit(0) + + if not opts.cfgfile: + opts.cfgfile = os.path.join( + str(pathlib.Path(inspect.getfile(Config)).parent), + 'config.auto.yaml', + ) + + logger.info('No configuration file specified. Using %s.', opts.cfgfile) + + return cls(**vars(opts)) diff --git a/platypush/cli.py b/platypush/cli.py index f3768b199c..b9bec5564c 100644 --- a/platypush/cli.py +++ b/platypush/cli.py @@ -9,7 +9,10 @@ def parse_cmdline(args: Sequence[str]) -> argparse.Namespace: """ Parse command-line arguments from a list of strings. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + 'platypush', + description='A general-purpose platform for automation. See https://docs.platypush.tech for more info.', + ) parser.add_argument( '--config', diff --git a/platypush/platydock/__init__.py b/platypush/platydock/__init__.py index 3f1a425aec..7ad3eb74c0 100755 --- a/platypush/platydock/__init__.py +++ b/platypush/platydock/__init__.py @@ -4,14 +4,19 @@ Dockerfile for Platypush starting from a configuration file. """ import argparse +from contextlib import contextmanager import inspect +import logging import os import pathlib import re +import subprocess import sys import textwrap -from typing import Iterable, Sequence +from typing import IO, Generator, Iterable +from typing_extensions import override +from platypush.builder import BaseBuilder from platypush.config import Config from platypush.utils.manifest import ( BaseImage, @@ -20,14 +25,12 @@ from platypush.utils.manifest import ( PackageManagers, ) +logger = logging.getLogger() -# pylint: disable=too-few-public-methods -class DockerfileGenerator: + +class DockerBuilder(BaseBuilder): """ - Generate a Dockerfile from on a configuration file. - - :param cfgfile: Path to the configuration file. - :param image: The base image to use. + Creates a Platypush Docker image from a configuration file. """ _pkg_manager_by_base_image = { @@ -49,7 +52,7 @@ class DockerfileGenerator: # docker run --rm --name platypush \\ # -v /path/to/your/config/dir:/etc/platypush \\ # -v /path/to/your/workdir:/var/lib/platypush \\ - # -p 8080:8080 \\ + # -p 8008:8008 \\ # platypush\n """ ) @@ -61,31 +64,63 @@ class DockerfileGenerator: """ ) - def __init__(self, cfgfile: str, image: BaseImage, gitref: str) -> None: - self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) + def __init__( + self, *args, image: BaseImage, tag: str, print_only: bool = False, **kwargs + ): + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) self.image = image - self.gitref = gitref + self.tag = tag + self.print_only = print_only # TODO - def generate(self) -> str: + @override + @classmethod + def get_name(cls): + return "platydock" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush Docker image from a configuration file." + + @property + def dockerfile_dir(self) -> str: """ - Generate a Dockerfile based on a configuration file. + Proxy property for the output Dockerfile directory. + """ + output = self.output + parent = os.path.dirname(output) - :return: The content of the generated Dockerfile. + if os.path.isfile(output): + return parent + + if os.path.isdir(output): + return output + + logger.info('%s directory does not exist, creating it', output) + pathlib.Path(output).mkdir(mode=0o750, parents=True, exist_ok=True) + return output + + @property + def dockerfile(self) -> str: + """ + Proxy property for the output Dockerfile. + """ + return os.path.join(self.dockerfile_dir, 'Dockerfile') + + @property + def pkg_manager(self) -> PackageManagers: + """ + Proxy property for the package manager to use. + """ + return self._pkg_manager_by_base_image[self.image] + + def _read_base_dockerfile_lines(self) -> Generator[str, None, None]: + """ + :return: The lines of the base Dockerfile. """ import platypush - Config.init(self.cfgfile) - new_file_lines = [] - ports = self._get_exposed_ports() - pkg_manager = self._pkg_manager_by_base_image[self.image] - deps = Dependencies.from_config( - self.cfgfile, - pkg_manager=pkg_manager, - install_context=InstallContext.DOCKER, - base_image=self.image, - ) - - is_after_expose_cmd = False base_file = os.path.join( str(pathlib.Path(inspect.getfile(platypush)).parent), 'install', @@ -94,40 +129,145 @@ class DockerfileGenerator: ) with open(base_file, 'r') as f: - file_lines = [line.rstrip() for line in f.readlines()] + for line in f: + yield line.rstrip() - new_file_lines.extend(self._header.split('\n')) + @property + @override + def deps(self) -> Dependencies: + return Dependencies.from_config( + self.cfgfile, + pkg_manager=self.pkg_manager, + install_context=InstallContext.DOCKER, + base_image=self.image, + ) - for line in file_lines: - if re.match( - r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', - line.strip(), - ): - new_file_lines.append(self._generate_git_clone_command()) - elif line.startswith('RUN cd /install '): - for new_line in deps.before: - new_file_lines.append('RUN ' + new_line) + def _create_dockerfile_parser(self): + """ + Closure for a context-aware parser for the default Dockerfile. + """ + is_after_expose_cmd = False + deps = self.deps + ports = self._get_exposed_ports() - for new_line in deps.to_pkg_install_commands(): - new_file_lines.append('RUN ' + new_line) - elif line == 'RUN rm -rf /install': - for new_line in deps.to_pip_install_commands(): - new_file_lines.append('RUN ' + new_line) + def parser(): + nonlocal is_after_expose_cmd - for new_line in deps.after: - new_file_lines.append('RUN' + new_line) - elif line.startswith('EXPOSE ') and ports: - if not is_after_expose_cmd: - new_file_lines.extend([f'EXPOSE {port}' for port in ports]) - is_after_expose_cmd = True + for line in self._read_base_dockerfile_lines(): + if re.match( + r'RUN /install/platypush/install/scripts/[A-Za-z0-9_-]+/install.sh', + line.strip(), + ): + yield self._generate_git_clone_command() + elif line.startswith('RUN cd /install '): + for new_line in deps.before: + yield 'RUN ' + new_line - continue - elif line.startswith('CMD'): - new_file_lines.extend(self._footer.split('\n')) + for new_line in deps.to_pkg_install_commands(): + yield 'RUN ' + new_line + elif line == 'RUN rm -rf /install': + for new_line in deps.to_pip_install_commands(): + yield 'RUN ' + new_line - new_file_lines.append(line) + for new_line in deps.after: + yield 'RUN' + new_line + elif line.startswith('EXPOSE ') and ports: + if not is_after_expose_cmd: + yield from [f'EXPOSE {port}' for port in ports] + is_after_expose_cmd = True - return '\n'.join(new_file_lines) + continue + elif line.startswith('CMD'): + yield from self._footer.split('\n') + + yield line + + if line.startswith('CMD') and self.device_id: + yield f'\t--device-id {self.device_id} \\' + + return parser + + @override + def build(self): + """ + Build a Dockerfile based on a configuration file. + + :return: The content of the generated Dockerfile. + """ + + # Set the DOCKER_CTX environment variable so any downstream logic knows + # that we're running in a Docker build context. + os.environ['DOCKER_CTX'] = '1' + + self._generate_dockerfile() + if self.print_only: + return + + self._build_image() + self._print_instructions( + textwrap.dedent( + f""" + A Docker image has been built from the configuration file {self.cfgfile}. + The Dockerfile is available under {self.dockerfile}. + You can run the Docker image with the following command: + + docker run \\ + --rm --name platypush \\ + -v {os.path.dirname(self.cfgfile)}:/etc/platypush \\ + -v /path/to/your/workdir:/var/lib/platypush \\ + -p 8008:8008 \\ + platypush + """ + ) + ) + + def _build_image(self): + """ + Build a Platypush Docker image from the generated Dockerfile. + """ + logger.info('Building Docker image...') + cmd = [ + 'docker', + 'build', + '-f', + self.dockerfile, + '-t', + self.tag, + '.', + ] + + subprocess.run(cmd, check=True) + + def _generate_dockerfile(self): + """ + Parses the configuration file and generates a Dockerfile based on it. + """ + + @contextmanager + def open_writer() -> Generator[IO, None, None]: + # flake8: noqa + f = sys.stdout if self.print_only else open(self.dockerfile, 'w') + + try: + yield f + finally: + if f is not sys.stdout: + f.close() + + if not self.print_only: + logger.info('Parsing configuration file %s...', self.cfgfile) + + Config.init(self.cfgfile) + + if not self.print_only: + logger.info('Generating Dockerfile %s...', self.dockerfile) + + parser = self._create_dockerfile_parser() + + with open_writer() as f: + f.write(self._header + '\n') + for line in parser(): + f.write(line + '\n') def _generate_git_clone_command(self) -> str: """ @@ -135,16 +275,15 @@ class DockerfileGenerator: and the right git reference, if the application sources aren't already available under /install. """ - pkg_manager = self._pkg_manager_by_base_image[self.image] - install_cmd = ' '.join(pkg_manager.value.install) - uninstall_cmd = ' '.join(pkg_manager.value.uninstall) + install_cmd = ' '.join(self.pkg_manager.value.install) + uninstall_cmd = ' '.join(self.pkg_manager.value.uninstall) return textwrap.dedent( f""" RUN if [ ! -f "/install/setup.py" ]; then \\ echo "Platypush source not found under the current directory, downloading it" && \\ {install_cmd} git && \\ rm -rf /install && \\ - git clone https://github.com/BlackLight/platypush.git /install && \\ + git clone --recursive https://github.com/BlackLight/platypush.git /install && \\ cd /install && \\ git checkout {self.gitref} && \\ {uninstall_cmd} git; \\ @@ -153,71 +292,41 @@ class DockerfileGenerator: ) @classmethod - def from_cmdline(cls, args: Sequence[str]) -> 'DockerfileGenerator': - """ - Create a DockerfileGenerator instance from command line arguments. - - :param args: Command line arguments. - :return: A DockerfileGenerator instance. - """ - parser = argparse.ArgumentParser( - prog='platydock', - add_help=False, - description='Create a Platypush Dockerfile from a config.yaml.', - ) + @override + def _get_arg_parser(cls) -> argparse.ArgumentParser: + parser = super()._get_arg_parser() parser.add_argument( - '-h', '--help', dest='show_usage', action='store_true', help='Show usage' - ) - - parser.add_argument( - 'cfgfile', - type=str, - nargs='?', - help='The path to the configuration file. If not specified a minimal ' - 'Dockerfile with no extra dependencies will be generated.', - ) - - parser.add_argument( - '--image', '-i', + '--image', dest='image', required=False, type=BaseImage, choices=list(BaseImage), default=BaseImage.ALPINE, - help='Base image to use for the Dockerfile.', + help='Base image to use for the Dockerfile (default: alpine).', ) parser.add_argument( - '--ref', - '-r', - dest='gitref', + '-t', + '--tag', + dest='tag', required=False, type=str, - default='master', - help='If platydock is not run from a Platypush installation directory, ' - 'it will clone the source via git. You can specify through this ' - 'option which branch, tag or commit hash to use. Defaults to master.', + default='platypush:latest', + help='Tag name to be used for the built image ' + '(default: "platypush:latest").', ) - opts, _ = parser.parse_known_args(args) - if opts.show_usage: - parser.print_help() - sys.exit(0) + parser.add_argument( + '--print', + dest='print_only', + action='store_true', + help='Use this flag if you only want to print the Dockerfile to ' + 'stdout instead of generating an image.', + ) - if not opts.cfgfile: - opts.cfgfile = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', - ) - - print( - f'No configuration file specified. Using {opts.cfgfile}.', - file=sys.stderr, - ) - - return cls(opts.cfgfile, image=opts.image, gitref=opts.gitref) + return parser @staticmethod def _get_exposed_ports() -> Iterable[int]: @@ -240,7 +349,7 @@ def main(): """ Generates a Dockerfile based on the configuration file. """ - print(DockerfileGenerator.from_cmdline(sys.argv[1:]).generate()) + DockerBuilder.from_cmdline(sys.argv[1:]).build() return 0 diff --git a/platypush/platyvenv/__init__.py b/platypush/platyvenv/__init__.py index ea161cf01b..0564b719f1 100755 --- a/platypush/platyvenv/__init__.py +++ b/platypush/platyvenv/__init__.py @@ -3,11 +3,9 @@ Platyvenv is a helper script that allows you to automatically create a virtual environment for Platypush starting from a configuration file. """ -import argparse from contextlib import contextmanager -import inspect +import logging import os -import pathlib import re import shutil import subprocess @@ -17,26 +15,36 @@ import textwrap from typing import Generator, Sequence import venv +from typing_extensions import override + +from platypush.builder import BaseBuilder from platypush.config import Config from platypush.utils.manifest import ( Dependencies, InstallContext, ) +logger = logging.getLogger() -# pylint: disable=too-few-public-methods -class VenvBuilder: + +class VenvBuilder(BaseBuilder): """ Build a virtual environment from on a configuration file. - - :param cfgfile: Path to the configuration file. - :param image: The base image to use. """ - def __init__(self, cfgfile: str, gitref: str, output_dir: str) -> None: - self.cfgfile = os.path.abspath(os.path.expanduser(cfgfile)) - self.output_dir = os.path.abspath(os.path.expanduser(output_dir)) - self.gitref = gitref + def __init__(self, *args, **kwargs) -> None: + kwargs['install_context'] = InstallContext.DOCKER + super().__init__(*args, **kwargs) + + @override + @classmethod + def get_name(cls): + return "platyvenv" + + @override + @classmethod + def get_description(cls): + return "Build a Platypush virtual environment from a configuration file." @property def _pip_cmd(self) -> Sequence[str]: @@ -44,7 +52,7 @@ class VenvBuilder: :return: The pip install command to use for the selected environment. """ return ( - os.path.join(self.output_dir, 'bin', 'python'), + os.path.join(self.output, 'bin', 'python'), '-m', 'pip', 'install', @@ -58,7 +66,7 @@ class VenvBuilder: Install the required system packages. """ for cmd in deps.to_pkg_install_commands(): - print(f'Installing system packages: {cmd}') + logger.info('Installing system packages: %s', cmd) subprocess.call(re.split(r'\s+', cmd.strip())) @contextmanager @@ -74,11 +82,11 @@ class VenvBuilder: """ setup_py_path = os.path.join(os.getcwd(), 'setup.py') if os.path.isfile(setup_py_path): - print('Using local checkout of the Platypush source code') + logger.info('Using local checkout of the Platypush source code') yield os.getcwd() else: checkout_dir = tempfile.mkdtemp(prefix='platypush-', suffix='.git') - print(f'Cloning Platypush source code from git into {checkout_dir}') + logger.info('Cloning Platypush source code from git into %s', checkout_dir) subprocess.call( [ 'git', @@ -95,72 +103,42 @@ class VenvBuilder: yield checkout_dir os.chdir(pwd) - print(f'Cleaning up {checkout_dir}') + logger.info('Cleaning up %s', checkout_dir) shutil.rmtree(checkout_dir, ignore_errors=True) def _prepare_venv(self) -> None: """ - Installs the virtual environment under the configured output_dir. + Install the virtual environment under the configured output. """ - print(f'Creating virtual environment under {self.output_dir}...') + logger.info('Creating virtual environment under %s...', self.output) venv.create( - self.output_dir, + self.output, symlinks=True, with_pip=True, upgrade_deps=True, ) - print( - f'Installing base Python dependencies under {self.output_dir}...', - ) - + logger.info('Installing base Python dependencies under %s...', self.output) subprocess.call([*self._pip_cmd, 'pip', '.']) def _install_extra_pip_packages(self, deps: Dependencies): """ - Install the extra pip dependencies parsed through the + Install the extra pip dependencies inferred from the configured + extensions. """ pip_deps = list(deps.to_pip_install_commands(full_command=False)) if not pip_deps: return - print( - f'Installing extra pip dependencies under {self.output_dir}: ' - + ' '.join(pip_deps) + logger.info( + 'Installing extra pip dependencies under %s: %s', + self.output, + ' '.join(pip_deps), ) subprocess.call([*self._pip_cmd, *pip_deps]) - def _generate_run_sh(self) -> str: - """ - Generate a ``run.sh`` script to run the application from a newly built - virtual environment. - - :return: The location of the generated ``run.sh`` script. - """ - run_sh_path = os.path.join(self.output_dir, 'bin', 'run.sh') - with open(run_sh_path, 'w') as run_sh: - run_sh.write( - textwrap.dedent( - f""" - #!/bin/bash - - cd {self.output_dir} - - # Activate the virtual environment - source bin/activate - - # Run the application with the configuration file passed - # during build - platypush -c {self.cfgfile} $* - """ - ) - ) - - os.chmod(run_sh_path, 0o750) - return run_sh_path - def build(self): """ Build a Dockerfile based on a configuration file. @@ -178,78 +156,26 @@ class VenvBuilder: self._prepare_venv() self._install_extra_pip_packages(deps) - run_sh_path = self._generate_run_sh() - print( - f'\nVirtual environment created under {self.output_dir}.\n' - f'You can run the application through the {run_sh_path} script.\n' - ) + self._print_instructions( + textwrap.dedent( + f""" + Virtual environment created under {self.output}. + To run the application: - @classmethod - def from_cmdline(cls, args: Sequence[str]) -> 'VenvBuilder': - """ - Create a DockerfileGenerator instance from command line arguments. + source {self.output}/bin/activate + platypush -c {self.cfgfile} { + "--device_id " + self.device_id if self.device_id else "" + } - :param args: Command line arguments. - :return: A DockerfileGenerator instance. - """ - parser = argparse.ArgumentParser( - prog='platyvenv', - add_help=False, - description='Create a Platypush virtual environment from a config.yaml.', - ) + Platypush requires a Redis instance. If you don't want to use a + stand-alone server, you can pass the --start-redis option to + the executable (optionally with --redis-port). - parser.add_argument( - '-h', '--help', dest='show_usage', action='store_true', help='Show usage' - ) - - parser.add_argument( - 'cfgfile', - type=str, - nargs='?', - help='The path to the configuration file. If not specified a minimal ' - 'virtual environment only with the base dependencies will be ' - 'generated.', - ) - - parser.add_argument( - '-o', - '--output', - dest='output_dir', - type=str, - required=False, - default='venv', - help='Target directory for the virtual environment (default: ./venv)', - ) - - parser.add_argument( - '--ref', - '-r', - dest='gitref', - required=False, - type=str, - default='master', - help='If platyvenv is not run from a Platypush installation directory, ' - 'it will clone the sources via git. You can specify through this ' - 'option which branch, tag or commit hash to use. Defaults to master.', - ) - - opts, _ = parser.parse_known_args(args) - if opts.show_usage: - parser.print_help() - sys.exit(0) - - if not opts.cfgfile: - opts.cfgfile = os.path.join( - str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', + Platypush will then start its own local instance and it will + terminate it once the application is stopped. + """ ) - - print( - f'No configuration file specified. Using {opts.cfgfile}.', - file=sys.stderr, - ) - - return cls(opts.cfgfile, gitref=opts.gitref, output_dir=opts.output_dir) + ) def main(): @@ -257,6 +183,7 @@ def main(): Generates a virtual environment based on the configuration file. """ VenvBuilder.from_cmdline(sys.argv[1:]).build() + return 0 if __name__ == '__main__': From e6b5abe909f3e8308d339f2e9dad24d85b1ea05c Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 02:09:23 +0200 Subject: [PATCH 47/64] Added SIGTERM handler for clean termination in Docker contexts. --- platypush/runner/_runner.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/platypush/runner/_runner.py b/platypush/runner/_runner.py index ce2a443abf..cf89b6a4b2 100644 --- a/platypush/runner/_runner.py +++ b/platypush/runner/_runner.py @@ -1,4 +1,6 @@ import logging +import os +import signal import sys from threading import Thread from typing import Optional @@ -23,6 +25,7 @@ class ApplicationRunner: logging.basicConfig(level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger('platypush:runner') self._proc: Optional[ApplicationProcess] = None + self._stream: Optional[CommandStream] = None def _listen(self, stream: CommandStream): """ @@ -48,12 +51,16 @@ class ApplicationRunner: if parsed_args.version: self._print_version() + signal.signal(signal.SIGTERM, lambda *_: self.stop()) + while True: - with CommandStream(parsed_args.ctrl_sock) as stream, ApplicationProcess( + with CommandStream( + parsed_args.ctrl_sock + ) as self._stream, ApplicationProcess( *args, pidfile=parsed_args.pidfile, timeout=self._default_timeout ) as self._proc: try: - self._listen(stream) + self._listen(self._stream) except KeyboardInterrupt: pass @@ -63,6 +70,8 @@ class ApplicationRunner: break + self._stream = None + def run(self, *args: str) -> None: try: self._run(*args) @@ -73,6 +82,10 @@ class ApplicationRunner: if self._proc is not None: self._proc.stop() + if self._stream and self._stream.pid: + os.kill(self._stream.pid, signal.SIGKILL) + self._stream = None + def restart(self): if self._proc is not None: self._proc.mark_for_restart() From ee955882bfab30da6ef5f6ee8920a1742612c732 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 02:11:50 +0200 Subject: [PATCH 48/64] Always rebase when pulling from the Github remote. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 45b8102cf4..780d69a46a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null - git config --global --add safe.directory $PWD - git remote add github git@github.com:/BlackLight/platypush.git - - git pull github master + - git pull --rebase github master - git push --all -v github - name: docs From 0e02e617b3aa9b8f219d28c3fa0ba204fb743ed3 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Wed, 30 Aug 2023 00:05:38 +0200 Subject: [PATCH 49/64] Skip checksum in platypush-git AUR package. The master branch can move fast and easily get out of sync with the released version. --- .drone.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 780d69a46a..7625973f1e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -183,7 +183,6 @@ steps: - export GIT_VERSION="$VERSION.r$(git log --pretty=oneline HEAD...v$VERSION | wc -l).$HEAD" - export TAG_URL="https://git.platypush.tech/platypush/platypush/archive/v$VERSION.tar.gz" - export TAG_ARCHIVE="platypush-$VERSION.tar.gz" - - export CHECKSUM=$(curl --silent https://git.platypush.tech/platypush/platypush/archive/master.tar.gz | sha512sum | awk '{print $1}') - echo "--- Preparing environment" - mkdir -p ~/.ssh @@ -208,7 +207,6 @@ steps: sed -i 'PKGBUILD' -r \ -e "s/^pkgver=.*/pkgver=$GIT_VERSION/" \ -e "s/^pkgrel=.*/pkgrel=1/" \ - -e "s/^sha512sums=.*/sha512sums=('$CHECKSUM')/" - sudo -u build makepkg --printsrcinfo > .SRCINFO - export FILES_CHANGED=$(git status --porcelain --untracked-files=no | wc -l) - | From 91fde717c93e6e5c869a57a6fafc9aea42a262e1 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 10:38:21 +0200 Subject: [PATCH 50/64] Dockerfile moved to application root --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..6e01d42c28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ + +# This Dockerfile was automatically generated by Platydock. +# +# You can build a Platypush image from it by running +# `docker build -t platypush .` in the same folder as this file, +# or copy it to the root a Platypush source folder to install the +# checked out version instead of downloading it first. +# +# You can then run your new image through: +# docker run --rm --name platypush \ +# -v /path/to/your/config/dir:/etc/platypush \ +# -v /path/to/your/workdir:/var/lib/platypush \ +# -p 8008:8008 \ +# platypush + + +FROM alpine + +ADD . /install +WORKDIR /var/lib/platypush + +ARG DOCKER_CTX=1 +ENV DOCKER_CTX=1 + +RUN apk update + +RUN if [ ! -f "/install/setup.py" ]; then \ + echo "Platypush source not found under the current directory, downloading it" && \ + apk add --update --no-interactive --no-cache git && \ + rm -rf /install && \ + git clone --recursive https://github.com/BlackLight/platypush.git /install && \ + cd /install && \ + git checkout master && \ + apk del --no-interactive git; \ +fi + +RUN /install/platypush/install/scripts/alpine/install.sh +RUN cd /install && pip install -U --no-input --no-cache-dir . +RUN rm -rf /install +RUN rm -rf /var/cache/apk + +EXPOSE 8008 + +VOLUME /etc/platypush +VOLUME /var/lib/platypush + + +# You can customize the name of your installation by passing +# --device-id=... to the launched command. + +CMD platypush \ + --start-redis \ + --config /etc/platypush/config.yaml \ + --workdir /var/lib/platypush From a88b57fff20e902d5621f3347e1883834011511c Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 17 Aug 2023 10:38:21 +0200 Subject: [PATCH 51/64] Dockerfile moved to application root --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e01d42c28..c1b11b8413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ - # This Dockerfile was automatically generated by Platydock. # # You can build a Platypush image from it by running From a87a713f5ec0b799d2f29e0a0ea104cfcd8b2cf5 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 20 Aug 2023 13:48:31 +0200 Subject: [PATCH 52/64] Ignore the Dockerfile in the root folder --- Dockerfile | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c1b11b8413..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# This Dockerfile was automatically generated by Platydock. -# -# You can build a Platypush image from it by running -# `docker build -t platypush .` in the same folder as this file, -# or copy it to the root a Platypush source folder to install the -# checked out version instead of downloading it first. -# -# You can then run your new image through: -# docker run --rm --name platypush \ -# -v /path/to/your/config/dir:/etc/platypush \ -# -v /path/to/your/workdir:/var/lib/platypush \ -# -p 8008:8008 \ -# platypush - - -FROM alpine - -ADD . /install -WORKDIR /var/lib/platypush - -ARG DOCKER_CTX=1 -ENV DOCKER_CTX=1 - -RUN apk update - -RUN if [ ! -f "/install/setup.py" ]; then \ - echo "Platypush source not found under the current directory, downloading it" && \ - apk add --update --no-interactive --no-cache git && \ - rm -rf /install && \ - git clone --recursive https://github.com/BlackLight/platypush.git /install && \ - cd /install && \ - git checkout master && \ - apk del --no-interactive git; \ -fi - -RUN /install/platypush/install/scripts/alpine/install.sh -RUN cd /install && pip install -U --no-input --no-cache-dir . -RUN rm -rf /install -RUN rm -rf /var/cache/apk - -EXPOSE 8008 - -VOLUME /etc/platypush -VOLUME /var/lib/platypush - - -# You can customize the name of your installation by passing -# --device-id=... to the launched command. - -CMD platypush \ - --start-redis \ - --config /etc/platypush/config.yaml \ - --workdir /var/lib/platypush From 867198a092d6c27ab6db8c3eecd77505f830b0c6 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 02:22:01 +0200 Subject: [PATCH 53/64] Try and force push the commits to Github. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 7625973f1e..001e93dbae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,7 +30,7 @@ steps: - git config --global --add safe.directory $PWD - git remote add github git@github.com:/BlackLight/platypush.git - git pull --rebase github master - - git push --all -v github + - git push -f --all -v github - name: docs image: alpine From 86ce2647e40117a1823927fabcc9de20ace47d7d Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 02:23:13 +0200 Subject: [PATCH 54/64] Removed the -f flag from git push. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 001e93dbae..7625973f1e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,7 +30,7 @@ steps: - git config --global --add safe.directory $PWD - git remote add github git@github.com:/BlackLight/platypush.git - git pull --rebase github master - - git push -f --all -v github + - git push --all -v github - name: docs image: alpine From 011f6d3a66394821292fd1bce3802ca0cf28d442 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 02:32:42 +0200 Subject: [PATCH 55/64] (Try and) pull the current branch from Github instead of master. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 7625973f1e..3ea4e54344 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: - ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null - git config --global --add safe.directory $PWD - git remote add github git@github.com:/BlackLight/platypush.git - - git pull --rebase github master + - git pull --rebase github "$(git branch | head -1 | awk '{print $2}')" || echo "No such branch on Github" - git push --all -v github - name: docs From 9aaf2559faace8789c4eacf5178ea3b6680fb11d Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 23:19:13 +0200 Subject: [PATCH 56/64] Added `utils.is_root` method. --- platypush/utils/__init__.py | 7 +++++++ platypush/utils/manifest.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/platypush/utils/__init__.py b/platypush/utils/__init__.py index 3ec6934e09..45ffbedb91 100644 --- a/platypush/utils/__init__.py +++ b/platypush/utils/__init__.py @@ -655,4 +655,11 @@ def get_src_root() -> str: return os.path.dirname(inspect.getfile(platypush)) +def is_root() -> bool: + """ + :return: True if the current user is root/administrator. + """ + return os.getuid() == 0 + + # vim:sw=4:ts=4:et: diff --git a/platypush/utils/manifest.py b/platypush/utils/manifest.py index 48a8a0ab28..b5f6c39b25 100644 --- a/platypush/utils/manifest.py +++ b/platypush/utils/manifest.py @@ -30,7 +30,7 @@ from typing_extensions import override import yaml from platypush.message.event import Event -from platypush.utils import get_src_root +from platypush.utils import get_src_root, is_root _available_package_manager = None logger = logging.getLogger(__name__) @@ -250,7 +250,7 @@ class Dependencies: """ :return: True if the system dependencies should be installed with sudo. """ - return not (self._is_docker or os.getuid() == 0) + return not (self._is_docker or is_root()) @staticmethod def _get_requirements_dir() -> str: @@ -359,7 +359,7 @@ class Dependencies: dependencies on the system. """ - wants_sudo = not (self._is_docker or os.getuid() == 0) + wants_sudo = not (self._is_docker or is_root()) pkg_manager = self.pkg_manager or PackageManagers.scan() if self.packages and pkg_manager: From 5481ae753d59a86f43e610346412352dd5273da7 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Thu, 31 Aug 2023 23:56:57 +0200 Subject: [PATCH 57/64] gitignore should only skip /config, not any config directories. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c446c8336d..20fcb3caf5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ package.sh platypush/backend/http/static/resources/* docs/build .idea/ -config +/config platypush/backend/http/static/css/*/.sass-cache/ .vscode platypush/backend/http/static/js/lib/vue.js From 759075f1d94dd8fc3cd90e90e93a553400493f91 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Fri, 1 Sep 2023 01:09:38 +0200 Subject: [PATCH 58/64] Updated sample nginx configuration. --- examples/nginx/nginx.sample.conf | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/examples/nginx/nginx.sample.conf b/examples/nginx/nginx.sample.conf index 321ae42d99..b7cc465ab0 100644 --- a/examples/nginx/nginx.sample.conf +++ b/examples/nginx/nginx.sample.conf @@ -1,12 +1,17 @@ # An nginx configuration that can be used to reverse proxy connections to your # Platypush' HTTP service. -server { - server_name my-platypush-host.domain.com; +upstream platypush { + # The address and port where the HTTP backend is listening + server 127.0.0.1:8008; +} - # Proxy standard HTTP connections to your Platypush IP +server { + server_name platypush.example.com; + + # Proxy standard HTTP connections location / { - proxy_pass http://my-platypush-host:8008/; + proxy_pass http://platypush; client_max_body_size 5M; proxy_read_timeout 60; @@ -18,21 +23,33 @@ server { } # Proxy websocket connections - location ~ ^/ws/(.*)$ { - proxy_pass http://10.0.0.2:8008/ws/$1; + location /ws/ { + proxy_pass http://platypush; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_redirect off; proxy_http_version 1.1; - client_max_body_size 200M; + client_max_body_size 5M; proxy_set_header Host $http_host; } # Optional SSL configuration - using Let's Encrypt certificates in this case # listen 443 ssl; - # ssl_certificate /etc/letsencrypt/live/my-platypush-host.domain.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/my-platypush-host.domain.com/privkey.pem; + # ssl_certificate /etc/letsencrypt/live/platypush.example.com/fullchain.pem; + # ssl_certificate_key /etc/letsencrypt/live/platypush.example.com/privkey.pem; # include /etc/letsencrypt/options-ssl-nginx.conf; # ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; } + +# Uncomment if you are using SSL and you want to force an HTTPS upgrade to +# clients connecting over the port 80 +# server { +# if ($host = platypush.example.com) { +# return 301 https://$host$request_uri; +# } +# +# server_name platypush.example.com; +# listen 80; +# return 404; +# } From 35416f3ee3649b730c3618859cc3c15695d93013 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 2 Sep 2023 01:01:16 +0200 Subject: [PATCH 59/64] Some LINT on the old `http.request.rss` plugin. --- platypush/plugins/http/request/rss/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platypush/plugins/http/request/rss/__init__.py b/platypush/plugins/http/request/rss/__init__.py index 498579a2e9..f1ec501f08 100644 --- a/platypush/plugins/http/request/rss/__init__.py +++ b/platypush/plugins/http/request/rss/__init__.py @@ -1,6 +1,7 @@ from platypush.plugins import action from platypush.plugins.http.request import HttpRequestPlugin + class HttpRequestRssPlugin(HttpRequestPlugin): """ Plugin to programmatically retrieve and parse an RSS feed URL. @@ -11,12 +12,12 @@ class HttpRequestRssPlugin(HttpRequestPlugin): """ @action - def get(self, url): + def get(self, url, **_): import feedparser + response = super().get(url, output='text').output feed = feedparser.parse(response) return feed.entries # vim:sw=4:ts=4:et: - From 669f2eb2d2e6b97b8d775de6a0d967f565fad9ec Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sat, 2 Sep 2023 12:40:34 +0200 Subject: [PATCH 60/64] LINT/black for `tts.mimic3` plugin. --- platypush/plugins/tts/mimic3/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platypush/plugins/tts/mimic3/__init__.py b/platypush/plugins/tts/mimic3/__init__.py index d5fdea2b01..0b0fdd0499 100644 --- a/platypush/plugins/tts/mimic3/__init__.py +++ b/platypush/plugins/tts/mimic3/__init__.py @@ -1,8 +1,9 @@ -import requests from typing import Optional from urllib.parse import urljoin, urlencode -from platypush.backend.http.app.utils import get_local_base_url +import requests + +from platypush.backend.http.app.utils import get_local_base_url from platypush.context import get_backend from platypush.plugins import action from platypush.plugins.tts import TtsPlugin @@ -10,7 +11,7 @@ from platypush.schemas.tts.mimic3 import Mimic3VoiceSchema class TtsMimic3Plugin(TtsPlugin): - """ + r""" TTS plugin that uses the `Mimic3 webserver `_ provided by `Mycroft `_ as a text-to-speech engine. @@ -42,7 +43,7 @@ class TtsMimic3Plugin(TtsPlugin): voice: str = 'en_UK/apope_low', media_plugin: Optional[str] = None, player_args: Optional[dict] = None, - **kwargs + **kwargs, ): """ :param server_url: Base URL of the web server that runs the Mimic3 engine. @@ -69,6 +70,7 @@ class TtsMimic3Plugin(TtsPlugin): def say( self, text: str, + *_, server_url: Optional[str] = None, voice: Optional[str] = None, player_args: Optional[dict] = None, From b6c0ff799bafaa4e59d5e2516fb7733a1cd09fa8 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 3 Sep 2023 17:33:25 +0200 Subject: [PATCH 61/64] Rewritten the `http.webpage` plugin. --- platypush/plugins/http/webpage/__init__.py | 353 ++++++++++++++++----- 1 file changed, 276 insertions(+), 77 deletions(-) diff --git a/platypush/plugins/http/webpage/__init__.py b/platypush/plugins/http/webpage/__init__.py index 33589b504d..425fbade4e 100644 --- a/platypush/plugins/http/webpage/__init__.py +++ b/platypush/plugins/http/webpage/__init__.py @@ -1,13 +1,66 @@ +from dataclasses import dataclass import datetime +from enum import Enum import json import os import re import subprocess import tempfile +import textwrap +from typing import Iterable, Optional, Union from urllib.parse import urlparse -from platypush.plugins import action -from platypush.plugins.http.request import Plugin +from platypush.plugins import Plugin, action + + +@dataclass +class OutputFormat: + """ + Definition of a supported output format. + """ + + name: str + cmd_fmt: str + extensions: Iterable[str] = () + + +class OutputFormats(Enum): + """ + Supported output formats. + """ + + HTML = OutputFormat('html', extensions=('html', 'htm'), cmd_fmt='html') + # PDF will first be exported to HTML and then converted to PDF + PDF = OutputFormat('pdf', extensions=('pdf',), cmd_fmt='html') + TEXT = OutputFormat('text', extensions=('txt',), cmd_fmt='text') + MARKDOWN = OutputFormat('markdown', extensions=('md',), cmd_fmt='markdown') + + @classmethod + def parse( + cls, + type: Union[str, "OutputFormats"], # pylint: disable=redefined-builtin + outfile: Optional[str] = None, + ) -> "OutputFormats": + """ + Parse the format given a type argument and and output file name. + """ + try: + fmt = ( + getattr(OutputFormats, type.upper()) if isinstance(type, str) else type + ) + except AttributeError as e: + raise AssertionError( + f'Unsupported output format: {type}. Supported formats: ' + + f'{[f.name for f in OutputFormats]}' + ) from e + + by_extension = {ext.lower(): f for f in cls for ext in f.value.extensions} + if outfile: + fmt_by_ext = by_extension.get(os.path.splitext(outfile)[1].lower()[1:]) + if fmt_by_ext: + return fmt_by_ext + + return fmt class HttpWebpagePlugin(Plugin): @@ -24,34 +77,71 @@ class HttpWebpagePlugin(Plugin): """ - _mercury_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js') + _mercury_script = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'mercury-parser.js' + ) @staticmethod def _parse(proc): + """ + Runs the mercury-parser script and returns the result as a string. + """ with subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=None) as parser: return parser.communicate()[0].decode() @staticmethod def _fix_relative_links(markdown: str, url: str) -> str: - url = urlparse(url) - base_url = f'{url.scheme}://{url.netloc}' + """ + Fix relative links to match the base URL of the page (Markdown only). + """ + parsed_url = urlparse(url) + base_url = f'{parsed_url.scheme}://{parsed_url.netloc}' return re.sub(r'(\[.+?])\((/.+?)\)', fr'\1({base_url}\2)', markdown) - # noinspection PyShadowingBuiltins @action - def simplify(self, url, type='html', html=None, outfile=None): + def simplify( + self, + url: str, + type: Union[ # pylint: disable=redefined-builtin + str, OutputFormats + ] = OutputFormats.HTML, + html: Optional[str] = None, + outfile: Optional[str] = None, + font_size: str = '19px', + font_family: Union[str, Iterable[str]] = ( + '-apple-system', + 'Segoe UI', + 'Roboto', + 'Oxygen', + 'Ubuntu', + 'Cantarell', + "Fira Sans", + 'Open Sans', + 'Droid Sans', + 'Helvetica Neue', + 'Helvetica', + 'Arial', + 'sans-serif', + ), + ): """ Parse the readable content of a web page removing any extra HTML elements using Mercury. :param url: URL to parse. - :param type: Output format. Supported types: ``html``, ``markdown``, ``text`` (default: ``html``). - :param html: Set this parameter if you want to parse some HTML content already fetched. Note - that URL is still required by Mercury to properly style the output, but it won't be used - to actually fetch the content. - - :param outfile: If set then the output will be written to the specified file. If the file extension - is ``.pdf`` then the content will be exported in PDF format. If the output ``type`` is not - specified then it can also be inferred from the extension of the output file. + :param type: Output format. Supported types: ``html``, ``markdown``, + ``text``, ``pdf`` (default: ``html``). + :param html: Set this parameter if you want to parse some HTML content + already fetched. Note that URL is still required by Mercury to + properly style the output, but it won't be used to actually fetch + the content. + :param outfile: If set then the output will be written to the specified + file. If the file extension is ``.pdf`` then the content will be + exported in PDF format. If the output ``type`` is not specified + then it can also be inferred from the extension of the output file. + :param font_size: Font size to use for the output (default: 19px). + :param font_family: Custom font family (or list of font families, in + decreasing order) to use for the output. It only applies to HTML + and PDF. :return: dict Example if outfile is not specified:: @@ -74,48 +164,46 @@ class HttpWebpagePlugin(Plugin): """ - self.logger.info('Parsing URL {}'.format(url)) - wants_pdf = False - - if outfile: - wants_pdf = outfile.lower().endswith('.pdf') - if ( - wants_pdf # HTML will be exported to PDF - or outfile.lower().split('.')[-1].startswith('htm') - ): - type = 'html' - elif outfile.lower().endswith('.md'): - type = 'markdown' - elif outfile.lower().endswith('.txt'): - type = 'text' - - proc = ['node', self._mercury_script, url, type] - f = None + self.logger.info('Parsing URL %s', url) + fmt = OutputFormats.parse(type=type, outfile=outfile) + proc = ['node', self._mercury_script, url, fmt.value.cmd_fmt] + tmp_file = None if html: - f = tempfile.NamedTemporaryFile('w+', delete=False) - f.write(html) - f.flush() - proc.append(f.name) + with tempfile.NamedTemporaryFile('w+', delete=False) as f: + tmp_file = f.name + f.write(html) + f.flush() + proc.append(f.name) try: response = self._parse(proc) finally: - if f: - os.unlink(f.name) + if tmp_file: + os.unlink(tmp_file) try: response = json.loads(response.strip()) except Exception as e: - raise RuntimeError('Could not parse JSON: {}. Response: {}'.format(str(e), response)) + raise RuntimeError( + f'Could not parse JSON: {e}. Response: {response}' + ) from e - if type == 'markdown': + if fmt == OutputFormats.MARKDOWN: response['content'] = self._fix_relative_links(response['content'], url) - self.logger.debug('Got response from Mercury API: {}'.format(response)) - title = response.get('title', '{} on {}'.format( - 'Published' if response.get('date_published') else 'Generated', - response.get('date_published', datetime.datetime.now().isoformat()))) + self.logger.debug('Got response from Mercury API: %s', response) + title = response.get( + 'title', + ( + ('Published' if response.get('date_published') else 'Generated') + + ' on ' + + ( + response.get('date_published') + or datetime.datetime.now().isoformat() + ) + ), + ) content = response.get('content', '') @@ -126,46 +214,134 @@ class HttpWebpagePlugin(Plugin): 'content': content, } - outfile = os.path.abspath(os.path.expanduser(outfile)) - style = ''' - body { - font-size: 22px; - font-family: 'Merriweather', Georgia, 'Times New Roman', Times, serif; - } - ''' + return self._process_outfile( + url=url, + fmt=fmt, + title=title, + content=content, + outfile=outfile, + font_size=font_size, + font_family=tuple( + font_family, + ) + if isinstance(font_family, str) + else tuple(font_family), + ) - if type == 'html': - content = ( + @staticmethod + def _style_by_format( + fmt: OutputFormats, + font_size: str, + font_family: Iterable[str], + ) -> str: + """ + :return: The CSS style to be used for the given output format. + """ + style = textwrap.dedent( + f''' + ._parsed-content-container {{ + font-size: {font_size}; + font-family: {', '.join(f'"{f}"' for f in font_family)}; + }} + + ._parsed-content {{ + text-align: justify; + }} + + pre {{ + white-space: pre-wrap; + }} + ''' + ) + + if fmt == OutputFormats.HTML: + style += textwrap.dedent( ''' + ._parsed-content-container { + margin: 1em; + display: flex; + flex-direction: column; + align-items: center; + } + + ._parsed-content { + max-width: 800px; + } + + h1 { + max-width: 800px; + } + ''' + ) + + return style + + @classmethod + def _process_outfile( + cls, + url: str, + fmt: OutputFormats, + title: str, + content: str, + outfile: str, + font_size: str, + font_family: Iterable[str], + ): + """ + Process the output file. + + :param url: URL to parse. + :param fmt: Output format. Supported types: ``html``, ``markdown``, + ``text``, ``pdf`` (default: ``html``). + :param title: Page title. + :param content: Page content. + :param outfile: Output file path. + :param font_size: Font size to use for the output (default: 19px). + :param font_family: Custom font family (or list of font families, in + decreasing order) to use for the output. It only applies to HTML + and PDF. + :return: dict + """ + outfile = os.path.abspath(os.path.expanduser(outfile)) + style = cls._style_by_format(fmt, font_size, font_family) + + if fmt in {OutputFormats.HTML, OutputFormats.PDF}: + content = textwrap.dedent( + f''' +

{title}

{content}
- '''.format(title=title, url=url, content=content) +
+ ''' ) - if not wants_pdf: - content = ''' - - {title} - - '''.format(title=title, style=style) + \ - '{{' + content + '}}' - elif type == 'markdown': - content = '# [{title}]({url})\n\n{content}'.format( - title=title, url=url, content=content - ) + if fmt == OutputFormats.PDF: + content = textwrap.dedent( + f''' + + + {title} + + + {content} + + + ''' + ) + else: + content = textwrap.dedent( + f''' + + {content} + ''' + ) + elif fmt == OutputFormats.MARKDOWN: + content = f'# [{title}]({url})\n\n{content}' - if wants_pdf: - import weasyprint - try: - from weasyprint.fonts import FontConfiguration - except ImportError: - from weasyprint.document import FontConfiguration - - font_config = FontConfiguration() - css = [weasyprint.CSS('https://fonts.googleapis.com/css?family=Merriweather'), - weasyprint.CSS(string=style, font_config=font_config)] - - weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css) + if fmt == OutputFormats.PDF: + cls._process_pdf(content, outfile, style) else: with open(outfile, 'w', encoding='utf-8') as f: f.write(content) @@ -176,5 +352,28 @@ class HttpWebpagePlugin(Plugin): 'outfile': outfile, } + @staticmethod + def _process_pdf(content: str, outfile: str, style: str): + """ + Convert the given HTML content to a PDF document. + + :param content: Page content. + :param outfile: Output file path. + :param style: CSS style to use for the output. + """ + import weasyprint + + try: + from weasyprint.fonts import FontConfiguration # pylint: disable + except ImportError: + from weasyprint.document import FontConfiguration + + font_config = FontConfiguration() + css = [ + weasyprint.CSS(string=style, font_config=font_config), + ] + + weasyprint.HTML(string=content).write_pdf(outfile, stylesheets=css) + # vim:sw=4:ts=4:et: From 07c2eee890ab6e9df6c50f909090e50700714dff Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Sep 2023 02:19:13 +0200 Subject: [PATCH 62/64] Changed (fixed) default location for config dir if not existing. Following some common UNIX conventions, if no configuration file is specified and none exists under the default locations, then a new configuration directory should be created under: ``` - if root: /etc/platypush - else: - if XDG_CONFIG_HOME: - $XDG_CONFIG_HOME/platypush - else: - ~/.config/platypush ``` --- platypush/config/__init__.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/platypush/config/__init__.py b/platypush/config/__init__.py index 28c6beaaa0..fce5fe92c1 100644 --- a/platypush/config/__init__.py +++ b/platypush/config/__init__.py @@ -20,6 +20,7 @@ from platypush.utils import ( is_functional_procedure, is_functional_hook, is_functional_cron, + is_root, ) @@ -212,11 +213,24 @@ class Config: def _create_default_config(self): cfg_mod_dir = os.path.dirname(os.path.abspath(__file__)) - cfgfile = self._cfgfile_locations[0] + # Use /etc/platypush/config.yaml if the user is running as root, + # otherwise ~/.config/platypush/config.yaml + cfgfile = ( + ( + os.path.join(os.environ['XDG_CONFIG_HOME'], 'config.yaml') + if os.environ.get('XDG_CONFIG_HOME') + else os.path.join( + os.path.expanduser('~'), '.config', 'platypush', 'config.yaml' + ) + ) + if not is_root() + else os.path.join(os.sep, 'etc', 'platypush', 'config.yaml') + ) + cfgdir = pathlib.Path(cfgfile).parent cfgdir.mkdir(parents=True, exist_ok=True) - for cfgfile in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')): - shutil.copy(cfgfile, str(cfgdir)) + for cf in glob.glob(os.path.join(cfg_mod_dir, 'config*.yaml')): + shutil.copy(cf, str(cfgdir)) return cfgfile From c69f97c0a56f5bf53e3ed55418d6e314dc4241ae Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Sep 2023 02:22:46 +0200 Subject: [PATCH 63/64] Updated default config.yaml. The new configuration: - Enables `backend.http` by default - Removes the extra `config.auto.yaml` dependency - Includes many more examples, lots of updates for existing examples, and extensive comments. --- examples/conf/config.yaml | 379 ---------- examples/config/config.yaml | 1 + examples/{conf => config}/dashboard.xml | 0 examples/{conf => config}/hook.py | 16 +- platypush/builder/_base.py | 2 +- platypush/config/config.auto.yaml | 6 - platypush/config/config.yaml | 922 +++++++++++++++++++++++- 7 files changed, 933 insertions(+), 393 deletions(-) delete mode 100644 examples/conf/config.yaml create mode 120000 examples/config/config.yaml rename examples/{conf => config}/dashboard.xml (100%) rename examples/{conf => config}/hook.py (88%) delete mode 100644 platypush/config/config.auto.yaml diff --git a/examples/conf/config.yaml b/examples/conf/config.yaml deleted file mode 100644 index 5c9561a562..0000000000 --- a/examples/conf/config.yaml +++ /dev/null @@ -1,379 +0,0 @@ -################################################################################# -# Sample platypush configuration file. -# Edit it and copy it to /etc/platypush/config.yaml for system installation or to -# ~/.config/platypush/config.yaml for user installation (recommended). -################################################################################# - -# -- -# include directive example -# -- -# -# You can split your configuration over multiple files -# and use the include directive to import them in your configuration. -# Relative paths are also supported, and computed using the config.yaml -# installation directory as base folder. Symlinks are also supported. -# -# Using multiple files is encouraged in the case of large configurations -# that can easily end up in a messy config.yaml file, as they help you -# keep your configuration more organized. -#include: -# - include/logging.yaml -# - include/media.yaml -# - include/sensors.yaml - -# platypush logs on stdout by default. You can use the logging section to specify -# an alternative file or change the logging level. -#logging: -# filename: ~/.local/log/platypush/platypush.log -# level: INFO - -# The device_id is used by many components of platypush and it should uniquely -# identify a device in your network. If nothing is specified then the hostname -# will be used. -#device_id: my_device - -## -- -## Plugin configuration examples -## -- -# -# Plugins configuration is very straightforward. Each plugin is mapped to -# a plugin class. The methods of the class with @action annotation will -# be exported as runnable actions, while the __init__ parameters are -# configuration attributes that you can initialize in your config.yaml. -# Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html -# -# In this example we'll configure the light.hue plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/light.hue.html -# for reference. You can easily install the required dependencies for the plugin through -# pip install 'platypush[hue]' -light.hue: - # IP address or hostname of the Hue bridge - bridge: 192.168.1.10 - # Groups that will be handled by default if nothing is specified on the request - groups: - - Living Room - -# Example configuration of music.mpd plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html -# You can easily install the dependencies through pip install 'platypush[mpd]' -music.mpd: - host: localhost - port: 6600 - -# Example configuration of media.chromecast plugin, see -# https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html -# You can easily install the dependencies through pip install 'platypush[chromecast]' -media.chromecast: - chromecast: Living Room TV - -# Plugins with empty configuration can also be explicitly enabled by specifying -# enabled=True or disabled=False (it's a good practice if you want the -# corresponding web panel to be enabled, if available) -camera.pi: - enabled: True - -# Support for calendars - in this case Google and Facebook calendars -# Installing the dependencies: pip install 'platypush[ical,google]' -calendar: - calendars: - - type: platypush.plugins.google.calendar.GoogleCalendarPlugin - - type: platypush.plugins.calendar.ical.CalendarIcalPlugin - url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key - -## -- -## Backends configuration examples -## -- -# -# Backends are basically threads that run in the background and listen for something -# to happen and either trigger events or provide additional services on top of platypush. -# Just like plugins, backends are classes whose configuration matches one-to-one the -# supported parameters on the __init__ methods. You can check the documentation for the -# available backends here: https://docs.platypush.tech/en/latest/backends.html. -# Moreover, most of the backends will generate events that you can react to through custom -# event hooks. Check here for the events documentation: -# https://docs.platypush.tech/en/latest/events.html -# -# You may usually want to enable the HTTP backend, as it provides many useful features on -# top of platypush. Among those: -# -# - Expose the /execute endpoint, that allows you to send requests to platypush through a -# JSON-RPC interface. -# - Web panel, one of the key additiona features of platypush. Many plugins will expose web -# panel tabs for e.g. accessing and controlling lights, music, media and sensors. -# - Dashboard: platypush can be configured to show a custom dashboard on your screens with -# e.g. music platypush and weather info, news, upcoming calendar events and photo carousel. -# - Streaming support - the HTTP backend makes it possible to stream local media to other -# devices - e.g. Chromecasts and external browsers. -# -# To install the HTTP backend dependencies simply run 'pip install "platypush[http]"' -backend.http: - # Listening port - port: 8008 - - # Through resource_dirs you can specify external folders whose content can be accessed on - # the web server through a custom URL. In the case below we have a Dropbox folder containing - # our pictures and we mount it to the '/carousel' endpoint. - resource_dirs: - carousel: /mnt/hd/photos/carousel - -# The HTTP poll backend is a versatile backend that can monitor for HTTP-based resources and -# trigger events whenever new entries are available. In the example below we show how to use -# the backend to listen for changes on a set of RSS feeds. New content will be stored by default -# on a SQLite database under ~/.local/share/platypush/feeds/rss.db. -# Install the required dependencies through 'pip install "platypush[rss,db]"' -backend.http.poll: - requests: - - type: platypush.backend.http.request.rss.RssUpdates # HTTP poll type (RSS) - # Remote URL - url: http://www.theguardian.com/rss/world - # Custom title - title: The Guardian - World News - # How often we should check for changes - poll_seconds: 600 - # Maximum number of new entries to be processed - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://www.physorg.com/rss-feed - title: Phys.org - poll_seconds: 600 - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://feeds.feedburner.com/Techcrunch - title: Tech Crunch - poll_seconds: 600 - max_entries: 10 - - - type: platypush.backend.http.request.rss.RssUpdates - url: http://www.nytimes.com/services/xml/rss/nyt/HomePage.xml - title: The New York Times - poll_seconds: 300 - max_entries: 10 - -# MQTT backend. Installed required dependencies through 'pip install "platypush[mqtt]"' -backend.mqtt: - # Remote MQTT server IP or hostname - host: mqtt-server - # By default the backend will listen for messages on the platypush_bus_mq/device_id - # topic, but you can change the prefix using the topic attribute -# topic: MyBus - -# Raw TCP socket backend. It can run commands sent as JSON over telnet or netcat -#backend.tcp: -# port: 3333 - -## -- -## Assistant configuration examples -## -- -# -# Both Google Assistant and Alexa voice assistant interfaces are supported by platypush. -# You can easily make your custom voice assistant with a RaspberryPi and a USB microphone, -# or on your laptop. Note however that the Alexa integration is still experimental -# (mostly because of glitches and bugs on the avs package provided by Amazon), while the -# Google Assistant support should be more robust. The recommended way of triggering a -# hotword ('OK Google', 'Alexa' or any custom hotword you like) is through the snowboy -# backend (install it through 'pip install "platypush[hotword]"'). You can download custom -# voice model files (.umdl) from https://snowboy.kitt.ai. -backend.assistant.snowboy: - # Microphone audio gain - audio_gain: 1.1 - - models: - # "Computer" hotword model - computer: - # UMDL file path - voice_model_file: ~/.local/share/snowboy/models/computer.umdl - # Plugin to use (Google Assistant) - assistant_plugin: assistant.google.pushtotalk - # Language assistant (Italian) - assistant_language: it-IT - # Sound to play when the hotword is detected - detect_sound: ~/.local/share/sounds/hotword.wav - # Model sensitivity - sensitivity: 0.4 - # "OK Google" hotword model - ok_google: - voice_model_file: ~/.local/share/snowboy/models/OK Google.pmdl - assistant_plugin: assistant.google.pushtotalk - assistant_language: en-US - detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav - sensitivity: 0.4 - # "Alexa" voice model - alexa: - voice_model_file: ~/.local/share/snowboy/models/Alexa.pmdl - assistant_plugin: assistant.echo - assistant_language: en-US - detect_sound: ~/.local/share/sounds/sci-fi/PremiumBeat_0013_cursor_selection_16.wav - sensitivity: 0.5 - -# Install Alexa dependencies with 'pip install "platypush[alexa]"' -assistant.echo: - audio_player: mplayer - -# Install Google Assistant dependencies with 'pip install "platypush[google-assistant-legacy]"' -assistant.google: - enabled: True - -backend.assistant.google: - enabled: True - -## -- -## Procedure examples -## -- -# -# Procedures are lists of actions that can be executed synchronously (default) or in parallel -# (procedure.async. prefix). Basic flow control operators (if/else/for) are also available. -# You can also access Python variables and evaluate Python expressions by using the ${} expressions. -# The 'context' special variable is a name->value dictionary containing the items returned from -# previous actions - for example if an action returned '{"status": "ok", "temperature":21.5}' then -# the following actions can access those variables through ${status} and ${temperature} respectively, -# and you can also add things like '- if ${temperature > 20.0}' or '- for ${temp in temperature_values}'. -# Alternatively, you can access those variable also through ${context.get('status')} or ${context.get('temperature')}. -# Other special variables that you can use in your procedures: -# -# - output: Will contain the parsed output of the previous action -# - errors: Will contain the errors of the previous action -# - event: If the procedure is executed within an event hook, it contains the event that triggered the hook -# -# An example procedure that can be called when you arrive home. You can run this procedure by sending a JSON -# message like this on whichever backend you like (HTTP, websocket, TCP, Redis, MQTT, Node-RED, Pushbullet...) -# {"type":"request", "action":"procedure.at_home"} -# You can for instance install Tasker+AutoLocation on your mobile and send this message whenever you enter -# your home area. -procedure.at_home: - # Set the db variable HOME to 1 - - action: variable.set - args: - HOME: 1 - - # Check the luminosity level from a connected LTR559 sensor - - action: gpio.sensor.ltr559.get_data - - # If it's below a certain threshold turn on the lights - - if ${int(light or 0) < 110}: - - action: light.hue.on - - # Say a welcome home message. Install dependencies through 'pip install "platypush[google-tts]"' - - action: tts.google.say - args: - text: Welcome home - - # Start the music - - action: music.mpd.play - -# Procedure that will be execute when you're outside of home -procedure.outside_home: - # Unset the db variable HOME - - action: variable.unset - args: - name: HOME - - # Stop the music - - action: music.mpd.stop - - # Turn off the lights - - action: light.hue.off - - # Start the camera streaming. Install the Pi Camera dependencies through - # 'pip install "platypush[picamera]"' - - action: camera.pi.start_streaming - args: - listen_port: 2222 - -# Procedures can also take optional arguments. The example below show a -# generic procedure to send a request to another platypush host over MQTT -# given target, action and args -procedure.send_request(target, action, args): - - action: mqtt.send_message - args: - topic: platypush_bus_mq/${target} - host: mqtt-server - port: 1883 - msg: - type: request - target: ${target} - action: ${action} - args: ${args} - -## -- -## Event hook examples -## -- -# -# Event hooks are procedures that are run when a certain condition is met. -# Check the documentation of the backends to see which events they can trigger. -# An event hook consists of two parts: an 'if' field that specifies on which -# event the hook will be triggered (type and attributes content), and a 'then' -# field that uses the same syntax as procedures to specify a list of actions to -# execute when the event is matched. -# -# The example below plays the music on mpd/mopidy when your voice assistant -# triggers a speech recognized event with "play the music" content. -event.hook.PlayMusicAssistantCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - # Note that basic regexes are supported, so the hook will be triggered - # both if you say "play the music" and "play music" - phrase: "play (the)? music" - then: - - action: music.mpd.play - -# This will turn on the lights when you say "turn on the lights" -event.hook.TurnOnLightsCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - phrase: "turn on (the)? lights?" - then: - - action: light.hue.on - -# This will play a song by a specified artist -event.hook.SearchSongVoiceCommand: - if: - type: platypush.message.event.assistant.SpeechRecognizedEvent - # Note that you can use the ${} operator in event matching to - # extract part of the matched string into context variables that - # can be accessed in your event hook. - phrase: "play ${title} by ${artist}" - then: - - action: music.mpd.clear - - action: music.mpd.search - args: - filter: - artist: ${artist} - title: ${title} - - # Play the first search result - - action: music.mpd.play - args: - resource: ${output[0]['file']} - -# This event will scrobble newly listened tracks on mpd/mopidy to last.fm -event.hook.ScrobbleNewTrack: - if: - type: platypush.message.event.music.NewPlayingTrackEvent - then: - - action: lastfm.scrobble - args: - artist: ${track['artist']} - title: ${track['title']} - - - action: lastfm.update_now_playing - args: - artist: ${track['artist']} - title: ${track['title']} - -## -- -## Cron examples -## -- -# -# Cronjobs allow you to execute procedures at periodic intervals. -# Standard UNIX cron syntax is supported, plus an optional 6th indicator -# at the end of the expression to run jobs with second granularity. -# The example below executes a script at intervals of 1 minute. -cron.TestCron: - cron_expression: '* * * * *' - actions: - - action: shell.exec - args: - cmd: ~/bin/myscript.sh - diff --git a/examples/config/config.yaml b/examples/config/config.yaml new file mode 120000 index 0000000000..6e3e62269a --- /dev/null +++ b/examples/config/config.yaml @@ -0,0 +1 @@ +../../platypush/config/config.yaml \ No newline at end of file diff --git a/examples/conf/dashboard.xml b/examples/config/dashboard.xml similarity index 100% rename from examples/conf/dashboard.xml rename to examples/config/dashboard.xml diff --git a/examples/conf/hook.py b/examples/config/hook.py similarity index 88% rename from examples/conf/hook.py rename to examples/config/hook.py index 2ad765f4b3..2f4081b815 100644 --- a/examples/conf/hook.py +++ b/examples/config/hook.py @@ -12,7 +12,10 @@ from platypush.utils import run from platypush.event.hook import hook # Event types that you want to react to -from platypush.message.event.assistant import ConversationStartEvent, SpeechRecognizedEvent +from platypush.message.event.assistant import ( + ConversationStartEvent, + SpeechRecognizedEvent, +) @hook(SpeechRecognizedEvent, phrase='play ${title} by ${artist}') @@ -23,10 +26,13 @@ def on_music_play_command(event, title=None, artist=None, **context): Note that in this specific case we can leverage the token-extraction feature of SpeechRecognizedEvent through ${} that operates on regex-like principles to extract any text that matches the pattern into context variables. """ - results = run('music.mpd.search', filter={ - 'artist': artist, - 'title': title, - }) + results = run( + 'music.mpd.search', + filter={ + 'artist': artist, + 'title': title, + }, + ) if results: run('music.mpd.play', results[0]['file']) diff --git a/platypush/builder/_base.py b/platypush/builder/_base.py index fe228fb258..73cf69c8a7 100644 --- a/platypush/builder/_base.py +++ b/platypush/builder/_base.py @@ -190,7 +190,7 @@ class BaseBuilder(ABC): if not opts.cfgfile: opts.cfgfile = os.path.join( str(pathlib.Path(inspect.getfile(Config)).parent), - 'config.auto.yaml', + 'config.yaml', ) logger.info('No configuration file specified. Using %s.', opts.cfgfile) diff --git a/platypush/config/config.auto.yaml b/platypush/config/config.auto.yaml deleted file mode 100644 index e9bbcd0ce6..0000000000 --- a/platypush/config/config.auto.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-generated configuration file. -# Do not edit manually - use the config.yaml file for manual modifications -# instead - -backend.http: - enabled: True diff --git a/platypush/config/config.yaml b/platypush/config/config.yaml index 1e36005fea..7f31c3cd3d 100644 --- a/platypush/config/config.yaml +++ b/platypush/config/config.yaml @@ -1,2 +1,920 @@ -include: - - config.auto.yaml +################################################################################ +# Sample Platypush configuration file. +# +# Edit it and: +# - Copy it to /etc/platypush/config.yaml for system installation. +# - Copy it to ~/.config/platypush/config.yaml for user installation. +# - Start the application with `-c `. +# +# Since the configuration file also includes the custom integrations, you can +# create a Platypush custom installation, with all the extra dependencies +# required by the configured integrations, using the `platydock` or `platyvenv` +# commands and passing this file as an argument. These commands will build a +# Docker image or a Python virtual environment respectively, with all the +# required extra dependencies inferred from your configuration file. +# +# A `scripts` directory with an empty `__init__.py` script will also be created +# under the same directory as the configuration file. This directory can be +# used to add custom scripts containing procedures, hooks and crons if you want +# a full Python interface to define your logic rather than a YAML file. +# +# Please refer to the `scripts` directory provided under this file's directory +# for some examples that use the Python API. +################################################################################ + +### ------------------ +### Include directives +### ------------------ + +# # You can split your configuration over multiple files and use the include +# # directive to import other files into your configuration. +# +# # Files referenced via relative paths will be searched in the directory of +# # the configuration file that references them. Symbolic links are also +# # supported. +# +# include: +# - logging.yaml +# - media.yaml +# - sensors.yaml + +### ----------------- +### Working directory +### ----------------- + +# # Working directory of the application. This is where the main database will be +# # stored by default (if the default SQLite configuration is used), and it's +# # where the integrations will store their state. +# +# # Note that the working directory can also be specified at runtime using the +# # -w/--workdir option. +# # +# # If not specified, then one of the following will be used: +# # +# # - $XDG_DATA_HOME/platypush if the XDG_DATA_HOME environment variable is set. +# # - $HOME/.local/share/platypush otherwise. +# +# workdir: ~/.local/share/platypush + +### ---------------------- +### Database configuration +### ---------------------- + +# # By default Platypush will use a SQLite database named `main.db` under the +# # `workdir`. You can specify any other engine string here - the application has +# # been tested against SQLite, Postgres and MariaDB/MySQL >= 8. +# # +# # NOTE: If you want to use a DBMS other than SQLite, then you will also need to +# # ensure that a compatible Python driver is installed on the system where +# # Platypush is running. For example, Postgres will require the Python pg8000, +# # psycopg or another compatible driver. +# +# main.db: +# engine: sqlite:///home/user/.local/share/platypush/main.db +# # OR, if you want to use e.g. Postgres with the pg8000 driver: +# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname + +### --------------------- +### Logging configuration +### --------------------- + +# # Platypush logs on stdout by default. You can use the logging section to +# # specify an alternative file or change the logging level. +# +# # Note that a custom logging directory can also be specified at runtime using +# # the -l/--logsdir option. +# +# logging: +# filename: ~/.local/log/platypush/platypush.log +# level: INFO + +### ----------------------- +### device_id configuration +### ----------------------- + +# # The device_id is used by many components of Platypush and it should uniquely +# # identify a device in your network. If nothing is specified then the hostname +# # will be used. +# +# # Note that a custom device ID can also be specified at runtime using the +# # -d/--device-id option. +# +# device_id: my_device + +### ------------------- +### Redis configuration +### ------------------- + +# # Platypush needs a Redis instance for inter-process communication. +# # +# # By default, the application will try and connect to a Redis server listening +# # on localhost:6379. +# # +# # Platypush can also start the service on the fly if instructed to do so +# # through the `--start-redis` option. You can also specify a custom port +# # through the `--redis-port` option. +# # +# # If you are running Platypush in a Docker image built through Platydock, then +# # `--start-redis` is the default behaviour and you won't need any extra +# # documentation here. +# +# redis: +# host: localhost +# port: 6379 +# username: user +# password: secret + +### ------------------------ +### Web server configuration +### ------------------------ + +# Platypush comes with a versatile Web server that is used to: +# +# - Serve the main UI and the UIs for the plugins that provide one. +# - Serve custom user-configured dashboards. +# - Expose the `/execute` RPC endpoint to send synchronous requests. +# - Expose the `/ws/events` and `/ws/requests` Websocket paths, which can be +# respectively by other clients to subscribe to the application's events or +# send asynchronous requests. +# - Stream media files provided by other plugins, as well as camera and audio +# feeds. +# - Serve custom directories of static files that can be accessed by other +# clients. +# - Provide a versatile API for hooks - the user can easily create custom HTTP +# hooks by creating a hook with their custom logic that reacts when a +# `platypush.message.event.http.hook.WebhookEvent` is received. The `hook` +# parameter of the event specifies which URL will be served by the hook. +# +# The Web server is enabled by default, but you can disable it simply by +# commenting/removing the `backend.http` section. The default listen port is +# 8008. +# +# After starting the application, you can access the UI at +# http://localhost:8008, set up your username and password, and also create an +# access or session token from the configuration panel. +# +# This token can be used to authenticate calls to the available APIs. +# For example, to turn on the lights using the /execute endpoint: +# +# curl -XPOST -H "Authorization: Bearer $TOKEN" \ +# -H "Content-Type: application/json" \ +# -d ' +# { +# "type": "request", +# "action": "light.hue.on", +# "args": { +# "lights": ["Bedroom"] +# } +# }' http://localhost:8008/execute +# +# If you want to serve the Web server behind a reverse proxy, you can copy the +# reference configuration available at +# https://git.platypush.tech/platypush/platypush/src/branch/master/examples/nginx/nginx.sample.conf + +backend.http: + # # Bind address (default: 0.0.0.0) + # bind_address: 0.0.0.0 + # # Listen port (default: 8008) + port: 8008 + + # # resource_dirs can be used to specify directories on the host system that + # # you want to expose through the Web server. For example, you may want to + # # expose directories that contain photos or images if you want to make a + # # carousel dashboard, or a directory containing some files that you want to + # # share with someone (or other systems) using a simple Web server. + # # + # # In the following example, we're exposing a directory with photos on an + # # external hard drive other the `/photos` URL. An image like e.g. + # # `/mnt/hd/photos/IMG_1234.jpg` will be served over e.g. + # # `http://localhost:8008/photos/IMG_1234.jpg` in this case. + # resource_dirs: + # photos: /mnt/hd/photos + + # # Number of WSGI workers. Default: (#cpus * 2) + 1 + # num_workers: 4 + +### ----------------------------- +### Plugin configuration examples +### ----------------------------- + +### +# # The configuration of a plugin matches one-to-one the parameters required by +# # its constructor. +# # +# # Plugin classes are documented at https://docs.platypush.tech/en/latest/plugins.html +# # +# # For example, there is a `light.hue` plugin +# # (https://docs.platypush.tech/platypush/plugins/light.hue.html) whose +# # constructor takes the following parameters: `bridge`, `lights` (default +# # target lights for the commands), `groups` (default target groups for the +# # commands) and `poll_interval` (how often the plugin should poll for updates). +# # +# # This means that the `light.hue` plugin can be configured here as: +# +# light.hue: +# # IP address or hostname of the Hue bridge +# # NOTE: The first run will require you to register the application with +# # your bridge - that's usually done by pressing a physical button on your +# # bridge while the application is pairing. +# bridge: 192.168.1.3 +# # Groups that will be handled by default if nothing is specified on the request +# groups: +# - Living Room +# +# # How often we should poll for updates (default: 20 seconds) +# poll_interval: 20 +### + +### +# # Example configuration of music.mpd plugin, a plugin to interact with MPD and +# # Mopidy music server instances. See +# # https://docs.platypush.tech/en/latest/platypush/plugins/music.mpd.html +# # You can easily install the dependencies through pip install 'platypush[mpd]' +# +# music.mpd: +# host: localhost +# port: 6600 +### + +### +# # Plugins with empty configuration can also be explicitly enabled by specifying +# # `enabled: true` or `disabled: false`. An integration with no items will be +# # enabled with no configuration. +# +# clipboard: +### + +### +# # Example configuration of the MQTT plugin. This specifies a server that the +# # application will use by default (if not specified on the request body). +# +# mqtt: +# host: 192.168.1.2 +# port: 1883 +# username: user +# password: secret +### + +### +# # Enable the system plugin if you want your device to periodically report +# # system statistics (CPU load, disk usage, memory usage etc.) +# # +# # When new data is gathered, an `EntityUpdateEvent` with `plugin='system'` will +# # be triggered with the new data, and you can subscribe a hook to these events +# # to run your custom logic. +# +# system: +# # How often we should poll for new data +# poll_interval: 60 +### + +### +# # Example configuration for the calendar plugin. In this case, we have +# # registered a Google calendar that uses the `google.calendar` integration, and +# # a Facebook plugin and a NextCloud (WebDAV) plugin exposed over iCal format. +# # Installing the dependencies: pip install 'platypush[ical,google]' +# calendar: +# calendars: +# - type: platypush.plugins.google.calendar.GoogleCalendarPlugin +# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin +# url: https://www.facebook.com/events/ical/upcoming/?uid=your_user_id&key=your_key +# - type: platypush.plugins.calendar.ical.CalendarIcalPlugin +# url: http://riemann/nextcloud/remote.php/dav/public-calendars/9JBWHR7iioM88Y4D?export +### + +### +# # Torrent plugin configuration, with the default directory that should be used +# # to store downloaded files. +# +# torrent: +# download_dir: ~/Downloads +### + +### +# # List of RSS/Atom subscriptions. These feeds will be monitored for changes and +# # a `platypush.message.event.rss.NewFeedEntryEvent` +# # (https://docs.platypush.tech/platypush/events/rss.html#platypush.message.event.rss.NewFeedEntryEvent) +# # will be triggered when one of these feeds has new entries - you can subscribe +# # the event to run your custom logic. +# +# rss: +# # How often we should check for updates (default: 5 minutes) +# poll_seconds: 300 +# # List of feeds to monitor +# subscriptions: +# - https://www.theguardian.com/rss/world +# - https://phys.org/rss-feed/ +# - https://news.ycombinator.com/rss +# - https://www.technologyreview.com/stories.rss +# - https://api.quantamagazine.org/feed/ +### + +### +# # Example configuration of a weather plugin +# +# weather.openweathermap: +# token: secret +# lat: lat +# long: long +### + +### +# # You can add IFTTT integrations to your routines quite easily. +# # +# # Register an API key for IFTTT, paste it here, and you can run an +# # `ifttt.trigger_event` action to fire an event on IFTTT. +# # +# # You can also create IFTTT routines that call your Platypush instance, by +# # using Web hooks (i.e. event hooks that subscribe to +# # `platypush.message.event.http.hook.WebhookEvent` events), provided that the +# # Web server is listening on a publicly accessible address. +# +# ifttt: +# ifttt_key: SECRET +### + +### +# # The `http.webpage` integration comes with the mercury-parser JavaScript library. +# # It allows you to "distill" the content of a Web page and export it in readable format (in simplified HTML, Markdown or PDF) through the +# +# http.webpage: +### + +### +# # Example configuration of the zigbee.mqtt integration. +# # This integration listens for the events pushed by zigbee2mqtt service to an +# # MQTT broker. It can forward those events to native Platypush events (see +# # https://docs.platypush.tech/platypush/events/zigbee.mqtt.html) that you can +# # build automation routines on. You can also use Platypush to control your +# # Zigbee devices, either through the Web interface or programmatically through +# # the available plugin actions. +# +# zigbee.mqtt: +# # Host of the MQTT broker +# host: riemann +# # Listen port of the MQTT broker +# port: 1883 +# # Base topic, as specified in `/data/configuration.yaml` +# base_topic: zigbee2mqtt +### + +### +# # Example configuration of the zwave.mqtt integration. +# # This integration listens for the events pushed by ZWaveJS service to an MQTT +# # broker. It can forward those events to native Platypush events (see +# # https://docs.platypush.tech/platypush/events/zwave.html) that you can build +# # automation routines on. +# # You can also use Platypush to control your Z-Wave devices, either through the +# # Web interface or programmatically through the available plugin actions. +# +# zwave.mqtt: +# # Host of the MQTT broker +# host: riemann +# # Listen port of the MQTT broker +# port: 1883 +# # Gateway name, usually configured in the ZWaveJS-UI through `Settings -> +# # MQTT -> Name` +# name: zwavejs2mqtt +# # The prefix of the published topics, usually configured in the ZWaveJS-UI +# # through `Settings -> MQTT -> Prefix`. +# topic_prefix: zwave +### + +### -------------------- +### Camera configuration +### -------------------- + +### +# # There are several providers for the camera integration - you can choose +# # between ffmpeg, gstreamer, PiCamera etc., and they all expose the same +# # interface/configuration options. +# # +# # It is advised to use the ffmpeg integration, as it's the one that provides +# # the highest degree of features and supported hardware. +# # +# # If the plugin is correctly configured, you can access your camera feed from +# # the Platypush Web panel, programmatically start/stop recording sessions, take +# # photos, get a feed stream URL etc. +# +# # The camera feed will be available at `/camera//video[.extension]`, +# # for example `/camera/ffmpeg/video.mjpeg` for MJPEG (usually faster), or +# # `camera/ffmpeg/video.mp4` for MP4. +# +# # You can also capture images by connecting to the +# # `/camera//photo[.extension]`, for example `/camera/ffmpeg/photo.jpg`. +# +# camera.ffmpeg: +# # Default video device to use +# device: /dev/video0 +# # Default resolution +# resolution: +# - 640 +# - 480 +# # The directory that will be used to store captured frames/images +# frames_dir: ~/Camera/Photos +# # Default image scaling factors (default: 1, no scaling) +# scale_x: 1.5 +# scale_y: 1.5 +# # Default rotation of the image, in degrees (default: 0, no rotation) +# rotate: 90 +# # Grayscale mode (default: False): +# grayscale: false +# # Default frames per second (default: 16) +# fps: 16 +# # Whether to flip the image along the horizontal axis (default: False) +# horizontal_flip: false +# # Whether to flip the image along the horizontal axis (default: False) +# vertical_flip: false +### + +### ----------------- +### Sound integration +### ----------------- + +### +# # The sound plugin allows you to stream from an audio source connected to the +# # machine, play audio files or synthetic audio waves or MIDI sounds. +# +# # After enabling the plugin, you can access the audio stream at +# # `/sound/stream[.extension]` (e.g. `/sound/stream.mp3`) if you want to get a +# # live recording of the captured sound from the configured audio +# # `input_device`. +# +# sound: +# enabled: true +### + +### ----------------------------------- +### Some examples of media integrations +### ----------------------------------- + +### +# # Example configuration for the media.vlc plugin. You can replace `vlc` with +# # `mpv`, `mplayer`, `omxplayer` or `gstreamer` if you want to use another +# # player - the supported configuration option are the same across all these +# # players. +# +# media.vlc: +# # Volume level, between 0 and 100 +# volume: 50 +# # Where to store downloaded files +# download_dir: ~/Downloads +# # Play videos in fullscreen by default +# fullscreen: True +# # If youtube-dl or any compatible application is installed, play requested +# # videos in this format by default. Default: `best`. +# youtube_format: 'mp4[height<=?480]' +# # Extra arguments to pass to the executable. --play-and-exit may be a good +# # idea with VLC, so the player terminates upon stop instead of lingering in +# # the background. +# args: +# - --play-and-exit +# # List of directories to search for media files. The media files in these +# # folders can be searched through the `media..search` command, or +# # through the Web interface. +# media_dirs: +# - /mnt/hd/media/movies +# - /mnt/hd/media/series +# - /mnt/hd/media/videos +# - ~/Downloads +### + +### +# # Example configuration for the media.chromecast plugin, see +# # https://docs.platypush.tech/en/latest/platypush/plugins/media.chromecast.html +# # You can easily install the dependencies through pip install 'platypush[chromecast]' +# +# media.chromecast: +# chromecast: Living Room TV +### + +### +# # Example Kodi configuration. This makes it possible to control and query a +# # Kodi instance, from your automation hooks, from the Platypush APIs or from +# # the Platypush Web interface. It requires you to enable the JSON API service +# # from Kodi's settings. +# +# media.kodi: +# host: localhost +# http_port: 8080 +# username: kodi +# password: secret +### + +### +# # Example configuration for a Plex media server. This integration makes it +# # possible to navigate and search media items from your Plex library in the +# # media UI. +# +# media.plex: +# server: localhost +# username: plex +# password: secret +### + +### +# # Jellyfin media server configuration. +# +# media.jellyfin: +# server: https://media.example.com +# api_key: secret +### + +### --------------------- +### Sensors configuration +### --------------------- + +### +# # The serial plugin can be used to read sensor data from a device connected +# # over serial/USB interface. +# # +# # It can be used, for example, to connect to an Arduino or ESP device over +# # serial port, where the remote microcontroller periodically sends sensor data +# # over the serial interface. +# # +# # The data can be sent on the wire either as raw string-encoded numbers (one +# # per line), or (better) in JSON format. For example, you can program your +# # microcontroller to periodically send JSON strings like these when you get new +# # readings from your sensors: +# # +# # {"temperature": 25.0, "humidity": 20.0, "smoke": 0.01, "luminosity": 45} +# # +# # The JSON will be automatically unpacked by the application, and the relevant +# # `platypush.message.event.sensor.SensorDataChangeEvent` events will be +# # triggered when the data changes - you can subscribe to them in your custom +# # hooks. +# +# serial: +# # The path to the USB interface with e.g. an Arduino or ESP microcontroller +# # connected. +# # A way to get a deterministic path name on Linux, instead of +# # `/dev/ttyUSB`, can be the following: +# # +# # - Get the vendor and product ID of your device via e.g. `lsusb`. For +# # example, for an Arduino-compatible microcontroller: +# # +# # Bus 001 Device 008: ID 1a86:7523 QinHeng Electronics CH340 serial converter +# # +# # - In the case above, `1a86` is the vendor ID and `7523` is the product +# # ID. Create a new udev rule for it, so every time the device is +# # connected it will also be symlinked to e.g. `/dev/arduino`: +# # +# # echo 'SUBSYSTEM=="tty", ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", SYMLINK+="arduino"' | \ +# # sudo tee /etc/udev/rules.d/98-usb-serial.rules +# device: /dev/ttyUSB0 +# # How often the interface should be polled for updates, in seconds +# poll_interval: 1 +# # The tolerance argument can be used to tune when you want to be notified +# # of data changes through `SensorDataChangeEvent` events. In the case +# # below, if the microcontroller sends two consecutive temperature reads, +# # one for 22.0 and one for 22.2, then only one `SensorDataChangeEvent` will +# # be triggered (for the first read), since the absolute value of the +# # difference between the two events is less than the configured tolerance. +# # However, if the device sends two temperature reads, one for 22.0 and one +# # for 22.7, then two `SensorDataChangeEvent` events will be triggered. +# # The tolerance for all the metrics is set to a value close to zero by +# # default - i.e. any read, unless it's exactly the same as the previous +# # one, will trigger a new event. +# tolerance: +# temperature: 0.5 +# humidity: 0.75 +# luminosity: 5 +# +# # If a threshold is defined for a sensor, and the value of that sensor goes +# # below/above that temperature between two reads, then a +# # `SensorDataBelowThresholdEvent` or a `SensorDataAboveThresholdEvent` will +# # be triggered respectively. +# thresholds: +# temperature: 25.0 +### + +### +# # Alternatively to the serial plugin, you can also use the arduino plugin if +# # you want to specifically interface with Arduino. +# # +# # This plugin won't require you to write any logic for your microcontroller. +# # However, it requires your microcontroller to be flash with the Firmata +# # firmware, which allows programmatic external control. +# # +# # Note that the interface of this plugin is basically the same as the serial +# # plugin, and any other plugin that extends `SensorPlugin` in general. +# # Therefore, poll_interval, tolerance and thresholds are supported here too. +# +# arduino: +# board: /dev/ttyUSB0 +# # name -> PIN number mapping (similar for digital_pins). +# # It allows you to pick a common name for your PINs that will be used in +# # the forwarded events. +# analog_pins: +# temperature: 7 +# +# tolerance: +# temperature: 0.5 +# +# thresholds: +# temperature: 25.0 +### + +### +# # Another example: the LTR559 is a common sensor for proximity and luminosity +# # that can be wired to a Raspberry Pi or similar devices over SPI or I2C +# # interface. It exposes the same base interface as all other sensor plugins. +# +# sensor.ltr559: +# poll_interval: 1.0 +# tolerance: +# light: 7.0 +# proximity: 5.0 +# +# thresholds: +# proximity: 10.0 +### + +### -------------------------------- +### Some text-to-speech integrations +### -------------------------------- + +### +# # `tts` is the simplest TTS integration. It leverages the Google Translate open +# # "say" endpoint to render text as audio speech. +# +# tts: +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +# # The default language of the voice +# language: en-gb +### + +### +# # `tts.google` leverages Google's official text-to-speech API to render audio +# # speech from text. +# # +# # Install its dependencies via 'pip install "platypush[google-tts]"'. +# # +# # Like all other Google integrations, it requires you to register an app on the +# # Google developers console, create an API key, and follow the instruction +# # logged on the next restart to give your app the required permissions to your +# # account. +# +# tts.google: +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +# # The default language of the voice +# language: en-US +# # The gender of the voice (MALE or FEMALE) +# gender: FEMALE +# # The path to the JSON file containing your Google API credentials +# credentials_file: '~/.credentials/platypush/google/platypush-tts.json' +### + +### +# # This TTS integration leverages mimic3, an open-source TTS Web server +# # developed by Mycroft (RIP). +# # +# # Follow the instructions at +# # https://docs.platypush.tech/platypush/plugins/tts.mimic3.html to quickly +# # bootstrap a mimic3 server. +# +# tts.mimic3: +# # The base URL of the mimic3 server +# server_url: http://riemann:59125 +# # Path of the default voice that should be used +# voice: 'en_UK/apope_low' +# # The media plugin that should be used to play the audio response +# media_plugin: media.vlc +### + +## ---------- +## Procedures +## ---------- + +# Procedures are lists of actions that are executed sequentially. +# +# This section shows how to define procedures directly in your YAML +# configuration file(s). However, you can also put your procedures into Python +# scripts inside of the `/scripts` directory if you want access to +# a full-blown Python syntax. They will be automatically discovered at startup +# and available to the application. +# +# You can also access Python variables and evaluate Python expressions by using +# `${}` context expressions. +# +# The `context` special variable is a name->value dictionary containing the +# items returned from previous actions. For example, if an action returned +# `{"status": "ok", "temperature": 21.5}`, then the following actions can access +# those variables through `${context["status"]}` or +# `${context["temperature"]}`, or simply `${status}` and `${temperature}`, +# respectively. +# +# You can also add statements like `- if ${temperature > 20.0}` or +# `- for ${temp in temperature_values}` in your procedures. +# +# Besides the `context` variables, the following special variables are also +# available to the `${}` constructs when running a procedure: +# +# - `output`: It contains the parsed output of the previous action. +# - `errors`: It contains the errors of the previous action +# - `event`: If the procedure is an event hook (or it is executed within an +# event hook), it contains the event that triggered the hook + +### +# # An example procedure that can be called when you arrive home. +# # +# # You can run this procedure from the Platypush `execute` Web panel, or +# # programmatically by sending a JSON request to your Web server (or to the +# # `/ws/requests` Websocket route, or to the TCP backend) +# # +# # curl -XPOST \ +# # -H "Authorization: Bearer $YOUR_TOKEN" \ +# # -d '{"type": "request", "action": "procedure.at_home"}' +# # +# # A use-case can be the one where you have a Tasker automation running on your +# # Android device that detects when your phone enters or exits a certain area, +# # and sends the appropriate request to your Platypush server. +# +# procedure.at_home: +# # Set the db variable AT_HOME to 1. +# # Variables are flexible entities with a name and a value that will be +# # stored on the database and persisted across sessions. +# # You can access them in other procedures, scripts or hooks and run +# # custom logic on the basis of their value. +# - action: variable.set +# args: +# AT_HOME: 1 +# +# # Check the luminosity level from e.g. a connected LTR559 sensor. +# # It could also be a Bluetooth, Zigbee, Z-Wave, serial etc. sensor. +# - action: sensor.ltr559.get_measurement +# +# # If it's below a certain threshold, turn on the lights. +# # In this case, `light` is a parameter returned by the previous response, +# # so we can directly access it here through the `${}` context operator. +# # ${light} in this case is equivalent to ${context["light"]} or +# # ${output["light"]}. +# - if ${int(light or 0) < 110}: +# - action: light.hue.on +# +# # Say a welcome home message +# - action: tts.mimic3.say +# args: +# text: Welcome home +# +# # Start the music +# - action: music.mpd.play +### + +### +# # Procedure that will be execute when you walk outside your home. +# +# procedure.outside_home: +# # Unset the db variable AT_HOME +# - action: variable.unset +# args: +# name: AT_HOME +# +# # Stop the music +# - action: music.mpd.stop +# +# # Turn off the lights +# - action: light.hue.off +### + +### +# # Procedures can also take optional arguments. The example below shows a +# # generic procedure that broadcasts measurements from a sensor through an +# MQTT broker. +# +# # A listener on this topic can react to an `MQTTMessageEvent` and, for +# # example, store the event on a centralized storage. +# # +# # See the event hook section below for a sample hook that listens for messages +# # sent by other clients using this procedure. +# +# procedure.send_sensor_data(name, value): +# - action: mqtt.send_message +# args: +# topic: platypush/sensors +# host: mqtt-server +# port: 1883 +# msg: +# name: ${name} +# value: ${value} +# source: ${Config.get("device_id")} +### + +## ------------------- +## Event hook examples +## ------------------- + +# Event hooks are procedures that are run when a certain condition is met. +# +# Check the documentation of your configured backends and plugins to see which +# events they can trigger, and check https://docs.platypush.tech/events.html +# for the full list of available events with their schemas. +# +# Just like procedures, event hooks can be defined either using the YAML +# syntax, or in Python snippets in your `scripts` folder. +# +# A YAML event hook consists of two parts: an `if` field that specifies on +# which event the hook will be triggered (type and attribute values), and a +# `then` field that uses the same syntax as procedures to specify a list of +# actions to execute when the event is matched. + +### +# # This example is a hook that reacts when an `MQTTMessageEvent` is received on +# # a topic named `platypush/sensor` (see `send_sensor_data` example from the +# # procedures section). +# # +# # It will store the event on a centralized Postgres database. +# # +# # Note that, for this event to be triggered, the application must first +# # subscribe to the `platypush/sensor` topic - e.g. by adding `platypush/sensor` +# # to the active subscriptions in the `mqtt` configurations. +# +# event.hook.OnSensorDataReceived: +# if: +# type: platypush.message.event.mqtt.MQTTMessageEvent +# topic: platypush/sensor +# then: +# - action: db.insert +# args: +# engine: postgresql+pg8000://dbuser:dbpass@dbhost/dbname +# table: sensor_data +# records: +# - name: ${msg["name"]} +# value: ${msg["value"]} +# source: ${msg["source"]} +### + +### +# # The example below plays the music on mpd/mopidy when your voice assistant +# # triggers a speech recognized event with "play the music" content. +# +# event.hook.PlayMusicAssistantCommand: +# if: +# type: platypush.message.event.assistant.SpeechRecognizedEvent +# # Note that basic regexes are supported for `SpeechRecognizedEvent`, +# # so the hook will be triggered both if you say "play the music" and +# # "play music" +# phrase: "play (the)? music" +# then: +# - action: music.mpd.play +### + +### +# # This will turn on the lights when you say "turn on the lights" +# +# event.hook.TurnOnLightsCommand: +# if: +# type: platypush.message.event.assistant.SpeechRecognizedEvent +# phrase: "turn on (the)? lights?" +# then: +# - action: light.hue.on +### + +### +# # The WebhookEvent is a special type of event. It allows you to dynamically +# # register a Web hook that can be invoked by other clients, if the HTTP backend +# # is active. +# # +# # In this case, we are registering a hook under `/hook/test-hook` that accepts +# # POST requests, gets the body of the requests and logs it. +# # +# # NOTE: Since Web hooks are supposed to be called by external (and potentially +# # untrusted) parties, they aren't designed to use the standard authentication +# # mechanism used by all other routes. +# # +# # By default they don't have an authentication layer at all. You are however +# # advised to create your custom passphrase and checks the request's headers or +# # query string for it - preferably one passphrase per endpoint. +# +# event.hook.WebhookExample: +# if: +# type: platypush.message.event.http.hook.WebhookEvent +# hook: test-hook +# method: POST +# then: +# # Check the token/passphrase +# - if ${args.get('headers', {}).get('X-Token') == 'SECRET': +# - action: logger.info +# args: +# msg: ${data} +### + +### ------------- +### Cron examples +### ------------- + +### +# # Cronjobs allow you to execute procedures at periodic intervals. +# # Standard UNIX cron syntax is supported, plus an optional 6th indicator +# # at the end of the expression to run jobs with second granularity. +# # The example below executes a script at intervals of 1 minute. +# +# cron.TestCron: +# cron_expression: '* * * * *' +# actions: +# - action: shell.exec +# args: +# cmd: ~/bin/myscript.sh +### From b3c82fe0d1e134fbd62c12647fee674a4a378960 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 4 Sep 2023 02:47:18 +0200 Subject: [PATCH 64/64] More resilient termination logic for `CommandStream`. --- platypush/commands/_stream.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/platypush/commands/_stream.py b/platypush/commands/_stream.py index 3ae185d3ea..b4b1201425 100644 --- a/platypush/commands/_stream.py +++ b/platypush/commands/_stream.py @@ -1,4 +1,4 @@ -from multiprocessing import Queue +from multiprocessing import RLock, Queue import os from queue import Empty import socket @@ -35,6 +35,7 @@ class CommandStream(ControllableProcess): self.path = os.path.abspath(os.path.expanduser(path or self._default_sock_path)) self._sock: Optional[socket.socket] = None self._cmd_queue: Queue["Command"] = Queue() + self._close_lock = RLock() def reset(self): if self._sock is not None: @@ -68,9 +69,18 @@ class CommandStream(ControllableProcess): return self def __exit__(self, *_, **__): - self.terminate() - self.join() - self.close() + with self._close_lock: + self.terminate() + + try: + self.close() + except Exception as e: + self.logger.warning(str(e)) + + try: + self.kill() + except Exception as e: + self.logger.warning(str(e)) def _serve(self, sock: socket.socket): """