diff --git a/docs/source/backends.rst b/docs/source/backends.rst index 9ac610bb..0523b397 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -8,8 +8,6 @@ Backends platypush/backend/adafruit.io.rst platypush/backend/alarm.rst - platypush/backend/assistant.google.rst - platypush/backend/assistant.snowboy.rst platypush/backend/button.flic.rst platypush/backend/camera.pi.rst platypush/backend/chat.telegram.rst diff --git a/docs/source/platypush/backend/assistant.google.rst b/docs/source/platypush/backend/assistant.google.rst deleted file mode 100644 index 1fecc4f5..00000000 --- a/docs/source/platypush/backend/assistant.google.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.google`` -====================================== - -.. automodule:: platypush.backend.assistant.google - :members: - diff --git a/docs/source/platypush/backend/assistant.snowboy.rst b/docs/source/platypush/backend/assistant.snowboy.rst deleted file mode 100644 index ac8b265a..00000000 --- a/docs/source/platypush/backend/assistant.snowboy.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.snowboy`` -======================================= - -.. automodule:: platypush.backend.assistant.snowboy - :members: - diff --git a/docs/source/platypush/plugins/assistant.echo.rst b/docs/source/platypush/plugins/assistant.echo.rst deleted file mode 100644 index f4002cd3..00000000 --- a/docs/source/platypush/plugins/assistant.echo.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.echo`` -==================================== - -.. automodule:: platypush.plugins.assistant.echo - :members: - diff --git a/docs/source/platypush/plugins/assistant.google.pushtotalk.rst b/docs/source/platypush/plugins/assistant.google.pushtotalk.rst deleted file mode 100644 index 4805e890..00000000 --- a/docs/source/platypush/plugins/assistant.google.pushtotalk.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.google.pushtotalk`` -================================================= - -.. automodule:: platypush.plugins.assistant.google.pushtotalk - :members: - diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index be9e6324..a602352f 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -10,9 +10,7 @@ Plugins platypush/plugins/alarm.rst platypush/plugins/application.rst platypush/plugins/arduino.rst - platypush/plugins/assistant.echo.rst platypush/plugins/assistant.google.rst - platypush/plugins/assistant.google.pushtotalk.rst platypush/plugins/autoremote.rst platypush/plugins/bluetooth.rst platypush/plugins/calendar.rst diff --git a/platypush/backend/assistant/__init__.py b/platypush/backend/assistant/__init__.py deleted file mode 100644 index 37589fd9..00000000 --- a/platypush/backend/assistant/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from abc import ABC -import threading -from typing import Optional, Dict, Any, Tuple - -from platypush.backend import Backend -from platypush.context import get_plugin -from platypush.plugins.tts import TtsPlugin - - -class AssistantBackend(Backend): - def __init__(self, tts_plugin: Optional[str] = None, tts_args: Optional[Dict[str, Any]] = None, **kwargs): - """ - Default assistant backend constructor. - - :param tts_plugin: If set, and if the assistant returns the processed response as text, then the processed - response will be played through the selected text-to-speech plugin (can be e.g. "``tts``", - "``tts.google``" or any other implementation of :class:`platypush.plugins.tts.TtsPlugin`). - :param tts_args: Extra parameters to pass to the ``say`` method of the selected TTS plugin (e.g. - language, voice or gender). - """ - super().__init__(**kwargs) - self._detection_paused = threading.Event() - self.tts_plugin = tts_plugin - self.tts_args = tts_args or {} - - def pause_detection(self): - self._detection_paused.set() - - def resume_detection(self): - self._detection_paused.clear() - - def is_detecting(self): - return not self._detection_paused.is_set() - - def _get_tts_plugin(self) -> Tuple[Optional[TtsPlugin], Dict[str, Any]]: - return get_plugin(self.tts_plugin) if self.tts_plugin else None, self.tts_args - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/google/__init__.py b/platypush/backend/assistant/google/__init__.py deleted file mode 100644 index f6967bcd..00000000 --- a/platypush/backend/assistant/google/__init__.py +++ /dev/null @@ -1,191 +0,0 @@ -import json -import os -import time - -from platypush.backend.assistant import AssistantBackend -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - ConversationTimeoutEvent, - ResponseEvent, - NoResponseEvent, - SpeechRecognizedEvent, - AlarmStartedEvent, - AlarmEndEvent, - TimerStartedEvent, - TimerEndEvent, - AlertStartedEvent, - AlertEndEvent, - MicMutedEvent, - MicUnmutedEvent, -) - - -class AssistantGoogleBackend(AssistantBackend): - """ - Google Assistant backend. - - It listens for voice commands and post conversation events on the bus. - - **WARNING**: The Google Assistant library used by this backend has officially been deprecated: - https://developers.google.com/assistant/sdk/reference/library/python/. This backend still works on most of the - devices where I use it, but its correct functioning is not guaranteed as the assistant library is no longer - maintained. - """ - - _default_credentials_file = os.path.join( - os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json' - ) - - def __init__( - self, - credentials_file=_default_credentials_file, - device_model_id='Platypush', - **kwargs - ): - """ - :param credentials_file: Path to the Google OAuth credentials file - (default: ~/.config/google-oauthlib-tool/credentials.json). - See - https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials - for instructions to get your own credentials file. - - :type credentials_file: str - - :param device_model_id: Device model ID to use for the assistant - (default: Platypush) - :type device_model_id: str - """ - - super().__init__(**kwargs) - self.credentials_file = credentials_file - self.device_model_id = device_model_id - self.credentials = None - self.assistant = None - self._has_error = False - self._is_muted = False - - self.logger.info('Initialized Google Assistant backend') - - def _process_event(self, event): - from google.assistant.library.event import EventType, AlertType - - self.logger.info('Received assistant event: {}'.format(event)) - self._has_error = False - - if event.type == EventType.ON_CONVERSATION_TURN_STARTED: - self.bus.post(ConversationStartEvent(assistant=self)) - elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED: - if not event.args.get('with_follow_on_turn'): - self.bus.post(ConversationEndEvent(assistant=self)) - elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT: - self.bus.post(ConversationTimeoutEvent(assistant=self)) - elif event.type == EventType.ON_NO_RESPONSE: - self.bus.post(NoResponseEvent(assistant=self)) - elif ( - hasattr(EventType, 'ON_RENDER_RESPONSE') - and event.type == EventType.ON_RENDER_RESPONSE - ): - self.bus.post( - ResponseEvent(assistant=self, response_text=event.args.get('text')) - ) - tts, args = self._get_tts_plugin() - - if tts and 'text' in event.args: - self.stop_conversation() - tts.say(text=event.args['text'], **args) - elif ( - hasattr(EventType, 'ON_RESPONDING_STARTED') - and event.type == EventType.ON_RESPONDING_STARTED - and event.args.get('is_error_response', False) is True - ): - self.logger.warning('Assistant response error') - elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED: - phrase = event.args['text'].lower().strip() - self.logger.info('Speech recognized: {}'.format(phrase)) - self.bus.post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) - elif event.type == EventType.ON_ALERT_STARTED: - if event.args.get('alert_type') == AlertType.ALARM: - self.bus.post(AlarmStartedEvent(assistant=self)) - elif event.args.get('alert_type') == AlertType.TIMER: - self.bus.post(TimerStartedEvent(assistant=self)) - else: - self.bus.post(AlertStartedEvent(assistant=self)) - elif event.type == EventType.ON_ALERT_FINISHED: - if event.args.get('alert_type') == AlertType.ALARM: - self.bus.post(AlarmEndEvent(assistant=self)) - elif event.args.get('alert_type') == AlertType.TIMER: - self.bus.post(TimerEndEvent(assistant=self)) - else: - self.bus.post(AlertEndEvent(assistant=self)) - elif event.type == EventType.ON_ASSISTANT_ERROR: - self._has_error = True - if event.args.get('is_fatal'): - self.logger.error('Fatal assistant error') - else: - self.logger.warning('Assistant error') - if event.type == EventType.ON_MUTED_CHANGED: - self._is_muted = event.args.get('is_muted') - event = MicMutedEvent() if self._is_muted else MicUnmutedEvent() - self.bus.post(event) - - def start_conversation(self): - """Starts a conversation.""" - if self.assistant: - self.assistant.start_conversation() - - def stop_conversation(self): - """Stops an active conversation.""" - if self.assistant: - self.assistant.stop_conversation() - - def set_mic_mute(self, muted): - if not self.assistant: - self.logger.warning('Assistant not running') - return - - self.assistant.set_mic_mute(muted) - - def is_muted(self) -> bool: - return self._is_muted - - def send_text_query(self, query): - if not self.assistant: - self.logger.warning('Assistant not running') - return - - self.assistant.send_text_query(query) - - def run(self): - import google.oauth2.credentials - from google.assistant.library import Assistant - - super().run() - - with open(self.credentials_file, 'r') as f: - self.credentials = google.oauth2.credentials.Credentials( - token=None, **json.load(f) - ) - - while not self.should_stop(): - self._has_error = False - - with Assistant(self.credentials, self.device_model_id) as assistant: - self.assistant = assistant - for event in assistant.start(): - if not self.is_detecting(): - self.logger.info( - 'Assistant event received but detection is currently paused' - ) - continue - - self._process_event(event) - if self._has_error: - self.logger.info( - 'Restarting the assistant after an unrecoverable error' - ) - time.sleep(5) - break - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/google/manifest.yaml b/platypush/backend/assistant/google/manifest.yaml deleted file mode 100644 index 271582af..00000000 --- a/platypush/backend/assistant/google/manifest.yaml +++ /dev/null @@ -1,39 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.AlarmEndEvent: when an alarm ends - platypush.message.event.assistant.AlarmStartedEvent: when an alarm starts - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.ConversationTimeoutEvent: when a conversation - times out - platypush.message.event.assistant.MicMutedEvent: when the microphone is muted. - platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted. - platypush.message.event.assistant.NoResponseEvent: when a conversation returned no - response - platypush.message.event.assistant.ResponseEvent: when the assistant is speaking - a response - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - platypush.message.event.assistant.TimerEndEvent: when a timer ends - platypush.message.event.assistant.TimerStartedEvent: when a timer starts - install: - apk: - - py3-grpcio - - py3-google-auth - apt: - - python3-grpcio - - python3-google-auth - dnf: - - python-grpcio - - python-google-auth - pacman: - - python-grpcio - - python-google-auth - pip: - - google-assistant-library - - google-assistant-sdk[samples] - - google-auth - package: platypush.backend.assistant.google - type: backend diff --git a/platypush/backend/assistant/snowboy/__init__.py b/platypush/backend/assistant/snowboy/__init__.py deleted file mode 100644 index 6d4bd4f1..00000000 --- a/platypush/backend/assistant/snowboy/__init__.py +++ /dev/null @@ -1,184 +0,0 @@ -import os -import threading - -from platypush.backend.assistant import AssistantBackend -from platypush.context import get_plugin -from platypush.message.event.assistant import HotwordDetectedEvent - - -class AssistantSnowboyBackend(AssistantBackend): - """ - Backend for detecting custom voice hotwords through Snowboy. The purpose of - this component is only to detect the hotword specified in your Snowboy voice - model. If you want to trigger proper assistant conversations or custom - speech recognition, you should create a hook in your configuration on - HotwordDetectedEvent to trigger the conversation on whichever assistant - plugin you're using (Google, Alexa...) - - Manual installation for snowboy and its Python bindings if the installation via package fails:: - - $ [sudo] apt-get install libatlas-base-dev swig - $ [sudo] pip install pyaudio - $ git clone https://github.com/Kitt-AI/snowboy - $ cd snowboy/swig/Python3 - $ make - $ cd ../.. - $ python3 setup.py build - $ [sudo] python setup.py install - - You will also need a voice model for the hotword detection. You can find - some under the ``resources/models`` directory of the Snowboy repository, - or train/download other models from https://snowboy.kitt.ai. - """ - - def __init__(self, models, audio_gain=1.0, **kwargs): - """ - :param models: Map (name -> configuration) of voice models to be used by - the assistant. See https://snowboy.kitt.ai/ for training/downloading - models. Sample format:: - - ok_google: # Hotword model name - voice_model_file: /path/models/OK Google.pmdl # Voice model file location - sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5) - assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google - # push-to-talk assistant plugin (optional) - assistant_language: en-US # The assistant will conversate in English when this hotword is - detected (optional) - detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional) - - ciao_google: # Hotword model name - voice_model_file: /path/models/Ciao Google.pmdl # Voice model file location - sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5) - assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google - # push-to-talk assistant plugin (optional) - assistant_language: it-IT # The assistant will conversate in Italian when this hotword is - # detected (optional) - detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional) - - :type models: dict - - :param audio_gain: Audio gain, between 0 and 1. Default: 1 - :type audio_gain: float - """ - - try: - import snowboydecoder - except ImportError: - import snowboy.snowboydecoder as snowboydecoder - - super().__init__(**kwargs) - - self.models = {} - self._init_models(models) - self.audio_gain = audio_gain - - self.detector = snowboydecoder.HotwordDetector( - [model['voice_model_file'] for model in self.models.values()], - sensitivity=[model['sensitivity'] for model in self.models.values()], - audio_gain=self.audio_gain, - ) - - self.logger.info( - 'Initialized Snowboy hotword detection with {} voice model configurations'.format( - len(self.models) - ) - ) - - def _init_models(self, models): - if not models: - raise AttributeError('Please specify at least one voice model') - - self.models = {} - for name, conf in models.items(): - if name in self.models: - raise AttributeError('Duplicate model key {}'.format(name)) - - model_file = conf.get('voice_model_file') - detect_sound = conf.get('detect_sound') - - if not model_file: - raise AttributeError( - 'No voice_model_file specified for model {}'.format(name) - ) - - model_file = os.path.abspath(os.path.expanduser(model_file)) - assistant_plugin_name = conf.get('assistant_plugin') - - if detect_sound: - detect_sound = os.path.abspath(os.path.expanduser(detect_sound)) - - if not os.path.isfile(model_file): - raise FileNotFoundError( - 'Voice model file {} does not exist or it not a regular file'.format( - model_file - ) - ) - - self.models[name] = { - 'voice_model_file': model_file, - 'sensitivity': conf.get('sensitivity', 0.5), - 'detect_sound': detect_sound, - 'assistant_plugin': get_plugin(assistant_plugin_name) - if assistant_plugin_name - else None, - 'assistant_language': conf.get('assistant_language'), - 'tts_plugin': conf.get('tts_plugin'), - 'tts_args': conf.get('tts_args', {}), - } - - def hotword_detected(self, hotword): - """ - Callback called on hotword detection - """ - try: - import snowboydecoder - except ImportError: - import snowboy.snowboydecoder as snowboydecoder - - def sound_thread(sound): - snowboydecoder.play_audio_file(sound) - - def callback(): - if not self.is_detecting(): - self.logger.info( - 'Hotword detected but assistant response currently paused' - ) - return - - self.bus.post(HotwordDetectedEvent(hotword=hotword)) - model = self.models[hotword] - - detect_sound = model.get('detect_sound') - assistant_plugin = model.get('assistant_plugin') - assistant_language = model.get('assistant_language') - tts_plugin = model.get('tts_plugin') - tts_args = model.get('tts_args') - - if detect_sound: - threading.Thread(target=sound_thread, args=(detect_sound,)).start() - - if assistant_plugin: - assistant_plugin.start_conversation( - language=assistant_language, - tts_plugin=tts_plugin, - tts_args=tts_args, - ) - - return callback - - def on_stop(self): - super().on_stop() - if self.detector: - self.detector.terminate() - self.detector = None - - def run(self): - super().run() - self.detector.start( - detected_callback=[ - self.hotword_detected(hotword) for hotword in self.models.keys() - ] - ) - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/snowboy/manifest.yaml b/platypush/backend/assistant/snowboy/manifest.yaml deleted file mode 100644 index 1a3fcebc..00000000 --- a/platypush/backend/assistant/snowboy/manifest.yaml +++ /dev/null @@ -1,9 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.HotwordDetectedEvent: whenever the hotword has - been detected - install: - pip: - - snowboy - package: platypush.backend.assistant.snowboy - type: backend diff --git a/platypush/backend/http/webapp/src/App.vue b/platypush/backend/http/webapp/src/App.vue index ce8171f0..8a8699a6 100644 --- a/platypush/backend/http/webapp/src/App.vue +++ b/platypush/backend/http/webapp/src/App.vue @@ -32,6 +32,7 @@ export default { return { config: {}, userAuthenticated: false, + connected: false, pwaInstallEvent: null, } }, @@ -97,6 +98,8 @@ export default { mounted() { bus.onNotification(this.onNotification) + bus.on('connect', () => this.connected = true) + bus.on('disconnect', () => this.connected = false) }, } diff --git a/platypush/backend/http/webapp/src/assets/icons.json b/platypush/backend/http/webapp/src/assets/icons.json index 6b7d050c..3a481c13 100644 --- a/platypush/backend/http/webapp/src/assets/icons.json +++ b/platypush/backend/http/webapp/src/assets/icons.json @@ -2,6 +2,9 @@ "arduino": { "class": "fas fa-microchip" }, + "assistant.google": { + "class": "fas fa-microphone-lines" + }, "bluetooth": { "class": "fab fa-bluetooth" }, diff --git a/platypush/backend/http/webapp/src/components/Nav.vue b/platypush/backend/http/webapp/src/components/Nav.vue index ebdee691..10c97760 100644 --- a/platypush/backend/http/webapp/src/components/Nav.vue +++ b/platypush/backend/http/webapp/src/components/Nav.vue @@ -163,6 +163,7 @@ export default { this.collapsed = this.collapsedDefault bus.on('connect', this.setConnected(true)) bus.on('disconnect', this.setConnected(false)) + this.setConnected(this.$root.connected) }, } @@ -192,6 +193,11 @@ nav { top: 0; left: 0; z-index: 5; + + .icon.status { + top: 0.75em !important; + left: 2em; + } } } @@ -318,6 +324,10 @@ nav { } } + .icon.status { + width: 1em; + } + &.collapsed { display: flex; flex-direction: column; @@ -353,15 +363,19 @@ nav { box-shadow: none; background: $nav-toggler-collapsed-bg; - @media screen and (max-width: #{$tablet - 1px}) { - background: $nav-toggler-collapsed-mobile-bg; - color: $nav-toggler-collapsed-mobile-fg; - } - .icon.status { top: 0.75em; left: 2em; } + + @include until($tablet) { + background: $nav-toggler-collapsed-mobile-bg; + color: $nav-toggler-collapsed-mobile-fg; + + .icon.status { + top: 0.75em !important; + } + } } .footer { diff --git a/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue b/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue new file mode 100644 index 00000000..e68aee55 --- /dev/null +++ b/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue @@ -0,0 +1,107 @@ + + + + + diff --git a/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue b/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue new file mode 100644 index 00000000..90818d82 --- /dev/null +++ b/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue @@ -0,0 +1,250 @@ + + + + + diff --git a/platypush/backend/http/webapp/src/components/panels/Entities/meta.json b/platypush/backend/http/webapp/src/components/panels/Entities/meta.json index 41a050a0..fb7fd046 100644 --- a/platypush/backend/http/webapp/src/components/panels/Entities/meta.json +++ b/platypush/backend/http/webapp/src/components/panels/Entities/meta.json @@ -1,4 +1,12 @@ { + "assistant": { + "name": "Assistant", + "name_plural": "Assistants", + "icon": { + "class": "fas fa-microphone-lines" + } + }, + "battery": { "name": "Battery", "name_plural": "Batteries", diff --git a/platypush/config/config.yaml b/platypush/config/config.yaml index df1ec68a..b0005cb9 100644 --- a/platypush/config/config.yaml +++ b/platypush/config/config.yaml @@ -280,6 +280,35 @@ backend.http: # - platypush/tests ### +### +# # Example configuration of a voice assistant. +# # Several voice assistant plugins and engines are available - Google +# # Assistant, Alexa, DeepSpeech, Picovoice etc. +# # +# # The Google Assistant is probably the most straightforward to configure and +# # the richest in terms of features provided out-of-the-box and speech +# # detection quality, while others may require further tinkering, may perform +# # worse than the Google model, and/or may run models on-device which could not +# # be within reach for some machines. +# # +# # Check the documentation of the `assistant.google` plugin for instructions on +# # how to get a credentials file that you can use with a custom assistant +# # installation. +# # +# # Note however that the Google Assistant plugin leverages the +# # `google-assistant-library`, which has been deprecated a while ago by Google +# # and it's unlikely to receive updates (although it still works and I'd still +# # expect it to work). +# +# assistant.google: +# # Path to your credentials file (by default it will look either under +# # ~/.config/google-oauthlib-tool/credentials.json or +# # /credentials/google/assistant.json +# # credentials_file: ~/credentials/assistant.json +# # If specified, then this sound will be played when a conversation starts +# # conversation_start_sound: ~/sounds/assistant-start.mp3 +### + ### # # Example configuration of music.mpd plugin, a plugin to interact with MPD and # # Mopidy music server instances. See diff --git a/platypush/entities/assistants.py b/platypush/entities/assistants.py new file mode 100644 index 00000000..db7bcff2 --- /dev/null +++ b/platypush/entities/assistants.py @@ -0,0 +1,29 @@ +from sqlalchemy import Column, Integer, ForeignKey, Boolean, String + +from platypush.common.db import is_defined + +from . import Entity + + +if not is_defined('assistant'): + + class Assistant(Entity): + """ + Base class for voice assistant entities. + """ + + __tablename__ = 'assistant' + + id = Column( + Integer, ForeignKey(Entity.id, ondelete='CASCADE'), primary_key=True + ) + last_query = Column(String) + last_response = Column(String) + conversation_running = Column(Boolean) + is_muted = Column(Boolean, default=False) + is_detecting = Column(Boolean, default=True) + + __table_args__ = {'extend_existing': True} + __mapper_args__ = { + 'polymorphic_identity': __tablename__, + } diff --git a/platypush/entities/managers/assistants.py b/platypush/entities/managers/assistants.py new file mode 100644 index 00000000..ae412837 --- /dev/null +++ b/platypush/entities/managers/assistants.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod + +from . import EntityManager + + +class AssistantEntityManager(EntityManager, ABC): + """ + Base class for voice assistant integrations that support entity management. + """ + + @abstractmethod + def start_conversation(self, *args, **kwargs): + """ + Programmatically starts a conversation. + """ + raise NotImplementedError() + + @abstractmethod + def stop_conversation(self, *args, **kwargs): + """ + Programmatically stops a conversation. + """ + raise NotImplementedError() + + @abstractmethod + def is_muted(self, *args, **kwargs) -> bool: + """ + :return: True if the microphone is muted, False otherwise. + """ + raise NotImplementedError() + + @abstractmethod + def mute(self, *args, **kwargs): + """ + Mute the microphone. + """ + raise NotImplementedError() + + @abstractmethod + def unmute(self, *args, **kwargs): + """ + Unmute the microphone. + """ + raise NotImplementedError() + + def toggle_mute(self, *_, **__): + """ + Toggle the mute state of the microphone. + """ + return self.mute() if self.is_muted() else self.unmute() + + @abstractmethod + def pause_detection(self, *args, **kwargs): + """ + Put the assistant on pause. No new conversation events will be triggered. + """ + raise NotImplementedError() + + @abstractmethod + def resume_detection(self, *args, **kwargs): + """ + Resume the assistant hotword detection from a paused state. + """ + raise NotImplementedError() + + @abstractmethod + def is_detecting(self, *args, **kwargs) -> bool: + """ + :return: True if the asistant is detecting, False otherwise. + """ + raise NotImplementedError() + + @abstractmethod + def send_text_query(self, *args, **kwargs): + """ + Send a text query to the assistant. + """ + raise NotImplementedError() diff --git a/platypush/message/event/assistant/__init__.py b/platypush/message/event/assistant/__init__.py index 01a21eb5..f2dea592 100644 --- a/platypush/message/event/assistant/__init__.py +++ b/platypush/message/event/assistant/__init__.py @@ -1,8 +1,6 @@ -import logging import re import sys -from platypush.context import get_backend, get_plugin from platypush.message.event import Event @@ -11,18 +9,7 @@ class AssistantEvent(Event): def __init__(self, *args, assistant=None, **kwargs): super().__init__(*args, **kwargs) - self.logger = logging.getLogger('platypush:assistant') - - if assistant: - self._assistant = assistant - else: - try: - self._assistant = get_backend('assistant.google') - if not self._assistant: - self._assistant = get_plugin('assistant.google.pushtotalk') - except Exception as e: - self.logger.debug('Could not initialize the assistant component: %s', e) - self._assistant = None + self._assistant = assistant class ConversationStartEvent(AssistantEvent): diff --git a/platypush/plugins/assistant/__init__.py b/platypush/plugins/assistant/__init__.py index 08b25e8c..6291b111 100644 --- a/platypush/plugins/assistant/__init__.py +++ b/platypush/plugins/assistant/__init__.py @@ -1,54 +1,286 @@ from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass +from enum import Enum +import os +from threading import Event +from typing import Any, Collection, Dict, Optional -from platypush.context import get_backend +from platypush.context import get_bus, get_plugin +from platypush.entities.assistants import Assistant +from platypush.entities.managers.assistants import AssistantEntityManager +from platypush.message.event.assistant import ( + AssistantEvent, + ConversationStartEvent, + ConversationEndEvent, + ConversationTimeoutEvent, + ResponseEvent, + NoResponseEvent, + SpeechRecognizedEvent, + AlarmStartedEvent, + AlarmEndEvent, + TimerStartedEvent, + TimerEndEvent, + AlertStartedEvent, + AlertEndEvent, + MicMutedEvent, + MicUnmutedEvent, +) from platypush.plugins import Plugin, action +from platypush.utils import get_plugin_name_by_class -class AssistantPlugin(ABC, Plugin): +@dataclass +class AlertType(Enum): """ - Base class for assistant plugins + Enum representing the type of an alert. """ + ALARM = 'alarm' + TIMER = 'timer' + ALERT = 'alert' + + +@dataclass +class AssistantState: + """ + Dataclass representing the state of an assistant. + """ + + last_query: Optional[str] = None + last_response: Optional[str] = None + conversation_running: bool = False + is_muted: bool = False + is_detecting: bool = True + alert_state: Optional[str] = None + + +class AssistantPlugin(Plugin, AssistantEntityManager, ABC): + """ + Base class for assistant plugins. + """ + + _entity_name = 'Assistant' + + def __init__( + self, + *args, + tts_plugin: Optional[str] = None, + tts_plugin_args: Optional[Dict[str, Any]] = None, + conversation_start_sound: Optional[str] = None, + **kwargs + ): + """ + :param tts_plugin: If set, the assistant will use this plugin (e.g. + ``tts``, ``tts.google`` or ``tts.mimic3``) to render the responses, + instead of using the built-in assistant voice. + + :param tts_plugin_args: Optional arguments to be passed to the TTS + ``say`` action, if ``tts_plugin`` is set. + + :param conversation_start_sound: If set, the assistant will play this + audio file when it detects a speech. The sound file will be played + on the default audio output device. If not set, the assistant won't + play any sound when it detects a speech. + """ + super().__init__(*args, **kwargs) + self.tts_plugin = tts_plugin + self.tts_plugin_args = tts_plugin_args or {} + if conversation_start_sound: + self._conversation_start_sound = os.path.abspath( + os.path.expanduser(conversation_start_sound) + ) + + self._detection_paused = Event() + self._conversation_running = Event() + self._is_muted = False + self._last_query: Optional[str] = None + self._last_response: Optional[str] = None + self._cur_alert_type: Optional[AlertType] = None + + @property + def _state(self) -> AssistantState: + return AssistantState( + last_query=self._last_query, + last_response=self._last_response, + conversation_running=self._conversation_running.is_set(), + is_muted=self._is_muted, + is_detecting=not self._detection_paused.is_set(), + alert_state=self._cur_alert_type.value if self._cur_alert_type else None, + ) + @abstractmethod - def start_conversation(self, *args, language=None, tts_plugin=None, tts_args=None, **kwargs): + def start_conversation(self, *_, **__): """ - Start a conversation. + Programmatically starts a conversation. """ raise NotImplementedError @abstractmethod - def stop_conversation(self, *args, **kwargs): + def stop_conversation(self, *_, **__): """ - Stop a conversation. + Programmatically stops a conversation. """ raise NotImplementedError - def _get_assistant(self): - return get_backend('assistant.snowboy') - @action - def pause_detection(self): + def pause_detection(self, *_, **__): """ Put the assistant on pause. No new conversation events will be triggered. """ - assistant = self._get_assistant() - assistant.pause_detection() + self._detection_paused.set() @action - def resume_detection(self): + def resume_detection(self, *_, **__): """ Resume the assistant hotword detection from a paused state. """ - assistant = self._get_assistant() - assistant.resume_detection() + self._detection_paused.clear() @action - def is_detecting(self) -> bool: + def is_detecting(self, *_, **__) -> bool: """ :return: True if the asistant is detecting, False otherwise. """ - assistant = self._get_assistant() - return assistant.is_detecting() + return not self._detection_paused.is_set() + + @action + def is_muted(self, *_, **__) -> bool: + """ + :return: True if the microphone is muted, False otherwise. + """ + return self._is_muted + + @action + def status(self, *_, **__): + """ + :return: The current assistant status: + + .. code-block:: json + + { + "last_query": "What time is it?", + "last_response": "It's 10:30 AM", + "conversation_running": true, + "is_muted": false, + "is_detecting": true + } + + """ + self.publish_entities([self]) + return asdict(self._state) + + def _get_tts_plugin(self): + if not self.tts_plugin: + return None + + return get_plugin(self.tts_plugin) + + def _play_conversation_start_sound(self): + if not self._conversation_start_sound: + return + + audio = get_plugin('sound') + if not audio: + self.logger.warning( + 'Unable to play conversation start sound: sound plugin not found' + ) + return + + audio.play(self._conversation_start_sound) + + def _send_event(self, event: AssistantEvent): + self.publish_entities([self]) + get_bus().post(event) + + def _on_conversation_start(self): + self._last_response = None + self._last_query = None + self._conversation_running.set() + self._send_event(ConversationStartEvent(assistant=self)) + self._play_conversation_start_sound() + + def _on_conversation_end(self): + self._conversation_running.clear() + self._send_event(ConversationEndEvent(assistant=self)) + + def _on_conversation_timeout(self): + self._last_response = None + self._last_query = None + self._conversation_running.clear() + self._send_event(ConversationTimeoutEvent(assistant=self)) + + def _on_no_response(self): + self._last_response = None + self._conversation_running.clear() + self._send_event(NoResponseEvent(assistant=self)) + + def _on_reponse_rendered(self, text: Optional[str]): + self._last_response = text + self._send_event(ResponseEvent(assistant=self, response_text=text)) + tts = self._get_tts_plugin() + + if tts and text: + self.stop_conversation() + tts.say(text=text, **self.tts_plugin_args) + + def _on_speech_recognized(self, phrase: Optional[str]): + phrase = (phrase or '').lower().strip() + self._last_query = phrase + self._send_event(SpeechRecognizedEvent(assistant=self, phrase=phrase)) + + def _on_alarm_start(self): + self._cur_alert_type = AlertType.ALARM + self._send_event(AlarmStartedEvent(assistant=self)) + + def _on_alarm_end(self): + self._cur_alert_type = None + self._send_event(AlarmEndEvent(assistant=self)) + + def _on_timer_start(self): + self._cur_alert_type = AlertType.TIMER + self._send_event(TimerStartedEvent(assistant=self)) + + def _on_timer_end(self): + self._cur_alert_type = None + self._send_event(TimerEndEvent(assistant=self)) + + def _on_alert_start(self): + self._cur_alert_type = AlertType.ALERT + self._send_event(AlertStartedEvent(assistant=self)) + + def _on_alert_end(self): + self._cur_alert_type = None + self._send_event(AlertEndEvent(assistant=self)) + + def _on_mute(self): + self._is_muted = True + self._send_event(MicMutedEvent(assistant=self)) + + def _on_unmute(self): + self._is_muted = False + self._send_event(MicUnmutedEvent(assistant=self)) + + def _on_mute_changed(self, value: bool): + if value: + self._on_mute() + else: + self._on_unmute() + + def transform_entities(self, entities: Collection['AssistantPlugin']): + return super().transform_entities( + [ + Assistant( + external_id=get_plugin_name_by_class(type(dev)), + name=self._entity_name, + last_query=dev._state.last_query, + last_response=dev._state.last_response, + conversation_running=dev._state.conversation_running, + is_muted=dev._state.is_muted, + is_detecting=dev._state.is_detecting, + ) + for dev in (entities or []) + ] + ) # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/echo/__init__.py b/platypush/plugins/assistant/echo/__init__.py deleted file mode 100644 index c9429ba8..00000000 --- a/platypush/plugins/assistant/echo/__init__.py +++ /dev/null @@ -1,133 +0,0 @@ -import os -from typing import Optional - -from platypush.context import get_bus -from platypush.plugins import action -from platypush.plugins.assistant import AssistantPlugin - -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - SpeechRecognizedEvent, - ResponseEvent, -) - - -class AssistantEchoPlugin(AssistantPlugin): - """ - Amazon Echo/Alexa assistant plugin. - - In order to activate the Echo service on your device follow these steps: - - 1. Install avs (``pip install git+https://github.com/BlackLight/avs.git``) - 2. Run the ``alexa-auth`` script. A local webservice will start on port 3000 - 3. If a browser instance doesn't open automatically then head to http://localhost:3000 - 4. Log in to your Amazon account - 5. The required credentials will be stored to ~/.avs.json - - """ - - def __init__( - self, - avs_config_file: Optional[str] = None, - audio_device: str = 'default', - audio_player: str = 'default', - **kwargs - ): - """ - :param avs_config_file: AVS credentials file - default: ~/.avs.json. If the file doesn't exist then - an instance of the AVS authentication service will be spawned. You can login through an Amazon - account either in the spawned browser window, if available, or by opening http://your-ip:3000 - in the browser on another machine. - - :param audio_device: Name of the input audio device (default: 'default') - :param audio_player: Player to be used for audio playback (default: 'default'). - Supported values: 'mpv', 'mpg123', 'gstreamer' - """ - from avs.alexa import Alexa - from avs.config import DEFAULT_CONFIG_FILE - from avs.mic import Audio - - super().__init__(**kwargs) - - if not avs_config_file: - avs_config_file = DEFAULT_CONFIG_FILE - - if not avs_config_file or not os.path.isfile(avs_config_file): - from avs.auth import auth - - auth(None, avs_config_file) - self.logger.warning( - 'Amazon Echo assistant credentials not configured. Open http://localhost:3000 ' - + 'to authenticate this client' - ) - - self.audio_device = audio_device - self.audio_player = audio_player - self.audio = Audio(device_name=audio_device) - self.alexa = Alexa(avs_config_file, audio_player=audio_player) - self._ready = False - - self.alexa.state_listener.on_ready = self._on_ready() - self.alexa.state_listener.on_listening = self._on_listening() - self.alexa.state_listener.on_speaking = self._on_speaking() - self.alexa.state_listener.on_thinking = self._on_thinking() - self.alexa.state_listener.on_finished = self._on_finished() - self.alexa.state_listener.on_disconnected = self._on_disconnected() - - self.audio.link(self.alexa) - self.alexa.start() - - def _on_ready(self): - def _callback(): - self._ready = True - - return _callback - - def _on_listening(self): - def _callback(): - get_bus().post(ConversationStartEvent(assistant=self)) - - return _callback - - def _on_speaking(self): - def _callback(): - # AVS doesn't provide a way to access the response text - get_bus().post(ResponseEvent(assistant=self, response_text='')) - - return _callback - - def _on_finished(self): - def _callback(): - get_bus().post(ConversationEndEvent(assistant=self)) - - return _callback - - def _on_disconnected(self): - def _callback(): - self._ready = False - - return _callback - - def _on_thinking(self): - def _callback(): - # AVS doesn't provide a way to access the detected text - get_bus().post(SpeechRecognizedEvent(assistant=self, phrase='')) - - return _callback - - @action - def start_conversation(self, **_): - if not self._ready: - raise RuntimeError('Echo assistant not ready') - - self.audio.start() - self.alexa.listen() - - @action - def stop_conversation(self): - self.audio.stop() - self._on_finished()() - - -# vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/echo/manifest.yaml b/platypush/plugins/assistant/echo/manifest.yaml deleted file mode 100644 index b19df7be..00000000 --- a/platypush/plugins/assistant/echo/manifest.yaml +++ /dev/null @@ -1,13 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - install: - pip: - - avs - package: platypush.plugins.assistant.echo - type: plugin diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 59ff8f4e..52758cb5 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -1,76 +1,313 @@ -from platypush.backend.assistant.google import AssistantGoogleBackend -from platypush.context import get_backend -from platypush.plugins import action +import json +import os +from typing import Optional + +from platypush.config import Config +from platypush.plugins import RunnablePlugin, action from platypush.plugins.assistant import AssistantPlugin -class AssistantGooglePlugin(AssistantPlugin): +class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): """ - Google assistant plugin. - It acts like a wrapper around the :mod:`platypush.backend.assistant.google` - backend to programmatically control the conversation status. + Google Assistant plugin. + + This plugin allows you to run the Google Assistant _directly_ on your + device. It requires you to have an audio microphone and a speaker connected + to the device. + + If you have multiple sound devices, you can specify which one(s) to use for + input and output through a ``~/.asoundrc`` configuration file like this: + + .. code-block:: text + + pcm.!default { + type asym + playback.pcm { + type plug + slave.pcm "hw:0,0" + } + capture.pcm { + type plug + slave.pcm "hw:1,0" + } + } + + You can use ``aplay -l`` and ``arecord -l`` to respectively list the + detected audio output and input devices with their indices. + + If you are using PulseAudio instead of bare ALSA, then you can: + + 1. Use the ``pavucontrol`` (GUI) tool to select the audio input and + output devices and volumes for the assistant. + 2. Use a program like ``pamix`` (ncurses) or ``pamixer`` (CLI). + 3. Run the ``pactl list sources`` and ``pactl list sinks`` commands to + respectively list the detected audio input and output devices. Take + note of their name, and specify which ones the assistant should use + by starting the application with the right ``PULSE_SOURCE`` and + ``PULSE_SINK`` environment variables. + + .. warning:: The Google Assistant library used by this backend has + been deprecated by Google: + https://developers.google.com/assistant/sdk/reference/library/python/. + This integration still works on all of my devices, but its future + functionality is not guaranteed - Google may decide to turn off the + API, the library may no longer be built against new architectures and + it's unlikely to be updated. + + .. note:: Since the Google Assistant library hasn't been updated in several + years, some of its dependencies are quite old and may break more recent + Python installations. Please refer to the comments in the [manifest + file](https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml) + for more information on how to install the required dependencies, if + the automated ways fail. """ - def __init__(self, **kwargs): + _entity_name = 'Google Assistant' + + _default_credentials_files = ( + os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'), + os.path.join( + os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json' + ), + ) + + def __init__( + self, + credentials_file: Optional[str] = None, + device_model_id: str = 'Platypush', + **kwargs, + ): + """ + :param credentials_file: Path to the Google OAuth credentials file. + See + https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials + for instructions to get your own credentials file. + By default, it will search for the credentials file under: + + * ``~/.config/google-oauthlib-tool/credentials.json``: default + location supported by the Google Assistant library + + * ``/credentials/google/assistant.json``: + recommended location, under the Platypush working directory. + + :param device_model_id: The device model ID that identifies the device + where the assistant is running (default: Platypush). It can be a + custom string. + """ + super().__init__(**kwargs) + self._credentials_file = credentials_file + self.device_model_id = device_model_id + self.credentials = None + self._assistant = None + self.logger.info('Initialized Google Assistant plugin') - def _get_assistant(self) -> AssistantGoogleBackend: - backend = get_backend('assistant.google') - assert backend, 'The assistant.google backend is not configured.' - return backend + @property + def credentials_file(self) -> str: + if self._credentials_file: + return os.path.abspath(os.path.expanduser(self._credentials_file)) + + f = None + for default_file in self._default_credentials_files: + f = default_file + if os.path.isfile(default_file): + break + + assert f, 'No credentials_file provided and no default file found' + return f + + @property + def assistant(self): + if not self._assistant: + self.logger.warning('The assistant is not running') + return + + return self._assistant + + def _process_event(self, event): + from google.assistant.library.event import EventType, AlertType + + self.logger.info('Received assistant event: %s', event) + + if event.type == EventType.ON_CONVERSATION_TURN_STARTED: + self._on_conversation_start() + elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED: + if not event.args.get('with_follow_on_turn'): + self._on_conversation_end() + elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT: + self._on_conversation_timeout() + elif event.type == EventType.ON_NO_RESPONSE: + self._on_no_response() + elif ( + hasattr(EventType, 'ON_RENDER_RESPONSE') + and event.type == EventType.ON_RENDER_RESPONSE + ): + self._on_reponse_rendered(event.args.get('text')) + elif ( + hasattr(EventType, 'ON_RESPONDING_STARTED') + and event.type == EventType.ON_RESPONDING_STARTED + and event.args.get('is_error_response') is True + ): + self.logger.warning('Assistant response error: %s', json.dumps(event.args)) + elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED: + self._on_speech_recognized(event.args.get('text')) + elif event.type == EventType.ON_ALERT_STARTED: + if event.args.get('alert_type') == AlertType.ALARM: + self._on_alarm_start() + elif event.args.get('alert_type') == AlertType.TIMER: + self._on_timer_start() + else: + self._on_alert_start() + elif event.type == EventType.ON_ALERT_FINISHED: + if event.args.get('alert_type') == AlertType.ALARM: + self._on_alarm_end() + elif event.args.get('alert_type') == AlertType.TIMER: + self._on_timer_end() + else: + self._on_alert_end() + elif event.type == EventType.ON_ASSISTANT_ERROR: + if event.args.get('is_fatal'): + raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}') + + self.logger.warning('Assistant error: %s', json.dumps(event.args)) + elif event.type == EventType.ON_MUTED_CHANGED: + self._on_mute_changed(event.args.get('is_muted', False)) @action - def start_conversation(self): + def start_conversation(self, *_, **__): """ Programmatically start a conversation with the assistant """ - assistant = self._get_assistant() - assistant.start_conversation() + if self.assistant: + self.assistant.start_conversation() @action - def stop_conversation(self): + def stop_conversation(self, *_, **__): """ Programmatically stop a running conversation with the assistant """ - assistant = self._get_assistant() - assistant.stop_conversation() + if self.assistant: + self.assistant.stop_conversation() @action - def set_mic_mute(self, muted: bool = True): + def mute(self, *_, **__): + """ + Mute the microphone. Alias for :meth:`.set_mic_mute` with + ``muted=True``. + """ + self.set_mic_mute(muted=True) + + @action + def unmute(self, *_, **__): + """ + Unmute the microphone. Alias for :meth:`.set_mic_mute` with + ``muted=False``. + """ + self.set_mic_mute(muted=False) + + @action + def set_mic_mute(self, muted: bool): """ Programmatically mute/unmute the microphone. :param muted: Set to True or False. """ - assistant = self._get_assistant() - assistant.set_mic_mute(muted) + if self.assistant: + self.assistant.set_mic_mute(muted) + + if muted: + self._on_mute() + else: + self._on_unmute() + + @action + def toggle_mute(self, *_, **__): + """ + Toggle the mic mute state. + """ + self.set_mic_mute(muted=not self._is_muted) @action def toggle_mic_mute(self): """ - Toggle the mic mute state. + Deprecated alias for :meth:`.toggle_mute`. """ - assistant = self._get_assistant() - is_muted = assistant.is_muted() - self.set_mic_mute(muted=not is_muted) + return self.toggle_mute() @action - def is_muted(self) -> bool: - """ - :return: True if the microphone is muted, False otherwise. - """ - assistant = self._get_assistant() - return assistant.is_muted() - - @action - def send_text_query(self, query: str): + def send_text_query(self, *_, query: str, **__): """ Send a text query to the assistant. + This is equivalent to saying something to the assistant. + :param query: Query to be sent. """ - assistant = self._get_assistant() - assistant.send_text_query(query) + if self.assistant: + self.assistant.send_text_query(query) + + def main(self): + import google.oauth2.credentials + from google.assistant.library import Assistant + + last_sleep = 0 + + while not self.should_stop(): + try: + with open(self.credentials_file, 'r') as f: + self.credentials = google.oauth2.credentials.Credentials( + token=None, **json.load(f) + ) + except Exception as e: + self.logger.error( + 'Error while loading Google Assistant credentials: %s', e + ) + self.logger.info( + 'Please follow the instructions at ' + 'https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample' + '#generate_credentials to get your own credentials file' + ) + self.logger.exception(e) + break + + try: + with Assistant( + self.credentials, self.device_model_id + ) as self._assistant: + self.publish_entities([self]) + for event in self._assistant.start(): + last_sleep = 0 + + if not self.is_detecting(): + self.logger.info( + 'Assistant event received but detection is currently paused' + ) + continue + + self._process_event(event) + except Exception as e: + self.logger.exception(e) + sleep_secs = min(60, max(5, last_sleep * 2)) + self.logger.warning( + 'Restarting the assistant in %d seconds after an unrecoverable error', + sleep_secs, + ) + + self.wait_stop(sleep_secs) + last_sleep = sleep_secs + continue + + def stop(self): + try: + self.stop_conversation() + except RuntimeError: + pass + + if self._assistant: + del self._assistant + self._assistant = None + + super().stop() # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/lib/__init__.py b/platypush/plugins/assistant/google/lib/__init__.py deleted file mode 100644 index 9953b93b..00000000 --- a/platypush/plugins/assistant/google/lib/__init__.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Based on Google pushtotalk.py sample.""" - -import concurrent.futures -import json -import logging - -import grpc - -from google.assistant.embedded.v1alpha2 import ( - embedded_assistant_pb2, - embedded_assistant_pb2_grpc -) -from tenacity import retry, stop_after_attempt, retry_if_exception - -try: - from googlesamples.assistant.grpc import ( - assistant_helpers, - audio_helpers, - browser_helpers, - device_helpers - ) -except (SystemError, ImportError): - import assistant_helpers - import audio_helpers - import browser_helpers - import device_helpers - - -ASSISTANT_API_ENDPOINT = 'embeddedassistant.googleapis.com' -END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE -DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON -CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE -PLAYING = embedded_assistant_pb2.ScreenOutConfig.PLAYING -DEFAULT_GRPC_DEADLINE = 60 * 3 + 5 - - -class SampleAssistant(object): - """Sample Assistant that supports conversations and device actions. - - Args: - device_model_id: identifier of the device model. - device_id: identifier of the registered device instance. - conversation_stream(ConversationStream): audio stream - for recording query and playing back assistant answer. - channel: authorized gRPC channel for connection to the - Google Assistant API. - deadline_sec: gRPC deadline in seconds for Google Assistant API call. - device_handler: callback for device actions. - """ - - def __init__(self, language_code, device_model_id, device_id, - conversation_stream, display, - channel, deadline_sec, device_handler, play_response=True, - on_conversation_start=None, on_conversation_end=None, - on_speech_recognized=None, on_volume_changed=None, - on_response=None): - self.language_code = language_code - self.device_model_id = device_model_id - self.device_id = device_id - self.conversation_stream = conversation_stream - self.display = display - self.play_response = play_response - - # Opaque blob provided in AssistResponse that, - # when provided in a follow-up AssistRequest, - # gives the Assistant a context marker within the current state - # of the multi-Assist()-RPC "conversation". - # This value, along with MicrophoneMode, supports a more natural - # "conversation" with the Assistant. - self.conversation_state = None - # Force reset of first conversation. - self.is_new_conversation = True - - # Create Google Assistant API gRPC client. - self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(channel) - self.deadline = deadline_sec - - self.device_handler = device_handler - self.detected_speech = None - - self.on_conversation_start = on_conversation_start - self.on_conversation_end = on_conversation_end - self.on_speech_recognized = on_speech_recognized - self.on_volume_changed = on_volume_changed - self.on_response = on_response - - def __enter__(self): - return self - - def __exit__(self, etype, e, traceback): - if e: - return False - self.conversation_stream.close() - - @staticmethod - def is_grpc_error_unavailable(e): - is_grpc_error = isinstance(e, grpc.RpcError) - if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE): - logging.error('grpc unavailable error: %s', e) - return True - return False - - @retry(reraise=True, stop=stop_after_attempt(3), - retry=retry_if_exception(is_grpc_error_unavailable)) - def assist(self): - """Send a voice request to the Assistant and playback the response. - - Returns: True if conversation should continue. - """ - continue_conversation = False - device_actions_futures = [] - - self.conversation_stream.start_recording() - if self.on_conversation_start: - self.on_conversation_start() - - logging.info('Recording audio request.') - - def iter_log_assist_requests(): - for c in self.gen_assist_requests(): - assistant_helpers.log_assist_request_without_audio(c) - yield c - logging.debug('Reached end of AssistRequest iteration.') - - # This generator yields AssistResponse proto messages - # received from the gRPC Google Assistant API. - for resp in self.assistant.Assist(iter_log_assist_requests(), self.deadline): - assistant_helpers.log_assist_response_without_audio(resp) - if resp.event_type == END_OF_UTTERANCE: - logging.info('End of audio request detected.') - logging.info('Stopping recording.') - self.conversation_stream.stop_recording() - - if self.detected_speech and self.on_speech_recognized: - self.on_speech_recognized(self.detected_speech) - - if resp.speech_results: - self.detected_speech = ' '.join( - r.transcript.strip() for r in resp.speech_results - if len(r.transcript.strip())).strip() - - logging.info('Transcript of user request: "%s".', self.detected_speech) - - if len(resp.audio_out.audio_data) > 0: - if not self.conversation_stream.playing: - self.conversation_stream.stop_recording() - - if self.play_response: - self.conversation_stream.start_playback() - logging.info('Playing assistant response.') - - if self.play_response and self.conversation_stream.playing: - self.conversation_stream.write(resp.audio_out.audio_data) - elif self.conversation_stream.playing: - self.conversation_stream.stop_playback() - - if resp.dialog_state_out.conversation_state: - conversation_state = resp.dialog_state_out.conversation_state - logging.debug('Updating conversation state.') - self.conversation_state = conversation_state - - if resp.dialog_state_out.volume_percentage != 0: - volume_percentage = resp.dialog_state_out.volume_percentage - logging.info('Setting volume to %s%%', volume_percentage) - self.conversation_stream.volume_percentage = volume_percentage - - if self.on_volume_changed: - self.on_volume_changed(volume_percentage) - - if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON: - continue_conversation = True - logging.info('Expecting follow-on query from user.') - elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE: - continue_conversation = False - - if resp.device_action.device_request_json: - device_request = json.loads( - resp.device_action.device_request_json - ) - fs = self.device_handler(device_request) - if fs: - device_actions_futures.extend(fs) - - if self.display and resp.screen_out.data: - system_browser = browser_helpers.system_browser - system_browser.display(resp.screen_out.data) - - if resp.dialog_state_out.supplemental_display_text and self.on_response: - self.on_response(resp.dialog_state_out.supplemental_display_text) - - if len(device_actions_futures): - logging.info('Waiting for device executions to complete.') - concurrent.futures.wait(device_actions_futures) - - logging.info('Finished playing assistant response.') - self.conversation_stream.stop_playback() - - if self.on_conversation_end: - self.on_conversation_end(continue_conversation) - - return continue_conversation - - def gen_assist_requests(self): - """Yields: AssistRequest messages to send to the API.""" - - config = embedded_assistant_pb2.AssistConfig( - audio_in_config=embedded_assistant_pb2.AudioInConfig( - encoding='LINEAR16', - sample_rate_hertz=self.conversation_stream.sample_rate, - ), - audio_out_config=embedded_assistant_pb2.AudioOutConfig( - encoding='LINEAR16', - sample_rate_hertz=self.conversation_stream.sample_rate, - volume_percentage=self.conversation_stream.volume_percentage, - ), - dialog_state_in=embedded_assistant_pb2.DialogStateIn( - language_code=self.language_code, - conversation_state=self.conversation_state, - is_new_conversation=self.is_new_conversation, - ), - device_config=embedded_assistant_pb2.DeviceConfig( - device_id=self.device_id, - device_model_id=self.device_model_id, - ) - ) - if self.display: - config.screen_out_config.screen_mode = PLAYING - # Continue current conversation with later requests. - self.is_new_conversation = False - # The first AssistRequest must contain the AssistConfig - # and no audio data. - yield embedded_assistant_pb2.AssistRequest(config=config) - for data in self.conversation_stream: - # Subsequent requests need audio data, but not config. - yield embedded_assistant_pb2.AssistRequest(audio_in=data) - diff --git a/platypush/plugins/assistant/google/manifest.yaml b/platypush/plugins/assistant/google/manifest.yaml index 0ec3fa8d..fab528bc 100644 --- a/platypush/plugins/assistant/google/manifest.yaml +++ b/platypush/plugins/assistant/google/manifest.yaml @@ -1,6 +1,74 @@ manifest: - events: {} - install: - pip: [] package: platypush.plugins.assistant.google type: plugin + events: + - platypush.message.event.assistant.AlarmEndEvent + - platypush.message.event.assistant.AlarmStartedEvent + - platypush.message.event.assistant.ConversationEndEvent + - platypush.message.event.assistant.ConversationStartEvent + - platypush.message.event.assistant.ConversationTimeoutEvent + - platypush.message.event.assistant.MicMutedEvent + - platypush.message.event.assistant.MicUnmutedEvent + - platypush.message.event.assistant.NoResponseEvent + - platypush.message.event.assistant.ResponseEvent + - platypush.message.event.assistant.SpeechRecognizedEvent + - platypush.message.event.assistant.TimerEndEvent + - platypush.message.event.assistant.TimerStartedEvent + + install: + apk: + - ffmpeg + - portaudio-dev + - py3-cachetools + - py3-grpcio + - py3-google-auth + - py3-numpy + - py3-pathlib2 + - py3-tenacity + - py3-urllib3 + apt: + - ffmpeg + - portaudio19-dev + - python3-cachetools + - python3-grpcio + - python3-google-auth + - python3-monotonic + - python3-tenacity + - python3-urllib3 + dnf: + - ffmpeg + - portaudio-devel + - python-cachetools + - python-grpcio + - python-google-auth + - python-monotonic + - python-numpy + - python-tenacity + - python-urllib3 + pacman: + - ffmpeg + - portaudio + - python-cachetools + - python-grpcio + - python-google-auth + - python-monotonic + - python-numpy + - python-sounddevice + - python-tenacity + - python-urllib3 + pip: + - google-assistant-library + - google-assistant-sdk[samples] + - google-auth + - numpy + - sounddevice + after: + # Uninstall old versions of packages that break things on recent versions + # of Python, when the new versions work just fine + - yes | pip uninstall --break-system-packages enum34 click urllib3 requests google-auth + # Upgrade the dependencies (back) to the latest version. + # NOTE: Be careful when running this command on older distros that may + # not ship the latest versions of all the packages! This is a workaround + # caused by the fact that google-assistant-library pulls in some old + # breaking dependencies that need to be surgically removed. + - pip install -U --no-input --break-system-packages click urllib3 requests google-auth diff --git a/platypush/plugins/assistant/google/pushtotalk/__init__.py b/platypush/plugins/assistant/google/pushtotalk/__init__.py deleted file mode 100644 index addcd0b8..00000000 --- a/platypush/plugins/assistant/google/pushtotalk/__init__.py +++ /dev/null @@ -1,330 +0,0 @@ -import json -import os -from typing import Optional, Dict, Any - -from platypush.context import get_bus, get_plugin -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - SpeechRecognizedEvent, - VolumeChangedEvent, - ResponseEvent, -) - -from platypush.message.event.google import GoogleDeviceOnOffEvent - -from platypush.plugins import action -from platypush.plugins.assistant import AssistantPlugin - - -class AssistantGooglePushtotalkPlugin(AssistantPlugin): - """ - Plugin for the Google Assistant push-to-talk API. - """ - - api_endpoint = 'embeddedassistant.googleapis.com' - grpc_deadline = 60 * 3 + 5 - device_handler = None - _default_credentials_file = os.path.join( - os.path.expanduser('~'), - '.config', - 'google-oauthlib-tool', - 'credentials.json', - ) - - _default_device_config = os.path.join( - os.path.expanduser('~'), - '.config', - 'googlesamples-assistant', - 'device_config.json', - ) - - def __init__( - self, - credentials_file=_default_credentials_file, - device_config=_default_device_config, - language='en-US', - play_response=True, - tts_plugin=None, - tts_args=None, - **kwargs - ): - """ - :param credentials_file: Path to the Google OAuth credentials file - (default: ~/.config/google-oauthlib-tool/credentials.json). - See - https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials - for instructions to get your own credentials file. - :type credentials_file: str - - :param device_config: Path to device_config.json. Register your device - (see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device) - and create a project, then run the pushtotalk.py script from - googlesamples to create your device_config.json - :type device_config: str - - :param language: Assistant language (default: en-US) - :type language: str - - :param play_response: If True (default) then the plugin will play the assistant response upon processed - response. Otherwise nothing will be played - but you may want to handle the ``ResponseEvent`` manually. - :type play_response: bool - - :param tts_plugin: Optional text-to-speech plugin to be used to process response text. - :type tts_plugin: str - - :param tts_args: Optional arguments for the TTS plugin ``say`` method. - :type tts_args: dict - """ - - import googlesamples.assistant.grpc.audio_helpers as audio_helpers - - super().__init__(**kwargs) - - self.audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE - self.audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH - self.audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE - self.audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE - self.audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE - - self.language = language - self.credentials_file = credentials_file - self.device_config = device_config - self.play_response = play_response - self.tts_plugin = tts_plugin - self.tts_args = tts_args or {} - self.assistant = None - self.interactions = [] - - with open(self.device_config) as f: - device = json.load(f) - self.device_id = device['id'] - self.device_model_id = device['model_id'] - - # Load OAuth 2.0 credentials. - try: - from google.oauth2.credentials import Credentials - from google.auth.transport.requests import Request - - with open(self.credentials_file, 'r') as f: - self.credentials = Credentials(token=None, **json.load(f)) - self.http_request = Request() - self.credentials.refresh(self.http_request) - except Exception as ex: - self.logger.error('Error loading credentials: %s', str(ex)) - self.logger.error( - 'Run google-oauthlib-tool to initialize ' 'new OAuth 2.0 credentials.' - ) - raise - - self.grpc_channel = None - self.conversation_stream = None - - def _init_assistant(self): - import googlesamples.assistant.grpc.audio_helpers as audio_helpers - from google.auth.transport.grpc import secure_authorized_channel - - self.interactions = [] - - # Create an authorized gRPC channel. - self.grpc_channel = secure_authorized_channel( - self.credentials, self.http_request, self.api_endpoint - ) - self.logger.info('Connecting to {}'.format(self.api_endpoint)) - - # Configure audio source and sink. - audio_device = None - audio_source = audio_device = audio_device or audio_helpers.SoundDeviceStream( - sample_rate=self.audio_sample_rate, - sample_width=self.audio_sample_width, - block_size=self.audio_block_size, - flush_size=self.audio_flush_size, - ) - - audio_sink = audio_device or audio_helpers.SoundDeviceStream( - sample_rate=self.audio_sample_rate, - sample_width=self.audio_sample_width, - block_size=self.audio_block_size, - flush_size=self.audio_flush_size, - ) - - # Create conversation stream with the given audio source and sink. - self.conversation_stream = audio_helpers.ConversationStream( - source=audio_source, - sink=audio_sink, - iter_size=self.audio_iter_size, - sample_width=self.audio_sample_width, - ) - - self._install_device_handlers() - - def on_conversation_start(self): - """Conversation start handler""" - - def handler(): - get_bus().post(ConversationStartEvent(assistant=self)) - - return handler - - def on_conversation_end(self): - """Conversation end handler""" - - def handler(with_follow_on_turn): - get_bus().post( - ConversationEndEvent( - assistant=self, with_follow_on_turn=with_follow_on_turn - ) - ) - - return handler - - def on_speech_recognized(self): - """Speech recognized handler""" - - def handler(phrase): - get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) - self.interactions.append({'request': phrase}) - - return handler - - def on_volume_changed(self): - """Volume changed event""" - - def handler(volume): - get_bus().post(VolumeChangedEvent(assistant=self, volume=volume)) - - return handler - - def on_response(self): - """Response handler""" - - def handler(response): - get_bus().post(ResponseEvent(assistant=self, response_text=response)) - - if not self.interactions: - self.interactions.append({'response': response}) - else: - self.interactions[-1]['response'] = response - - if self.tts_plugin: - tts = get_plugin(self.tts_plugin) - tts.say(response, **self.tts_args) - - return handler - - @action - def start_conversation( - self, - *_, - language: Optional[str] = None, - tts_plugin: Optional[str] = None, - tts_args: Optional[Dict[str, Any]] = None, - **__ - ): - """ - Start a conversation - - :param language: Language code override (default: default configured language). - :param tts_plugin: Optional text-to-speech plugin to be used for rendering text. - :param tts_args: Optional arguments for the TTS plugin say method. - :returns: A list of the interactions that happen within the conversation. - - .. code-block:: json - - [ - { - "request": "request 1", - "response": "response 1" - }, - { - "request": "request 2", - "response": "response 2" - } - ] - - """ - - from platypush.plugins.assistant.google.lib import SampleAssistant - - self.tts_plugin = tts_plugin - self.tts_args = tts_args - language = language or self.language - play_response = False if self.tts_plugin else self.play_response - - self._init_assistant() - self.on_conversation_start() - - with SampleAssistant( - language_code=language, - device_model_id=self.device_model_id, - device_id=self.device_id, - conversation_stream=self.conversation_stream, - display=None, - channel=self.grpc_channel, - deadline_sec=self.grpc_deadline, - play_response=play_response, - device_handler=self.device_handler, - on_conversation_start=self.on_conversation_start(), - on_conversation_end=self.on_conversation_end(), - on_volume_changed=self.on_volume_changed(), - on_response=self.on_response(), - on_speech_recognized=self.on_speech_recognized(), - ) as self.assistant: - continue_conversation = True - - while continue_conversation: - try: - continue_conversation = self.assistant.assist() - except Exception as e: - self.logger.warning( - 'Unhandled assistant exception: {}'.format(str(e)) - ) - self.logger.exception(e) - self._init_assistant() - - return self.interactions - - @action - def stop_conversation(self): - if self.assistant: - self.assistant.play_response = False - - if self.conversation_stream: - self.conversation_stream.stop_playback() - self.conversation_stream.stop_recording() - - get_bus().post(ConversationEndEvent(assistant=self)) - - @action - def set_mic_mute(self, muted: bool = True): - """ - Programmatically mute/unmute the microphone. - - :param muted: Set to True or False. - """ - if not self.conversation_stream: - self.logger.warning('The assistant is not running') - return - - if muted: - self.conversation_stream.stop_recording() - else: - self.conversation_stream.start_recording() - - def _install_device_handlers(self): - import googlesamples.assistant.grpc.device_helpers as device_helpers - - self.device_handler = device_helpers.DeviceRequestHandler(self.device_id) - - @self.device_handler.command('action.devices.commands.OnOff') - def handler(on): # type: ignore - get_bus().post( - GoogleDeviceOnOffEvent( - device_id=self.device_id, - device_model_id=self.device_model_id, - on=on, - ) - ) - - -# vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/pushtotalk/manifest.yaml b/platypush/plugins/assistant/google/pushtotalk/manifest.yaml deleted file mode 100644 index dfb859d2..00000000 --- a/platypush/plugins/assistant/google/pushtotalk/manifest.yaml +++ /dev/null @@ -1,27 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - install: - apk: - - py3-tenacity - - py3-google-auth - apt: - - python3-tenacity - - python3-google-auth - dnf: - - python-tenacity - - python-google-auth - pacman: - - python-tenacity - - python-google-auth - pip: - - tenacity - - google-assistant-sdk - - google-auth - package: platypush.plugins.assistant.google.pushtotalk - type: plugin diff --git a/setup.py b/setup.py index e9efd0cb..129ee380 100755 --- a/setup.py +++ b/setup.py @@ -139,9 +139,6 @@ setup( ], # Support for Last.FM scrobbler plugin 'lastfm': ['pylast'], - # Support for custom hotword detection - 'hotword': ['snowboy'], - 'snowboy': ['snowboy'], # Support for real-time MIDI events 'midi': ['rtmidi'], # Support for RaspberryPi GPIO @@ -193,8 +190,6 @@ setup( 'flic': [ 'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master' ], - # Support for Alexa/Echo plugin - 'alexa': ['avs @ https://github.com/BlackLight/avs/tarball/master'], # Support for Bluetooth devices 'bluetooth': [ 'bleak',