From cce6c4c5ad3a5d7f1ae28395a75b3288835f863c Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 22 Oct 2023 19:55:11 +0200 Subject: [PATCH 1/9] [#284] Merged `assistant.google` plugin and backend. This removes the deprecated `assistant.google` backend and also adds a new `conversation_start_sound` parameter. --- docs/source/backends.rst | 1 - .../platypush/backend/assistant.google.rst | 6 - .../backend/assistant/google/__init__.py | 191 ----------- .../backend/assistant/google/manifest.yaml | 39 --- platypush/message/event/assistant/__init__.py | 15 +- platypush/plugins/assistant/__init__.py | 54 ++- .../plugins/assistant/google/__init__.py | 320 ++++++++++++++++-- .../plugins/assistant/google/manifest.yaml | 74 +++- 8 files changed, 397 insertions(+), 303 deletions(-) delete mode 100644 docs/source/platypush/backend/assistant.google.rst delete mode 100644 platypush/backend/assistant/google/__init__.py delete mode 100644 platypush/backend/assistant/google/manifest.yaml diff --git a/docs/source/backends.rst b/docs/source/backends.rst index 9ac610bb..44f1cbb8 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -8,7 +8,6 @@ Backends platypush/backend/adafruit.io.rst platypush/backend/alarm.rst - platypush/backend/assistant.google.rst platypush/backend/assistant.snowboy.rst platypush/backend/button.flic.rst platypush/backend/camera.pi.rst diff --git a/docs/source/platypush/backend/assistant.google.rst b/docs/source/platypush/backend/assistant.google.rst deleted file mode 100644 index 1fecc4f5..00000000 --- a/docs/source/platypush/backend/assistant.google.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.google`` -====================================== - -.. automodule:: platypush.backend.assistant.google - :members: - diff --git a/platypush/backend/assistant/google/__init__.py b/platypush/backend/assistant/google/__init__.py deleted file mode 100644 index f6967bcd..00000000 --- a/platypush/backend/assistant/google/__init__.py +++ /dev/null @@ -1,191 +0,0 @@ -import json -import os -import time - -from platypush.backend.assistant import AssistantBackend -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - ConversationTimeoutEvent, - ResponseEvent, - NoResponseEvent, - SpeechRecognizedEvent, - AlarmStartedEvent, - AlarmEndEvent, - TimerStartedEvent, - TimerEndEvent, - AlertStartedEvent, - AlertEndEvent, - MicMutedEvent, - MicUnmutedEvent, -) - - -class AssistantGoogleBackend(AssistantBackend): - """ - Google Assistant backend. - - It listens for voice commands and post conversation events on the bus. - - **WARNING**: The Google Assistant library used by this backend has officially been deprecated: - https://developers.google.com/assistant/sdk/reference/library/python/. This backend still works on most of the - devices where I use it, but its correct functioning is not guaranteed as the assistant library is no longer - maintained. - """ - - _default_credentials_file = os.path.join( - os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json' - ) - - def __init__( - self, - credentials_file=_default_credentials_file, - device_model_id='Platypush', - **kwargs - ): - """ - :param credentials_file: Path to the Google OAuth credentials file - (default: ~/.config/google-oauthlib-tool/credentials.json). - See - https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials - for instructions to get your own credentials file. - - :type credentials_file: str - - :param device_model_id: Device model ID to use for the assistant - (default: Platypush) - :type device_model_id: str - """ - - super().__init__(**kwargs) - self.credentials_file = credentials_file - self.device_model_id = device_model_id - self.credentials = None - self.assistant = None - self._has_error = False - self._is_muted = False - - self.logger.info('Initialized Google Assistant backend') - - def _process_event(self, event): - from google.assistant.library.event import EventType, AlertType - - self.logger.info('Received assistant event: {}'.format(event)) - self._has_error = False - - if event.type == EventType.ON_CONVERSATION_TURN_STARTED: - self.bus.post(ConversationStartEvent(assistant=self)) - elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED: - if not event.args.get('with_follow_on_turn'): - self.bus.post(ConversationEndEvent(assistant=self)) - elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT: - self.bus.post(ConversationTimeoutEvent(assistant=self)) - elif event.type == EventType.ON_NO_RESPONSE: - self.bus.post(NoResponseEvent(assistant=self)) - elif ( - hasattr(EventType, 'ON_RENDER_RESPONSE') - and event.type == EventType.ON_RENDER_RESPONSE - ): - self.bus.post( - ResponseEvent(assistant=self, response_text=event.args.get('text')) - ) - tts, args = self._get_tts_plugin() - - if tts and 'text' in event.args: - self.stop_conversation() - tts.say(text=event.args['text'], **args) - elif ( - hasattr(EventType, 'ON_RESPONDING_STARTED') - and event.type == EventType.ON_RESPONDING_STARTED - and event.args.get('is_error_response', False) is True - ): - self.logger.warning('Assistant response error') - elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED: - phrase = event.args['text'].lower().strip() - self.logger.info('Speech recognized: {}'.format(phrase)) - self.bus.post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) - elif event.type == EventType.ON_ALERT_STARTED: - if event.args.get('alert_type') == AlertType.ALARM: - self.bus.post(AlarmStartedEvent(assistant=self)) - elif event.args.get('alert_type') == AlertType.TIMER: - self.bus.post(TimerStartedEvent(assistant=self)) - else: - self.bus.post(AlertStartedEvent(assistant=self)) - elif event.type == EventType.ON_ALERT_FINISHED: - if event.args.get('alert_type') == AlertType.ALARM: - self.bus.post(AlarmEndEvent(assistant=self)) - elif event.args.get('alert_type') == AlertType.TIMER: - self.bus.post(TimerEndEvent(assistant=self)) - else: - self.bus.post(AlertEndEvent(assistant=self)) - elif event.type == EventType.ON_ASSISTANT_ERROR: - self._has_error = True - if event.args.get('is_fatal'): - self.logger.error('Fatal assistant error') - else: - self.logger.warning('Assistant error') - if event.type == EventType.ON_MUTED_CHANGED: - self._is_muted = event.args.get('is_muted') - event = MicMutedEvent() if self._is_muted else MicUnmutedEvent() - self.bus.post(event) - - def start_conversation(self): - """Starts a conversation.""" - if self.assistant: - self.assistant.start_conversation() - - def stop_conversation(self): - """Stops an active conversation.""" - if self.assistant: - self.assistant.stop_conversation() - - def set_mic_mute(self, muted): - if not self.assistant: - self.logger.warning('Assistant not running') - return - - self.assistant.set_mic_mute(muted) - - def is_muted(self) -> bool: - return self._is_muted - - def send_text_query(self, query): - if not self.assistant: - self.logger.warning('Assistant not running') - return - - self.assistant.send_text_query(query) - - def run(self): - import google.oauth2.credentials - from google.assistant.library import Assistant - - super().run() - - with open(self.credentials_file, 'r') as f: - self.credentials = google.oauth2.credentials.Credentials( - token=None, **json.load(f) - ) - - while not self.should_stop(): - self._has_error = False - - with Assistant(self.credentials, self.device_model_id) as assistant: - self.assistant = assistant - for event in assistant.start(): - if not self.is_detecting(): - self.logger.info( - 'Assistant event received but detection is currently paused' - ) - continue - - self._process_event(event) - if self._has_error: - self.logger.info( - 'Restarting the assistant after an unrecoverable error' - ) - time.sleep(5) - break - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/google/manifest.yaml b/platypush/backend/assistant/google/manifest.yaml deleted file mode 100644 index 271582af..00000000 --- a/platypush/backend/assistant/google/manifest.yaml +++ /dev/null @@ -1,39 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.AlarmEndEvent: when an alarm ends - platypush.message.event.assistant.AlarmStartedEvent: when an alarm starts - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.ConversationTimeoutEvent: when a conversation - times out - platypush.message.event.assistant.MicMutedEvent: when the microphone is muted. - platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted. - platypush.message.event.assistant.NoResponseEvent: when a conversation returned no - response - platypush.message.event.assistant.ResponseEvent: when the assistant is speaking - a response - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - platypush.message.event.assistant.TimerEndEvent: when a timer ends - platypush.message.event.assistant.TimerStartedEvent: when a timer starts - install: - apk: - - py3-grpcio - - py3-google-auth - apt: - - python3-grpcio - - python3-google-auth - dnf: - - python-grpcio - - python-google-auth - pacman: - - python-grpcio - - python-google-auth - pip: - - google-assistant-library - - google-assistant-sdk[samples] - - google-auth - package: platypush.backend.assistant.google - type: backend diff --git a/platypush/message/event/assistant/__init__.py b/platypush/message/event/assistant/__init__.py index 01a21eb5..f2dea592 100644 --- a/platypush/message/event/assistant/__init__.py +++ b/platypush/message/event/assistant/__init__.py @@ -1,8 +1,6 @@ -import logging import re import sys -from platypush.context import get_backend, get_plugin from platypush.message.event import Event @@ -11,18 +9,7 @@ class AssistantEvent(Event): def __init__(self, *args, assistant=None, **kwargs): super().__init__(*args, **kwargs) - self.logger = logging.getLogger('platypush:assistant') - - if assistant: - self._assistant = assistant - else: - try: - self._assistant = get_backend('assistant.google') - if not self._assistant: - self._assistant = get_plugin('assistant.google.pushtotalk') - except Exception as e: - self.logger.debug('Could not initialize the assistant component: %s', e) - self._assistant = None + self._assistant = assistant class ConversationStartEvent(AssistantEvent): diff --git a/platypush/plugins/assistant/__init__.py b/platypush/plugins/assistant/__init__.py index 08b25e8c..40f35d32 100644 --- a/platypush/plugins/assistant/__init__.py +++ b/platypush/plugins/assistant/__init__.py @@ -1,54 +1,76 @@ from abc import ABC, abstractmethod +from threading import Event +from typing import Any, Dict, Optional -from platypush.context import get_backend +from platypush.context import get_plugin from platypush.plugins import Plugin, action class AssistantPlugin(ABC, Plugin): """ - Base class for assistant plugins + Base class for assistant plugins. """ - @abstractmethod - def start_conversation(self, *args, language=None, tts_plugin=None, tts_args=None, **kwargs): + def __init__( + self, + *args, + tts_plugin: Optional[str] = None, + tts_plugin_args: Optional[Dict[str, Any]] = None, + **kwargs + ): """ - Start a conversation. + :param tts_plugin: If set, the assistant will use this plugin (e.g. + ``tts``, ``tts.google`` or ``tts.mimic3``) to render the responses, + instead of using the built-in assistant voice. + + :param tts_plugin_args: Optional arguments to be passed to the TTS + ``say`` action, if ``tts_plugin`` is set. + """ + super().__init__(*args, **kwargs) + self.tts_plugin = tts_plugin + self.tts_plugin_args = tts_plugin_args or {} + self._detection_paused = Event() + + @abstractmethod + def start_conversation(self, *_, **__): + """ + Programmatically starts a conversation. """ raise NotImplementedError @abstractmethod - def stop_conversation(self, *args, **kwargs): + def stop_conversation(self, *_, **__): """ - Stop a conversation. + Programmatically stops a conversation. """ raise NotImplementedError - def _get_assistant(self): - return get_backend('assistant.snowboy') - @action def pause_detection(self): """ Put the assistant on pause. No new conversation events will be triggered. """ - assistant = self._get_assistant() - assistant.pause_detection() + self._detection_paused.set() @action def resume_detection(self): """ Resume the assistant hotword detection from a paused state. """ - assistant = self._get_assistant() - assistant.resume_detection() + self._detection_paused.clear() @action def is_detecting(self) -> bool: """ :return: True if the asistant is detecting, False otherwise. """ - assistant = self._get_assistant() - return assistant.is_detecting() + return not self._detection_paused.is_set() + + def _get_tts_plugin(self): + if not self.tts_plugin: + return None + + return get_plugin(self.tts_plugin) # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 59ff8f4e..5623d278 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -1,76 +1,330 @@ -from platypush.backend.assistant.google import AssistantGoogleBackend -from platypush.context import get_backend -from platypush.plugins import action +import json +import os +from typing import Optional + +from platypush.config import Config +from platypush.context import get_bus, get_plugin +from platypush.message.event.assistant import ( + ConversationStartEvent, + ConversationEndEvent, + ConversationTimeoutEvent, + ResponseEvent, + NoResponseEvent, + SpeechRecognizedEvent, + AlarmStartedEvent, + AlarmEndEvent, + TimerStartedEvent, + TimerEndEvent, + AlertStartedEvent, + AlertEndEvent, + MicMutedEvent, + MicUnmutedEvent, +) +from platypush.plugins import RunnablePlugin, action from platypush.plugins.assistant import AssistantPlugin -class AssistantGooglePlugin(AssistantPlugin): +class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): """ - Google assistant plugin. - It acts like a wrapper around the :mod:`platypush.backend.assistant.google` - backend to programmatically control the conversation status. + Google Assistant plugin. + + This plugin allows you to run the Google Assistant _directly_ on your + device. It requires you to have an audio microphone and a speaker connected + to the device. + + .. warning:: The Google Assistant library used by this backend has + been deprecated by Google: + https://developers.google.com/assistant/sdk/reference/library/python/. + This integration still works on all of my devices, but its future + functionality is not guaranteed - Google may decide to turn off the + API, the library may no longer be built against new architectures and + it's unlikely to be updated. + + .. note:: Since the Google Assistant library hasn't been updated in several + years, some of its dependencies are quite old and may break more recent + Python installations. Please refer to the comments in the [manifest + file](https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml) + for more information on how to install the required dependencies, if + the automated ways fail. """ - def __init__(self, **kwargs): + _default_credentials_files = ( + os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'), + os.path.join( + os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json' + ), + ) + + def __init__( + self, + credentials_file: Optional[str] = None, + device_model_id: str = 'Platypush', + conversation_start_sound: Optional[str] = None, + **kwargs, + ): + """ + :param credentials_file: Path to the Google OAuth credentials file. + See + https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials + for instructions to get your own credentials file. + By default, it will search for the credentials file under: + + * ``~/.config/google-oauthlib-tool/credentials.json``: default + location supported by the Google Assistant library + + * ``/credentials/google/assistant.json``: + recommended location, under the Platypush working directory. + + :param device_model_id: The device model ID that identifies the device + where the assistant is running (default: Platypush). It can be a + custom string. + + :param conversation_start_sound: If set, the assistant will play this + audio file when it detects a speech. The sound file will be played + on the default audio output device. If not set, the assistant won't + play any sound when it detects a speech. + """ + super().__init__(**kwargs) + self._credentials_file = credentials_file + self.device_model_id = device_model_id + self.credentials = None + self._assistant = None + self._is_muted = False - def _get_assistant(self) -> AssistantGoogleBackend: - backend = get_backend('assistant.google') - assert backend, 'The assistant.google backend is not configured.' - return backend + if conversation_start_sound: + self._conversation_start_sound = os.path.abspath( + os.path.expanduser(conversation_start_sound) + ) + + self.logger.info('Initialized Google Assistant plugin') + + @property + def credentials_file(self) -> str: + if self._credentials_file: + return self._credentials_file + + f = None + for default_file in self._default_credentials_files: + f = default_file + if os.path.isfile(default_file): + break + + assert f, 'No credentials_file provided and no default file found' + return f + + @property + def assistant(self): + if not self._assistant: + self.logger.warning('The assistant is not running') + return + + return self._assistant + + def _play_conversation_start_sound(self): + if not self._conversation_start_sound: + return + + audio = get_plugin('sound') + if not audio: + self.logger.warning( + 'Unable to play conversation start sound: sound plugin not found' + ) + return + + audio.play(self._conversation_start_sound) + + def _process_event(self, event): + from google.assistant.library.event import EventType, AlertType + + self.logger.info('Received assistant event: %s', event) + + if event.type == EventType.ON_CONVERSATION_TURN_STARTED: + get_bus().post(ConversationStartEvent(assistant=self)) + self._play_conversation_start_sound() + elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED: + if not event.args.get('with_follow_on_turn'): + get_bus().post(ConversationEndEvent(assistant=self)) + elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT: + get_bus().post(ConversationTimeoutEvent(assistant=self)) + elif event.type == EventType.ON_NO_RESPONSE: + get_bus().post(NoResponseEvent(assistant=self)) + elif ( + hasattr(EventType, 'ON_RENDER_RESPONSE') + and event.type == EventType.ON_RENDER_RESPONSE + ): + get_bus().post( + ResponseEvent(assistant=self, response_text=event.args.get('text')) + ) + tts = self._get_tts_plugin() + + if tts and event.args.get('text'): + self.stop_conversation() + tts.say(text=event.args['text'], **self.tts_plugin_args) + elif ( + hasattr(EventType, 'ON_RESPONDING_STARTED') + and event.type == EventType.ON_RESPONDING_STARTED + and event.args.get('is_error_response', False) is True + ): + self.logger.warning('Assistant response error') + elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED: + phrase = event.args['text'].lower().strip() + self.logger.info('Speech recognized: %s', phrase) + get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) + elif event.type == EventType.ON_ALERT_STARTED: + if event.args.get('alert_type') == AlertType.ALARM: + get_bus().post(AlarmStartedEvent(assistant=self)) + elif event.args.get('alert_type') == AlertType.TIMER: + get_bus().post(TimerStartedEvent(assistant=self)) + else: + get_bus().post(AlertStartedEvent(assistant=self)) + elif event.type == EventType.ON_ALERT_FINISHED: + if event.args.get('alert_type') == AlertType.ALARM: + get_bus().post(AlarmEndEvent(assistant=self)) + elif event.args.get('alert_type') == AlertType.TIMER: + get_bus().post(TimerEndEvent(assistant=self)) + else: + get_bus().post(AlertEndEvent(assistant=self)) + elif event.type == EventType.ON_ASSISTANT_ERROR: + if event.args.get('is_fatal'): + raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}') + + self.logger.warning('Assistant error: %s', json.dumps(event.args)) + elif event.type == EventType.ON_MUTED_CHANGED: + self._is_muted = event.args.get('is_muted') + event = MicMutedEvent() if self._is_muted else MicUnmutedEvent() + get_bus().post(event) @action - def start_conversation(self): + def start_conversation(self, *_, **__): """ Programmatically start a conversation with the assistant """ - assistant = self._get_assistant() - assistant.start_conversation() + if self.assistant: + self.assistant.start_conversation() @action def stop_conversation(self): """ Programmatically stop a running conversation with the assistant """ - assistant = self._get_assistant() - assistant.stop_conversation() + if self.assistant: + self.assistant.stop_conversation() @action - def set_mic_mute(self, muted: bool = True): + def mute(self): + """ + Mute the microphone. Alias for :meth:`.set_mic_mute` with + ``muted=True``. + """ + return self.set_mic_mute(muted=True) + + @action + def unmute(self): + """ + Unmute the microphone. Alias for :meth:`.set_mic_mute` with + ``muted=False``. + """ + return self.set_mic_mute(muted=False) + + @action + def set_mic_mute(self, muted: bool): """ Programmatically mute/unmute the microphone. :param muted: Set to True or False. """ - assistant = self._get_assistant() - assistant.set_mic_mute(muted) - - @action - def toggle_mic_mute(self): - """ - Toggle the mic mute state. - """ - assistant = self._get_assistant() - is_muted = assistant.is_muted() - self.set_mic_mute(muted=not is_muted) + if self.assistant: + self.assistant.set_mic_mute(muted) @action def is_muted(self) -> bool: """ :return: True if the microphone is muted, False otherwise. """ - assistant = self._get_assistant() - return assistant.is_muted() + return self._is_muted + + @action + def toggle_mic_mute(self): + """ + Toggle the mic mute state. + """ + is_muted = self.is_muted() + self.set_mic_mute(muted=not is_muted) @action def send_text_query(self, query: str): """ Send a text query to the assistant. + This is equivalent to saying something to the assistant. + :param query: Query to be sent. """ - assistant = self._get_assistant() - assistant.send_text_query(query) + if self.assistant: + self.assistant.send_text_query(query) + + def main(self): + import google.oauth2.credentials + from google.assistant.library import Assistant + + last_sleep = 0 + + while not self.should_stop(): + try: + with open(self.credentials_file, 'r') as f: + self.credentials = google.oauth2.credentials.Credentials( + token=None, **json.load(f) + ) + except Exception as e: + self.logger.error( + 'Error while loading Google Assistant credentials: %s', e + ) + self.logger.info( + 'Please follow the instructions at ' + 'https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample' + '#generate_credentials to get your own credentials file' + ) + self.logger.exception(e) + break + + try: + with Assistant( + self.credentials, self.device_model_id + ) as self._assistant: + for event in self._assistant.start(): + last_sleep = 0 + + if not self.is_detecting(): + self.logger.info( + 'Assistant event received but detection is currently paused' + ) + continue + + self._process_event(event) + except Exception as e: + self.logger.exception(e) + sleep_secs = min(60, max(5, last_sleep * 2)) + self.logger.warning( + 'Restarting the assistant in %d seconds after an unrecoverable error', + sleep_secs, + ) + + self.wait_stop(sleep_secs) + last_sleep = sleep_secs + continue + + def stop(self): + try: + self.stop_conversation() + except RuntimeError: + pass + + if self._assistant: + del self._assistant + self._assistant = None + + super().stop() # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/manifest.yaml b/platypush/plugins/assistant/google/manifest.yaml index 0ec3fa8d..fab528bc 100644 --- a/platypush/plugins/assistant/google/manifest.yaml +++ b/platypush/plugins/assistant/google/manifest.yaml @@ -1,6 +1,74 @@ manifest: - events: {} - install: - pip: [] package: platypush.plugins.assistant.google type: plugin + events: + - platypush.message.event.assistant.AlarmEndEvent + - platypush.message.event.assistant.AlarmStartedEvent + - platypush.message.event.assistant.ConversationEndEvent + - platypush.message.event.assistant.ConversationStartEvent + - platypush.message.event.assistant.ConversationTimeoutEvent + - platypush.message.event.assistant.MicMutedEvent + - platypush.message.event.assistant.MicUnmutedEvent + - platypush.message.event.assistant.NoResponseEvent + - platypush.message.event.assistant.ResponseEvent + - platypush.message.event.assistant.SpeechRecognizedEvent + - platypush.message.event.assistant.TimerEndEvent + - platypush.message.event.assistant.TimerStartedEvent + + install: + apk: + - ffmpeg + - portaudio-dev + - py3-cachetools + - py3-grpcio + - py3-google-auth + - py3-numpy + - py3-pathlib2 + - py3-tenacity + - py3-urllib3 + apt: + - ffmpeg + - portaudio19-dev + - python3-cachetools + - python3-grpcio + - python3-google-auth + - python3-monotonic + - python3-tenacity + - python3-urllib3 + dnf: + - ffmpeg + - portaudio-devel + - python-cachetools + - python-grpcio + - python-google-auth + - python-monotonic + - python-numpy + - python-tenacity + - python-urllib3 + pacman: + - ffmpeg + - portaudio + - python-cachetools + - python-grpcio + - python-google-auth + - python-monotonic + - python-numpy + - python-sounddevice + - python-tenacity + - python-urllib3 + pip: + - google-assistant-library + - google-assistant-sdk[samples] + - google-auth + - numpy + - sounddevice + after: + # Uninstall old versions of packages that break things on recent versions + # of Python, when the new versions work just fine + - yes | pip uninstall --break-system-packages enum34 click urllib3 requests google-auth + # Upgrade the dependencies (back) to the latest version. + # NOTE: Be careful when running this command on older distros that may + # not ship the latest versions of all the packages! This is a workaround + # caused by the fact that google-assistant-library pulls in some old + # breaking dependencies that need to be surgically removed. + - pip install -U --no-input --break-system-packages click urllib3 requests google-auth -- 2.45.1 From b46c00f0152ddb241eea173a93857ab32fcc548e Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 22 Oct 2023 19:57:55 +0200 Subject: [PATCH 2/9] Removed deprecated `google.assistant.pushtotalk` plugin. It only existed as a back-compatibility layer with armv6, since there was no build of the assistant library that worked on Raspberry Pi Zero. But that API layer has been discontinued by Google and it's no longer functional, so only the `assistant.google` integration (on x86_64 and armv7) is currently supported. --- .../plugins/assistant.google.pushtotalk.rst | 6 - docs/source/plugins.rst | 1 - .../plugins/assistant/google/lib/__init__.py | 236 ------------- .../assistant/google/pushtotalk/__init__.py | 330 ------------------ .../assistant/google/pushtotalk/manifest.yaml | 27 -- 5 files changed, 600 deletions(-) delete mode 100644 docs/source/platypush/plugins/assistant.google.pushtotalk.rst delete mode 100644 platypush/plugins/assistant/google/lib/__init__.py delete mode 100644 platypush/plugins/assistant/google/pushtotalk/__init__.py delete mode 100644 platypush/plugins/assistant/google/pushtotalk/manifest.yaml diff --git a/docs/source/platypush/plugins/assistant.google.pushtotalk.rst b/docs/source/platypush/plugins/assistant.google.pushtotalk.rst deleted file mode 100644 index 4805e890..00000000 --- a/docs/source/platypush/plugins/assistant.google.pushtotalk.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.google.pushtotalk`` -================================================= - -.. automodule:: platypush.plugins.assistant.google.pushtotalk - :members: - diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index be9e6324..c43fd3cd 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -12,7 +12,6 @@ Plugins platypush/plugins/arduino.rst platypush/plugins/assistant.echo.rst platypush/plugins/assistant.google.rst - platypush/plugins/assistant.google.pushtotalk.rst platypush/plugins/autoremote.rst platypush/plugins/bluetooth.rst platypush/plugins/calendar.rst diff --git a/platypush/plugins/assistant/google/lib/__init__.py b/platypush/plugins/assistant/google/lib/__init__.py deleted file mode 100644 index 9953b93b..00000000 --- a/platypush/plugins/assistant/google/lib/__init__.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Based on Google pushtotalk.py sample.""" - -import concurrent.futures -import json -import logging - -import grpc - -from google.assistant.embedded.v1alpha2 import ( - embedded_assistant_pb2, - embedded_assistant_pb2_grpc -) -from tenacity import retry, stop_after_attempt, retry_if_exception - -try: - from googlesamples.assistant.grpc import ( - assistant_helpers, - audio_helpers, - browser_helpers, - device_helpers - ) -except (SystemError, ImportError): - import assistant_helpers - import audio_helpers - import browser_helpers - import device_helpers - - -ASSISTANT_API_ENDPOINT = 'embeddedassistant.googleapis.com' -END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE -DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON -CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE -PLAYING = embedded_assistant_pb2.ScreenOutConfig.PLAYING -DEFAULT_GRPC_DEADLINE = 60 * 3 + 5 - - -class SampleAssistant(object): - """Sample Assistant that supports conversations and device actions. - - Args: - device_model_id: identifier of the device model. - device_id: identifier of the registered device instance. - conversation_stream(ConversationStream): audio stream - for recording query and playing back assistant answer. - channel: authorized gRPC channel for connection to the - Google Assistant API. - deadline_sec: gRPC deadline in seconds for Google Assistant API call. - device_handler: callback for device actions. - """ - - def __init__(self, language_code, device_model_id, device_id, - conversation_stream, display, - channel, deadline_sec, device_handler, play_response=True, - on_conversation_start=None, on_conversation_end=None, - on_speech_recognized=None, on_volume_changed=None, - on_response=None): - self.language_code = language_code - self.device_model_id = device_model_id - self.device_id = device_id - self.conversation_stream = conversation_stream - self.display = display - self.play_response = play_response - - # Opaque blob provided in AssistResponse that, - # when provided in a follow-up AssistRequest, - # gives the Assistant a context marker within the current state - # of the multi-Assist()-RPC "conversation". - # This value, along with MicrophoneMode, supports a more natural - # "conversation" with the Assistant. - self.conversation_state = None - # Force reset of first conversation. - self.is_new_conversation = True - - # Create Google Assistant API gRPC client. - self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(channel) - self.deadline = deadline_sec - - self.device_handler = device_handler - self.detected_speech = None - - self.on_conversation_start = on_conversation_start - self.on_conversation_end = on_conversation_end - self.on_speech_recognized = on_speech_recognized - self.on_volume_changed = on_volume_changed - self.on_response = on_response - - def __enter__(self): - return self - - def __exit__(self, etype, e, traceback): - if e: - return False - self.conversation_stream.close() - - @staticmethod - def is_grpc_error_unavailable(e): - is_grpc_error = isinstance(e, grpc.RpcError) - if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE): - logging.error('grpc unavailable error: %s', e) - return True - return False - - @retry(reraise=True, stop=stop_after_attempt(3), - retry=retry_if_exception(is_grpc_error_unavailable)) - def assist(self): - """Send a voice request to the Assistant and playback the response. - - Returns: True if conversation should continue. - """ - continue_conversation = False - device_actions_futures = [] - - self.conversation_stream.start_recording() - if self.on_conversation_start: - self.on_conversation_start() - - logging.info('Recording audio request.') - - def iter_log_assist_requests(): - for c in self.gen_assist_requests(): - assistant_helpers.log_assist_request_without_audio(c) - yield c - logging.debug('Reached end of AssistRequest iteration.') - - # This generator yields AssistResponse proto messages - # received from the gRPC Google Assistant API. - for resp in self.assistant.Assist(iter_log_assist_requests(), self.deadline): - assistant_helpers.log_assist_response_without_audio(resp) - if resp.event_type == END_OF_UTTERANCE: - logging.info('End of audio request detected.') - logging.info('Stopping recording.') - self.conversation_stream.stop_recording() - - if self.detected_speech and self.on_speech_recognized: - self.on_speech_recognized(self.detected_speech) - - if resp.speech_results: - self.detected_speech = ' '.join( - r.transcript.strip() for r in resp.speech_results - if len(r.transcript.strip())).strip() - - logging.info('Transcript of user request: "%s".', self.detected_speech) - - if len(resp.audio_out.audio_data) > 0: - if not self.conversation_stream.playing: - self.conversation_stream.stop_recording() - - if self.play_response: - self.conversation_stream.start_playback() - logging.info('Playing assistant response.') - - if self.play_response and self.conversation_stream.playing: - self.conversation_stream.write(resp.audio_out.audio_data) - elif self.conversation_stream.playing: - self.conversation_stream.stop_playback() - - if resp.dialog_state_out.conversation_state: - conversation_state = resp.dialog_state_out.conversation_state - logging.debug('Updating conversation state.') - self.conversation_state = conversation_state - - if resp.dialog_state_out.volume_percentage != 0: - volume_percentage = resp.dialog_state_out.volume_percentage - logging.info('Setting volume to %s%%', volume_percentage) - self.conversation_stream.volume_percentage = volume_percentage - - if self.on_volume_changed: - self.on_volume_changed(volume_percentage) - - if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON: - continue_conversation = True - logging.info('Expecting follow-on query from user.') - elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE: - continue_conversation = False - - if resp.device_action.device_request_json: - device_request = json.loads( - resp.device_action.device_request_json - ) - fs = self.device_handler(device_request) - if fs: - device_actions_futures.extend(fs) - - if self.display and resp.screen_out.data: - system_browser = browser_helpers.system_browser - system_browser.display(resp.screen_out.data) - - if resp.dialog_state_out.supplemental_display_text and self.on_response: - self.on_response(resp.dialog_state_out.supplemental_display_text) - - if len(device_actions_futures): - logging.info('Waiting for device executions to complete.') - concurrent.futures.wait(device_actions_futures) - - logging.info('Finished playing assistant response.') - self.conversation_stream.stop_playback() - - if self.on_conversation_end: - self.on_conversation_end(continue_conversation) - - return continue_conversation - - def gen_assist_requests(self): - """Yields: AssistRequest messages to send to the API.""" - - config = embedded_assistant_pb2.AssistConfig( - audio_in_config=embedded_assistant_pb2.AudioInConfig( - encoding='LINEAR16', - sample_rate_hertz=self.conversation_stream.sample_rate, - ), - audio_out_config=embedded_assistant_pb2.AudioOutConfig( - encoding='LINEAR16', - sample_rate_hertz=self.conversation_stream.sample_rate, - volume_percentage=self.conversation_stream.volume_percentage, - ), - dialog_state_in=embedded_assistant_pb2.DialogStateIn( - language_code=self.language_code, - conversation_state=self.conversation_state, - is_new_conversation=self.is_new_conversation, - ), - device_config=embedded_assistant_pb2.DeviceConfig( - device_id=self.device_id, - device_model_id=self.device_model_id, - ) - ) - if self.display: - config.screen_out_config.screen_mode = PLAYING - # Continue current conversation with later requests. - self.is_new_conversation = False - # The first AssistRequest must contain the AssistConfig - # and no audio data. - yield embedded_assistant_pb2.AssistRequest(config=config) - for data in self.conversation_stream: - # Subsequent requests need audio data, but not config. - yield embedded_assistant_pb2.AssistRequest(audio_in=data) - diff --git a/platypush/plugins/assistant/google/pushtotalk/__init__.py b/platypush/plugins/assistant/google/pushtotalk/__init__.py deleted file mode 100644 index addcd0b8..00000000 --- a/platypush/plugins/assistant/google/pushtotalk/__init__.py +++ /dev/null @@ -1,330 +0,0 @@ -import json -import os -from typing import Optional, Dict, Any - -from platypush.context import get_bus, get_plugin -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - SpeechRecognizedEvent, - VolumeChangedEvent, - ResponseEvent, -) - -from platypush.message.event.google import GoogleDeviceOnOffEvent - -from platypush.plugins import action -from platypush.plugins.assistant import AssistantPlugin - - -class AssistantGooglePushtotalkPlugin(AssistantPlugin): - """ - Plugin for the Google Assistant push-to-talk API. - """ - - api_endpoint = 'embeddedassistant.googleapis.com' - grpc_deadline = 60 * 3 + 5 - device_handler = None - _default_credentials_file = os.path.join( - os.path.expanduser('~'), - '.config', - 'google-oauthlib-tool', - 'credentials.json', - ) - - _default_device_config = os.path.join( - os.path.expanduser('~'), - '.config', - 'googlesamples-assistant', - 'device_config.json', - ) - - def __init__( - self, - credentials_file=_default_credentials_file, - device_config=_default_device_config, - language='en-US', - play_response=True, - tts_plugin=None, - tts_args=None, - **kwargs - ): - """ - :param credentials_file: Path to the Google OAuth credentials file - (default: ~/.config/google-oauthlib-tool/credentials.json). - See - https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials - for instructions to get your own credentials file. - :type credentials_file: str - - :param device_config: Path to device_config.json. Register your device - (see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device) - and create a project, then run the pushtotalk.py script from - googlesamples to create your device_config.json - :type device_config: str - - :param language: Assistant language (default: en-US) - :type language: str - - :param play_response: If True (default) then the plugin will play the assistant response upon processed - response. Otherwise nothing will be played - but you may want to handle the ``ResponseEvent`` manually. - :type play_response: bool - - :param tts_plugin: Optional text-to-speech plugin to be used to process response text. - :type tts_plugin: str - - :param tts_args: Optional arguments for the TTS plugin ``say`` method. - :type tts_args: dict - """ - - import googlesamples.assistant.grpc.audio_helpers as audio_helpers - - super().__init__(**kwargs) - - self.audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE - self.audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH - self.audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE - self.audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE - self.audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE - - self.language = language - self.credentials_file = credentials_file - self.device_config = device_config - self.play_response = play_response - self.tts_plugin = tts_plugin - self.tts_args = tts_args or {} - self.assistant = None - self.interactions = [] - - with open(self.device_config) as f: - device = json.load(f) - self.device_id = device['id'] - self.device_model_id = device['model_id'] - - # Load OAuth 2.0 credentials. - try: - from google.oauth2.credentials import Credentials - from google.auth.transport.requests import Request - - with open(self.credentials_file, 'r') as f: - self.credentials = Credentials(token=None, **json.load(f)) - self.http_request = Request() - self.credentials.refresh(self.http_request) - except Exception as ex: - self.logger.error('Error loading credentials: %s', str(ex)) - self.logger.error( - 'Run google-oauthlib-tool to initialize ' 'new OAuth 2.0 credentials.' - ) - raise - - self.grpc_channel = None - self.conversation_stream = None - - def _init_assistant(self): - import googlesamples.assistant.grpc.audio_helpers as audio_helpers - from google.auth.transport.grpc import secure_authorized_channel - - self.interactions = [] - - # Create an authorized gRPC channel. - self.grpc_channel = secure_authorized_channel( - self.credentials, self.http_request, self.api_endpoint - ) - self.logger.info('Connecting to {}'.format(self.api_endpoint)) - - # Configure audio source and sink. - audio_device = None - audio_source = audio_device = audio_device or audio_helpers.SoundDeviceStream( - sample_rate=self.audio_sample_rate, - sample_width=self.audio_sample_width, - block_size=self.audio_block_size, - flush_size=self.audio_flush_size, - ) - - audio_sink = audio_device or audio_helpers.SoundDeviceStream( - sample_rate=self.audio_sample_rate, - sample_width=self.audio_sample_width, - block_size=self.audio_block_size, - flush_size=self.audio_flush_size, - ) - - # Create conversation stream with the given audio source and sink. - self.conversation_stream = audio_helpers.ConversationStream( - source=audio_source, - sink=audio_sink, - iter_size=self.audio_iter_size, - sample_width=self.audio_sample_width, - ) - - self._install_device_handlers() - - def on_conversation_start(self): - """Conversation start handler""" - - def handler(): - get_bus().post(ConversationStartEvent(assistant=self)) - - return handler - - def on_conversation_end(self): - """Conversation end handler""" - - def handler(with_follow_on_turn): - get_bus().post( - ConversationEndEvent( - assistant=self, with_follow_on_turn=with_follow_on_turn - ) - ) - - return handler - - def on_speech_recognized(self): - """Speech recognized handler""" - - def handler(phrase): - get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) - self.interactions.append({'request': phrase}) - - return handler - - def on_volume_changed(self): - """Volume changed event""" - - def handler(volume): - get_bus().post(VolumeChangedEvent(assistant=self, volume=volume)) - - return handler - - def on_response(self): - """Response handler""" - - def handler(response): - get_bus().post(ResponseEvent(assistant=self, response_text=response)) - - if not self.interactions: - self.interactions.append({'response': response}) - else: - self.interactions[-1]['response'] = response - - if self.tts_plugin: - tts = get_plugin(self.tts_plugin) - tts.say(response, **self.tts_args) - - return handler - - @action - def start_conversation( - self, - *_, - language: Optional[str] = None, - tts_plugin: Optional[str] = None, - tts_args: Optional[Dict[str, Any]] = None, - **__ - ): - """ - Start a conversation - - :param language: Language code override (default: default configured language). - :param tts_plugin: Optional text-to-speech plugin to be used for rendering text. - :param tts_args: Optional arguments for the TTS plugin say method. - :returns: A list of the interactions that happen within the conversation. - - .. code-block:: json - - [ - { - "request": "request 1", - "response": "response 1" - }, - { - "request": "request 2", - "response": "response 2" - } - ] - - """ - - from platypush.plugins.assistant.google.lib import SampleAssistant - - self.tts_plugin = tts_plugin - self.tts_args = tts_args - language = language or self.language - play_response = False if self.tts_plugin else self.play_response - - self._init_assistant() - self.on_conversation_start() - - with SampleAssistant( - language_code=language, - device_model_id=self.device_model_id, - device_id=self.device_id, - conversation_stream=self.conversation_stream, - display=None, - channel=self.grpc_channel, - deadline_sec=self.grpc_deadline, - play_response=play_response, - device_handler=self.device_handler, - on_conversation_start=self.on_conversation_start(), - on_conversation_end=self.on_conversation_end(), - on_volume_changed=self.on_volume_changed(), - on_response=self.on_response(), - on_speech_recognized=self.on_speech_recognized(), - ) as self.assistant: - continue_conversation = True - - while continue_conversation: - try: - continue_conversation = self.assistant.assist() - except Exception as e: - self.logger.warning( - 'Unhandled assistant exception: {}'.format(str(e)) - ) - self.logger.exception(e) - self._init_assistant() - - return self.interactions - - @action - def stop_conversation(self): - if self.assistant: - self.assistant.play_response = False - - if self.conversation_stream: - self.conversation_stream.stop_playback() - self.conversation_stream.stop_recording() - - get_bus().post(ConversationEndEvent(assistant=self)) - - @action - def set_mic_mute(self, muted: bool = True): - """ - Programmatically mute/unmute the microphone. - - :param muted: Set to True or False. - """ - if not self.conversation_stream: - self.logger.warning('The assistant is not running') - return - - if muted: - self.conversation_stream.stop_recording() - else: - self.conversation_stream.start_recording() - - def _install_device_handlers(self): - import googlesamples.assistant.grpc.device_helpers as device_helpers - - self.device_handler = device_helpers.DeviceRequestHandler(self.device_id) - - @self.device_handler.command('action.devices.commands.OnOff') - def handler(on): # type: ignore - get_bus().post( - GoogleDeviceOnOffEvent( - device_id=self.device_id, - device_model_id=self.device_model_id, - on=on, - ) - ) - - -# vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/pushtotalk/manifest.yaml b/platypush/plugins/assistant/google/pushtotalk/manifest.yaml deleted file mode 100644 index dfb859d2..00000000 --- a/platypush/plugins/assistant/google/pushtotalk/manifest.yaml +++ /dev/null @@ -1,27 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - install: - apk: - - py3-tenacity - - py3-google-auth - apt: - - python3-tenacity - - python3-google-auth - dnf: - - python-tenacity - - python-google-auth - pacman: - - python-tenacity - - python-google-auth - pip: - - tenacity - - google-assistant-sdk - - google-auth - package: platypush.plugins.assistant.google.pushtotalk - type: plugin -- 2.45.1 From 645e8c8f7701d98f7beb3a067e8cc5e9fde3cef8 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 22 Oct 2023 21:53:15 +0200 Subject: [PATCH 3/9] Added updated configuration snippet for `assistant.google` plugin. --- platypush/config/config.yaml | 29 +++++++++++++++++++ .../plugins/assistant/google/__init__.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/platypush/config/config.yaml b/platypush/config/config.yaml index df1ec68a..b0005cb9 100644 --- a/platypush/config/config.yaml +++ b/platypush/config/config.yaml @@ -280,6 +280,35 @@ backend.http: # - platypush/tests ### +### +# # Example configuration of a voice assistant. +# # Several voice assistant plugins and engines are available - Google +# # Assistant, Alexa, DeepSpeech, Picovoice etc. +# # +# # The Google Assistant is probably the most straightforward to configure and +# # the richest in terms of features provided out-of-the-box and speech +# # detection quality, while others may require further tinkering, may perform +# # worse than the Google model, and/or may run models on-device which could not +# # be within reach for some machines. +# # +# # Check the documentation of the `assistant.google` plugin for instructions on +# # how to get a credentials file that you can use with a custom assistant +# # installation. +# # +# # Note however that the Google Assistant plugin leverages the +# # `google-assistant-library`, which has been deprecated a while ago by Google +# # and it's unlikely to receive updates (although it still works and I'd still +# # expect it to work). +# +# assistant.google: +# # Path to your credentials file (by default it will look either under +# # ~/.config/google-oauthlib-tool/credentials.json or +# # /credentials/google/assistant.json +# # credentials_file: ~/credentials/assistant.json +# # If specified, then this sound will be played when a conversation starts +# # conversation_start_sound: ~/sounds/assistant-start.mp3 +### + ### # # Example configuration of music.mpd plugin, a plugin to interact with MPD and # # Mopidy music server instances. See diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 5623d278..07c02d06 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -102,7 +102,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): @property def credentials_file(self) -> str: if self._credentials_file: - return self._credentials_file + return os.path.abspath(os.path.expanduser(self._credentials_file)) f = None for default_file in self._default_credentials_files: -- 2.45.1 From 2c8b06e47114c8d9e64bc8fe3d2495766fcea595 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 22 Oct 2023 22:33:22 +0200 Subject: [PATCH 4/9] Remove Snowboy integration. It hurts to see it go, as I really believed in this project. But the website of the project went away in 2020, the Github project hasn't seen any activity since 2021, and the fork that is supposed to be used as a replacement for training .pmdl models hasn't been updated since 2021 - and it only supports Python 2 on Ubuntu 16.04 or 18.04. One day I may dedicate some efforts to bring Snowboy back to life, but until then it's definitely not in a state where it's usable for a Platypush integration. --- docs/source/backends.rst | 1 - .../platypush/backend/assistant.snowboy.rst | 6 - platypush/backend/assistant/__init__.py | 39 ---- .../backend/assistant/snowboy/__init__.py | 184 ------------------ .../backend/assistant/snowboy/manifest.yaml | 9 - setup.py | 3 - 6 files changed, 242 deletions(-) delete mode 100644 docs/source/platypush/backend/assistant.snowboy.rst delete mode 100644 platypush/backend/assistant/__init__.py delete mode 100644 platypush/backend/assistant/snowboy/__init__.py delete mode 100644 platypush/backend/assistant/snowboy/manifest.yaml diff --git a/docs/source/backends.rst b/docs/source/backends.rst index 44f1cbb8..0523b397 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -8,7 +8,6 @@ Backends platypush/backend/adafruit.io.rst platypush/backend/alarm.rst - platypush/backend/assistant.snowboy.rst platypush/backend/button.flic.rst platypush/backend/camera.pi.rst platypush/backend/chat.telegram.rst diff --git a/docs/source/platypush/backend/assistant.snowboy.rst b/docs/source/platypush/backend/assistant.snowboy.rst deleted file mode 100644 index ac8b265a..00000000 --- a/docs/source/platypush/backend/assistant.snowboy.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.snowboy`` -======================================= - -.. automodule:: platypush.backend.assistant.snowboy - :members: - diff --git a/platypush/backend/assistant/__init__.py b/platypush/backend/assistant/__init__.py deleted file mode 100644 index 37589fd9..00000000 --- a/platypush/backend/assistant/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from abc import ABC -import threading -from typing import Optional, Dict, Any, Tuple - -from platypush.backend import Backend -from platypush.context import get_plugin -from platypush.plugins.tts import TtsPlugin - - -class AssistantBackend(Backend): - def __init__(self, tts_plugin: Optional[str] = None, tts_args: Optional[Dict[str, Any]] = None, **kwargs): - """ - Default assistant backend constructor. - - :param tts_plugin: If set, and if the assistant returns the processed response as text, then the processed - response will be played through the selected text-to-speech plugin (can be e.g. "``tts``", - "``tts.google``" or any other implementation of :class:`platypush.plugins.tts.TtsPlugin`). - :param tts_args: Extra parameters to pass to the ``say`` method of the selected TTS plugin (e.g. - language, voice or gender). - """ - super().__init__(**kwargs) - self._detection_paused = threading.Event() - self.tts_plugin = tts_plugin - self.tts_args = tts_args or {} - - def pause_detection(self): - self._detection_paused.set() - - def resume_detection(self): - self._detection_paused.clear() - - def is_detecting(self): - return not self._detection_paused.is_set() - - def _get_tts_plugin(self) -> Tuple[Optional[TtsPlugin], Dict[str, Any]]: - return get_plugin(self.tts_plugin) if self.tts_plugin else None, self.tts_args - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/snowboy/__init__.py b/platypush/backend/assistant/snowboy/__init__.py deleted file mode 100644 index 6d4bd4f1..00000000 --- a/platypush/backend/assistant/snowboy/__init__.py +++ /dev/null @@ -1,184 +0,0 @@ -import os -import threading - -from platypush.backend.assistant import AssistantBackend -from platypush.context import get_plugin -from platypush.message.event.assistant import HotwordDetectedEvent - - -class AssistantSnowboyBackend(AssistantBackend): - """ - Backend for detecting custom voice hotwords through Snowboy. The purpose of - this component is only to detect the hotword specified in your Snowboy voice - model. If you want to trigger proper assistant conversations or custom - speech recognition, you should create a hook in your configuration on - HotwordDetectedEvent to trigger the conversation on whichever assistant - plugin you're using (Google, Alexa...) - - Manual installation for snowboy and its Python bindings if the installation via package fails:: - - $ [sudo] apt-get install libatlas-base-dev swig - $ [sudo] pip install pyaudio - $ git clone https://github.com/Kitt-AI/snowboy - $ cd snowboy/swig/Python3 - $ make - $ cd ../.. - $ python3 setup.py build - $ [sudo] python setup.py install - - You will also need a voice model for the hotword detection. You can find - some under the ``resources/models`` directory of the Snowboy repository, - or train/download other models from https://snowboy.kitt.ai. - """ - - def __init__(self, models, audio_gain=1.0, **kwargs): - """ - :param models: Map (name -> configuration) of voice models to be used by - the assistant. See https://snowboy.kitt.ai/ for training/downloading - models. Sample format:: - - ok_google: # Hotword model name - voice_model_file: /path/models/OK Google.pmdl # Voice model file location - sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5) - assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google - # push-to-talk assistant plugin (optional) - assistant_language: en-US # The assistant will conversate in English when this hotword is - detected (optional) - detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional) - - ciao_google: # Hotword model name - voice_model_file: /path/models/Ciao Google.pmdl # Voice model file location - sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5) - assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google - # push-to-talk assistant plugin (optional) - assistant_language: it-IT # The assistant will conversate in Italian when this hotword is - # detected (optional) - detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional) - - :type models: dict - - :param audio_gain: Audio gain, between 0 and 1. Default: 1 - :type audio_gain: float - """ - - try: - import snowboydecoder - except ImportError: - import snowboy.snowboydecoder as snowboydecoder - - super().__init__(**kwargs) - - self.models = {} - self._init_models(models) - self.audio_gain = audio_gain - - self.detector = snowboydecoder.HotwordDetector( - [model['voice_model_file'] for model in self.models.values()], - sensitivity=[model['sensitivity'] for model in self.models.values()], - audio_gain=self.audio_gain, - ) - - self.logger.info( - 'Initialized Snowboy hotword detection with {} voice model configurations'.format( - len(self.models) - ) - ) - - def _init_models(self, models): - if not models: - raise AttributeError('Please specify at least one voice model') - - self.models = {} - for name, conf in models.items(): - if name in self.models: - raise AttributeError('Duplicate model key {}'.format(name)) - - model_file = conf.get('voice_model_file') - detect_sound = conf.get('detect_sound') - - if not model_file: - raise AttributeError( - 'No voice_model_file specified for model {}'.format(name) - ) - - model_file = os.path.abspath(os.path.expanduser(model_file)) - assistant_plugin_name = conf.get('assistant_plugin') - - if detect_sound: - detect_sound = os.path.abspath(os.path.expanduser(detect_sound)) - - if not os.path.isfile(model_file): - raise FileNotFoundError( - 'Voice model file {} does not exist or it not a regular file'.format( - model_file - ) - ) - - self.models[name] = { - 'voice_model_file': model_file, - 'sensitivity': conf.get('sensitivity', 0.5), - 'detect_sound': detect_sound, - 'assistant_plugin': get_plugin(assistant_plugin_name) - if assistant_plugin_name - else None, - 'assistant_language': conf.get('assistant_language'), - 'tts_plugin': conf.get('tts_plugin'), - 'tts_args': conf.get('tts_args', {}), - } - - def hotword_detected(self, hotword): - """ - Callback called on hotword detection - """ - try: - import snowboydecoder - except ImportError: - import snowboy.snowboydecoder as snowboydecoder - - def sound_thread(sound): - snowboydecoder.play_audio_file(sound) - - def callback(): - if not self.is_detecting(): - self.logger.info( - 'Hotword detected but assistant response currently paused' - ) - return - - self.bus.post(HotwordDetectedEvent(hotword=hotword)) - model = self.models[hotword] - - detect_sound = model.get('detect_sound') - assistant_plugin = model.get('assistant_plugin') - assistant_language = model.get('assistant_language') - tts_plugin = model.get('tts_plugin') - tts_args = model.get('tts_args') - - if detect_sound: - threading.Thread(target=sound_thread, args=(detect_sound,)).start() - - if assistant_plugin: - assistant_plugin.start_conversation( - language=assistant_language, - tts_plugin=tts_plugin, - tts_args=tts_args, - ) - - return callback - - def on_stop(self): - super().on_stop() - if self.detector: - self.detector.terminate() - self.detector = None - - def run(self): - super().run() - self.detector.start( - detected_callback=[ - self.hotword_detected(hotword) for hotword in self.models.keys() - ] - ) - - -# vim:sw=4:ts=4:et: diff --git a/platypush/backend/assistant/snowboy/manifest.yaml b/platypush/backend/assistant/snowboy/manifest.yaml deleted file mode 100644 index 1a3fcebc..00000000 --- a/platypush/backend/assistant/snowboy/manifest.yaml +++ /dev/null @@ -1,9 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.HotwordDetectedEvent: whenever the hotword has - been detected - install: - pip: - - snowboy - package: platypush.backend.assistant.snowboy - type: backend diff --git a/setup.py b/setup.py index e9efd0cb..fee1bf08 100755 --- a/setup.py +++ b/setup.py @@ -139,9 +139,6 @@ setup( ], # Support for Last.FM scrobbler plugin 'lastfm': ['pylast'], - # Support for custom hotword detection - 'hotword': ['snowboy'], - 'snowboy': ['snowboy'], # Support for real-time MIDI events 'midi': ['rtmidi'], # Support for RaspberryPi GPIO -- 2.45.1 From fb017a8b35dee1a0ddd918ac87e1243d10d9e9cc Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Sun, 22 Oct 2023 22:49:41 +0200 Subject: [PATCH 5/9] [`assistant.google`] Added doc on how to configure audio devices. --- .../plugins/assistant/google/__init__.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 07c02d06..89ed6a58 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -32,6 +32,37 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): device. It requires you to have an audio microphone and a speaker connected to the device. + If you have multiple sound devices, you can specify which one(s) to use for + input and output through a ``~/.asoundrc`` configuration file like this: + + .. code-block:: text + + pcm.!default { + type asym + playback.pcm { + type plug + slave.pcm "hw:0,0" + } + capture.pcm { + type plug + slave.pcm "hw:1,0" + } + } + + You can use ``aplay -l`` and ``arecord -l`` to respectively list the + detected audio output and input devices with their indices. + + If you are using PulseAudio instead of bare ALSA, then you can: + + 1. Use the ``pavucontrol`` (GUI) tool to select the audio input and + output devices and volumes for the assistant. + 2. Use a program like ``pamix`` (ncurses) or ``pamixer`` (CLI). + 3. Run the ``pactl list sources`` and ``pactl list sinks`` commands to + respectively list the detected audio input and output devices. Take + note of their name, and specify which ones the assistant should use + by starting the application with the right ``PULSE_SOURCE`` and + ``PULSE_SINK`` environment variables. + .. warning:: The Google Assistant library used by this backend has been deprecated by Google: https://developers.google.com/assistant/sdk/reference/library/python/. -- 2.45.1 From 4c0e3a16b6be98fcc8f15ddea688adc7ec99fa88 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 23 Oct 2023 00:23:10 +0200 Subject: [PATCH 6/9] Removed Alexa/AVS integration. The integration was based on my old fork of the AVS service, which is no longer functional given the changes the the Amazon's backend side. A new `avs-device-sdk` is now available, but it seems that it requires lengthy compilation processes which are RaspberryPi-specific. Further investigation is needed for a new Alexa plugin - see #334. --- .../platypush/plugins/assistant.echo.rst | 6 - docs/source/plugins.rst | 1 - platypush/plugins/assistant/echo/__init__.py | 133 ------------------ .../plugins/assistant/echo/manifest.yaml | 13 -- setup.py | 2 - 5 files changed, 155 deletions(-) delete mode 100644 docs/source/platypush/plugins/assistant.echo.rst delete mode 100644 platypush/plugins/assistant/echo/__init__.py delete mode 100644 platypush/plugins/assistant/echo/manifest.yaml diff --git a/docs/source/platypush/plugins/assistant.echo.rst b/docs/source/platypush/plugins/assistant.echo.rst deleted file mode 100644 index f4002cd3..00000000 --- a/docs/source/platypush/plugins/assistant.echo.rst +++ /dev/null @@ -1,6 +0,0 @@ -``assistant.echo`` -==================================== - -.. automodule:: platypush.plugins.assistant.echo - :members: - diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index c43fd3cd..a602352f 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -10,7 +10,6 @@ Plugins platypush/plugins/alarm.rst platypush/plugins/application.rst platypush/plugins/arduino.rst - platypush/plugins/assistant.echo.rst platypush/plugins/assistant.google.rst platypush/plugins/autoremote.rst platypush/plugins/bluetooth.rst diff --git a/platypush/plugins/assistant/echo/__init__.py b/platypush/plugins/assistant/echo/__init__.py deleted file mode 100644 index c9429ba8..00000000 --- a/platypush/plugins/assistant/echo/__init__.py +++ /dev/null @@ -1,133 +0,0 @@ -import os -from typing import Optional - -from platypush.context import get_bus -from platypush.plugins import action -from platypush.plugins.assistant import AssistantPlugin - -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - SpeechRecognizedEvent, - ResponseEvent, -) - - -class AssistantEchoPlugin(AssistantPlugin): - """ - Amazon Echo/Alexa assistant plugin. - - In order to activate the Echo service on your device follow these steps: - - 1. Install avs (``pip install git+https://github.com/BlackLight/avs.git``) - 2. Run the ``alexa-auth`` script. A local webservice will start on port 3000 - 3. If a browser instance doesn't open automatically then head to http://localhost:3000 - 4. Log in to your Amazon account - 5. The required credentials will be stored to ~/.avs.json - - """ - - def __init__( - self, - avs_config_file: Optional[str] = None, - audio_device: str = 'default', - audio_player: str = 'default', - **kwargs - ): - """ - :param avs_config_file: AVS credentials file - default: ~/.avs.json. If the file doesn't exist then - an instance of the AVS authentication service will be spawned. You can login through an Amazon - account either in the spawned browser window, if available, or by opening http://your-ip:3000 - in the browser on another machine. - - :param audio_device: Name of the input audio device (default: 'default') - :param audio_player: Player to be used for audio playback (default: 'default'). - Supported values: 'mpv', 'mpg123', 'gstreamer' - """ - from avs.alexa import Alexa - from avs.config import DEFAULT_CONFIG_FILE - from avs.mic import Audio - - super().__init__(**kwargs) - - if not avs_config_file: - avs_config_file = DEFAULT_CONFIG_FILE - - if not avs_config_file or not os.path.isfile(avs_config_file): - from avs.auth import auth - - auth(None, avs_config_file) - self.logger.warning( - 'Amazon Echo assistant credentials not configured. Open http://localhost:3000 ' - + 'to authenticate this client' - ) - - self.audio_device = audio_device - self.audio_player = audio_player - self.audio = Audio(device_name=audio_device) - self.alexa = Alexa(avs_config_file, audio_player=audio_player) - self._ready = False - - self.alexa.state_listener.on_ready = self._on_ready() - self.alexa.state_listener.on_listening = self._on_listening() - self.alexa.state_listener.on_speaking = self._on_speaking() - self.alexa.state_listener.on_thinking = self._on_thinking() - self.alexa.state_listener.on_finished = self._on_finished() - self.alexa.state_listener.on_disconnected = self._on_disconnected() - - self.audio.link(self.alexa) - self.alexa.start() - - def _on_ready(self): - def _callback(): - self._ready = True - - return _callback - - def _on_listening(self): - def _callback(): - get_bus().post(ConversationStartEvent(assistant=self)) - - return _callback - - def _on_speaking(self): - def _callback(): - # AVS doesn't provide a way to access the response text - get_bus().post(ResponseEvent(assistant=self, response_text='')) - - return _callback - - def _on_finished(self): - def _callback(): - get_bus().post(ConversationEndEvent(assistant=self)) - - return _callback - - def _on_disconnected(self): - def _callback(): - self._ready = False - - return _callback - - def _on_thinking(self): - def _callback(): - # AVS doesn't provide a way to access the detected text - get_bus().post(SpeechRecognizedEvent(assistant=self, phrase='')) - - return _callback - - @action - def start_conversation(self, **_): - if not self._ready: - raise RuntimeError('Echo assistant not ready') - - self.audio.start() - self.alexa.listen() - - @action - def stop_conversation(self): - self.audio.stop() - self._on_finished()() - - -# vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/echo/manifest.yaml b/platypush/plugins/assistant/echo/manifest.yaml deleted file mode 100644 index b19df7be..00000000 --- a/platypush/plugins/assistant/echo/manifest.yaml +++ /dev/null @@ -1,13 +0,0 @@ -manifest: - events: - platypush.message.event.assistant.ConversationEndEvent: when a new conversation - ends - platypush.message.event.assistant.ConversationStartEvent: when a new conversation - starts - platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command - is recognized - install: - pip: - - avs - package: platypush.plugins.assistant.echo - type: plugin diff --git a/setup.py b/setup.py index fee1bf08..129ee380 100755 --- a/setup.py +++ b/setup.py @@ -190,8 +190,6 @@ setup( 'flic': [ 'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master' ], - # Support for Alexa/Echo plugin - 'alexa': ['avs @ https://github.com/BlackLight/avs/tarball/master'], # Support for Bluetooth devices 'bluetooth': [ 'bleak', -- 2.45.1 From 5c22271e88be55474f206746bc7a0f105ae8394b Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 23 Oct 2023 15:03:36 +0200 Subject: [PATCH 7/9] [UI] Expose the `connected` flag on the root element level. --- platypush/backend/http/webapp/src/App.vue | 3 +++ platypush/backend/http/webapp/src/components/Nav.vue | 1 + 2 files changed, 4 insertions(+) diff --git a/platypush/backend/http/webapp/src/App.vue b/platypush/backend/http/webapp/src/App.vue index ce8171f0..8a8699a6 100644 --- a/platypush/backend/http/webapp/src/App.vue +++ b/platypush/backend/http/webapp/src/App.vue @@ -32,6 +32,7 @@ export default { return { config: {}, userAuthenticated: false, + connected: false, pwaInstallEvent: null, } }, @@ -97,6 +98,8 @@ export default { mounted() { bus.onNotification(this.onNotification) + bus.on('connect', () => this.connected = true) + bus.on('disconnect', () => this.connected = false) }, } diff --git a/platypush/backend/http/webapp/src/components/Nav.vue b/platypush/backend/http/webapp/src/components/Nav.vue index ebdee691..f012705a 100644 --- a/platypush/backend/http/webapp/src/components/Nav.vue +++ b/platypush/backend/http/webapp/src/components/Nav.vue @@ -163,6 +163,7 @@ export default { this.collapsed = this.collapsedDefault bus.on('connect', this.setConnected(true)) bus.on('disconnect', this.setConnected(false)) + this.setConnected(this.$root.connected) }, } -- 2.45.1 From fd26f7fef337b43b232ecf62add7d7de849fa314 Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 23 Oct 2023 15:57:47 +0200 Subject: [PATCH 8/9] [UI Nav] Fixed position of the connected status dot. --- .../http/webapp/src/components/Nav.vue | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/platypush/backend/http/webapp/src/components/Nav.vue b/platypush/backend/http/webapp/src/components/Nav.vue index f012705a..10c97760 100644 --- a/platypush/backend/http/webapp/src/components/Nav.vue +++ b/platypush/backend/http/webapp/src/components/Nav.vue @@ -193,6 +193,11 @@ nav { top: 0; left: 0; z-index: 5; + + .icon.status { + top: 0.75em !important; + left: 2em; + } } } @@ -319,6 +324,10 @@ nav { } } + .icon.status { + width: 1em; + } + &.collapsed { display: flex; flex-direction: column; @@ -354,15 +363,19 @@ nav { box-shadow: none; background: $nav-toggler-collapsed-bg; - @media screen and (max-width: #{$tablet - 1px}) { - background: $nav-toggler-collapsed-mobile-bg; - color: $nav-toggler-collapsed-mobile-fg; - } - .icon.status { top: 0.75em; left: 2em; } + + @include until($tablet) { + background: $nav-toggler-collapsed-mobile-bg; + color: $nav-toggler-collapsed-mobile-fg; + + .icon.status { + top: 0.75em !important; + } + } } .footer { -- 2.45.1 From a0059583175b67021e165b3291a9bf149602b83a Mon Sep 17 00:00:00 2001 From: Fabio Manganiello Date: Mon, 23 Oct 2023 16:02:05 +0200 Subject: [PATCH 9/9] [Assistant] Added `assistant` entity type. --- .../backend/http/webapp/src/assets/icons.json | 3 + .../src/components/elements/TextPrompt.vue | 107 ++++++++ .../components/panels/Entities/Assistant.vue | 250 ++++++++++++++++++ .../src/components/panels/Entities/meta.json | 8 + platypush/entities/assistants.py | 29 ++ platypush/entities/managers/assistants.py | 78 ++++++ platypush/plugins/assistant/__init__.py | 222 +++++++++++++++- .../plugins/assistant/google/__init__.py | 116 +++----- 8 files changed, 725 insertions(+), 88 deletions(-) create mode 100644 platypush/backend/http/webapp/src/components/elements/TextPrompt.vue create mode 100644 platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue create mode 100644 platypush/entities/assistants.py create mode 100644 platypush/entities/managers/assistants.py diff --git a/platypush/backend/http/webapp/src/assets/icons.json b/platypush/backend/http/webapp/src/assets/icons.json index 6b7d050c..3a481c13 100644 --- a/platypush/backend/http/webapp/src/assets/icons.json +++ b/platypush/backend/http/webapp/src/assets/icons.json @@ -2,6 +2,9 @@ "arduino": { "class": "fas fa-microchip" }, + "assistant.google": { + "class": "fas fa-microphone-lines" + }, "bluetooth": { "class": "fab fa-bluetooth" }, diff --git a/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue b/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue new file mode 100644 index 00000000..e68aee55 --- /dev/null +++ b/platypush/backend/http/webapp/src/components/elements/TextPrompt.vue @@ -0,0 +1,107 @@ + + + + + diff --git a/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue b/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue new file mode 100644 index 00000000..90818d82 --- /dev/null +++ b/platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue @@ -0,0 +1,250 @@ + + + + + diff --git a/platypush/backend/http/webapp/src/components/panels/Entities/meta.json b/platypush/backend/http/webapp/src/components/panels/Entities/meta.json index 41a050a0..fb7fd046 100644 --- a/platypush/backend/http/webapp/src/components/panels/Entities/meta.json +++ b/platypush/backend/http/webapp/src/components/panels/Entities/meta.json @@ -1,4 +1,12 @@ { + "assistant": { + "name": "Assistant", + "name_plural": "Assistants", + "icon": { + "class": "fas fa-microphone-lines" + } + }, + "battery": { "name": "Battery", "name_plural": "Batteries", diff --git a/platypush/entities/assistants.py b/platypush/entities/assistants.py new file mode 100644 index 00000000..db7bcff2 --- /dev/null +++ b/platypush/entities/assistants.py @@ -0,0 +1,29 @@ +from sqlalchemy import Column, Integer, ForeignKey, Boolean, String + +from platypush.common.db import is_defined + +from . import Entity + + +if not is_defined('assistant'): + + class Assistant(Entity): + """ + Base class for voice assistant entities. + """ + + __tablename__ = 'assistant' + + id = Column( + Integer, ForeignKey(Entity.id, ondelete='CASCADE'), primary_key=True + ) + last_query = Column(String) + last_response = Column(String) + conversation_running = Column(Boolean) + is_muted = Column(Boolean, default=False) + is_detecting = Column(Boolean, default=True) + + __table_args__ = {'extend_existing': True} + __mapper_args__ = { + 'polymorphic_identity': __tablename__, + } diff --git a/platypush/entities/managers/assistants.py b/platypush/entities/managers/assistants.py new file mode 100644 index 00000000..ae412837 --- /dev/null +++ b/platypush/entities/managers/assistants.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod + +from . import EntityManager + + +class AssistantEntityManager(EntityManager, ABC): + """ + Base class for voice assistant integrations that support entity management. + """ + + @abstractmethod + def start_conversation(self, *args, **kwargs): + """ + Programmatically starts a conversation. + """ + raise NotImplementedError() + + @abstractmethod + def stop_conversation(self, *args, **kwargs): + """ + Programmatically stops a conversation. + """ + raise NotImplementedError() + + @abstractmethod + def is_muted(self, *args, **kwargs) -> bool: + """ + :return: True if the microphone is muted, False otherwise. + """ + raise NotImplementedError() + + @abstractmethod + def mute(self, *args, **kwargs): + """ + Mute the microphone. + """ + raise NotImplementedError() + + @abstractmethod + def unmute(self, *args, **kwargs): + """ + Unmute the microphone. + """ + raise NotImplementedError() + + def toggle_mute(self, *_, **__): + """ + Toggle the mute state of the microphone. + """ + return self.mute() if self.is_muted() else self.unmute() + + @abstractmethod + def pause_detection(self, *args, **kwargs): + """ + Put the assistant on pause. No new conversation events will be triggered. + """ + raise NotImplementedError() + + @abstractmethod + def resume_detection(self, *args, **kwargs): + """ + Resume the assistant hotword detection from a paused state. + """ + raise NotImplementedError() + + @abstractmethod + def is_detecting(self, *args, **kwargs) -> bool: + """ + :return: True if the asistant is detecting, False otherwise. + """ + raise NotImplementedError() + + @abstractmethod + def send_text_query(self, *args, **kwargs): + """ + Send a text query to the assistant. + """ + raise NotImplementedError() diff --git a/platypush/plugins/assistant/__init__.py b/platypush/plugins/assistant/__init__.py index 40f35d32..6291b111 100644 --- a/platypush/plugins/assistant/__init__.py +++ b/platypush/plugins/assistant/__init__.py @@ -1,21 +1,72 @@ from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass +from enum import Enum +import os from threading import Event -from typing import Any, Dict, Optional +from typing import Any, Collection, Dict, Optional -from platypush.context import get_plugin +from platypush.context import get_bus, get_plugin +from platypush.entities.assistants import Assistant +from platypush.entities.managers.assistants import AssistantEntityManager +from platypush.message.event.assistant import ( + AssistantEvent, + ConversationStartEvent, + ConversationEndEvent, + ConversationTimeoutEvent, + ResponseEvent, + NoResponseEvent, + SpeechRecognizedEvent, + AlarmStartedEvent, + AlarmEndEvent, + TimerStartedEvent, + TimerEndEvent, + AlertStartedEvent, + AlertEndEvent, + MicMutedEvent, + MicUnmutedEvent, +) from platypush.plugins import Plugin, action +from platypush.utils import get_plugin_name_by_class -class AssistantPlugin(ABC, Plugin): +@dataclass +class AlertType(Enum): + """ + Enum representing the type of an alert. + """ + + ALARM = 'alarm' + TIMER = 'timer' + ALERT = 'alert' + + +@dataclass +class AssistantState: + """ + Dataclass representing the state of an assistant. + """ + + last_query: Optional[str] = None + last_response: Optional[str] = None + conversation_running: bool = False + is_muted: bool = False + is_detecting: bool = True + alert_state: Optional[str] = None + + +class AssistantPlugin(Plugin, AssistantEntityManager, ABC): """ Base class for assistant plugins. """ + _entity_name = 'Assistant' + def __init__( self, *args, tts_plugin: Optional[str] = None, tts_plugin_args: Optional[Dict[str, Any]] = None, + conversation_start_sound: Optional[str] = None, **kwargs ): """ @@ -25,11 +76,37 @@ class AssistantPlugin(ABC, Plugin): :param tts_plugin_args: Optional arguments to be passed to the TTS ``say`` action, if ``tts_plugin`` is set. + + :param conversation_start_sound: If set, the assistant will play this + audio file when it detects a speech. The sound file will be played + on the default audio output device. If not set, the assistant won't + play any sound when it detects a speech. """ super().__init__(*args, **kwargs) self.tts_plugin = tts_plugin self.tts_plugin_args = tts_plugin_args or {} + if conversation_start_sound: + self._conversation_start_sound = os.path.abspath( + os.path.expanduser(conversation_start_sound) + ) + self._detection_paused = Event() + self._conversation_running = Event() + self._is_muted = False + self._last_query: Optional[str] = None + self._last_response: Optional[str] = None + self._cur_alert_type: Optional[AlertType] = None + + @property + def _state(self) -> AssistantState: + return AssistantState( + last_query=self._last_query, + last_response=self._last_response, + conversation_running=self._conversation_running.is_set(), + is_muted=self._is_muted, + is_detecting=not self._detection_paused.is_set(), + alert_state=self._cur_alert_type.value if self._cur_alert_type else None, + ) @abstractmethod def start_conversation(self, *_, **__): @@ -46,31 +123,164 @@ class AssistantPlugin(ABC, Plugin): raise NotImplementedError @action - def pause_detection(self): + def pause_detection(self, *_, **__): """ Put the assistant on pause. No new conversation events will be triggered. """ self._detection_paused.set() @action - def resume_detection(self): + def resume_detection(self, *_, **__): """ Resume the assistant hotword detection from a paused state. """ self._detection_paused.clear() @action - def is_detecting(self) -> bool: + def is_detecting(self, *_, **__) -> bool: """ :return: True if the asistant is detecting, False otherwise. """ return not self._detection_paused.is_set() + @action + def is_muted(self, *_, **__) -> bool: + """ + :return: True if the microphone is muted, False otherwise. + """ + return self._is_muted + + @action + def status(self, *_, **__): + """ + :return: The current assistant status: + + .. code-block:: json + + { + "last_query": "What time is it?", + "last_response": "It's 10:30 AM", + "conversation_running": true, + "is_muted": false, + "is_detecting": true + } + + """ + self.publish_entities([self]) + return asdict(self._state) + def _get_tts_plugin(self): if not self.tts_plugin: return None return get_plugin(self.tts_plugin) + def _play_conversation_start_sound(self): + if not self._conversation_start_sound: + return + + audio = get_plugin('sound') + if not audio: + self.logger.warning( + 'Unable to play conversation start sound: sound plugin not found' + ) + return + + audio.play(self._conversation_start_sound) + + def _send_event(self, event: AssistantEvent): + self.publish_entities([self]) + get_bus().post(event) + + def _on_conversation_start(self): + self._last_response = None + self._last_query = None + self._conversation_running.set() + self._send_event(ConversationStartEvent(assistant=self)) + self._play_conversation_start_sound() + + def _on_conversation_end(self): + self._conversation_running.clear() + self._send_event(ConversationEndEvent(assistant=self)) + + def _on_conversation_timeout(self): + self._last_response = None + self._last_query = None + self._conversation_running.clear() + self._send_event(ConversationTimeoutEvent(assistant=self)) + + def _on_no_response(self): + self._last_response = None + self._conversation_running.clear() + self._send_event(NoResponseEvent(assistant=self)) + + def _on_reponse_rendered(self, text: Optional[str]): + self._last_response = text + self._send_event(ResponseEvent(assistant=self, response_text=text)) + tts = self._get_tts_plugin() + + if tts and text: + self.stop_conversation() + tts.say(text=text, **self.tts_plugin_args) + + def _on_speech_recognized(self, phrase: Optional[str]): + phrase = (phrase or '').lower().strip() + self._last_query = phrase + self._send_event(SpeechRecognizedEvent(assistant=self, phrase=phrase)) + + def _on_alarm_start(self): + self._cur_alert_type = AlertType.ALARM + self._send_event(AlarmStartedEvent(assistant=self)) + + def _on_alarm_end(self): + self._cur_alert_type = None + self._send_event(AlarmEndEvent(assistant=self)) + + def _on_timer_start(self): + self._cur_alert_type = AlertType.TIMER + self._send_event(TimerStartedEvent(assistant=self)) + + def _on_timer_end(self): + self._cur_alert_type = None + self._send_event(TimerEndEvent(assistant=self)) + + def _on_alert_start(self): + self._cur_alert_type = AlertType.ALERT + self._send_event(AlertStartedEvent(assistant=self)) + + def _on_alert_end(self): + self._cur_alert_type = None + self._send_event(AlertEndEvent(assistant=self)) + + def _on_mute(self): + self._is_muted = True + self._send_event(MicMutedEvent(assistant=self)) + + def _on_unmute(self): + self._is_muted = False + self._send_event(MicUnmutedEvent(assistant=self)) + + def _on_mute_changed(self, value: bool): + if value: + self._on_mute() + else: + self._on_unmute() + + def transform_entities(self, entities: Collection['AssistantPlugin']): + return super().transform_entities( + [ + Assistant( + external_id=get_plugin_name_by_class(type(dev)), + name=self._entity_name, + last_query=dev._state.last_query, + last_response=dev._state.last_response, + conversation_running=dev._state.conversation_running, + is_muted=dev._state.is_muted, + is_detecting=dev._state.is_detecting, + ) + for dev in (entities or []) + ] + ) + # vim:sw=4:ts=4:et: diff --git a/platypush/plugins/assistant/google/__init__.py b/platypush/plugins/assistant/google/__init__.py index 89ed6a58..52758cb5 100644 --- a/platypush/plugins/assistant/google/__init__.py +++ b/platypush/plugins/assistant/google/__init__.py @@ -3,23 +3,6 @@ import os from typing import Optional from platypush.config import Config -from platypush.context import get_bus, get_plugin -from platypush.message.event.assistant import ( - ConversationStartEvent, - ConversationEndEvent, - ConversationTimeoutEvent, - ResponseEvent, - NoResponseEvent, - SpeechRecognizedEvent, - AlarmStartedEvent, - AlarmEndEvent, - TimerStartedEvent, - TimerEndEvent, - AlertStartedEvent, - AlertEndEvent, - MicMutedEvent, - MicUnmutedEvent, -) from platypush.plugins import RunnablePlugin, action from platypush.plugins.assistant import AssistantPlugin @@ -79,6 +62,8 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): the automated ways fail. """ + _entity_name = 'Google Assistant' + _default_credentials_files = ( os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'), os.path.join( @@ -90,7 +75,6 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): self, credentials_file: Optional[str] = None, device_model_id: str = 'Platypush', - conversation_start_sound: Optional[str] = None, **kwargs, ): """ @@ -109,11 +93,6 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): :param device_model_id: The device model ID that identifies the device where the assistant is running (default: Platypush). It can be a custom string. - - :param conversation_start_sound: If set, the assistant will play this - audio file when it detects a speech. The sound file will be played - on the default audio output device. If not set, the assistant won't - play any sound when it detects a speech. """ super().__init__(**kwargs) @@ -121,13 +100,6 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): self.device_model_id = device_model_id self.credentials = None self._assistant = None - self._is_muted = False - - if conversation_start_sound: - self._conversation_start_sound = os.path.abspath( - os.path.expanduser(conversation_start_sound) - ) - self.logger.info('Initialized Google Assistant plugin') @property @@ -152,79 +124,54 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): return self._assistant - def _play_conversation_start_sound(self): - if not self._conversation_start_sound: - return - - audio = get_plugin('sound') - if not audio: - self.logger.warning( - 'Unable to play conversation start sound: sound plugin not found' - ) - return - - audio.play(self._conversation_start_sound) - def _process_event(self, event): from google.assistant.library.event import EventType, AlertType self.logger.info('Received assistant event: %s', event) if event.type == EventType.ON_CONVERSATION_TURN_STARTED: - get_bus().post(ConversationStartEvent(assistant=self)) - self._play_conversation_start_sound() + self._on_conversation_start() elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED: if not event.args.get('with_follow_on_turn'): - get_bus().post(ConversationEndEvent(assistant=self)) + self._on_conversation_end() elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT: - get_bus().post(ConversationTimeoutEvent(assistant=self)) + self._on_conversation_timeout() elif event.type == EventType.ON_NO_RESPONSE: - get_bus().post(NoResponseEvent(assistant=self)) + self._on_no_response() elif ( hasattr(EventType, 'ON_RENDER_RESPONSE') and event.type == EventType.ON_RENDER_RESPONSE ): - get_bus().post( - ResponseEvent(assistant=self, response_text=event.args.get('text')) - ) - tts = self._get_tts_plugin() - - if tts and event.args.get('text'): - self.stop_conversation() - tts.say(text=event.args['text'], **self.tts_plugin_args) + self._on_reponse_rendered(event.args.get('text')) elif ( hasattr(EventType, 'ON_RESPONDING_STARTED') and event.type == EventType.ON_RESPONDING_STARTED - and event.args.get('is_error_response', False) is True + and event.args.get('is_error_response') is True ): - self.logger.warning('Assistant response error') + self.logger.warning('Assistant response error: %s', json.dumps(event.args)) elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED: - phrase = event.args['text'].lower().strip() - self.logger.info('Speech recognized: %s', phrase) - get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase)) + self._on_speech_recognized(event.args.get('text')) elif event.type == EventType.ON_ALERT_STARTED: if event.args.get('alert_type') == AlertType.ALARM: - get_bus().post(AlarmStartedEvent(assistant=self)) + self._on_alarm_start() elif event.args.get('alert_type') == AlertType.TIMER: - get_bus().post(TimerStartedEvent(assistant=self)) + self._on_timer_start() else: - get_bus().post(AlertStartedEvent(assistant=self)) + self._on_alert_start() elif event.type == EventType.ON_ALERT_FINISHED: if event.args.get('alert_type') == AlertType.ALARM: - get_bus().post(AlarmEndEvent(assistant=self)) + self._on_alarm_end() elif event.args.get('alert_type') == AlertType.TIMER: - get_bus().post(TimerEndEvent(assistant=self)) + self._on_timer_end() else: - get_bus().post(AlertEndEvent(assistant=self)) + self._on_alert_end() elif event.type == EventType.ON_ASSISTANT_ERROR: if event.args.get('is_fatal'): raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}') self.logger.warning('Assistant error: %s', json.dumps(event.args)) elif event.type == EventType.ON_MUTED_CHANGED: - self._is_muted = event.args.get('is_muted') - event = MicMutedEvent() if self._is_muted else MicUnmutedEvent() - get_bus().post(event) + self._on_mute_changed(event.args.get('is_muted', False)) @action def start_conversation(self, *_, **__): @@ -235,7 +182,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): self.assistant.start_conversation() @action - def stop_conversation(self): + def stop_conversation(self, *_, **__): """ Programmatically stop a running conversation with the assistant """ @@ -243,20 +190,20 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): self.assistant.stop_conversation() @action - def mute(self): + def mute(self, *_, **__): """ Mute the microphone. Alias for :meth:`.set_mic_mute` with ``muted=True``. """ - return self.set_mic_mute(muted=True) + self.set_mic_mute(muted=True) @action - def unmute(self): + def unmute(self, *_, **__): """ Unmute the microphone. Alias for :meth:`.set_mic_mute` with ``muted=False``. """ - return self.set_mic_mute(muted=False) + self.set_mic_mute(muted=False) @action def set_mic_mute(self, muted: bool): @@ -268,23 +215,27 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): if self.assistant: self.assistant.set_mic_mute(muted) + if muted: + self._on_mute() + else: + self._on_unmute() + @action - def is_muted(self) -> bool: + def toggle_mute(self, *_, **__): """ - :return: True if the microphone is muted, False otherwise. + Toggle the mic mute state. """ - return self._is_muted + self.set_mic_mute(muted=not self._is_muted) @action def toggle_mic_mute(self): """ - Toggle the mic mute state. + Deprecated alias for :meth:`.toggle_mute`. """ - is_muted = self.is_muted() - self.set_mic_mute(muted=not is_muted) + return self.toggle_mute() @action - def send_text_query(self, query: str): + def send_text_query(self, *_, query: str, **__): """ Send a text query to the assistant. @@ -323,6 +274,7 @@ class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin): with Assistant( self.credentials, self.device_model_id ) as self._assistant: + self.publish_entities([self]) for event in self._assistant.start(): last_sleep = 0 -- 2.45.1