[#284] Merged assistant.google plugin and backend.

This removes the deprecated `assistant.google` backend and also adds a
new `conversation_start_sound` parameter.
This commit is contained in:
Fabio Manganiello 2023-10-22 19:55:11 +02:00
parent debb4f6f36
commit cce6c4c5ad
Signed by: blacklight
GPG key ID: D90FBA7F76362774
8 changed files with 397 additions and 303 deletions

View file

@ -8,7 +8,6 @@ Backends
platypush/backend/adafruit.io.rst platypush/backend/adafruit.io.rst
platypush/backend/alarm.rst platypush/backend/alarm.rst
platypush/backend/assistant.google.rst
platypush/backend/assistant.snowboy.rst platypush/backend/assistant.snowboy.rst
platypush/backend/button.flic.rst platypush/backend/button.flic.rst
platypush/backend/camera.pi.rst platypush/backend/camera.pi.rst

View file

@ -1,6 +0,0 @@
``assistant.google``
======================================
.. automodule:: platypush.backend.assistant.google
:members:

View file

@ -1,191 +0,0 @@
import json
import os
import time
from platypush.backend.assistant import AssistantBackend
from platypush.message.event.assistant import (
ConversationStartEvent,
ConversationEndEvent,
ConversationTimeoutEvent,
ResponseEvent,
NoResponseEvent,
SpeechRecognizedEvent,
AlarmStartedEvent,
AlarmEndEvent,
TimerStartedEvent,
TimerEndEvent,
AlertStartedEvent,
AlertEndEvent,
MicMutedEvent,
MicUnmutedEvent,
)
class AssistantGoogleBackend(AssistantBackend):
"""
Google Assistant backend.
It listens for voice commands and post conversation events on the bus.
**WARNING**: The Google Assistant library used by this backend has officially been deprecated:
https://developers.google.com/assistant/sdk/reference/library/python/. This backend still works on most of the
devices where I use it, but its correct functioning is not guaranteed as the assistant library is no longer
maintained.
"""
_default_credentials_file = os.path.join(
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
)
def __init__(
self,
credentials_file=_default_credentials_file,
device_model_id='Platypush',
**kwargs
):
"""
:param credentials_file: Path to the Google OAuth credentials file
(default: ~/.config/google-oauthlib-tool/credentials.json).
See
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
for instructions to get your own credentials file.
:type credentials_file: str
:param device_model_id: Device model ID to use for the assistant
(default: Platypush)
:type device_model_id: str
"""
super().__init__(**kwargs)
self.credentials_file = credentials_file
self.device_model_id = device_model_id
self.credentials = None
self.assistant = None
self._has_error = False
self._is_muted = False
self.logger.info('Initialized Google Assistant backend')
def _process_event(self, event):
from google.assistant.library.event import EventType, AlertType
self.logger.info('Received assistant event: {}'.format(event))
self._has_error = False
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
self.bus.post(ConversationStartEvent(assistant=self))
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
if not event.args.get('with_follow_on_turn'):
self.bus.post(ConversationEndEvent(assistant=self))
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
self.bus.post(ConversationTimeoutEvent(assistant=self))
elif event.type == EventType.ON_NO_RESPONSE:
self.bus.post(NoResponseEvent(assistant=self))
elif (
hasattr(EventType, 'ON_RENDER_RESPONSE')
and event.type == EventType.ON_RENDER_RESPONSE
):
self.bus.post(
ResponseEvent(assistant=self, response_text=event.args.get('text'))
)
tts, args = self._get_tts_plugin()
if tts and 'text' in event.args:
self.stop_conversation()
tts.say(text=event.args['text'], **args)
elif (
hasattr(EventType, 'ON_RESPONDING_STARTED')
and event.type == EventType.ON_RESPONDING_STARTED
and event.args.get('is_error_response', False) is True
):
self.logger.warning('Assistant response error')
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
phrase = event.args['text'].lower().strip()
self.logger.info('Speech recognized: {}'.format(phrase))
self.bus.post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
elif event.type == EventType.ON_ALERT_STARTED:
if event.args.get('alert_type') == AlertType.ALARM:
self.bus.post(AlarmStartedEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
self.bus.post(TimerStartedEvent(assistant=self))
else:
self.bus.post(AlertStartedEvent(assistant=self))
elif event.type == EventType.ON_ALERT_FINISHED:
if event.args.get('alert_type') == AlertType.ALARM:
self.bus.post(AlarmEndEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
self.bus.post(TimerEndEvent(assistant=self))
else:
self.bus.post(AlertEndEvent(assistant=self))
elif event.type == EventType.ON_ASSISTANT_ERROR:
self._has_error = True
if event.args.get('is_fatal'):
self.logger.error('Fatal assistant error')
else:
self.logger.warning('Assistant error')
if event.type == EventType.ON_MUTED_CHANGED:
self._is_muted = event.args.get('is_muted')
event = MicMutedEvent() if self._is_muted else MicUnmutedEvent()
self.bus.post(event)
def start_conversation(self):
"""Starts a conversation."""
if self.assistant:
self.assistant.start_conversation()
def stop_conversation(self):
"""Stops an active conversation."""
if self.assistant:
self.assistant.stop_conversation()
def set_mic_mute(self, muted):
if not self.assistant:
self.logger.warning('Assistant not running')
return
self.assistant.set_mic_mute(muted)
def is_muted(self) -> bool:
return self._is_muted
def send_text_query(self, query):
if not self.assistant:
self.logger.warning('Assistant not running')
return
self.assistant.send_text_query(query)
def run(self):
import google.oauth2.credentials
from google.assistant.library import Assistant
super().run()
with open(self.credentials_file, 'r') as f:
self.credentials = google.oauth2.credentials.Credentials(
token=None, **json.load(f)
)
while not self.should_stop():
self._has_error = False
with Assistant(self.credentials, self.device_model_id) as assistant:
self.assistant = assistant
for event in assistant.start():
if not self.is_detecting():
self.logger.info(
'Assistant event received but detection is currently paused'
)
continue
self._process_event(event)
if self._has_error:
self.logger.info(
'Restarting the assistant after an unrecoverable error'
)
time.sleep(5)
break
# vim:sw=4:ts=4:et:

View file

@ -1,39 +0,0 @@
manifest:
events:
platypush.message.event.assistant.AlarmEndEvent: when an alarm ends
platypush.message.event.assistant.AlarmStartedEvent: when an alarm starts
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
ends
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
starts
platypush.message.event.assistant.ConversationTimeoutEvent: when a conversation
times out
platypush.message.event.assistant.MicMutedEvent: when the microphone is muted.
platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted.
platypush.message.event.assistant.NoResponseEvent: when a conversation returned no
response
platypush.message.event.assistant.ResponseEvent: when the assistant is speaking
a response
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
is recognized
platypush.message.event.assistant.TimerEndEvent: when a timer ends
platypush.message.event.assistant.TimerStartedEvent: when a timer starts
install:
apk:
- py3-grpcio
- py3-google-auth
apt:
- python3-grpcio
- python3-google-auth
dnf:
- python-grpcio
- python-google-auth
pacman:
- python-grpcio
- python-google-auth
pip:
- google-assistant-library
- google-assistant-sdk[samples]
- google-auth
package: platypush.backend.assistant.google
type: backend

View file

@ -1,8 +1,6 @@
import logging
import re import re
import sys import sys
from platypush.context import get_backend, get_plugin
from platypush.message.event import Event from platypush.message.event import Event
@ -11,18 +9,7 @@ class AssistantEvent(Event):
def __init__(self, *args, assistant=None, **kwargs): def __init__(self, *args, assistant=None, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.logger = logging.getLogger('platypush:assistant')
if assistant:
self._assistant = assistant self._assistant = assistant
else:
try:
self._assistant = get_backend('assistant.google')
if not self._assistant:
self._assistant = get_plugin('assistant.google.pushtotalk')
except Exception as e:
self.logger.debug('Could not initialize the assistant component: %s', e)
self._assistant = None
class ConversationStartEvent(AssistantEvent): class ConversationStartEvent(AssistantEvent):

View file

@ -1,54 +1,76 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from threading import Event
from typing import Any, Dict, Optional
from platypush.context import get_backend from platypush.context import get_plugin
from platypush.plugins import Plugin, action from platypush.plugins import Plugin, action
class AssistantPlugin(ABC, Plugin): class AssistantPlugin(ABC, Plugin):
""" """
Base class for assistant plugins Base class for assistant plugins.
""" """
@abstractmethod def __init__(
def start_conversation(self, *args, language=None, tts_plugin=None, tts_args=None, **kwargs): self,
*args,
tts_plugin: Optional[str] = None,
tts_plugin_args: Optional[Dict[str, Any]] = None,
**kwargs
):
""" """
Start a conversation. :param tts_plugin: If set, the assistant will use this plugin (e.g.
``tts``, ``tts.google`` or ``tts.mimic3``) to render the responses,
instead of using the built-in assistant voice.
:param tts_plugin_args: Optional arguments to be passed to the TTS
``say`` action, if ``tts_plugin`` is set.
"""
super().__init__(*args, **kwargs)
self.tts_plugin = tts_plugin
self.tts_plugin_args = tts_plugin_args or {}
self._detection_paused = Event()
@abstractmethod
def start_conversation(self, *_, **__):
"""
Programmatically starts a conversation.
""" """
raise NotImplementedError raise NotImplementedError
@abstractmethod @abstractmethod
def stop_conversation(self, *args, **kwargs): def stop_conversation(self, *_, **__):
""" """
Stop a conversation. Programmatically stops a conversation.
""" """
raise NotImplementedError raise NotImplementedError
def _get_assistant(self):
return get_backend('assistant.snowboy')
@action @action
def pause_detection(self): def pause_detection(self):
""" """
Put the assistant on pause. No new conversation events will be triggered. Put the assistant on pause. No new conversation events will be triggered.
""" """
assistant = self._get_assistant() self._detection_paused.set()
assistant.pause_detection()
@action @action
def resume_detection(self): def resume_detection(self):
""" """
Resume the assistant hotword detection from a paused state. Resume the assistant hotword detection from a paused state.
""" """
assistant = self._get_assistant() self._detection_paused.clear()
assistant.resume_detection()
@action @action
def is_detecting(self) -> bool: def is_detecting(self) -> bool:
""" """
:return: True if the asistant is detecting, False otherwise. :return: True if the asistant is detecting, False otherwise.
""" """
assistant = self._get_assistant() return not self._detection_paused.is_set()
return assistant.is_detecting()
def _get_tts_plugin(self):
if not self.tts_plugin:
return None
return get_plugin(self.tts_plugin)
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -1,76 +1,330 @@
from platypush.backend.assistant.google import AssistantGoogleBackend import json
from platypush.context import get_backend import os
from platypush.plugins import action from typing import Optional
from platypush.config import Config
from platypush.context import get_bus, get_plugin
from platypush.message.event.assistant import (
ConversationStartEvent,
ConversationEndEvent,
ConversationTimeoutEvent,
ResponseEvent,
NoResponseEvent,
SpeechRecognizedEvent,
AlarmStartedEvent,
AlarmEndEvent,
TimerStartedEvent,
TimerEndEvent,
AlertStartedEvent,
AlertEndEvent,
MicMutedEvent,
MicUnmutedEvent,
)
from platypush.plugins import RunnablePlugin, action
from platypush.plugins.assistant import AssistantPlugin from platypush.plugins.assistant import AssistantPlugin
class AssistantGooglePlugin(AssistantPlugin): class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
""" """
Google assistant plugin. Google Assistant plugin.
It acts like a wrapper around the :mod:`platypush.backend.assistant.google`
backend to programmatically control the conversation status. This plugin allows you to run the Google Assistant _directly_ on your
device. It requires you to have an audio microphone and a speaker connected
to the device.
.. warning:: The Google Assistant library used by this backend has
been deprecated by Google:
https://developers.google.com/assistant/sdk/reference/library/python/.
This integration still works on all of my devices, but its future
functionality is not guaranteed - Google may decide to turn off the
API, the library may no longer be built against new architectures and
it's unlikely to be updated.
.. note:: Since the Google Assistant library hasn't been updated in several
years, some of its dependencies are quite old and may break more recent
Python installations. Please refer to the comments in the [manifest
file](https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml)
for more information on how to install the required dependencies, if
the automated ways fail.
"""
_default_credentials_files = (
os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'),
os.path.join(
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
),
)
def __init__(
self,
credentials_file: Optional[str] = None,
device_model_id: str = 'Platypush',
conversation_start_sound: Optional[str] = None,
**kwargs,
):
"""
:param credentials_file: Path to the Google OAuth credentials file.
See
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
for instructions to get your own credentials file.
By default, it will search for the credentials file under:
* ``~/.config/google-oauthlib-tool/credentials.json``: default
location supported by the Google Assistant library
* ``<PLATYPUSH_WORKDIR>/credentials/google/assistant.json``:
recommended location, under the Platypush working directory.
:param device_model_id: The device model ID that identifies the device
where the assistant is running (default: Platypush). It can be a
custom string.
:param conversation_start_sound: If set, the assistant will play this
audio file when it detects a speech. The sound file will be played
on the default audio output device. If not set, the assistant won't
play any sound when it detects a speech.
""" """
def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
self._credentials_file = credentials_file
self.device_model_id = device_model_id
self.credentials = None
self._assistant = None
self._is_muted = False
def _get_assistant(self) -> AssistantGoogleBackend: if conversation_start_sound:
backend = get_backend('assistant.google') self._conversation_start_sound = os.path.abspath(
assert backend, 'The assistant.google backend is not configured.' os.path.expanduser(conversation_start_sound)
return backend )
self.logger.info('Initialized Google Assistant plugin')
@property
def credentials_file(self) -> str:
if self._credentials_file:
return self._credentials_file
f = None
for default_file in self._default_credentials_files:
f = default_file
if os.path.isfile(default_file):
break
assert f, 'No credentials_file provided and no default file found'
return f
@property
def assistant(self):
if not self._assistant:
self.logger.warning('The assistant is not running')
return
return self._assistant
def _play_conversation_start_sound(self):
if not self._conversation_start_sound:
return
audio = get_plugin('sound')
if not audio:
self.logger.warning(
'Unable to play conversation start sound: sound plugin not found'
)
return
audio.play(self._conversation_start_sound)
def _process_event(self, event):
from google.assistant.library.event import EventType, AlertType
self.logger.info('Received assistant event: %s', event)
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
get_bus().post(ConversationStartEvent(assistant=self))
self._play_conversation_start_sound()
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
if not event.args.get('with_follow_on_turn'):
get_bus().post(ConversationEndEvent(assistant=self))
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
get_bus().post(ConversationTimeoutEvent(assistant=self))
elif event.type == EventType.ON_NO_RESPONSE:
get_bus().post(NoResponseEvent(assistant=self))
elif (
hasattr(EventType, 'ON_RENDER_RESPONSE')
and event.type == EventType.ON_RENDER_RESPONSE
):
get_bus().post(
ResponseEvent(assistant=self, response_text=event.args.get('text'))
)
tts = self._get_tts_plugin()
if tts and event.args.get('text'):
self.stop_conversation()
tts.say(text=event.args['text'], **self.tts_plugin_args)
elif (
hasattr(EventType, 'ON_RESPONDING_STARTED')
and event.type == EventType.ON_RESPONDING_STARTED
and event.args.get('is_error_response', False) is True
):
self.logger.warning('Assistant response error')
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
phrase = event.args['text'].lower().strip()
self.logger.info('Speech recognized: %s', phrase)
get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
elif event.type == EventType.ON_ALERT_STARTED:
if event.args.get('alert_type') == AlertType.ALARM:
get_bus().post(AlarmStartedEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
get_bus().post(TimerStartedEvent(assistant=self))
else:
get_bus().post(AlertStartedEvent(assistant=self))
elif event.type == EventType.ON_ALERT_FINISHED:
if event.args.get('alert_type') == AlertType.ALARM:
get_bus().post(AlarmEndEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
get_bus().post(TimerEndEvent(assistant=self))
else:
get_bus().post(AlertEndEvent(assistant=self))
elif event.type == EventType.ON_ASSISTANT_ERROR:
if event.args.get('is_fatal'):
raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}')
self.logger.warning('Assistant error: %s', json.dumps(event.args))
elif event.type == EventType.ON_MUTED_CHANGED:
self._is_muted = event.args.get('is_muted')
event = MicMutedEvent() if self._is_muted else MicUnmutedEvent()
get_bus().post(event)
@action @action
def start_conversation(self): def start_conversation(self, *_, **__):
""" """
Programmatically start a conversation with the assistant Programmatically start a conversation with the assistant
""" """
assistant = self._get_assistant() if self.assistant:
assistant.start_conversation() self.assistant.start_conversation()
@action @action
def stop_conversation(self): def stop_conversation(self):
""" """
Programmatically stop a running conversation with the assistant Programmatically stop a running conversation with the assistant
""" """
assistant = self._get_assistant() if self.assistant:
assistant.stop_conversation() self.assistant.stop_conversation()
@action @action
def set_mic_mute(self, muted: bool = True): def mute(self):
"""
Mute the microphone. Alias for :meth:`.set_mic_mute` with
``muted=True``.
"""
return self.set_mic_mute(muted=True)
@action
def unmute(self):
"""
Unmute the microphone. Alias for :meth:`.set_mic_mute` with
``muted=False``.
"""
return self.set_mic_mute(muted=False)
@action
def set_mic_mute(self, muted: bool):
""" """
Programmatically mute/unmute the microphone. Programmatically mute/unmute the microphone.
:param muted: Set to True or False. :param muted: Set to True or False.
""" """
assistant = self._get_assistant() if self.assistant:
assistant.set_mic_mute(muted) self.assistant.set_mic_mute(muted)
@action
def toggle_mic_mute(self):
"""
Toggle the mic mute state.
"""
assistant = self._get_assistant()
is_muted = assistant.is_muted()
self.set_mic_mute(muted=not is_muted)
@action @action
def is_muted(self) -> bool: def is_muted(self) -> bool:
""" """
:return: True if the microphone is muted, False otherwise. :return: True if the microphone is muted, False otherwise.
""" """
assistant = self._get_assistant() return self._is_muted
return assistant.is_muted()
@action
def toggle_mic_mute(self):
"""
Toggle the mic mute state.
"""
is_muted = self.is_muted()
self.set_mic_mute(muted=not is_muted)
@action @action
def send_text_query(self, query: str): def send_text_query(self, query: str):
""" """
Send a text query to the assistant. Send a text query to the assistant.
This is equivalent to saying something to the assistant.
:param query: Query to be sent. :param query: Query to be sent.
""" """
assistant = self._get_assistant() if self.assistant:
assistant.send_text_query(query) self.assistant.send_text_query(query)
def main(self):
import google.oauth2.credentials
from google.assistant.library import Assistant
last_sleep = 0
while not self.should_stop():
try:
with open(self.credentials_file, 'r') as f:
self.credentials = google.oauth2.credentials.Credentials(
token=None, **json.load(f)
)
except Exception as e:
self.logger.error(
'Error while loading Google Assistant credentials: %s', e
)
self.logger.info(
'Please follow the instructions at '
'https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample'
'#generate_credentials to get your own credentials file'
)
self.logger.exception(e)
break
try:
with Assistant(
self.credentials, self.device_model_id
) as self._assistant:
for event in self._assistant.start():
last_sleep = 0
if not self.is_detecting():
self.logger.info(
'Assistant event received but detection is currently paused'
)
continue
self._process_event(event)
except Exception as e:
self.logger.exception(e)
sleep_secs = min(60, max(5, last_sleep * 2))
self.logger.warning(
'Restarting the assistant in %d seconds after an unrecoverable error',
sleep_secs,
)
self.wait_stop(sleep_secs)
last_sleep = sleep_secs
continue
def stop(self):
try:
self.stop_conversation()
except RuntimeError:
pass
if self._assistant:
del self._assistant
self._assistant = None
super().stop()
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -1,6 +1,74 @@
manifest: manifest:
events: {}
install:
pip: []
package: platypush.plugins.assistant.google package: platypush.plugins.assistant.google
type: plugin type: plugin
events:
- platypush.message.event.assistant.AlarmEndEvent
- platypush.message.event.assistant.AlarmStartedEvent
- platypush.message.event.assistant.ConversationEndEvent
- platypush.message.event.assistant.ConversationStartEvent
- platypush.message.event.assistant.ConversationTimeoutEvent
- platypush.message.event.assistant.MicMutedEvent
- platypush.message.event.assistant.MicUnmutedEvent
- platypush.message.event.assistant.NoResponseEvent
- platypush.message.event.assistant.ResponseEvent
- platypush.message.event.assistant.SpeechRecognizedEvent
- platypush.message.event.assistant.TimerEndEvent
- platypush.message.event.assistant.TimerStartedEvent
install:
apk:
- ffmpeg
- portaudio-dev
- py3-cachetools
- py3-grpcio
- py3-google-auth
- py3-numpy
- py3-pathlib2
- py3-tenacity
- py3-urllib3
apt:
- ffmpeg
- portaudio19-dev
- python3-cachetools
- python3-grpcio
- python3-google-auth
- python3-monotonic
- python3-tenacity
- python3-urllib3
dnf:
- ffmpeg
- portaudio-devel
- python-cachetools
- python-grpcio
- python-google-auth
- python-monotonic
- python-numpy
- python-tenacity
- python-urllib3
pacman:
- ffmpeg
- portaudio
- python-cachetools
- python-grpcio
- python-google-auth
- python-monotonic
- python-numpy
- python-sounddevice
- python-tenacity
- python-urllib3
pip:
- google-assistant-library
- google-assistant-sdk[samples]
- google-auth
- numpy
- sounddevice
after:
# Uninstall old versions of packages that break things on recent versions
# of Python, when the new versions work just fine
- yes | pip uninstall --break-system-packages enum34 click urllib3 requests google-auth
# Upgrade the dependencies (back) to the latest version.
# NOTE: Be careful when running this command on older distros that may
# not ship the latest versions of all the packages! This is a workaround
# caused by the fact that google-assistant-library pulls in some old
# breaking dependencies that need to be surgically removed.
- pip install -U --no-input --break-system-packages click urllib3 requests google-auth