New assistant.google
integration #335
|
@ -8,8 +8,6 @@ Backends
|
||||||
|
|
||||||
platypush/backend/adafruit.io.rst
|
platypush/backend/adafruit.io.rst
|
||||||
platypush/backend/alarm.rst
|
platypush/backend/alarm.rst
|
||||||
platypush/backend/assistant.google.rst
|
|
||||||
platypush/backend/assistant.snowboy.rst
|
|
||||||
platypush/backend/button.flic.rst
|
platypush/backend/button.flic.rst
|
||||||
platypush/backend/camera.pi.rst
|
platypush/backend/camera.pi.rst
|
||||||
platypush/backend/chat.telegram.rst
|
platypush/backend/chat.telegram.rst
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
``assistant.google``
|
|
||||||
======================================
|
|
||||||
|
|
||||||
.. automodule:: platypush.backend.assistant.google
|
|
||||||
:members:
|
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
``assistant.snowboy``
|
|
||||||
=======================================
|
|
||||||
|
|
||||||
.. automodule:: platypush.backend.assistant.snowboy
|
|
||||||
:members:
|
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
``assistant.echo``
|
|
||||||
====================================
|
|
||||||
|
|
||||||
.. automodule:: platypush.plugins.assistant.echo
|
|
||||||
:members:
|
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
``assistant.google.pushtotalk``
|
|
||||||
=================================================
|
|
||||||
|
|
||||||
.. automodule:: platypush.plugins.assistant.google.pushtotalk
|
|
||||||
:members:
|
|
||||||
|
|
|
@ -10,9 +10,7 @@ Plugins
|
||||||
platypush/plugins/alarm.rst
|
platypush/plugins/alarm.rst
|
||||||
platypush/plugins/application.rst
|
platypush/plugins/application.rst
|
||||||
platypush/plugins/arduino.rst
|
platypush/plugins/arduino.rst
|
||||||
platypush/plugins/assistant.echo.rst
|
|
||||||
platypush/plugins/assistant.google.rst
|
platypush/plugins/assistant.google.rst
|
||||||
platypush/plugins/assistant.google.pushtotalk.rst
|
|
||||||
platypush/plugins/autoremote.rst
|
platypush/plugins/autoremote.rst
|
||||||
platypush/plugins/bluetooth.rst
|
platypush/plugins/bluetooth.rst
|
||||||
platypush/plugins/calendar.rst
|
platypush/plugins/calendar.rst
|
||||||
|
|
|
@ -1,39 +0,0 @@
|
||||||
from abc import ABC
|
|
||||||
import threading
|
|
||||||
from typing import Optional, Dict, Any, Tuple
|
|
||||||
|
|
||||||
from platypush.backend import Backend
|
|
||||||
from platypush.context import get_plugin
|
|
||||||
from platypush.plugins.tts import TtsPlugin
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantBackend(Backend):
|
|
||||||
def __init__(self, tts_plugin: Optional[str] = None, tts_args: Optional[Dict[str, Any]] = None, **kwargs):
|
|
||||||
"""
|
|
||||||
Default assistant backend constructor.
|
|
||||||
|
|
||||||
:param tts_plugin: If set, and if the assistant returns the processed response as text, then the processed
|
|
||||||
response will be played through the selected text-to-speech plugin (can be e.g. "``tts``",
|
|
||||||
"``tts.google``" or any other implementation of :class:`platypush.plugins.tts.TtsPlugin`).
|
|
||||||
:param tts_args: Extra parameters to pass to the ``say`` method of the selected TTS plugin (e.g.
|
|
||||||
language, voice or gender).
|
|
||||||
"""
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self._detection_paused = threading.Event()
|
|
||||||
self.tts_plugin = tts_plugin
|
|
||||||
self.tts_args = tts_args or {}
|
|
||||||
|
|
||||||
def pause_detection(self):
|
|
||||||
self._detection_paused.set()
|
|
||||||
|
|
||||||
def resume_detection(self):
|
|
||||||
self._detection_paused.clear()
|
|
||||||
|
|
||||||
def is_detecting(self):
|
|
||||||
return not self._detection_paused.is_set()
|
|
||||||
|
|
||||||
def _get_tts_plugin(self) -> Tuple[Optional[TtsPlugin], Dict[str, Any]]:
|
|
||||||
return get_plugin(self.tts_plugin) if self.tts_plugin else None, self.tts_args
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
|
@ -1,191 +0,0 @@
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
|
|
||||||
from platypush.backend.assistant import AssistantBackend
|
|
||||||
from platypush.message.event.assistant import (
|
|
||||||
ConversationStartEvent,
|
|
||||||
ConversationEndEvent,
|
|
||||||
ConversationTimeoutEvent,
|
|
||||||
ResponseEvent,
|
|
||||||
NoResponseEvent,
|
|
||||||
SpeechRecognizedEvent,
|
|
||||||
AlarmStartedEvent,
|
|
||||||
AlarmEndEvent,
|
|
||||||
TimerStartedEvent,
|
|
||||||
TimerEndEvent,
|
|
||||||
AlertStartedEvent,
|
|
||||||
AlertEndEvent,
|
|
||||||
MicMutedEvent,
|
|
||||||
MicUnmutedEvent,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantGoogleBackend(AssistantBackend):
|
|
||||||
"""
|
|
||||||
Google Assistant backend.
|
|
||||||
|
|
||||||
It listens for voice commands and post conversation events on the bus.
|
|
||||||
|
|
||||||
**WARNING**: The Google Assistant library used by this backend has officially been deprecated:
|
|
||||||
https://developers.google.com/assistant/sdk/reference/library/python/. This backend still works on most of the
|
|
||||||
devices where I use it, but its correct functioning is not guaranteed as the assistant library is no longer
|
|
||||||
maintained.
|
|
||||||
"""
|
|
||||||
|
|
||||||
_default_credentials_file = os.path.join(
|
|
||||||
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
credentials_file=_default_credentials_file,
|
|
||||||
device_model_id='Platypush',
|
|
||||||
**kwargs
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
:param credentials_file: Path to the Google OAuth credentials file
|
|
||||||
(default: ~/.config/google-oauthlib-tool/credentials.json).
|
|
||||||
See
|
|
||||||
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
|
|
||||||
for instructions to get your own credentials file.
|
|
||||||
|
|
||||||
:type credentials_file: str
|
|
||||||
|
|
||||||
:param device_model_id: Device model ID to use for the assistant
|
|
||||||
(default: Platypush)
|
|
||||||
:type device_model_id: str
|
|
||||||
"""
|
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.credentials_file = credentials_file
|
|
||||||
self.device_model_id = device_model_id
|
|
||||||
self.credentials = None
|
|
||||||
self.assistant = None
|
|
||||||
self._has_error = False
|
|
||||||
self._is_muted = False
|
|
||||||
|
|
||||||
self.logger.info('Initialized Google Assistant backend')
|
|
||||||
|
|
||||||
def _process_event(self, event):
|
|
||||||
from google.assistant.library.event import EventType, AlertType
|
|
||||||
|
|
||||||
self.logger.info('Received assistant event: {}'.format(event))
|
|
||||||
self._has_error = False
|
|
||||||
|
|
||||||
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
|
|
||||||
self.bus.post(ConversationStartEvent(assistant=self))
|
|
||||||
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
|
|
||||||
if not event.args.get('with_follow_on_turn'):
|
|
||||||
self.bus.post(ConversationEndEvent(assistant=self))
|
|
||||||
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
|
|
||||||
self.bus.post(ConversationTimeoutEvent(assistant=self))
|
|
||||||
elif event.type == EventType.ON_NO_RESPONSE:
|
|
||||||
self.bus.post(NoResponseEvent(assistant=self))
|
|
||||||
elif (
|
|
||||||
hasattr(EventType, 'ON_RENDER_RESPONSE')
|
|
||||||
and event.type == EventType.ON_RENDER_RESPONSE
|
|
||||||
):
|
|
||||||
self.bus.post(
|
|
||||||
ResponseEvent(assistant=self, response_text=event.args.get('text'))
|
|
||||||
)
|
|
||||||
tts, args = self._get_tts_plugin()
|
|
||||||
|
|
||||||
if tts and 'text' in event.args:
|
|
||||||
self.stop_conversation()
|
|
||||||
tts.say(text=event.args['text'], **args)
|
|
||||||
elif (
|
|
||||||
hasattr(EventType, 'ON_RESPONDING_STARTED')
|
|
||||||
and event.type == EventType.ON_RESPONDING_STARTED
|
|
||||||
and event.args.get('is_error_response', False) is True
|
|
||||||
):
|
|
||||||
self.logger.warning('Assistant response error')
|
|
||||||
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
|
|
||||||
phrase = event.args['text'].lower().strip()
|
|
||||||
self.logger.info('Speech recognized: {}'.format(phrase))
|
|
||||||
self.bus.post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
|
|
||||||
elif event.type == EventType.ON_ALERT_STARTED:
|
|
||||||
if event.args.get('alert_type') == AlertType.ALARM:
|
|
||||||
self.bus.post(AlarmStartedEvent(assistant=self))
|
|
||||||
elif event.args.get('alert_type') == AlertType.TIMER:
|
|
||||||
self.bus.post(TimerStartedEvent(assistant=self))
|
|
||||||
else:
|
|
||||||
self.bus.post(AlertStartedEvent(assistant=self))
|
|
||||||
elif event.type == EventType.ON_ALERT_FINISHED:
|
|
||||||
if event.args.get('alert_type') == AlertType.ALARM:
|
|
||||||
self.bus.post(AlarmEndEvent(assistant=self))
|
|
||||||
elif event.args.get('alert_type') == AlertType.TIMER:
|
|
||||||
self.bus.post(TimerEndEvent(assistant=self))
|
|
||||||
else:
|
|
||||||
self.bus.post(AlertEndEvent(assistant=self))
|
|
||||||
elif event.type == EventType.ON_ASSISTANT_ERROR:
|
|
||||||
self._has_error = True
|
|
||||||
if event.args.get('is_fatal'):
|
|
||||||
self.logger.error('Fatal assistant error')
|
|
||||||
else:
|
|
||||||
self.logger.warning('Assistant error')
|
|
||||||
if event.type == EventType.ON_MUTED_CHANGED:
|
|
||||||
self._is_muted = event.args.get('is_muted')
|
|
||||||
event = MicMutedEvent() if self._is_muted else MicUnmutedEvent()
|
|
||||||
self.bus.post(event)
|
|
||||||
|
|
||||||
def start_conversation(self):
|
|
||||||
"""Starts a conversation."""
|
|
||||||
if self.assistant:
|
|
||||||
self.assistant.start_conversation()
|
|
||||||
|
|
||||||
def stop_conversation(self):
|
|
||||||
"""Stops an active conversation."""
|
|
||||||
if self.assistant:
|
|
||||||
self.assistant.stop_conversation()
|
|
||||||
|
|
||||||
def set_mic_mute(self, muted):
|
|
||||||
if not self.assistant:
|
|
||||||
self.logger.warning('Assistant not running')
|
|
||||||
return
|
|
||||||
|
|
||||||
self.assistant.set_mic_mute(muted)
|
|
||||||
|
|
||||||
def is_muted(self) -> bool:
|
|
||||||
return self._is_muted
|
|
||||||
|
|
||||||
def send_text_query(self, query):
|
|
||||||
if not self.assistant:
|
|
||||||
self.logger.warning('Assistant not running')
|
|
||||||
return
|
|
||||||
|
|
||||||
self.assistant.send_text_query(query)
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
import google.oauth2.credentials
|
|
||||||
from google.assistant.library import Assistant
|
|
||||||
|
|
||||||
super().run()
|
|
||||||
|
|
||||||
with open(self.credentials_file, 'r') as f:
|
|
||||||
self.credentials = google.oauth2.credentials.Credentials(
|
|
||||||
token=None, **json.load(f)
|
|
||||||
)
|
|
||||||
|
|
||||||
while not self.should_stop():
|
|
||||||
self._has_error = False
|
|
||||||
|
|
||||||
with Assistant(self.credentials, self.device_model_id) as assistant:
|
|
||||||
self.assistant = assistant
|
|
||||||
for event in assistant.start():
|
|
||||||
if not self.is_detecting():
|
|
||||||
self.logger.info(
|
|
||||||
'Assistant event received but detection is currently paused'
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
self._process_event(event)
|
|
||||||
if self._has_error:
|
|
||||||
self.logger.info(
|
|
||||||
'Restarting the assistant after an unrecoverable error'
|
|
||||||
)
|
|
||||||
time.sleep(5)
|
|
||||||
break
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
|
@ -1,39 +0,0 @@
|
||||||
manifest:
|
|
||||||
events:
|
|
||||||
platypush.message.event.assistant.AlarmEndEvent: when an alarm ends
|
|
||||||
platypush.message.event.assistant.AlarmStartedEvent: when an alarm starts
|
|
||||||
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
|
|
||||||
ends
|
|
||||||
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
|
|
||||||
starts
|
|
||||||
platypush.message.event.assistant.ConversationTimeoutEvent: when a conversation
|
|
||||||
times out
|
|
||||||
platypush.message.event.assistant.MicMutedEvent: when the microphone is muted.
|
|
||||||
platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted.
|
|
||||||
platypush.message.event.assistant.NoResponseEvent: when a conversation returned no
|
|
||||||
response
|
|
||||||
platypush.message.event.assistant.ResponseEvent: when the assistant is speaking
|
|
||||||
a response
|
|
||||||
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
|
|
||||||
is recognized
|
|
||||||
platypush.message.event.assistant.TimerEndEvent: when a timer ends
|
|
||||||
platypush.message.event.assistant.TimerStartedEvent: when a timer starts
|
|
||||||
install:
|
|
||||||
apk:
|
|
||||||
- py3-grpcio
|
|
||||||
- py3-google-auth
|
|
||||||
apt:
|
|
||||||
- python3-grpcio
|
|
||||||
- python3-google-auth
|
|
||||||
dnf:
|
|
||||||
- python-grpcio
|
|
||||||
- python-google-auth
|
|
||||||
pacman:
|
|
||||||
- python-grpcio
|
|
||||||
- python-google-auth
|
|
||||||
pip:
|
|
||||||
- google-assistant-library
|
|
||||||
- google-assistant-sdk[samples]
|
|
||||||
- google-auth
|
|
||||||
package: platypush.backend.assistant.google
|
|
||||||
type: backend
|
|
|
@ -1,184 +0,0 @@
|
||||||
import os
|
|
||||||
import threading
|
|
||||||
|
|
||||||
from platypush.backend.assistant import AssistantBackend
|
|
||||||
from platypush.context import get_plugin
|
|
||||||
from platypush.message.event.assistant import HotwordDetectedEvent
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantSnowboyBackend(AssistantBackend):
|
|
||||||
"""
|
|
||||||
Backend for detecting custom voice hotwords through Snowboy. The purpose of
|
|
||||||
this component is only to detect the hotword specified in your Snowboy voice
|
|
||||||
model. If you want to trigger proper assistant conversations or custom
|
|
||||||
speech recognition, you should create a hook in your configuration on
|
|
||||||
HotwordDetectedEvent to trigger the conversation on whichever assistant
|
|
||||||
plugin you're using (Google, Alexa...)
|
|
||||||
|
|
||||||
Manual installation for snowboy and its Python bindings if the installation via package fails::
|
|
||||||
|
|
||||||
$ [sudo] apt-get install libatlas-base-dev swig
|
|
||||||
$ [sudo] pip install pyaudio
|
|
||||||
$ git clone https://github.com/Kitt-AI/snowboy
|
|
||||||
$ cd snowboy/swig/Python3
|
|
||||||
$ make
|
|
||||||
$ cd ../..
|
|
||||||
$ python3 setup.py build
|
|
||||||
$ [sudo] python setup.py install
|
|
||||||
|
|
||||||
You will also need a voice model for the hotword detection. You can find
|
|
||||||
some under the ``resources/models`` directory of the Snowboy repository,
|
|
||||||
or train/download other models from https://snowboy.kitt.ai.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, models, audio_gain=1.0, **kwargs):
|
|
||||||
"""
|
|
||||||
:param models: Map (name -> configuration) of voice models to be used by
|
|
||||||
the assistant. See https://snowboy.kitt.ai/ for training/downloading
|
|
||||||
models. Sample format::
|
|
||||||
|
|
||||||
ok_google: # Hotword model name
|
|
||||||
voice_model_file: /path/models/OK Google.pmdl # Voice model file location
|
|
||||||
sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5)
|
|
||||||
assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google
|
|
||||||
# push-to-talk assistant plugin (optional)
|
|
||||||
assistant_language: en-US # The assistant will conversate in English when this hotword is
|
|
||||||
detected (optional)
|
|
||||||
detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional)
|
|
||||||
|
|
||||||
ciao_google: # Hotword model name
|
|
||||||
voice_model_file: /path/models/Ciao Google.pmdl # Voice model file location
|
|
||||||
sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5)
|
|
||||||
assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google
|
|
||||||
# push-to-talk assistant plugin (optional)
|
|
||||||
assistant_language: it-IT # The assistant will conversate in Italian when this hotword is
|
|
||||||
# detected (optional)
|
|
||||||
detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional)
|
|
||||||
|
|
||||||
:type models: dict
|
|
||||||
|
|
||||||
:param audio_gain: Audio gain, between 0 and 1. Default: 1
|
|
||||||
:type audio_gain: float
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
import snowboydecoder
|
|
||||||
except ImportError:
|
|
||||||
import snowboy.snowboydecoder as snowboydecoder
|
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
self.models = {}
|
|
||||||
self._init_models(models)
|
|
||||||
self.audio_gain = audio_gain
|
|
||||||
|
|
||||||
self.detector = snowboydecoder.HotwordDetector(
|
|
||||||
[model['voice_model_file'] for model in self.models.values()],
|
|
||||||
sensitivity=[model['sensitivity'] for model in self.models.values()],
|
|
||||||
audio_gain=self.audio_gain,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.info(
|
|
||||||
'Initialized Snowboy hotword detection with {} voice model configurations'.format(
|
|
||||||
len(self.models)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def _init_models(self, models):
|
|
||||||
if not models:
|
|
||||||
raise AttributeError('Please specify at least one voice model')
|
|
||||||
|
|
||||||
self.models = {}
|
|
||||||
for name, conf in models.items():
|
|
||||||
if name in self.models:
|
|
||||||
raise AttributeError('Duplicate model key {}'.format(name))
|
|
||||||
|
|
||||||
model_file = conf.get('voice_model_file')
|
|
||||||
detect_sound = conf.get('detect_sound')
|
|
||||||
|
|
||||||
if not model_file:
|
|
||||||
raise AttributeError(
|
|
||||||
'No voice_model_file specified for model {}'.format(name)
|
|
||||||
)
|
|
||||||
|
|
||||||
model_file = os.path.abspath(os.path.expanduser(model_file))
|
|
||||||
assistant_plugin_name = conf.get('assistant_plugin')
|
|
||||||
|
|
||||||
if detect_sound:
|
|
||||||
detect_sound = os.path.abspath(os.path.expanduser(detect_sound))
|
|
||||||
|
|
||||||
if not os.path.isfile(model_file):
|
|
||||||
raise FileNotFoundError(
|
|
||||||
'Voice model file {} does not exist or it not a regular file'.format(
|
|
||||||
model_file
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.models[name] = {
|
|
||||||
'voice_model_file': model_file,
|
|
||||||
'sensitivity': conf.get('sensitivity', 0.5),
|
|
||||||
'detect_sound': detect_sound,
|
|
||||||
'assistant_plugin': get_plugin(assistant_plugin_name)
|
|
||||||
if assistant_plugin_name
|
|
||||||
else None,
|
|
||||||
'assistant_language': conf.get('assistant_language'),
|
|
||||||
'tts_plugin': conf.get('tts_plugin'),
|
|
||||||
'tts_args': conf.get('tts_args', {}),
|
|
||||||
}
|
|
||||||
|
|
||||||
def hotword_detected(self, hotword):
|
|
||||||
"""
|
|
||||||
Callback called on hotword detection
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
import snowboydecoder
|
|
||||||
except ImportError:
|
|
||||||
import snowboy.snowboydecoder as snowboydecoder
|
|
||||||
|
|
||||||
def sound_thread(sound):
|
|
||||||
snowboydecoder.play_audio_file(sound)
|
|
||||||
|
|
||||||
def callback():
|
|
||||||
if not self.is_detecting():
|
|
||||||
self.logger.info(
|
|
||||||
'Hotword detected but assistant response currently paused'
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
self.bus.post(HotwordDetectedEvent(hotword=hotword))
|
|
||||||
model = self.models[hotword]
|
|
||||||
|
|
||||||
detect_sound = model.get('detect_sound')
|
|
||||||
assistant_plugin = model.get('assistant_plugin')
|
|
||||||
assistant_language = model.get('assistant_language')
|
|
||||||
tts_plugin = model.get('tts_plugin')
|
|
||||||
tts_args = model.get('tts_args')
|
|
||||||
|
|
||||||
if detect_sound:
|
|
||||||
threading.Thread(target=sound_thread, args=(detect_sound,)).start()
|
|
||||||
|
|
||||||
if assistant_plugin:
|
|
||||||
assistant_plugin.start_conversation(
|
|
||||||
language=assistant_language,
|
|
||||||
tts_plugin=tts_plugin,
|
|
||||||
tts_args=tts_args,
|
|
||||||
)
|
|
||||||
|
|
||||||
return callback
|
|
||||||
|
|
||||||
def on_stop(self):
|
|
||||||
super().on_stop()
|
|
||||||
if self.detector:
|
|
||||||
self.detector.terminate()
|
|
||||||
self.detector = None
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
super().run()
|
|
||||||
self.detector.start(
|
|
||||||
detected_callback=[
|
|
||||||
self.hotword_detected(hotword) for hotword in self.models.keys()
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
|
@ -1,9 +0,0 @@
|
||||||
manifest:
|
|
||||||
events:
|
|
||||||
platypush.message.event.assistant.HotwordDetectedEvent: whenever the hotword has
|
|
||||||
been detected
|
|
||||||
install:
|
|
||||||
pip:
|
|
||||||
- snowboy
|
|
||||||
package: platypush.backend.assistant.snowboy
|
|
||||||
type: backend
|
|
|
@ -32,6 +32,7 @@ export default {
|
||||||
return {
|
return {
|
||||||
config: {},
|
config: {},
|
||||||
userAuthenticated: false,
|
userAuthenticated: false,
|
||||||
|
connected: false,
|
||||||
pwaInstallEvent: null,
|
pwaInstallEvent: null,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -97,6 +98,8 @@ export default {
|
||||||
|
|
||||||
mounted() {
|
mounted() {
|
||||||
bus.onNotification(this.onNotification)
|
bus.onNotification(this.onNotification)
|
||||||
|
bus.on('connect', () => this.connected = true)
|
||||||
|
bus.on('disconnect', () => this.connected = false)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
|
@ -2,6 +2,9 @@
|
||||||
"arduino": {
|
"arduino": {
|
||||||
"class": "fas fa-microchip"
|
"class": "fas fa-microchip"
|
||||||
},
|
},
|
||||||
|
"assistant.google": {
|
||||||
|
"class": "fas fa-microphone-lines"
|
||||||
|
},
|
||||||
"bluetooth": {
|
"bluetooth": {
|
||||||
"class": "fab fa-bluetooth"
|
"class": "fab fa-bluetooth"
|
||||||
},
|
},
|
||||||
|
|
|
@ -163,6 +163,7 @@ export default {
|
||||||
this.collapsed = this.collapsedDefault
|
this.collapsed = this.collapsedDefault
|
||||||
bus.on('connect', this.setConnected(true))
|
bus.on('connect', this.setConnected(true))
|
||||||
bus.on('disconnect', this.setConnected(false))
|
bus.on('disconnect', this.setConnected(false))
|
||||||
|
this.setConnected(this.$root.connected)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
@ -192,6 +193,11 @@ nav {
|
||||||
top: 0;
|
top: 0;
|
||||||
left: 0;
|
left: 0;
|
||||||
z-index: 5;
|
z-index: 5;
|
||||||
|
|
||||||
|
.icon.status {
|
||||||
|
top: 0.75em !important;
|
||||||
|
left: 2em;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -318,6 +324,10 @@ nav {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.icon.status {
|
||||||
|
width: 1em;
|
||||||
|
}
|
||||||
|
|
||||||
&.collapsed {
|
&.collapsed {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
@ -353,15 +363,19 @@ nav {
|
||||||
box-shadow: none;
|
box-shadow: none;
|
||||||
background: $nav-toggler-collapsed-bg;
|
background: $nav-toggler-collapsed-bg;
|
||||||
|
|
||||||
@media screen and (max-width: #{$tablet - 1px}) {
|
|
||||||
background: $nav-toggler-collapsed-mobile-bg;
|
|
||||||
color: $nav-toggler-collapsed-mobile-fg;
|
|
||||||
}
|
|
||||||
|
|
||||||
.icon.status {
|
.icon.status {
|
||||||
top: 0.75em;
|
top: 0.75em;
|
||||||
left: 2em;
|
left: 2em;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@include until($tablet) {
|
||||||
|
background: $nav-toggler-collapsed-mobile-bg;
|
||||||
|
color: $nav-toggler-collapsed-mobile-fg;
|
||||||
|
|
||||||
|
.icon.status {
|
||||||
|
top: 0.75em !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.footer {
|
.footer {
|
||||||
|
|
|
@ -0,0 +1,107 @@
|
||||||
|
<template>
|
||||||
|
<Modal ref="modal" :title="title">
|
||||||
|
<form @submit.prevent="onConfirm">
|
||||||
|
<div class="dialog-content">
|
||||||
|
<slot />
|
||||||
|
<input type="text" ref="input" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="buttons">
|
||||||
|
<button type="submit" class="ok-btn" @click="onConfirm" @touch="onConfirm">
|
||||||
|
<i class="fas fa-check" /> {{ confirmText }}
|
||||||
|
</button>
|
||||||
|
<button type="button" class="cancel-btn" @click="close" @touch="close">
|
||||||
|
<i class="fas fa-xmark" /> {{ cancelText }}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</Modal>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
import Modal from "@/components/Modal";
|
||||||
|
|
||||||
|
export default {
|
||||||
|
emits: ['input'],
|
||||||
|
components: {Modal},
|
||||||
|
props: {
|
||||||
|
title: {
|
||||||
|
type: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
confirmText: {
|
||||||
|
type: String,
|
||||||
|
default: "OK",
|
||||||
|
},
|
||||||
|
|
||||||
|
cancelText: {
|
||||||
|
type: String,
|
||||||
|
default: "Cancel",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
methods: {
|
||||||
|
onConfirm() {
|
||||||
|
this.$emit('input', this.$refs.input.value)
|
||||||
|
this.close()
|
||||||
|
},
|
||||||
|
|
||||||
|
show() {
|
||||||
|
this.$refs.modal.show()
|
||||||
|
},
|
||||||
|
|
||||||
|
close() {
|
||||||
|
this.$refs.modal.hide()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
mounted() {
|
||||||
|
this.$nextTick(() => {
|
||||||
|
this.$refs.input.value = ""
|
||||||
|
this.$refs.input.focus()
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style lang="scss" scoped>
|
||||||
|
:deep(.modal) {
|
||||||
|
.dialog-content {
|
||||||
|
padding: 1em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.body {
|
||||||
|
padding: 1.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.buttons {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
justify-content: right;
|
||||||
|
margin-bottom: 1em;
|
||||||
|
border: 0;
|
||||||
|
border-radius: 0;
|
||||||
|
|
||||||
|
button {
|
||||||
|
margin-right: 1em;
|
||||||
|
padding: 0.5em 1em;
|
||||||
|
border: 1px solid $border-color-2;
|
||||||
|
border-radius: 1em;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
background: $hover-bg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
form {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column !important;
|
||||||
|
|
||||||
|
.buttons {
|
||||||
|
flex-direction: row !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
|
@ -0,0 +1,250 @@
|
||||||
|
<template>
|
||||||
|
<div class="entity assistant-container">
|
||||||
|
<TextPrompt ref="prompt">
|
||||||
|
Enter a text query to send to the assistant.
|
||||||
|
</TextPrompt>
|
||||||
|
|
||||||
|
<div class="head" @click="onHeadClick">
|
||||||
|
<div class="col-1 icon entity-icon" ref="icon">
|
||||||
|
<EntityIcon
|
||||||
|
:entity="value"
|
||||||
|
:class="{active: value.conversation_running}"
|
||||||
|
:loading="loading"
|
||||||
|
:error="error" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="label">
|
||||||
|
<div class="name" ref="name" v-text="value.name" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="value-container">
|
||||||
|
<button @click.stop="collapsed = !collapsed">
|
||||||
|
<i class="fas"
|
||||||
|
:class="{'fa-angle-up': !collapsed, 'fa-angle-down': collapsed}" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="body" ref="body" v-if="!collapsed" @click.stop="prevent">
|
||||||
|
<div class="row" @click.stop="stopConversation" v-if="value.conversation_running">
|
||||||
|
<div class="icon">
|
||||||
|
<i class="fas fa-comment-slash" />
|
||||||
|
</div>
|
||||||
|
<div class="label">
|
||||||
|
<div class="name">Stop Conversation</div>
|
||||||
|
</div>
|
||||||
|
<div class="value">
|
||||||
|
<ToggleSwitch
|
||||||
|
@click.stop="stopConversation"
|
||||||
|
:value="false"
|
||||||
|
:disabled="loading" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row" @click.stop="startConversation" v-else>
|
||||||
|
<div class="icon">
|
||||||
|
<i class="fas fa-comment" />
|
||||||
|
</div>
|
||||||
|
<div class="label">
|
||||||
|
<div class="name">Start Conversation</div>
|
||||||
|
</div>
|
||||||
|
<div class="value">
|
||||||
|
<ToggleSwitch
|
||||||
|
@click.stop="startConversation"
|
||||||
|
:value="false"
|
||||||
|
:disabled="loading" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row" @click.stop="toggleMute">
|
||||||
|
<div class="icon">
|
||||||
|
<i class="fas fa-microphone-lines-slash" />
|
||||||
|
</div>
|
||||||
|
<div class="label">
|
||||||
|
<div class="name">Muted</div>
|
||||||
|
</div>
|
||||||
|
<div class="value">
|
||||||
|
<ToggleSwitch
|
||||||
|
@click.stop="toggleMute"
|
||||||
|
:value="value.is_muted"
|
||||||
|
:disabled="loading" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row" @click.stop="showPrompt">
|
||||||
|
<div class="icon">
|
||||||
|
<i class="fas fa-keyboard" />
|
||||||
|
</div>
|
||||||
|
<div class="label">
|
||||||
|
<div class="name">Send query from text prompt</div>
|
||||||
|
</div>
|
||||||
|
<div class="value">
|
||||||
|
<ToggleSwitch
|
||||||
|
@click.stop="showPrompt"
|
||||||
|
:value="false"
|
||||||
|
:disabled="loading" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
import TextPrompt from "@/components/elements/TextPrompt"
|
||||||
|
import ToggleSwitch from "@/components/elements/ToggleSwitch"
|
||||||
|
import EntityIcon from "./EntityIcon"
|
||||||
|
import EntityMixin from "./EntityMixin"
|
||||||
|
|
||||||
|
export default {
|
||||||
|
name: 'Assistant',
|
||||||
|
mixins: [EntityMixin],
|
||||||
|
components: {
|
||||||
|
EntityIcon,
|
||||||
|
TextPrompt,
|
||||||
|
ToggleSwitch,
|
||||||
|
},
|
||||||
|
|
||||||
|
data() {
|
||||||
|
return {
|
||||||
|
collapsed: true,
|
||||||
|
showTextQueryPrompt: false,
|
||||||
|
modalId: 'assistant-text-prompt-modal',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
methods: {
|
||||||
|
hidePrompt() {
|
||||||
|
document.body.querySelector(`#${this.modalId}`)?.remove()
|
||||||
|
},
|
||||||
|
|
||||||
|
showPrompt() {
|
||||||
|
const modalElement = this.$refs.prompt.$el
|
||||||
|
this.hidePrompt()
|
||||||
|
|
||||||
|
modalElement.id = this.modalId
|
||||||
|
modalElement.classList.remove('hidden')
|
||||||
|
|
||||||
|
const input = modalElement.querySelector('input[type="text"]')
|
||||||
|
const form = modalElement.querySelector('form')
|
||||||
|
if (form) {
|
||||||
|
form.addEventListener('submit', (event) => {
|
||||||
|
event.stopPropagation()
|
||||||
|
this.onTextPrompt(input?.value)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const cancelBtn = modalElement.querySelector('.cancel-btn')
|
||||||
|
if (cancelBtn) {
|
||||||
|
cancelBtn.onclick = (event) => {
|
||||||
|
this.hidePrompt()
|
||||||
|
event.stopPropagation()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
modalElement.onclick = (event) => {
|
||||||
|
const modalContent = modalElement.querySelector('.modal')
|
||||||
|
if (modalContent?.contains(event.target)) {
|
||||||
|
event.stopPropagation()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
this.hidePrompt()
|
||||||
|
}
|
||||||
|
|
||||||
|
document.body.appendChild(modalElement)
|
||||||
|
this.$nextTick(() => {
|
||||||
|
modalElement.querySelector('input[type="text"]').focus()
|
||||||
|
})
|
||||||
|
},
|
||||||
|
|
||||||
|
onHeadClick(event) {
|
||||||
|
if (
|
||||||
|
this.$refs.name.contains(event.target) ||
|
||||||
|
this.$refs.icon.contains(event.target)
|
||||||
|
) {
|
||||||
|
// Propagate the event upwards and let it open the entity modal
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toggle the collapse state if the click is outside of the entity
|
||||||
|
// name/icon
|
||||||
|
this.collapsed = !this.collapsed
|
||||||
|
event.stopPropagation()
|
||||||
|
},
|
||||||
|
|
||||||
|
async toggleMute() {
|
||||||
|
await this.request('entities.execute', {
|
||||||
|
id: this.value.id,
|
||||||
|
action: 'toggle_mute',
|
||||||
|
})
|
||||||
|
},
|
||||||
|
|
||||||
|
async startConversation() {
|
||||||
|
await this.request('entities.execute', {
|
||||||
|
id: this.value.id,
|
||||||
|
action: 'start_conversation',
|
||||||
|
})
|
||||||
|
},
|
||||||
|
|
||||||
|
async stopConversation() {
|
||||||
|
await this.request('entities.execute', {
|
||||||
|
id: this.value.id,
|
||||||
|
action: 'stop_conversation',
|
||||||
|
})
|
||||||
|
},
|
||||||
|
|
||||||
|
async onTextPrompt(query) {
|
||||||
|
await this.request('entities.execute', {
|
||||||
|
id: this.value.id,
|
||||||
|
action: 'send_text_query',
|
||||||
|
query: query,
|
||||||
|
})
|
||||||
|
|
||||||
|
this.hidePrompt()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style lang="scss" scoped>
|
||||||
|
@import "common";
|
||||||
|
|
||||||
|
$icon-size: 2em;
|
||||||
|
|
||||||
|
.assistant-container {
|
||||||
|
.body {
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.row {
|
||||||
|
margin: 0;
|
||||||
|
padding: 1em 0.5em;
|
||||||
|
display: flex;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
background: $hover-bg;
|
||||||
|
}
|
||||||
|
|
||||||
|
&:not(:last-child) {
|
||||||
|
border-bottom: 1px solid $border-color-1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.icon {
|
||||||
|
flex: 0 0 $icon-size;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.label {
|
||||||
|
width: calc(100% - $icon-size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.entity-icon) {
|
||||||
|
.active {
|
||||||
|
color: $selected-fg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</style>
|
|
@ -1,4 +1,12 @@
|
||||||
{
|
{
|
||||||
|
"assistant": {
|
||||||
|
"name": "Assistant",
|
||||||
|
"name_plural": "Assistants",
|
||||||
|
"icon": {
|
||||||
|
"class": "fas fa-microphone-lines"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
"battery": {
|
"battery": {
|
||||||
"name": "Battery",
|
"name": "Battery",
|
||||||
"name_plural": "Batteries",
|
"name_plural": "Batteries",
|
||||||
|
|
|
@ -280,6 +280,35 @@ backend.http:
|
||||||
# - platypush/tests
|
# - platypush/tests
|
||||||
###
|
###
|
||||||
|
|
||||||
|
###
|
||||||
|
# # Example configuration of a voice assistant.
|
||||||
|
# # Several voice assistant plugins and engines are available - Google
|
||||||
|
# # Assistant, Alexa, DeepSpeech, Picovoice etc.
|
||||||
|
# #
|
||||||
|
# # The Google Assistant is probably the most straightforward to configure and
|
||||||
|
# # the richest in terms of features provided out-of-the-box and speech
|
||||||
|
# # detection quality, while others may require further tinkering, may perform
|
||||||
|
# # worse than the Google model, and/or may run models on-device which could not
|
||||||
|
# # be within reach for some machines.
|
||||||
|
# #
|
||||||
|
# # Check the documentation of the `assistant.google` plugin for instructions on
|
||||||
|
# # how to get a credentials file that you can use with a custom assistant
|
||||||
|
# # installation.
|
||||||
|
# #
|
||||||
|
# # Note however that the Google Assistant plugin leverages the
|
||||||
|
# # `google-assistant-library`, which has been deprecated a while ago by Google
|
||||||
|
# # and it's unlikely to receive updates (although it still works and I'd still
|
||||||
|
# # expect it to work).
|
||||||
|
#
|
||||||
|
# assistant.google:
|
||||||
|
# # Path to your credentials file (by default it will look either under
|
||||||
|
# # ~/.config/google-oauthlib-tool/credentials.json or
|
||||||
|
# # <WORKDIR>/credentials/google/assistant.json
|
||||||
|
# # credentials_file: ~/credentials/assistant.json
|
||||||
|
# # If specified, then this sound will be played when a conversation starts
|
||||||
|
# # conversation_start_sound: ~/sounds/assistant-start.mp3
|
||||||
|
###
|
||||||
|
|
||||||
###
|
###
|
||||||
# # Example configuration of music.mpd plugin, a plugin to interact with MPD and
|
# # Example configuration of music.mpd plugin, a plugin to interact with MPD and
|
||||||
# # Mopidy music server instances. See
|
# # Mopidy music server instances. See
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
from sqlalchemy import Column, Integer, ForeignKey, Boolean, String
|
||||||
|
|
||||||
|
from platypush.common.db import is_defined
|
||||||
|
|
||||||
|
from . import Entity
|
||||||
|
|
||||||
|
|
||||||
|
if not is_defined('assistant'):
|
||||||
|
|
||||||
|
class Assistant(Entity):
|
||||||
|
"""
|
||||||
|
Base class for voice assistant entities.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = 'assistant'
|
||||||
|
|
||||||
|
id = Column(
|
||||||
|
Integer, ForeignKey(Entity.id, ondelete='CASCADE'), primary_key=True
|
||||||
|
)
|
||||||
|
last_query = Column(String)
|
||||||
|
last_response = Column(String)
|
||||||
|
conversation_running = Column(Boolean)
|
||||||
|
is_muted = Column(Boolean, default=False)
|
||||||
|
is_detecting = Column(Boolean, default=True)
|
||||||
|
|
||||||
|
__table_args__ = {'extend_existing': True}
|
||||||
|
__mapper_args__ = {
|
||||||
|
'polymorphic_identity': __tablename__,
|
||||||
|
}
|
|
@ -0,0 +1,78 @@
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
from . import EntityManager
|
||||||
|
|
||||||
|
|
||||||
|
class AssistantEntityManager(EntityManager, ABC):
|
||||||
|
"""
|
||||||
|
Base class for voice assistant integrations that support entity management.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def start_conversation(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Programmatically starts a conversation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def stop_conversation(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Programmatically stops a conversation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def is_muted(self, *args, **kwargs) -> bool:
|
||||||
|
"""
|
||||||
|
:return: True if the microphone is muted, False otherwise.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def mute(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Mute the microphone.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def unmute(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Unmute the microphone.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def toggle_mute(self, *_, **__):
|
||||||
|
"""
|
||||||
|
Toggle the mute state of the microphone.
|
||||||
|
"""
|
||||||
|
return self.mute() if self.is_muted() else self.unmute()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def pause_detection(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Put the assistant on pause. No new conversation events will be triggered.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def resume_detection(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Resume the assistant hotword detection from a paused state.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def is_detecting(self, *args, **kwargs) -> bool:
|
||||||
|
"""
|
||||||
|
:return: True if the asistant is detecting, False otherwise.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def send_text_query(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Send a text query to the assistant.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
|
@ -1,8 +1,6 @@
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from platypush.context import get_backend, get_plugin
|
|
||||||
from platypush.message.event import Event
|
from platypush.message.event import Event
|
||||||
|
|
||||||
|
|
||||||
|
@ -11,18 +9,7 @@ class AssistantEvent(Event):
|
||||||
|
|
||||||
def __init__(self, *args, assistant=None, **kwargs):
|
def __init__(self, *args, assistant=None, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.logger = logging.getLogger('platypush:assistant')
|
self._assistant = assistant
|
||||||
|
|
||||||
if assistant:
|
|
||||||
self._assistant = assistant
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
self._assistant = get_backend('assistant.google')
|
|
||||||
if not self._assistant:
|
|
||||||
self._assistant = get_plugin('assistant.google.pushtotalk')
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.debug('Could not initialize the assistant component: %s', e)
|
|
||||||
self._assistant = None
|
|
||||||
|
|
||||||
|
|
||||||
class ConversationStartEvent(AssistantEvent):
|
class ConversationStartEvent(AssistantEvent):
|
||||||
|
|
|
@ -1,54 +1,286 @@
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from enum import Enum
|
||||||
|
import os
|
||||||
|
from threading import Event
|
||||||
|
from typing import Any, Collection, Dict, Optional
|
||||||
|
|
||||||
from platypush.context import get_backend
|
from platypush.context import get_bus, get_plugin
|
||||||
|
from platypush.entities.assistants import Assistant
|
||||||
|
from platypush.entities.managers.assistants import AssistantEntityManager
|
||||||
|
from platypush.message.event.assistant import (
|
||||||
|
AssistantEvent,
|
||||||
|
ConversationStartEvent,
|
||||||
|
ConversationEndEvent,
|
||||||
|
ConversationTimeoutEvent,
|
||||||
|
ResponseEvent,
|
||||||
|
NoResponseEvent,
|
||||||
|
SpeechRecognizedEvent,
|
||||||
|
AlarmStartedEvent,
|
||||||
|
AlarmEndEvent,
|
||||||
|
TimerStartedEvent,
|
||||||
|
TimerEndEvent,
|
||||||
|
AlertStartedEvent,
|
||||||
|
AlertEndEvent,
|
||||||
|
MicMutedEvent,
|
||||||
|
MicUnmutedEvent,
|
||||||
|
)
|
||||||
from platypush.plugins import Plugin, action
|
from platypush.plugins import Plugin, action
|
||||||
|
from platypush.utils import get_plugin_name_by_class
|
||||||
|
|
||||||
|
|
||||||
class AssistantPlugin(ABC, Plugin):
|
@dataclass
|
||||||
|
class AlertType(Enum):
|
||||||
"""
|
"""
|
||||||
Base class for assistant plugins
|
Enum representing the type of an alert.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
ALARM = 'alarm'
|
||||||
|
TIMER = 'timer'
|
||||||
|
ALERT = 'alert'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AssistantState:
|
||||||
|
"""
|
||||||
|
Dataclass representing the state of an assistant.
|
||||||
|
"""
|
||||||
|
|
||||||
|
last_query: Optional[str] = None
|
||||||
|
last_response: Optional[str] = None
|
||||||
|
conversation_running: bool = False
|
||||||
|
is_muted: bool = False
|
||||||
|
is_detecting: bool = True
|
||||||
|
alert_state: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class AssistantPlugin(Plugin, AssistantEntityManager, ABC):
|
||||||
|
"""
|
||||||
|
Base class for assistant plugins.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_entity_name = 'Assistant'
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*args,
|
||||||
|
tts_plugin: Optional[str] = None,
|
||||||
|
tts_plugin_args: Optional[Dict[str, Any]] = None,
|
||||||
|
conversation_start_sound: Optional[str] = None,
|
||||||
|
**kwargs
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
:param tts_plugin: If set, the assistant will use this plugin (e.g.
|
||||||
|
``tts``, ``tts.google`` or ``tts.mimic3``) to render the responses,
|
||||||
|
instead of using the built-in assistant voice.
|
||||||
|
|
||||||
|
:param tts_plugin_args: Optional arguments to be passed to the TTS
|
||||||
|
``say`` action, if ``tts_plugin`` is set.
|
||||||
|
|
||||||
|
:param conversation_start_sound: If set, the assistant will play this
|
||||||
|
audio file when it detects a speech. The sound file will be played
|
||||||
|
on the default audio output device. If not set, the assistant won't
|
||||||
|
play any sound when it detects a speech.
|
||||||
|
"""
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.tts_plugin = tts_plugin
|
||||||
|
self.tts_plugin_args = tts_plugin_args or {}
|
||||||
|
if conversation_start_sound:
|
||||||
|
self._conversation_start_sound = os.path.abspath(
|
||||||
|
os.path.expanduser(conversation_start_sound)
|
||||||
|
)
|
||||||
|
|
||||||
|
self._detection_paused = Event()
|
||||||
|
self._conversation_running = Event()
|
||||||
|
self._is_muted = False
|
||||||
|
self._last_query: Optional[str] = None
|
||||||
|
self._last_response: Optional[str] = None
|
||||||
|
self._cur_alert_type: Optional[AlertType] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _state(self) -> AssistantState:
|
||||||
|
return AssistantState(
|
||||||
|
last_query=self._last_query,
|
||||||
|
last_response=self._last_response,
|
||||||
|
conversation_running=self._conversation_running.is_set(),
|
||||||
|
is_muted=self._is_muted,
|
||||||
|
is_detecting=not self._detection_paused.is_set(),
|
||||||
|
alert_state=self._cur_alert_type.value if self._cur_alert_type else None,
|
||||||
|
)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def start_conversation(self, *args, language=None, tts_plugin=None, tts_args=None, **kwargs):
|
def start_conversation(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Start a conversation.
|
Programmatically starts a conversation.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def stop_conversation(self, *args, **kwargs):
|
def stop_conversation(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Stop a conversation.
|
Programmatically stops a conversation.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def _get_assistant(self):
|
|
||||||
return get_backend('assistant.snowboy')
|
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def pause_detection(self):
|
def pause_detection(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Put the assistant on pause. No new conversation events will be triggered.
|
Put the assistant on pause. No new conversation events will be triggered.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
self._detection_paused.set()
|
||||||
assistant.pause_detection()
|
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def resume_detection(self):
|
def resume_detection(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Resume the assistant hotword detection from a paused state.
|
Resume the assistant hotword detection from a paused state.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
self._detection_paused.clear()
|
||||||
assistant.resume_detection()
|
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def is_detecting(self) -> bool:
|
def is_detecting(self, *_, **__) -> bool:
|
||||||
"""
|
"""
|
||||||
:return: True if the asistant is detecting, False otherwise.
|
:return: True if the asistant is detecting, False otherwise.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
return not self._detection_paused.is_set()
|
||||||
return assistant.is_detecting()
|
|
||||||
|
@action
|
||||||
|
def is_muted(self, *_, **__) -> bool:
|
||||||
|
"""
|
||||||
|
:return: True if the microphone is muted, False otherwise.
|
||||||
|
"""
|
||||||
|
return self._is_muted
|
||||||
|
|
||||||
|
@action
|
||||||
|
def status(self, *_, **__):
|
||||||
|
"""
|
||||||
|
:return: The current assistant status:
|
||||||
|
|
||||||
|
.. code-block:: json
|
||||||
|
|
||||||
|
{
|
||||||
|
"last_query": "What time is it?",
|
||||||
|
"last_response": "It's 10:30 AM",
|
||||||
|
"conversation_running": true,
|
||||||
|
"is_muted": false,
|
||||||
|
"is_detecting": true
|
||||||
|
}
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.publish_entities([self])
|
||||||
|
return asdict(self._state)
|
||||||
|
|
||||||
|
def _get_tts_plugin(self):
|
||||||
|
if not self.tts_plugin:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return get_plugin(self.tts_plugin)
|
||||||
|
|
||||||
|
def _play_conversation_start_sound(self):
|
||||||
|
if not self._conversation_start_sound:
|
||||||
|
return
|
||||||
|
|
||||||
|
audio = get_plugin('sound')
|
||||||
|
if not audio:
|
||||||
|
self.logger.warning(
|
||||||
|
'Unable to play conversation start sound: sound plugin not found'
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
audio.play(self._conversation_start_sound)
|
||||||
|
|
||||||
|
def _send_event(self, event: AssistantEvent):
|
||||||
|
self.publish_entities([self])
|
||||||
|
get_bus().post(event)
|
||||||
|
|
||||||
|
def _on_conversation_start(self):
|
||||||
|
self._last_response = None
|
||||||
|
self._last_query = None
|
||||||
|
self._conversation_running.set()
|
||||||
|
self._send_event(ConversationStartEvent(assistant=self))
|
||||||
|
self._play_conversation_start_sound()
|
||||||
|
|
||||||
|
def _on_conversation_end(self):
|
||||||
|
self._conversation_running.clear()
|
||||||
|
self._send_event(ConversationEndEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_conversation_timeout(self):
|
||||||
|
self._last_response = None
|
||||||
|
self._last_query = None
|
||||||
|
self._conversation_running.clear()
|
||||||
|
self._send_event(ConversationTimeoutEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_no_response(self):
|
||||||
|
self._last_response = None
|
||||||
|
self._conversation_running.clear()
|
||||||
|
self._send_event(NoResponseEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_reponse_rendered(self, text: Optional[str]):
|
||||||
|
self._last_response = text
|
||||||
|
self._send_event(ResponseEvent(assistant=self, response_text=text))
|
||||||
|
tts = self._get_tts_plugin()
|
||||||
|
|
||||||
|
if tts and text:
|
||||||
|
self.stop_conversation()
|
||||||
|
tts.say(text=text, **self.tts_plugin_args)
|
||||||
|
|
||||||
|
def _on_speech_recognized(self, phrase: Optional[str]):
|
||||||
|
phrase = (phrase or '').lower().strip()
|
||||||
|
self._last_query = phrase
|
||||||
|
self._send_event(SpeechRecognizedEvent(assistant=self, phrase=phrase))
|
||||||
|
|
||||||
|
def _on_alarm_start(self):
|
||||||
|
self._cur_alert_type = AlertType.ALARM
|
||||||
|
self._send_event(AlarmStartedEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_alarm_end(self):
|
||||||
|
self._cur_alert_type = None
|
||||||
|
self._send_event(AlarmEndEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_timer_start(self):
|
||||||
|
self._cur_alert_type = AlertType.TIMER
|
||||||
|
self._send_event(TimerStartedEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_timer_end(self):
|
||||||
|
self._cur_alert_type = None
|
||||||
|
self._send_event(TimerEndEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_alert_start(self):
|
||||||
|
self._cur_alert_type = AlertType.ALERT
|
||||||
|
self._send_event(AlertStartedEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_alert_end(self):
|
||||||
|
self._cur_alert_type = None
|
||||||
|
self._send_event(AlertEndEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_mute(self):
|
||||||
|
self._is_muted = True
|
||||||
|
self._send_event(MicMutedEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_unmute(self):
|
||||||
|
self._is_muted = False
|
||||||
|
self._send_event(MicUnmutedEvent(assistant=self))
|
||||||
|
|
||||||
|
def _on_mute_changed(self, value: bool):
|
||||||
|
if value:
|
||||||
|
self._on_mute()
|
||||||
|
else:
|
||||||
|
self._on_unmute()
|
||||||
|
|
||||||
|
def transform_entities(self, entities: Collection['AssistantPlugin']):
|
||||||
|
return super().transform_entities(
|
||||||
|
[
|
||||||
|
Assistant(
|
||||||
|
external_id=get_plugin_name_by_class(type(dev)),
|
||||||
|
name=self._entity_name,
|
||||||
|
last_query=dev._state.last_query,
|
||||||
|
last_response=dev._state.last_response,
|
||||||
|
conversation_running=dev._state.conversation_running,
|
||||||
|
is_muted=dev._state.is_muted,
|
||||||
|
is_detecting=dev._state.is_detecting,
|
||||||
|
)
|
||||||
|
for dev in (entities or [])
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
|
@ -1,133 +0,0 @@
|
||||||
import os
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from platypush.context import get_bus
|
|
||||||
from platypush.plugins import action
|
|
||||||
from platypush.plugins.assistant import AssistantPlugin
|
|
||||||
|
|
||||||
from platypush.message.event.assistant import (
|
|
||||||
ConversationStartEvent,
|
|
||||||
ConversationEndEvent,
|
|
||||||
SpeechRecognizedEvent,
|
|
||||||
ResponseEvent,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantEchoPlugin(AssistantPlugin):
|
|
||||||
"""
|
|
||||||
Amazon Echo/Alexa assistant plugin.
|
|
||||||
|
|
||||||
In order to activate the Echo service on your device follow these steps:
|
|
||||||
|
|
||||||
1. Install avs (``pip install git+https://github.com/BlackLight/avs.git``)
|
|
||||||
2. Run the ``alexa-auth`` script. A local webservice will start on port 3000
|
|
||||||
3. If a browser instance doesn't open automatically then head to http://localhost:3000
|
|
||||||
4. Log in to your Amazon account
|
|
||||||
5. The required credentials will be stored to ~/.avs.json
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
avs_config_file: Optional[str] = None,
|
|
||||||
audio_device: str = 'default',
|
|
||||||
audio_player: str = 'default',
|
|
||||||
**kwargs
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
:param avs_config_file: AVS credentials file - default: ~/.avs.json. If the file doesn't exist then
|
|
||||||
an instance of the AVS authentication service will be spawned. You can login through an Amazon
|
|
||||||
account either in the spawned browser window, if available, or by opening http://your-ip:3000
|
|
||||||
in the browser on another machine.
|
|
||||||
|
|
||||||
:param audio_device: Name of the input audio device (default: 'default')
|
|
||||||
:param audio_player: Player to be used for audio playback (default: 'default').
|
|
||||||
Supported values: 'mpv', 'mpg123', 'gstreamer'
|
|
||||||
"""
|
|
||||||
from avs.alexa import Alexa
|
|
||||||
from avs.config import DEFAULT_CONFIG_FILE
|
|
||||||
from avs.mic import Audio
|
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
if not avs_config_file:
|
|
||||||
avs_config_file = DEFAULT_CONFIG_FILE
|
|
||||||
|
|
||||||
if not avs_config_file or not os.path.isfile(avs_config_file):
|
|
||||||
from avs.auth import auth
|
|
||||||
|
|
||||||
auth(None, avs_config_file)
|
|
||||||
self.logger.warning(
|
|
||||||
'Amazon Echo assistant credentials not configured. Open http://localhost:3000 '
|
|
||||||
+ 'to authenticate this client'
|
|
||||||
)
|
|
||||||
|
|
||||||
self.audio_device = audio_device
|
|
||||||
self.audio_player = audio_player
|
|
||||||
self.audio = Audio(device_name=audio_device)
|
|
||||||
self.alexa = Alexa(avs_config_file, audio_player=audio_player)
|
|
||||||
self._ready = False
|
|
||||||
|
|
||||||
self.alexa.state_listener.on_ready = self._on_ready()
|
|
||||||
self.alexa.state_listener.on_listening = self._on_listening()
|
|
||||||
self.alexa.state_listener.on_speaking = self._on_speaking()
|
|
||||||
self.alexa.state_listener.on_thinking = self._on_thinking()
|
|
||||||
self.alexa.state_listener.on_finished = self._on_finished()
|
|
||||||
self.alexa.state_listener.on_disconnected = self._on_disconnected()
|
|
||||||
|
|
||||||
self.audio.link(self.alexa)
|
|
||||||
self.alexa.start()
|
|
||||||
|
|
||||||
def _on_ready(self):
|
|
||||||
def _callback():
|
|
||||||
self._ready = True
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
def _on_listening(self):
|
|
||||||
def _callback():
|
|
||||||
get_bus().post(ConversationStartEvent(assistant=self))
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
def _on_speaking(self):
|
|
||||||
def _callback():
|
|
||||||
# AVS doesn't provide a way to access the response text
|
|
||||||
get_bus().post(ResponseEvent(assistant=self, response_text=''))
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
def _on_finished(self):
|
|
||||||
def _callback():
|
|
||||||
get_bus().post(ConversationEndEvent(assistant=self))
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
def _on_disconnected(self):
|
|
||||||
def _callback():
|
|
||||||
self._ready = False
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
def _on_thinking(self):
|
|
||||||
def _callback():
|
|
||||||
# AVS doesn't provide a way to access the detected text
|
|
||||||
get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=''))
|
|
||||||
|
|
||||||
return _callback
|
|
||||||
|
|
||||||
@action
|
|
||||||
def start_conversation(self, **_):
|
|
||||||
if not self._ready:
|
|
||||||
raise RuntimeError('Echo assistant not ready')
|
|
||||||
|
|
||||||
self.audio.start()
|
|
||||||
self.alexa.listen()
|
|
||||||
|
|
||||||
@action
|
|
||||||
def stop_conversation(self):
|
|
||||||
self.audio.stop()
|
|
||||||
self._on_finished()()
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
|
@ -1,13 +0,0 @@
|
||||||
manifest:
|
|
||||||
events:
|
|
||||||
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
|
|
||||||
ends
|
|
||||||
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
|
|
||||||
starts
|
|
||||||
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
|
|
||||||
is recognized
|
|
||||||
install:
|
|
||||||
pip:
|
|
||||||
- avs
|
|
||||||
package: platypush.plugins.assistant.echo
|
|
||||||
type: plugin
|
|
|
@ -1,76 +1,313 @@
|
||||||
from platypush.backend.assistant.google import AssistantGoogleBackend
|
import json
|
||||||
from platypush.context import get_backend
|
import os
|
||||||
from platypush.plugins import action
|
from typing import Optional
|
||||||
|
|
||||||
|
from platypush.config import Config
|
||||||
|
from platypush.plugins import RunnablePlugin, action
|
||||||
from platypush.plugins.assistant import AssistantPlugin
|
from platypush.plugins.assistant import AssistantPlugin
|
||||||
|
|
||||||
|
|
||||||
class AssistantGooglePlugin(AssistantPlugin):
|
class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
|
||||||
"""
|
"""
|
||||||
Google assistant plugin.
|
Google Assistant plugin.
|
||||||
It acts like a wrapper around the :mod:`platypush.backend.assistant.google`
|
|
||||||
backend to programmatically control the conversation status.
|
This plugin allows you to run the Google Assistant _directly_ on your
|
||||||
|
device. It requires you to have an audio microphone and a speaker connected
|
||||||
|
to the device.
|
||||||
|
|
||||||
|
If you have multiple sound devices, you can specify which one(s) to use for
|
||||||
|
input and output through a ``~/.asoundrc`` configuration file like this:
|
||||||
|
|
||||||
|
.. code-block:: text
|
||||||
|
|
||||||
|
pcm.!default {
|
||||||
|
type asym
|
||||||
|
playback.pcm {
|
||||||
|
type plug
|
||||||
|
slave.pcm "hw:0,0"
|
||||||
|
}
|
||||||
|
capture.pcm {
|
||||||
|
type plug
|
||||||
|
slave.pcm "hw:1,0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
You can use ``aplay -l`` and ``arecord -l`` to respectively list the
|
||||||
|
detected audio output and input devices with their indices.
|
||||||
|
|
||||||
|
If you are using PulseAudio instead of bare ALSA, then you can:
|
||||||
|
|
||||||
|
1. Use the ``pavucontrol`` (GUI) tool to select the audio input and
|
||||||
|
output devices and volumes for the assistant.
|
||||||
|
2. Use a program like ``pamix`` (ncurses) or ``pamixer`` (CLI).
|
||||||
|
3. Run the ``pactl list sources`` and ``pactl list sinks`` commands to
|
||||||
|
respectively list the detected audio input and output devices. Take
|
||||||
|
note of their name, and specify which ones the assistant should use
|
||||||
|
by starting the application with the right ``PULSE_SOURCE`` and
|
||||||
|
``PULSE_SINK`` environment variables.
|
||||||
|
|
||||||
|
.. warning:: The Google Assistant library used by this backend has
|
||||||
|
been deprecated by Google:
|
||||||
|
https://developers.google.com/assistant/sdk/reference/library/python/.
|
||||||
|
This integration still works on all of my devices, but its future
|
||||||
|
functionality is not guaranteed - Google may decide to turn off the
|
||||||
|
API, the library may no longer be built against new architectures and
|
||||||
|
it's unlikely to be updated.
|
||||||
|
|
||||||
|
.. note:: Since the Google Assistant library hasn't been updated in several
|
||||||
|
years, some of its dependencies are quite old and may break more recent
|
||||||
|
Python installations. Please refer to the comments in the [manifest
|
||||||
|
file](https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml)
|
||||||
|
for more information on how to install the required dependencies, if
|
||||||
|
the automated ways fail.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
_entity_name = 'Google Assistant'
|
||||||
|
|
||||||
|
_default_credentials_files = (
|
||||||
|
os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'),
|
||||||
|
os.path.join(
|
||||||
|
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
credentials_file: Optional[str] = None,
|
||||||
|
device_model_id: str = 'Platypush',
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
:param credentials_file: Path to the Google OAuth credentials file.
|
||||||
|
See
|
||||||
|
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
|
||||||
|
for instructions to get your own credentials file.
|
||||||
|
By default, it will search for the credentials file under:
|
||||||
|
|
||||||
|
* ``~/.config/google-oauthlib-tool/credentials.json``: default
|
||||||
|
location supported by the Google Assistant library
|
||||||
|
|
||||||
|
* ``<PLATYPUSH_WORKDIR>/credentials/google/assistant.json``:
|
||||||
|
recommended location, under the Platypush working directory.
|
||||||
|
|
||||||
|
:param device_model_id: The device model ID that identifies the device
|
||||||
|
where the assistant is running (default: Platypush). It can be a
|
||||||
|
custom string.
|
||||||
|
"""
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
self._credentials_file = credentials_file
|
||||||
|
self.device_model_id = device_model_id
|
||||||
|
self.credentials = None
|
||||||
|
self._assistant = None
|
||||||
|
self.logger.info('Initialized Google Assistant plugin')
|
||||||
|
|
||||||
def _get_assistant(self) -> AssistantGoogleBackend:
|
@property
|
||||||
backend = get_backend('assistant.google')
|
def credentials_file(self) -> str:
|
||||||
assert backend, 'The assistant.google backend is not configured.'
|
if self._credentials_file:
|
||||||
return backend
|
return os.path.abspath(os.path.expanduser(self._credentials_file))
|
||||||
|
|
||||||
|
f = None
|
||||||
|
for default_file in self._default_credentials_files:
|
||||||
|
f = default_file
|
||||||
|
if os.path.isfile(default_file):
|
||||||
|
break
|
||||||
|
|
||||||
|
assert f, 'No credentials_file provided and no default file found'
|
||||||
|
return f
|
||||||
|
|
||||||
|
@property
|
||||||
|
def assistant(self):
|
||||||
|
if not self._assistant:
|
||||||
|
self.logger.warning('The assistant is not running')
|
||||||
|
return
|
||||||
|
|
||||||
|
return self._assistant
|
||||||
|
|
||||||
|
def _process_event(self, event):
|
||||||
|
from google.assistant.library.event import EventType, AlertType
|
||||||
|
|
||||||
|
self.logger.info('Received assistant event: %s', event)
|
||||||
|
|
||||||
|
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
|
||||||
|
self._on_conversation_start()
|
||||||
|
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
|
||||||
|
if not event.args.get('with_follow_on_turn'):
|
||||||
|
self._on_conversation_end()
|
||||||
|
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
|
||||||
|
self._on_conversation_timeout()
|
||||||
|
elif event.type == EventType.ON_NO_RESPONSE:
|
||||||
|
self._on_no_response()
|
||||||
|
elif (
|
||||||
|
hasattr(EventType, 'ON_RENDER_RESPONSE')
|
||||||
|
and event.type == EventType.ON_RENDER_RESPONSE
|
||||||
|
):
|
||||||
|
self._on_reponse_rendered(event.args.get('text'))
|
||||||
|
elif (
|
||||||
|
hasattr(EventType, 'ON_RESPONDING_STARTED')
|
||||||
|
and event.type == EventType.ON_RESPONDING_STARTED
|
||||||
|
and event.args.get('is_error_response') is True
|
||||||
|
):
|
||||||
|
self.logger.warning('Assistant response error: %s', json.dumps(event.args))
|
||||||
|
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
|
||||||
|
self._on_speech_recognized(event.args.get('text'))
|
||||||
|
elif event.type == EventType.ON_ALERT_STARTED:
|
||||||
|
if event.args.get('alert_type') == AlertType.ALARM:
|
||||||
|
self._on_alarm_start()
|
||||||
|
elif event.args.get('alert_type') == AlertType.TIMER:
|
||||||
|
self._on_timer_start()
|
||||||
|
else:
|
||||||
|
self._on_alert_start()
|
||||||
|
elif event.type == EventType.ON_ALERT_FINISHED:
|
||||||
|
if event.args.get('alert_type') == AlertType.ALARM:
|
||||||
|
self._on_alarm_end()
|
||||||
|
elif event.args.get('alert_type') == AlertType.TIMER:
|
||||||
|
self._on_timer_end()
|
||||||
|
else:
|
||||||
|
self._on_alert_end()
|
||||||
|
elif event.type == EventType.ON_ASSISTANT_ERROR:
|
||||||
|
if event.args.get('is_fatal'):
|
||||||
|
raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}')
|
||||||
|
|
||||||
|
self.logger.warning('Assistant error: %s', json.dumps(event.args))
|
||||||
|
elif event.type == EventType.ON_MUTED_CHANGED:
|
||||||
|
self._on_mute_changed(event.args.get('is_muted', False))
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def start_conversation(self):
|
def start_conversation(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Programmatically start a conversation with the assistant
|
Programmatically start a conversation with the assistant
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
if self.assistant:
|
||||||
assistant.start_conversation()
|
self.assistant.start_conversation()
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def stop_conversation(self):
|
def stop_conversation(self, *_, **__):
|
||||||
"""
|
"""
|
||||||
Programmatically stop a running conversation with the assistant
|
Programmatically stop a running conversation with the assistant
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
if self.assistant:
|
||||||
assistant.stop_conversation()
|
self.assistant.stop_conversation()
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def set_mic_mute(self, muted: bool = True):
|
def mute(self, *_, **__):
|
||||||
|
"""
|
||||||
|
Mute the microphone. Alias for :meth:`.set_mic_mute` with
|
||||||
|
``muted=True``.
|
||||||
|
"""
|
||||||
|
self.set_mic_mute(muted=True)
|
||||||
|
|
||||||
|
@action
|
||||||
|
def unmute(self, *_, **__):
|
||||||
|
"""
|
||||||
|
Unmute the microphone. Alias for :meth:`.set_mic_mute` with
|
||||||
|
``muted=False``.
|
||||||
|
"""
|
||||||
|
self.set_mic_mute(muted=False)
|
||||||
|
|
||||||
|
@action
|
||||||
|
def set_mic_mute(self, muted: bool):
|
||||||
"""
|
"""
|
||||||
Programmatically mute/unmute the microphone.
|
Programmatically mute/unmute the microphone.
|
||||||
|
|
||||||
:param muted: Set to True or False.
|
:param muted: Set to True or False.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
if self.assistant:
|
||||||
assistant.set_mic_mute(muted)
|
self.assistant.set_mic_mute(muted)
|
||||||
|
|
||||||
|
if muted:
|
||||||
|
self._on_mute()
|
||||||
|
else:
|
||||||
|
self._on_unmute()
|
||||||
|
|
||||||
|
@action
|
||||||
|
def toggle_mute(self, *_, **__):
|
||||||
|
"""
|
||||||
|
Toggle the mic mute state.
|
||||||
|
"""
|
||||||
|
self.set_mic_mute(muted=not self._is_muted)
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def toggle_mic_mute(self):
|
def toggle_mic_mute(self):
|
||||||
"""
|
"""
|
||||||
Toggle the mic mute state.
|
Deprecated alias for :meth:`.toggle_mute`.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
return self.toggle_mute()
|
||||||
is_muted = assistant.is_muted()
|
|
||||||
self.set_mic_mute(muted=not is_muted)
|
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def is_muted(self) -> bool:
|
def send_text_query(self, *_, query: str, **__):
|
||||||
"""
|
|
||||||
:return: True if the microphone is muted, False otherwise.
|
|
||||||
"""
|
|
||||||
assistant = self._get_assistant()
|
|
||||||
return assistant.is_muted()
|
|
||||||
|
|
||||||
@action
|
|
||||||
def send_text_query(self, query: str):
|
|
||||||
"""
|
"""
|
||||||
Send a text query to the assistant.
|
Send a text query to the assistant.
|
||||||
|
|
||||||
|
This is equivalent to saying something to the assistant.
|
||||||
|
|
||||||
:param query: Query to be sent.
|
:param query: Query to be sent.
|
||||||
"""
|
"""
|
||||||
assistant = self._get_assistant()
|
if self.assistant:
|
||||||
assistant.send_text_query(query)
|
self.assistant.send_text_query(query)
|
||||||
|
|
||||||
|
def main(self):
|
||||||
|
import google.oauth2.credentials
|
||||||
|
from google.assistant.library import Assistant
|
||||||
|
|
||||||
|
last_sleep = 0
|
||||||
|
|
||||||
|
while not self.should_stop():
|
||||||
|
try:
|
||||||
|
with open(self.credentials_file, 'r') as f:
|
||||||
|
self.credentials = google.oauth2.credentials.Credentials(
|
||||||
|
token=None, **json.load(f)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.error(
|
||||||
|
'Error while loading Google Assistant credentials: %s', e
|
||||||
|
)
|
||||||
|
self.logger.info(
|
||||||
|
'Please follow the instructions at '
|
||||||
|
'https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample'
|
||||||
|
'#generate_credentials to get your own credentials file'
|
||||||
|
)
|
||||||
|
self.logger.exception(e)
|
||||||
|
break
|
||||||
|
|
||||||
|
try:
|
||||||
|
with Assistant(
|
||||||
|
self.credentials, self.device_model_id
|
||||||
|
) as self._assistant:
|
||||||
|
self.publish_entities([self])
|
||||||
|
for event in self._assistant.start():
|
||||||
|
last_sleep = 0
|
||||||
|
|
||||||
|
if not self.is_detecting():
|
||||||
|
self.logger.info(
|
||||||
|
'Assistant event received but detection is currently paused'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
self._process_event(event)
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.exception(e)
|
||||||
|
sleep_secs = min(60, max(5, last_sleep * 2))
|
||||||
|
self.logger.warning(
|
||||||
|
'Restarting the assistant in %d seconds after an unrecoverable error',
|
||||||
|
sleep_secs,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.wait_stop(sleep_secs)
|
||||||
|
last_sleep = sleep_secs
|
||||||
|
continue
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
try:
|
||||||
|
self.stop_conversation()
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if self._assistant:
|
||||||
|
del self._assistant
|
||||||
|
self._assistant = None
|
||||||
|
|
||||||
|
super().stop()
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
|
@ -1,236 +0,0 @@
|
||||||
"""Based on Google pushtotalk.py sample."""
|
|
||||||
|
|
||||||
import concurrent.futures
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
|
|
||||||
import grpc
|
|
||||||
|
|
||||||
from google.assistant.embedded.v1alpha2 import (
|
|
||||||
embedded_assistant_pb2,
|
|
||||||
embedded_assistant_pb2_grpc
|
|
||||||
)
|
|
||||||
from tenacity import retry, stop_after_attempt, retry_if_exception
|
|
||||||
|
|
||||||
try:
|
|
||||||
from googlesamples.assistant.grpc import (
|
|
||||||
assistant_helpers,
|
|
||||||
audio_helpers,
|
|
||||||
browser_helpers,
|
|
||||||
device_helpers
|
|
||||||
)
|
|
||||||
except (SystemError, ImportError):
|
|
||||||
import assistant_helpers
|
|
||||||
import audio_helpers
|
|
||||||
import browser_helpers
|
|
||||||
import device_helpers
|
|
||||||
|
|
||||||
|
|
||||||
ASSISTANT_API_ENDPOINT = 'embeddedassistant.googleapis.com'
|
|
||||||
END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE
|
|
||||||
DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON
|
|
||||||
CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE
|
|
||||||
PLAYING = embedded_assistant_pb2.ScreenOutConfig.PLAYING
|
|
||||||
DEFAULT_GRPC_DEADLINE = 60 * 3 + 5
|
|
||||||
|
|
||||||
|
|
||||||
class SampleAssistant(object):
|
|
||||||
"""Sample Assistant that supports conversations and device actions.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
device_model_id: identifier of the device model.
|
|
||||||
device_id: identifier of the registered device instance.
|
|
||||||
conversation_stream(ConversationStream): audio stream
|
|
||||||
for recording query and playing back assistant answer.
|
|
||||||
channel: authorized gRPC channel for connection to the
|
|
||||||
Google Assistant API.
|
|
||||||
deadline_sec: gRPC deadline in seconds for Google Assistant API call.
|
|
||||||
device_handler: callback for device actions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, language_code, device_model_id, device_id,
|
|
||||||
conversation_stream, display,
|
|
||||||
channel, deadline_sec, device_handler, play_response=True,
|
|
||||||
on_conversation_start=None, on_conversation_end=None,
|
|
||||||
on_speech_recognized=None, on_volume_changed=None,
|
|
||||||
on_response=None):
|
|
||||||
self.language_code = language_code
|
|
||||||
self.device_model_id = device_model_id
|
|
||||||
self.device_id = device_id
|
|
||||||
self.conversation_stream = conversation_stream
|
|
||||||
self.display = display
|
|
||||||
self.play_response = play_response
|
|
||||||
|
|
||||||
# Opaque blob provided in AssistResponse that,
|
|
||||||
# when provided in a follow-up AssistRequest,
|
|
||||||
# gives the Assistant a context marker within the current state
|
|
||||||
# of the multi-Assist()-RPC "conversation".
|
|
||||||
# This value, along with MicrophoneMode, supports a more natural
|
|
||||||
# "conversation" with the Assistant.
|
|
||||||
self.conversation_state = None
|
|
||||||
# Force reset of first conversation.
|
|
||||||
self.is_new_conversation = True
|
|
||||||
|
|
||||||
# Create Google Assistant API gRPC client.
|
|
||||||
self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(channel)
|
|
||||||
self.deadline = deadline_sec
|
|
||||||
|
|
||||||
self.device_handler = device_handler
|
|
||||||
self.detected_speech = None
|
|
||||||
|
|
||||||
self.on_conversation_start = on_conversation_start
|
|
||||||
self.on_conversation_end = on_conversation_end
|
|
||||||
self.on_speech_recognized = on_speech_recognized
|
|
||||||
self.on_volume_changed = on_volume_changed
|
|
||||||
self.on_response = on_response
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, etype, e, traceback):
|
|
||||||
if e:
|
|
||||||
return False
|
|
||||||
self.conversation_stream.close()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_grpc_error_unavailable(e):
|
|
||||||
is_grpc_error = isinstance(e, grpc.RpcError)
|
|
||||||
if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE):
|
|
||||||
logging.error('grpc unavailable error: %s', e)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
@retry(reraise=True, stop=stop_after_attempt(3),
|
|
||||||
retry=retry_if_exception(is_grpc_error_unavailable))
|
|
||||||
def assist(self):
|
|
||||||
"""Send a voice request to the Assistant and playback the response.
|
|
||||||
|
|
||||||
Returns: True if conversation should continue.
|
|
||||||
"""
|
|
||||||
continue_conversation = False
|
|
||||||
device_actions_futures = []
|
|
||||||
|
|
||||||
self.conversation_stream.start_recording()
|
|
||||||
if self.on_conversation_start:
|
|
||||||
self.on_conversation_start()
|
|
||||||
|
|
||||||
logging.info('Recording audio request.')
|
|
||||||
|
|
||||||
def iter_log_assist_requests():
|
|
||||||
for c in self.gen_assist_requests():
|
|
||||||
assistant_helpers.log_assist_request_without_audio(c)
|
|
||||||
yield c
|
|
||||||
logging.debug('Reached end of AssistRequest iteration.')
|
|
||||||
|
|
||||||
# This generator yields AssistResponse proto messages
|
|
||||||
# received from the gRPC Google Assistant API.
|
|
||||||
for resp in self.assistant.Assist(iter_log_assist_requests(), self.deadline):
|
|
||||||
assistant_helpers.log_assist_response_without_audio(resp)
|
|
||||||
if resp.event_type == END_OF_UTTERANCE:
|
|
||||||
logging.info('End of audio request detected.')
|
|
||||||
logging.info('Stopping recording.')
|
|
||||||
self.conversation_stream.stop_recording()
|
|
||||||
|
|
||||||
if self.detected_speech and self.on_speech_recognized:
|
|
||||||
self.on_speech_recognized(self.detected_speech)
|
|
||||||
|
|
||||||
if resp.speech_results:
|
|
||||||
self.detected_speech = ' '.join(
|
|
||||||
r.transcript.strip() for r in resp.speech_results
|
|
||||||
if len(r.transcript.strip())).strip()
|
|
||||||
|
|
||||||
logging.info('Transcript of user request: "%s".', self.detected_speech)
|
|
||||||
|
|
||||||
if len(resp.audio_out.audio_data) > 0:
|
|
||||||
if not self.conversation_stream.playing:
|
|
||||||
self.conversation_stream.stop_recording()
|
|
||||||
|
|
||||||
if self.play_response:
|
|
||||||
self.conversation_stream.start_playback()
|
|
||||||
logging.info('Playing assistant response.')
|
|
||||||
|
|
||||||
if self.play_response and self.conversation_stream.playing:
|
|
||||||
self.conversation_stream.write(resp.audio_out.audio_data)
|
|
||||||
elif self.conversation_stream.playing:
|
|
||||||
self.conversation_stream.stop_playback()
|
|
||||||
|
|
||||||
if resp.dialog_state_out.conversation_state:
|
|
||||||
conversation_state = resp.dialog_state_out.conversation_state
|
|
||||||
logging.debug('Updating conversation state.')
|
|
||||||
self.conversation_state = conversation_state
|
|
||||||
|
|
||||||
if resp.dialog_state_out.volume_percentage != 0:
|
|
||||||
volume_percentage = resp.dialog_state_out.volume_percentage
|
|
||||||
logging.info('Setting volume to %s%%', volume_percentage)
|
|
||||||
self.conversation_stream.volume_percentage = volume_percentage
|
|
||||||
|
|
||||||
if self.on_volume_changed:
|
|
||||||
self.on_volume_changed(volume_percentage)
|
|
||||||
|
|
||||||
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
|
|
||||||
continue_conversation = True
|
|
||||||
logging.info('Expecting follow-on query from user.')
|
|
||||||
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
|
|
||||||
continue_conversation = False
|
|
||||||
|
|
||||||
if resp.device_action.device_request_json:
|
|
||||||
device_request = json.loads(
|
|
||||||
resp.device_action.device_request_json
|
|
||||||
)
|
|
||||||
fs = self.device_handler(device_request)
|
|
||||||
if fs:
|
|
||||||
device_actions_futures.extend(fs)
|
|
||||||
|
|
||||||
if self.display and resp.screen_out.data:
|
|
||||||
system_browser = browser_helpers.system_browser
|
|
||||||
system_browser.display(resp.screen_out.data)
|
|
||||||
|
|
||||||
if resp.dialog_state_out.supplemental_display_text and self.on_response:
|
|
||||||
self.on_response(resp.dialog_state_out.supplemental_display_text)
|
|
||||||
|
|
||||||
if len(device_actions_futures):
|
|
||||||
logging.info('Waiting for device executions to complete.')
|
|
||||||
concurrent.futures.wait(device_actions_futures)
|
|
||||||
|
|
||||||
logging.info('Finished playing assistant response.')
|
|
||||||
self.conversation_stream.stop_playback()
|
|
||||||
|
|
||||||
if self.on_conversation_end:
|
|
||||||
self.on_conversation_end(continue_conversation)
|
|
||||||
|
|
||||||
return continue_conversation
|
|
||||||
|
|
||||||
def gen_assist_requests(self):
|
|
||||||
"""Yields: AssistRequest messages to send to the API."""
|
|
||||||
|
|
||||||
config = embedded_assistant_pb2.AssistConfig(
|
|
||||||
audio_in_config=embedded_assistant_pb2.AudioInConfig(
|
|
||||||
encoding='LINEAR16',
|
|
||||||
sample_rate_hertz=self.conversation_stream.sample_rate,
|
|
||||||
),
|
|
||||||
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
|
|
||||||
encoding='LINEAR16',
|
|
||||||
sample_rate_hertz=self.conversation_stream.sample_rate,
|
|
||||||
volume_percentage=self.conversation_stream.volume_percentage,
|
|
||||||
),
|
|
||||||
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
|
|
||||||
language_code=self.language_code,
|
|
||||||
conversation_state=self.conversation_state,
|
|
||||||
is_new_conversation=self.is_new_conversation,
|
|
||||||
),
|
|
||||||
device_config=embedded_assistant_pb2.DeviceConfig(
|
|
||||||
device_id=self.device_id,
|
|
||||||
device_model_id=self.device_model_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if self.display:
|
|
||||||
config.screen_out_config.screen_mode = PLAYING
|
|
||||||
# Continue current conversation with later requests.
|
|
||||||
self.is_new_conversation = False
|
|
||||||
# The first AssistRequest must contain the AssistConfig
|
|
||||||
# and no audio data.
|
|
||||||
yield embedded_assistant_pb2.AssistRequest(config=config)
|
|
||||||
for data in self.conversation_stream:
|
|
||||||
# Subsequent requests need audio data, but not config.
|
|
||||||
yield embedded_assistant_pb2.AssistRequest(audio_in=data)
|
|
||||||
|
|
|
@ -1,6 +1,74 @@
|
||||||
manifest:
|
manifest:
|
||||||
events: {}
|
|
||||||
install:
|
|
||||||
pip: []
|
|
||||||
package: platypush.plugins.assistant.google
|
package: platypush.plugins.assistant.google
|
||||||
type: plugin
|
type: plugin
|
||||||
|
events:
|
||||||
|
- platypush.message.event.assistant.AlarmEndEvent
|
||||||
|
- platypush.message.event.assistant.AlarmStartedEvent
|
||||||
|
- platypush.message.event.assistant.ConversationEndEvent
|
||||||
|
- platypush.message.event.assistant.ConversationStartEvent
|
||||||
|
- platypush.message.event.assistant.ConversationTimeoutEvent
|
||||||
|
- platypush.message.event.assistant.MicMutedEvent
|
||||||
|
- platypush.message.event.assistant.MicUnmutedEvent
|
||||||
|
- platypush.message.event.assistant.NoResponseEvent
|
||||||
|
- platypush.message.event.assistant.ResponseEvent
|
||||||
|
- platypush.message.event.assistant.SpeechRecognizedEvent
|
||||||
|
- platypush.message.event.assistant.TimerEndEvent
|
||||||
|
- platypush.message.event.assistant.TimerStartedEvent
|
||||||
|
|
||||||
|
install:
|
||||||
|
apk:
|
||||||
|
- ffmpeg
|
||||||
|
- portaudio-dev
|
||||||
|
- py3-cachetools
|
||||||
|
- py3-grpcio
|
||||||
|
- py3-google-auth
|
||||||
|
- py3-numpy
|
||||||
|
- py3-pathlib2
|
||||||
|
- py3-tenacity
|
||||||
|
- py3-urllib3
|
||||||
|
apt:
|
||||||
|
- ffmpeg
|
||||||
|
- portaudio19-dev
|
||||||
|
- python3-cachetools
|
||||||
|
- python3-grpcio
|
||||||
|
- python3-google-auth
|
||||||
|
- python3-monotonic
|
||||||
|
- python3-tenacity
|
||||||
|
- python3-urllib3
|
||||||
|
dnf:
|
||||||
|
- ffmpeg
|
||||||
|
- portaudio-devel
|
||||||
|
- python-cachetools
|
||||||
|
- python-grpcio
|
||||||
|
- python-google-auth
|
||||||
|
- python-monotonic
|
||||||
|
- python-numpy
|
||||||
|
- python-tenacity
|
||||||
|
- python-urllib3
|
||||||
|
pacman:
|
||||||
|
- ffmpeg
|
||||||
|
- portaudio
|
||||||
|
- python-cachetools
|
||||||
|
- python-grpcio
|
||||||
|
- python-google-auth
|
||||||
|
- python-monotonic
|
||||||
|
- python-numpy
|
||||||
|
- python-sounddevice
|
||||||
|
- python-tenacity
|
||||||
|
- python-urllib3
|
||||||
|
pip:
|
||||||
|
- google-assistant-library
|
||||||
|
- google-assistant-sdk[samples]
|
||||||
|
- google-auth
|
||||||
|
- numpy
|
||||||
|
- sounddevice
|
||||||
|
after:
|
||||||
|
# Uninstall old versions of packages that break things on recent versions
|
||||||
|
# of Python, when the new versions work just fine
|
||||||
|
- yes | pip uninstall --break-system-packages enum34 click urllib3 requests google-auth
|
||||||
|
# Upgrade the dependencies (back) to the latest version.
|
||||||
|
# NOTE: Be careful when running this command on older distros that may
|
||||||
|
# not ship the latest versions of all the packages! This is a workaround
|
||||||
|
# caused by the fact that google-assistant-library pulls in some old
|
||||||
|
# breaking dependencies that need to be surgically removed.
|
||||||
|
- pip install -U --no-input --break-system-packages click urllib3 requests google-auth
|
||||||
|
|
|
@ -1,330 +0,0 @@
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from typing import Optional, Dict, Any
|
|
||||||
|
|
||||||
from platypush.context import get_bus, get_plugin
|
|
||||||
from platypush.message.event.assistant import (
|
|
||||||
ConversationStartEvent,
|
|
||||||
ConversationEndEvent,
|
|
||||||
SpeechRecognizedEvent,
|
|
||||||
VolumeChangedEvent,
|
|
||||||
ResponseEvent,
|
|
||||||
)
|
|
||||||
|
|
||||||
from platypush.message.event.google import GoogleDeviceOnOffEvent
|
|
||||||
|
|
||||||
from platypush.plugins import action
|
|
||||||
from platypush.plugins.assistant import AssistantPlugin
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantGooglePushtotalkPlugin(AssistantPlugin):
|
|
||||||
"""
|
|
||||||
Plugin for the Google Assistant push-to-talk API.
|
|
||||||
"""
|
|
||||||
|
|
||||||
api_endpoint = 'embeddedassistant.googleapis.com'
|
|
||||||
grpc_deadline = 60 * 3 + 5
|
|
||||||
device_handler = None
|
|
||||||
_default_credentials_file = os.path.join(
|
|
||||||
os.path.expanduser('~'),
|
|
||||||
'.config',
|
|
||||||
'google-oauthlib-tool',
|
|
||||||
'credentials.json',
|
|
||||||
)
|
|
||||||
|
|
||||||
_default_device_config = os.path.join(
|
|
||||||
os.path.expanduser('~'),
|
|
||||||
'.config',
|
|
||||||
'googlesamples-assistant',
|
|
||||||
'device_config.json',
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
credentials_file=_default_credentials_file,
|
|
||||||
device_config=_default_device_config,
|
|
||||||
language='en-US',
|
|
||||||
play_response=True,
|
|
||||||
tts_plugin=None,
|
|
||||||
tts_args=None,
|
|
||||||
**kwargs
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
:param credentials_file: Path to the Google OAuth credentials file
|
|
||||||
(default: ~/.config/google-oauthlib-tool/credentials.json).
|
|
||||||
See
|
|
||||||
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
|
|
||||||
for instructions to get your own credentials file.
|
|
||||||
:type credentials_file: str
|
|
||||||
|
|
||||||
:param device_config: Path to device_config.json. Register your device
|
|
||||||
(see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device)
|
|
||||||
and create a project, then run the pushtotalk.py script from
|
|
||||||
googlesamples to create your device_config.json
|
|
||||||
:type device_config: str
|
|
||||||
|
|
||||||
:param language: Assistant language (default: en-US)
|
|
||||||
:type language: str
|
|
||||||
|
|
||||||
:param play_response: If True (default) then the plugin will play the assistant response upon processed
|
|
||||||
response. Otherwise nothing will be played - but you may want to handle the ``ResponseEvent`` manually.
|
|
||||||
:type play_response: bool
|
|
||||||
|
|
||||||
:param tts_plugin: Optional text-to-speech plugin to be used to process response text.
|
|
||||||
:type tts_plugin: str
|
|
||||||
|
|
||||||
:param tts_args: Optional arguments for the TTS plugin ``say`` method.
|
|
||||||
:type tts_args: dict
|
|
||||||
"""
|
|
||||||
|
|
||||||
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
|
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
self.audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE
|
|
||||||
self.audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH
|
|
||||||
self.audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE
|
|
||||||
self.audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE
|
|
||||||
self.audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE
|
|
||||||
|
|
||||||
self.language = language
|
|
||||||
self.credentials_file = credentials_file
|
|
||||||
self.device_config = device_config
|
|
||||||
self.play_response = play_response
|
|
||||||
self.tts_plugin = tts_plugin
|
|
||||||
self.tts_args = tts_args or {}
|
|
||||||
self.assistant = None
|
|
||||||
self.interactions = []
|
|
||||||
|
|
||||||
with open(self.device_config) as f:
|
|
||||||
device = json.load(f)
|
|
||||||
self.device_id = device['id']
|
|
||||||
self.device_model_id = device['model_id']
|
|
||||||
|
|
||||||
# Load OAuth 2.0 credentials.
|
|
||||||
try:
|
|
||||||
from google.oauth2.credentials import Credentials
|
|
||||||
from google.auth.transport.requests import Request
|
|
||||||
|
|
||||||
with open(self.credentials_file, 'r') as f:
|
|
||||||
self.credentials = Credentials(token=None, **json.load(f))
|
|
||||||
self.http_request = Request()
|
|
||||||
self.credentials.refresh(self.http_request)
|
|
||||||
except Exception as ex:
|
|
||||||
self.logger.error('Error loading credentials: %s', str(ex))
|
|
||||||
self.logger.error(
|
|
||||||
'Run google-oauthlib-tool to initialize ' 'new OAuth 2.0 credentials.'
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
|
|
||||||
self.grpc_channel = None
|
|
||||||
self.conversation_stream = None
|
|
||||||
|
|
||||||
def _init_assistant(self):
|
|
||||||
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
|
|
||||||
from google.auth.transport.grpc import secure_authorized_channel
|
|
||||||
|
|
||||||
self.interactions = []
|
|
||||||
|
|
||||||
# Create an authorized gRPC channel.
|
|
||||||
self.grpc_channel = secure_authorized_channel(
|
|
||||||
self.credentials, self.http_request, self.api_endpoint
|
|
||||||
)
|
|
||||||
self.logger.info('Connecting to {}'.format(self.api_endpoint))
|
|
||||||
|
|
||||||
# Configure audio source and sink.
|
|
||||||
audio_device = None
|
|
||||||
audio_source = audio_device = audio_device or audio_helpers.SoundDeviceStream(
|
|
||||||
sample_rate=self.audio_sample_rate,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
block_size=self.audio_block_size,
|
|
||||||
flush_size=self.audio_flush_size,
|
|
||||||
)
|
|
||||||
|
|
||||||
audio_sink = audio_device or audio_helpers.SoundDeviceStream(
|
|
||||||
sample_rate=self.audio_sample_rate,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
block_size=self.audio_block_size,
|
|
||||||
flush_size=self.audio_flush_size,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create conversation stream with the given audio source and sink.
|
|
||||||
self.conversation_stream = audio_helpers.ConversationStream(
|
|
||||||
source=audio_source,
|
|
||||||
sink=audio_sink,
|
|
||||||
iter_size=self.audio_iter_size,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
)
|
|
||||||
|
|
||||||
self._install_device_handlers()
|
|
||||||
|
|
||||||
def on_conversation_start(self):
|
|
||||||
"""Conversation start handler"""
|
|
||||||
|
|
||||||
def handler():
|
|
||||||
get_bus().post(ConversationStartEvent(assistant=self))
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
def on_conversation_end(self):
|
|
||||||
"""Conversation end handler"""
|
|
||||||
|
|
||||||
def handler(with_follow_on_turn):
|
|
||||||
get_bus().post(
|
|
||||||
ConversationEndEvent(
|
|
||||||
assistant=self, with_follow_on_turn=with_follow_on_turn
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
def on_speech_recognized(self):
|
|
||||||
"""Speech recognized handler"""
|
|
||||||
|
|
||||||
def handler(phrase):
|
|
||||||
get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
|
|
||||||
self.interactions.append({'request': phrase})
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
def on_volume_changed(self):
|
|
||||||
"""Volume changed event"""
|
|
||||||
|
|
||||||
def handler(volume):
|
|
||||||
get_bus().post(VolumeChangedEvent(assistant=self, volume=volume))
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
def on_response(self):
|
|
||||||
"""Response handler"""
|
|
||||||
|
|
||||||
def handler(response):
|
|
||||||
get_bus().post(ResponseEvent(assistant=self, response_text=response))
|
|
||||||
|
|
||||||
if not self.interactions:
|
|
||||||
self.interactions.append({'response': response})
|
|
||||||
else:
|
|
||||||
self.interactions[-1]['response'] = response
|
|
||||||
|
|
||||||
if self.tts_plugin:
|
|
||||||
tts = get_plugin(self.tts_plugin)
|
|
||||||
tts.say(response, **self.tts_args)
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
@action
|
|
||||||
def start_conversation(
|
|
||||||
self,
|
|
||||||
*_,
|
|
||||||
language: Optional[str] = None,
|
|
||||||
tts_plugin: Optional[str] = None,
|
|
||||||
tts_args: Optional[Dict[str, Any]] = None,
|
|
||||||
**__
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Start a conversation
|
|
||||||
|
|
||||||
:param language: Language code override (default: default configured language).
|
|
||||||
:param tts_plugin: Optional text-to-speech plugin to be used for rendering text.
|
|
||||||
:param tts_args: Optional arguments for the TTS plugin say method.
|
|
||||||
:returns: A list of the interactions that happen within the conversation.
|
|
||||||
|
|
||||||
.. code-block:: json
|
|
||||||
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"request": "request 1",
|
|
||||||
"response": "response 1"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"request": "request 2",
|
|
||||||
"response": "response 2"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
from platypush.plugins.assistant.google.lib import SampleAssistant
|
|
||||||
|
|
||||||
self.tts_plugin = tts_plugin
|
|
||||||
self.tts_args = tts_args
|
|
||||||
language = language or self.language
|
|
||||||
play_response = False if self.tts_plugin else self.play_response
|
|
||||||
|
|
||||||
self._init_assistant()
|
|
||||||
self.on_conversation_start()
|
|
||||||
|
|
||||||
with SampleAssistant(
|
|
||||||
language_code=language,
|
|
||||||
device_model_id=self.device_model_id,
|
|
||||||
device_id=self.device_id,
|
|
||||||
conversation_stream=self.conversation_stream,
|
|
||||||
display=None,
|
|
||||||
channel=self.grpc_channel,
|
|
||||||
deadline_sec=self.grpc_deadline,
|
|
||||||
play_response=play_response,
|
|
||||||
device_handler=self.device_handler,
|
|
||||||
on_conversation_start=self.on_conversation_start(),
|
|
||||||
on_conversation_end=self.on_conversation_end(),
|
|
||||||
on_volume_changed=self.on_volume_changed(),
|
|
||||||
on_response=self.on_response(),
|
|
||||||
on_speech_recognized=self.on_speech_recognized(),
|
|
||||||
) as self.assistant:
|
|
||||||
continue_conversation = True
|
|
||||||
|
|
||||||
while continue_conversation:
|
|
||||||
try:
|
|
||||||
continue_conversation = self.assistant.assist()
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.warning(
|
|
||||||
'Unhandled assistant exception: {}'.format(str(e))
|
|
||||||
)
|
|
||||||
self.logger.exception(e)
|
|
||||||
self._init_assistant()
|
|
||||||
|
|
||||||
return self.interactions
|
|
||||||
|
|
||||||
@action
|
|
||||||
def stop_conversation(self):
|
|
||||||
if self.assistant:
|
|
||||||
self.assistant.play_response = False
|
|
||||||
|
|
||||||
if self.conversation_stream:
|
|
||||||
self.conversation_stream.stop_playback()
|
|
||||||
self.conversation_stream.stop_recording()
|
|
||||||
|
|
||||||
get_bus().post(ConversationEndEvent(assistant=self))
|
|
||||||
|
|
||||||
@action
|
|
||||||
def set_mic_mute(self, muted: bool = True):
|
|
||||||
"""
|
|
||||||
Programmatically mute/unmute the microphone.
|
|
||||||
|
|
||||||
:param muted: Set to True or False.
|
|
||||||
"""
|
|
||||||
if not self.conversation_stream:
|
|
||||||
self.logger.warning('The assistant is not running')
|
|
||||||
return
|
|
||||||
|
|
||||||
if muted:
|
|
||||||
self.conversation_stream.stop_recording()
|
|
||||||
else:
|
|
||||||
self.conversation_stream.start_recording()
|
|
||||||
|
|
||||||
def _install_device_handlers(self):
|
|
||||||
import googlesamples.assistant.grpc.device_helpers as device_helpers
|
|
||||||
|
|
||||||
self.device_handler = device_helpers.DeviceRequestHandler(self.device_id)
|
|
||||||
|
|
||||||
@self.device_handler.command('action.devices.commands.OnOff')
|
|
||||||
def handler(on): # type: ignore
|
|
||||||
get_bus().post(
|
|
||||||
GoogleDeviceOnOffEvent(
|
|
||||||
device_id=self.device_id,
|
|
||||||
device_model_id=self.device_model_id,
|
|
||||||
on=on,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
|
@ -1,27 +0,0 @@
|
||||||
manifest:
|
|
||||||
events:
|
|
||||||
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
|
|
||||||
ends
|
|
||||||
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
|
|
||||||
starts
|
|
||||||
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
|
|
||||||
is recognized
|
|
||||||
install:
|
|
||||||
apk:
|
|
||||||
- py3-tenacity
|
|
||||||
- py3-google-auth
|
|
||||||
apt:
|
|
||||||
- python3-tenacity
|
|
||||||
- python3-google-auth
|
|
||||||
dnf:
|
|
||||||
- python-tenacity
|
|
||||||
- python-google-auth
|
|
||||||
pacman:
|
|
||||||
- python-tenacity
|
|
||||||
- python-google-auth
|
|
||||||
pip:
|
|
||||||
- tenacity
|
|
||||||
- google-assistant-sdk
|
|
||||||
- google-auth
|
|
||||||
package: platypush.plugins.assistant.google.pushtotalk
|
|
||||||
type: plugin
|
|
5
setup.py
5
setup.py
|
@ -139,9 +139,6 @@ setup(
|
||||||
],
|
],
|
||||||
# Support for Last.FM scrobbler plugin
|
# Support for Last.FM scrobbler plugin
|
||||||
'lastfm': ['pylast'],
|
'lastfm': ['pylast'],
|
||||||
# Support for custom hotword detection
|
|
||||||
'hotword': ['snowboy'],
|
|
||||||
'snowboy': ['snowboy'],
|
|
||||||
# Support for real-time MIDI events
|
# Support for real-time MIDI events
|
||||||
'midi': ['rtmidi'],
|
'midi': ['rtmidi'],
|
||||||
# Support for RaspberryPi GPIO
|
# Support for RaspberryPi GPIO
|
||||||
|
@ -193,8 +190,6 @@ setup(
|
||||||
'flic': [
|
'flic': [
|
||||||
'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
|
'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
|
||||||
],
|
],
|
||||||
# Support for Alexa/Echo plugin
|
|
||||||
'alexa': ['avs @ https://github.com/BlackLight/avs/tarball/master'],
|
|
||||||
# Support for Bluetooth devices
|
# Support for Bluetooth devices
|
||||||
'bluetooth': [
|
'bluetooth': [
|
||||||
'bleak',
|
'bleak',
|
||||||
|
|
Loading…
Reference in New Issue