forked from platypush/platypush
Rewritten snowboy and Google push-to-talk plugins to make them a better replacement for the deprecated google-assistant-library
This commit is contained in:
Fabio Manganiello
parent
30c7f7f955
commit
5766367402
10 changed files with 547 additions and 412 deletions
|
|
@ -28,6 +28,13 @@ class AssistantGoogleBackend(Backend):
|
||||||
|
|
||||||
It listens for voice commands and post conversation events on the bus.
|
It listens for voice commands and post conversation events on the bus.
|
||||||
|
|
||||||
|
**WARNING**: This backend is deprecated, as the underlying Google Assistant
|
||||||
|
library has been deprecated too: https://developers.google.com/assistant/sdk/reference/library/python/
|
||||||
|
The old library might still work on some systems but its proper functioning
|
||||||
|
is not guaranteed.
|
||||||
|
Please use the Snowboy backend for hotword detection and the Google Assistant
|
||||||
|
push-to-talk plugin for assistant interaction instead.
|
||||||
|
|
||||||
Triggers:
|
Triggers:
|
||||||
|
|
||||||
* :class:`platypush.message.event.assistant.ConversationStartEvent` \
|
* :class:`platypush.message.event.assistant.ConversationStartEvent` \
|
||||||
|
|
@ -1,385 +0,0 @@
|
||||||
"""
|
|
||||||
.. moduleauthor:: Fabio Manganiello <blacklight86@gmail.com>
|
|
||||||
.. license: MIT
|
|
||||||
"""
|
|
||||||
|
|
||||||
import concurrent
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
|
|
||||||
import grpc
|
|
||||||
import google.auth.transport.grpc
|
|
||||||
import google.auth.transport.requests
|
|
||||||
import google.oauth2.credentials
|
|
||||||
from google.assistant.embedded.v1alpha2 import embedded_assistant_pb2, \
|
|
||||||
embedded_assistant_pb2_grpc
|
|
||||||
|
|
||||||
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
|
|
||||||
import googlesamples.assistant.grpc.device_helpers as device_helpers
|
|
||||||
import googlesamples.assistant.grpc.assistant_helpers as assistant_helpers
|
|
||||||
|
|
||||||
from tenacity import retry, stop_after_attempt, retry_if_exception
|
|
||||||
|
|
||||||
|
|
||||||
from platypush.backend import Backend
|
|
||||||
from platypush.message.event.assistant import \
|
|
||||||
ConversationStartEvent, ConversationEndEvent, SpeechRecognizedEvent
|
|
||||||
|
|
||||||
|
|
||||||
class AssistantGooglePushtotalkBackend(Backend):
|
|
||||||
"""
|
|
||||||
Google Assistant pushtotalk backend. Instead of listening for the "OK
|
|
||||||
Google" hotword like the assistant.google backend, this implementation
|
|
||||||
programmatically starts a conversation upon start_conversation() method
|
|
||||||
call. Use this backend on devices that don't have an Assistant SDK package
|
|
||||||
(e.g. arm6 devices like the RaspberryPi Zero or the RaspberryPi 1).
|
|
||||||
|
|
||||||
Triggers:
|
|
||||||
|
|
||||||
* :class:`platypush.message.event.assistant.ConversationStartEvent` \
|
|
||||||
when a new conversation starts
|
|
||||||
* :class:`platypush.message.event.assistant.SpeechRecognizedEvent` \
|
|
||||||
when a new voice command is recognized
|
|
||||||
* :class:`platypush.message.event.assistant.ConversationEndEvent` \
|
|
||||||
when a new conversation ends
|
|
||||||
|
|
||||||
Requires:
|
|
||||||
|
|
||||||
* **tenacity** (``pip install tenacity``)
|
|
||||||
* **grpc** (``pip install grpc``)
|
|
||||||
* **google-assistant-grpc** (``pip install google-assistant-grpc``)
|
|
||||||
"""
|
|
||||||
|
|
||||||
api_endpoint = 'embeddedassistant.googleapis.com'
|
|
||||||
audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE
|
|
||||||
audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH
|
|
||||||
audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE
|
|
||||||
audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE
|
|
||||||
audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE
|
|
||||||
grpc_deadline = 60 * 3 + 5
|
|
||||||
|
|
||||||
def __init__(self, *args,
|
|
||||||
credentials_file=os.path.join(
|
|
||||||
os.path.expanduser('~'), '.config',
|
|
||||||
'google-oauthlib-tool', 'credentials.json'),
|
|
||||||
device_config=os.path.join(
|
|
||||||
os.path.expanduser('~'), '.config', 'googlesamples-assistant',
|
|
||||||
'device_config.json'),
|
|
||||||
lang='en-US',
|
|
||||||
conversation_start_fifo=os.path.join(os.path.sep, 'tmp', 'pushtotalk.fifo'),
|
|
||||||
**kwargs):
|
|
||||||
"""
|
|
||||||
:param credentials_file: Path to the Google OAuth credentials file \
|
|
||||||
(default: ~/.config/google-oauthlib-tool/credentials.json). \
|
|
||||||
See https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials \
|
|
||||||
for instructions to get your own credentials file.
|
|
||||||
:type credentials_file: str
|
|
||||||
|
|
||||||
:param device_config: Path to device_config.json. Register your device \
|
|
||||||
(see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device) \
|
|
||||||
and create a project, then run the pushtotalk.py script from \
|
|
||||||
googlesamples to create your device_config.json
|
|
||||||
:type device_config: str
|
|
||||||
|
|
||||||
:param lang: Assistant language (default: en-US)
|
|
||||||
:type lang: str
|
|
||||||
"""
|
|
||||||
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
|
|
||||||
self.lang = lang
|
|
||||||
self.credentials_file = credentials_file
|
|
||||||
self.device_config = device_config
|
|
||||||
self.conversation_start_fifo = conversation_start_fifo
|
|
||||||
self.assistant = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
os.mkfifo(self.conversation_start_fifo)
|
|
||||||
except FileExistsError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
with open(self.device_config) as f:
|
|
||||||
device = json.load(f)
|
|
||||||
self.device_id = device['id']
|
|
||||||
self.device_model_id = device['model_id']
|
|
||||||
|
|
||||||
# Load OAuth 2.0 credentials.
|
|
||||||
try:
|
|
||||||
with open(self.credentials_file, 'r') as f:
|
|
||||||
credentials = google.oauth2.credentials.Credentials(token=None,
|
|
||||||
**json.load(f))
|
|
||||||
http_request = google.auth.transport.requests.Request()
|
|
||||||
credentials.refresh(http_request)
|
|
||||||
except Exception as ex:
|
|
||||||
self.logger.error('Error loading credentials: %s', str(ex))
|
|
||||||
self.logger.error('Run google-oauthlib-tool to initialize '
|
|
||||||
'new OAuth 2.0 credentials.')
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Create an authorized gRPC channel.
|
|
||||||
self.grpc_channel = google.auth.transport.grpc.secure_authorized_channel(
|
|
||||||
credentials, http_request, self.api_endpoint)
|
|
||||||
self.logger.info('Connecting to %s', self.api_endpoint)
|
|
||||||
|
|
||||||
# Configure audio source and sink.
|
|
||||||
audio_device = None
|
|
||||||
audio_source = audio_device = (
|
|
||||||
audio_device or audio_helpers.SoundDeviceStream(
|
|
||||||
sample_rate=self.audio_sample_rate,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
block_size=self.audio_block_size,
|
|
||||||
flush_size=self.audio_flush_size
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
audio_sink = audio_device = (
|
|
||||||
audio_device or audio_helpers.SoundDeviceStream(
|
|
||||||
sample_rate=self.audio_sample_rate,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
block_size=self.audio_block_size,
|
|
||||||
flush_size=self.audio_flush_size
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create conversation stream with the given audio source and sink.
|
|
||||||
self.conversation_stream = audio_helpers.ConversationStream(
|
|
||||||
source=audio_source,
|
|
||||||
sink=audio_sink,
|
|
||||||
iter_size=self.audio_iter_size,
|
|
||||||
sample_width=self.audio_sample_width,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.device_handler = device_helpers.DeviceRequestHandler(self.device_id)
|
|
||||||
|
|
||||||
def start_conversation(self):
|
|
||||||
""" Start a conversation """
|
|
||||||
if self.assistant:
|
|
||||||
with open(self.conversation_start_fifo, 'w') as f:
|
|
||||||
f.write('1')
|
|
||||||
|
|
||||||
def stop_conversation(self):
|
|
||||||
""" Stop a conversation """
|
|
||||||
if self.assistant:
|
|
||||||
self.conversation_stream.stop_playback()
|
|
||||||
self.bus.post(ConversationEndEvent())
|
|
||||||
|
|
||||||
def on_conversation_start(self):
|
|
||||||
""" Conversation start handler """
|
|
||||||
self.bus.post(ConversationStartEvent())
|
|
||||||
|
|
||||||
def on_conversation_end(self):
|
|
||||||
""" Conversation end handler """
|
|
||||||
self.bus.post(ConversationEndEvent())
|
|
||||||
|
|
||||||
def on_speech_recognized(self, speech):
|
|
||||||
""" Speech recognized handler """
|
|
||||||
self.bus.post(SpeechRecognizedEvent(phrase=speech))
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
""" Backend executor """
|
|
||||||
super().run()
|
|
||||||
|
|
||||||
with SampleAssistant(self.lang, self.device_model_id, self.device_id,
|
|
||||||
self.conversation_stream,
|
|
||||||
self.grpc_channel, self.grpc_deadline,
|
|
||||||
self.device_handler,
|
|
||||||
on_conversation_start=self.on_conversation_start,
|
|
||||||
on_conversation_end=self.on_conversation_end,
|
|
||||||
on_speech_recognized=self.on_speech_recognized) as self.assistant:
|
|
||||||
while not self.should_stop():
|
|
||||||
with open(self.conversation_start_fifo, 'r') as f:
|
|
||||||
f.read()
|
|
||||||
|
|
||||||
self.logger.info('Received conversation start event')
|
|
||||||
continue_conversation = True
|
|
||||||
|
|
||||||
while continue_conversation:
|
|
||||||
(user_request, continue_conversation) = self.assistant.assist()
|
|
||||||
self.logger('User request: {}'.format(user_request))
|
|
||||||
|
|
||||||
self.on_conversation_end()
|
|
||||||
|
|
||||||
|
|
||||||
class SampleAssistant:
|
|
||||||
"""Sample Assistant that supports conversations and device actions.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
device_model_id: identifier of the device model.
|
|
||||||
device_id: identifier of the registered device instance.
|
|
||||||
conversation_stream(ConversationStream): audio stream for recording \
|
|
||||||
query and playing back assistant answer.
|
|
||||||
channel: authorized gRPC channel for connection to the Google Assistant API.
|
|
||||||
deadline_sec: gRPC deadline in seconds for Google Assistant API call.
|
|
||||||
device_handler: callback for device actions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE
|
|
||||||
DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON
|
|
||||||
CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE
|
|
||||||
|
|
||||||
def __init__(self, language_code, device_model_id, device_id,
|
|
||||||
conversation_stream,
|
|
||||||
channel, deadline_sec, device_handler,
|
|
||||||
on_conversation_start=None,
|
|
||||||
on_conversation_end=None,
|
|
||||||
on_speech_recognized=None):
|
|
||||||
self.language_code = language_code
|
|
||||||
self.device_model_id = device_model_id
|
|
||||||
self.device_id = device_id
|
|
||||||
self.conversation_stream = conversation_stream
|
|
||||||
self.logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
self.on_conversation_start = on_conversation_start
|
|
||||||
self.on_conversation_end = on_conversation_end
|
|
||||||
self.on_speech_recognized = on_speech_recognized
|
|
||||||
|
|
||||||
# Opaque blob provided in AssistResponse that,
|
|
||||||
# when provided in a follow-up AssistRequest,
|
|
||||||
# gives the Assistant a context marker within the current state
|
|
||||||
# of the multi-Assist()-RPC "conversation".
|
|
||||||
# This value, along with MicrophoneMode, supports a more natural
|
|
||||||
# "conversation" with the Assistant.
|
|
||||||
self.conversation_state = None
|
|
||||||
|
|
||||||
# Create Google Assistant API gRPC client.
|
|
||||||
self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(
|
|
||||||
channel
|
|
||||||
)
|
|
||||||
self.deadline = deadline_sec
|
|
||||||
|
|
||||||
self.device_handler = device_handler
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(self, etype, e, traceback):
|
|
||||||
if e:
|
|
||||||
return False
|
|
||||||
self.conversation_stream.close()
|
|
||||||
return True
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_grpc_error_unavailable(e):
|
|
||||||
""" Returns True if the gRPC is not available """
|
|
||||||
is_grpc_error = isinstance(e, grpc.RpcError)
|
|
||||||
if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE):
|
|
||||||
print('grpc unavailable error: {}'.format(e))
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
@retry(reraise=True, stop=stop_after_attempt(3),
|
|
||||||
retry=retry_if_exception(is_grpc_error_unavailable))
|
|
||||||
def assist(self):
|
|
||||||
"""Send a voice request to the Assistant and playback the response.
|
|
||||||
|
|
||||||
Returns: True if conversation should continue.
|
|
||||||
"""
|
|
||||||
continue_conversation = False
|
|
||||||
device_actions_futures = []
|
|
||||||
|
|
||||||
self.conversation_stream.start_recording()
|
|
||||||
self.logger.info('Recording audio request.')
|
|
||||||
|
|
||||||
if self.on_conversation_start:
|
|
||||||
self.on_conversation_start()
|
|
||||||
|
|
||||||
def iter_assist_requests():
|
|
||||||
for c in self.gen_assist_requests():
|
|
||||||
assistant_helpers.log_assist_request_without_audio(c)
|
|
||||||
yield c
|
|
||||||
self.conversation_stream.start_playback()
|
|
||||||
|
|
||||||
user_request = None
|
|
||||||
|
|
||||||
# This generator yields AssistResponse proto messages
|
|
||||||
# received from the gRPC Google Assistant API.
|
|
||||||
for resp in self.assistant.Assist(iter_assist_requests(),
|
|
||||||
self.deadline):
|
|
||||||
assistant_helpers.log_assist_response_without_audio(resp)
|
|
||||||
if resp.event_type == self.END_OF_UTTERANCE:
|
|
||||||
self.logger.info('End of audio request detected')
|
|
||||||
self.conversation_stream.stop_recording()
|
|
||||||
if resp.speech_results:
|
|
||||||
user_request = ' '.join(
|
|
||||||
r.transcript for r in resp.speech_results)
|
|
||||||
|
|
||||||
self.logger.info('Transcript of user request: "%s".', user_request)
|
|
||||||
self.logger.info('Playing assistant response.')
|
|
||||||
if resp.audio_out.audio_data:
|
|
||||||
self.conversation_stream.write(resp.audio_out.audio_data)
|
|
||||||
if resp.dialog_state_out.conversation_state:
|
|
||||||
conversation_state = resp.dialog_state_out.conversation_state
|
|
||||||
self.logger.debug('Updating conversation state.')
|
|
||||||
self.conversation_state = conversation_state
|
|
||||||
if resp.dialog_state_out.volume_percentage != 0:
|
|
||||||
volume_percentage = resp.dialog_state_out.volume_percentage
|
|
||||||
self.logger.info('Setting volume to %s%%', volume_percentage)
|
|
||||||
self.conversation_stream.volume_percentage = volume_percentage
|
|
||||||
if resp.dialog_state_out.microphone_mode == self.DIALOG_FOLLOW_ON:
|
|
||||||
continue_conversation = True
|
|
||||||
self.logger.info('Expecting follow-on query from user.')
|
|
||||||
elif resp.dialog_state_out.microphone_mode == self.CLOSE_MICROPHONE:
|
|
||||||
continue_conversation = False
|
|
||||||
if resp.device_action.device_request_json:
|
|
||||||
device_request = json.loads(
|
|
||||||
resp.device_action.device_request_json
|
|
||||||
)
|
|
||||||
fs = self.device_handler(device_request)
|
|
||||||
if fs:
|
|
||||||
device_actions_futures.extend(fs)
|
|
||||||
|
|
||||||
if device_actions_futures:
|
|
||||||
self.logger.info('Waiting for device executions to complete.')
|
|
||||||
concurrent.futures.wait(device_actions_futures)
|
|
||||||
|
|
||||||
self.logger.info('Finished playing assistant response.')
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.conversation_stream.stop_playback()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if user_request and self.on_speech_recognized:
|
|
||||||
self.on_speech_recognized(user_request)
|
|
||||||
|
|
||||||
return (user_request, continue_conversation)
|
|
||||||
|
|
||||||
def gen_assist_requests(self):
|
|
||||||
"""Yields: AssistRequest messages to send to the API."""
|
|
||||||
|
|
||||||
dialog_state_in = embedded_assistant_pb2.DialogStateIn(
|
|
||||||
language_code=self.language_code,
|
|
||||||
conversation_state=b''
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.conversation_state:
|
|
||||||
self.logger.debug('Sending conversation state.')
|
|
||||||
dialog_state_in.conversation_state = self.conversation_state
|
|
||||||
|
|
||||||
config = embedded_assistant_pb2.AssistConfig(
|
|
||||||
audio_in_config=embedded_assistant_pb2.AudioInConfig(
|
|
||||||
encoding='LINEAR16',
|
|
||||||
sample_rate_hertz=self.conversation_stream.sample_rate,
|
|
||||||
),
|
|
||||||
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
|
|
||||||
encoding='LINEAR16',
|
|
||||||
sample_rate_hertz=self.conversation_stream.sample_rate,
|
|
||||||
volume_percentage=self.conversation_stream.volume_percentage,
|
|
||||||
),
|
|
||||||
dialog_state_in=dialog_state_in,
|
|
||||||
device_config=embedded_assistant_pb2.DeviceConfig(
|
|
||||||
device_id=self.device_id,
|
|
||||||
device_model_id=self.device_model_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# The first AssistRequest must contain the AssistConfig
|
|
||||||
# and no audio data.
|
|
||||||
yield embedded_assistant_pb2.AssistRequest(config=config)
|
|
||||||
for data in self.conversation_stream:
|
|
||||||
# Subsequent requests need audio data, but not config.
|
|
||||||
yield embedded_assistant_pb2.AssistRequest(audio_in=data)
|
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
|
||||||
|
|
@ -9,10 +9,12 @@ import subprocess
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from platypush.backend import Backend
|
from platypush.backend import Backend
|
||||||
|
from platypush.context import get_plugin
|
||||||
from platypush.message.event.assistant import \
|
from platypush.message.event.assistant import \
|
||||||
ConversationStartEvent, ConversationEndEvent, \
|
ConversationStartEvent, ConversationEndEvent, \
|
||||||
SpeechRecognizedEvent, HotwordDetectedEvent
|
SpeechRecognizedEvent, HotwordDetectedEvent
|
||||||
|
|
||||||
|
|
||||||
class AssistantSnowboyBackend(Backend):
|
class AssistantSnowboyBackend(Backend):
|
||||||
"""
|
"""
|
||||||
Backend for detecting custom voice hotwords through Snowboy. The purpose of
|
Backend for detecting custom voice hotwords through Snowboy. The purpose of
|
||||||
|
|
@ -30,10 +32,26 @@ class AssistantSnowboyBackend(Backend):
|
||||||
Requires:
|
Requires:
|
||||||
|
|
||||||
* **snowboy** (``pip install snowboy``)
|
* **snowboy** (``pip install snowboy``)
|
||||||
|
|
||||||
|
Manual installation for snowboy and its Python bindings if the command above
|
||||||
|
fails:
|
||||||
|
|
||||||
|
$ [sudo] apt-get install libatlas-base-dev swig
|
||||||
|
$ [sudo] pip install pyaudio
|
||||||
|
$ git clone https://github.com/Kitt-AI/snowboy
|
||||||
|
$ cd snowboy/swig/Python3
|
||||||
|
$ make
|
||||||
|
$ cd ../..
|
||||||
|
$ python3 setup.py build
|
||||||
|
$ [sudo] python setup.py install
|
||||||
|
|
||||||
|
You will also need a voice model for the hotword detection. You can find
|
||||||
|
some under the ``resources/models`` directory of the Snowboy repository,
|
||||||
|
or train/download other models from https://snowboy.kitt.ai.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, voice_model_file, hotword=None, sensitivity=0.5,
|
def __init__(self, voice_model_file, hotword=None, sensitivity=0.5,
|
||||||
audio_gain=1.0, **kwargs):
|
audio_gain=1.0, assistant_plugin=None, **kwargs):
|
||||||
"""
|
"""
|
||||||
:param voice_model_file: Snowboy voice model file - \
|
:param voice_model_file: Snowboy voice model file - \
|
||||||
see https://snowboy.kitt.ai/
|
see https://snowboy.kitt.ai/
|
||||||
|
|
@ -42,20 +60,33 @@ class AssistantSnowboyBackend(Backend):
|
||||||
:param hotword: Name of the hotword
|
:param hotword: Name of the hotword
|
||||||
:type hotword: str
|
:type hotword: str
|
||||||
|
|
||||||
:param sensitivity: Hotword recognition sensitivity, between 0 and 1
|
:param sensitivity: Hotword recognition sensitivity, between 0 and 1.
|
||||||
|
Default: 0.5.
|
||||||
:type sensitivity: float
|
:type sensitivity: float
|
||||||
|
|
||||||
:param audio_gain: Audio gain, between 0 and 1
|
:param audio_gain: Audio gain, between 0 and 1
|
||||||
:type audio_gain: float
|
:type audio_gain: float
|
||||||
|
|
||||||
|
:param assistant_plugin: By default Snowboy fires a
|
||||||
|
:class:`platypush.message.event.assistant.HotwordDetectedEvent` event
|
||||||
|
whenever the hotword is detected. You can also pass the plugin name of
|
||||||
|
a :class:`platypush.plugins.assistant.AssistantPlugin` instance
|
||||||
|
(for example ``assistant.google.pushtotalk``). If set, then the
|
||||||
|
assistant plugin will be invoked to start a conversation.
|
||||||
|
:type assistant_plugin: str
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from snowboy import snowboydecoder
|
try:
|
||||||
|
import snowboydecoder
|
||||||
|
except ImportError:
|
||||||
|
import snowboy.snowboydecoder as snowboydecoder
|
||||||
|
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.voice_model_file = voice_model_file
|
self.voice_model_file = os.path.abspath(os.path.expanduser(voice_model_file))
|
||||||
self.hotword = hotword
|
self.hotword = hotword
|
||||||
self.sensitivity = sensitivity
|
self.sensitivity = sensitivity
|
||||||
self.audio_gain = audio_gain
|
self.audio_gain = audio_gain
|
||||||
|
self.assistant_plugin = assistant_plugin
|
||||||
|
|
||||||
self.detector = snowboydecoder.HotwordDetector(
|
self.detector = snowboydecoder.HotwordDetector(
|
||||||
self.voice_model_file, sensitivity=self.sensitivity,
|
self.voice_model_file, sensitivity=self.sensitivity,
|
||||||
|
|
@ -70,8 +101,18 @@ class AssistantSnowboyBackend(Backend):
|
||||||
|
|
||||||
def callback():
|
def callback():
|
||||||
self.bus.post(HotwordDetectedEvent(hotword=self.hotword))
|
self.bus.post(HotwordDetectedEvent(hotword=self.hotword))
|
||||||
|
|
||||||
|
if self.assistant_plugin:
|
||||||
|
# Trigger assistant conversation
|
||||||
|
get_plugin(self.assistant_plugin).start_conversation()
|
||||||
|
|
||||||
return callback
|
return callback
|
||||||
|
|
||||||
|
def on_stop(self):
|
||||||
|
if self.detector:
|
||||||
|
self.detector.terminate()
|
||||||
|
self.detector = None
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
super().run()
|
super().run()
|
||||||
self.detector.start(self.hotword_detected())
|
self.detector.start(self.hotword_detected())
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,7 @@ const Assistant = Vue.extend({
|
||||||
responseText: '',
|
responseText: '',
|
||||||
phrase: '',
|
phrase: '',
|
||||||
visible: false,
|
visible: false,
|
||||||
|
hideTimeout: undefined,
|
||||||
|
|
||||||
state: {
|
state: {
|
||||||
listening: false,
|
listening: false,
|
||||||
|
|
@ -51,11 +52,21 @@ const Assistant = Vue.extend({
|
||||||
this.reset();
|
this.reset();
|
||||||
this.state.listening = true;
|
this.state.listening = true;
|
||||||
this.visible = true;
|
this.visible = true;
|
||||||
|
|
||||||
|
if (this.hideTimeout) {
|
||||||
|
clearTimeout(this.hideTimeout);
|
||||||
|
this.hideTimeout = undefined;
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
conversationEnd: function() {
|
conversationEnd: function() {
|
||||||
this.reset();
|
const self = this;
|
||||||
this.visible = false;
|
|
||||||
|
this.hideTimeout = setTimeout(() => {
|
||||||
|
this.reset();
|
||||||
|
self.visible = false;
|
||||||
|
self.hideTimeout = undefined;
|
||||||
|
}, 4000);
|
||||||
},
|
},
|
||||||
|
|
||||||
speechRecognized: function(event) {
|
speechRecognized: function(event) {
|
||||||
|
|
@ -86,7 +97,6 @@ const Assistant = Vue.extend({
|
||||||
|
|
||||||
registerHandlers: function() {
|
registerHandlers: function() {
|
||||||
registerEventHandler(this.conversationStart, 'platypush.message.event.assistant.ConversationStartEvent');
|
registerEventHandler(this.conversationStart, 'platypush.message.event.assistant.ConversationStartEvent');
|
||||||
registerEventHandler(this.conversationStart, 'platypush.message.event.assistant.HotwordDetectedEvent');
|
|
||||||
registerEventHandler(this.alertOn, 'platypush.message.event.assistant.AlertStartedEvent');
|
registerEventHandler(this.alertOn, 'platypush.message.event.assistant.AlertStartedEvent');
|
||||||
registerEventHandler(this.alertOff, 'platypush.message.event.assistant.AlertEndEvent');
|
registerEventHandler(this.alertOff, 'platypush.message.event.assistant.AlertEndEvent');
|
||||||
registerEventHandler(this.speechRecognized, 'platypush.message.event.assistant.SpeechRecognizedEvent');
|
registerEventHandler(this.speechRecognized, 'platypush.message.event.assistant.SpeechRecognizedEvent');
|
||||||
|
|
|
||||||
|
|
@ -118,6 +118,15 @@ class HotwordDetectedEvent(AssistantEvent):
|
||||||
super().__init__(*args, hotword=hotword, **kwargs)
|
super().__init__(*args, hotword=hotword, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class VolumeChangedEvent(AssistantEvent):
|
||||||
|
"""
|
||||||
|
Event triggered when the volume of the assistant changes
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, volume, *args, **kwargs):
|
||||||
|
super().__init__(*args, volume=volume, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
class AlertStartedEvent(AssistantEvent):
|
class AlertStartedEvent(AssistantEvent):
|
||||||
"""
|
"""
|
||||||
Event triggered when an alert starts on the assistant
|
Event triggered when an alert starts on the assistant
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
from platypush.plugins import Plugin
|
||||||
|
|
||||||
|
class AssistantPlugin(ABC, Plugin):
|
||||||
|
"""
|
||||||
|
Base class for assistant plugins
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def start_conversation(self, *args, **kwargs):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def stop_conversation(self, *args, **kwargs):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
@ -33,4 +33,3 @@ class AssistantGooglePlugin(Plugin):
|
||||||
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
||||||
|
|
|
||||||
225
platypush/plugins/assistant/google/lib/__init__.py
Normal file
225
platypush/plugins/assistant/google/lib/__init__.py
Normal file
|
|
@ -0,0 +1,225 @@
|
||||||
|
"""Based on Google pushtotalk.py sample."""
|
||||||
|
|
||||||
|
import concurrent.futures
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import grpc
|
||||||
|
|
||||||
|
from google.assistant.embedded.v1alpha2 import (
|
||||||
|
embedded_assistant_pb2,
|
||||||
|
embedded_assistant_pb2_grpc
|
||||||
|
)
|
||||||
|
from tenacity import retry, stop_after_attempt, retry_if_exception
|
||||||
|
|
||||||
|
try:
|
||||||
|
from googlesamples.assistant.grpc import (
|
||||||
|
assistant_helpers,
|
||||||
|
audio_helpers,
|
||||||
|
browser_helpers,
|
||||||
|
device_helpers
|
||||||
|
)
|
||||||
|
except (SystemError, ImportError):
|
||||||
|
import assistant_helpers
|
||||||
|
import audio_helpers
|
||||||
|
import browser_helpers
|
||||||
|
import device_helpers
|
||||||
|
|
||||||
|
|
||||||
|
ASSISTANT_API_ENDPOINT = 'embeddedassistant.googleapis.com'
|
||||||
|
END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE
|
||||||
|
DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON
|
||||||
|
CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE
|
||||||
|
PLAYING = embedded_assistant_pb2.ScreenOutConfig.PLAYING
|
||||||
|
DEFAULT_GRPC_DEADLINE = 60 * 3 + 5
|
||||||
|
|
||||||
|
|
||||||
|
class SampleAssistant(object):
|
||||||
|
"""Sample Assistant that supports conversations and device actions.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
device_model_id: identifier of the device model.
|
||||||
|
device_id: identifier of the registered device instance.
|
||||||
|
conversation_stream(ConversationStream): audio stream
|
||||||
|
for recording query and playing back assistant answer.
|
||||||
|
channel: authorized gRPC channel for connection to the
|
||||||
|
Google Assistant API.
|
||||||
|
deadline_sec: gRPC deadline in seconds for Google Assistant API call.
|
||||||
|
device_handler: callback for device actions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, language_code, device_model_id, device_id,
|
||||||
|
conversation_stream, display,
|
||||||
|
channel, deadline_sec, device_handler,
|
||||||
|
on_conversation_start=None, on_conversation_end=None,
|
||||||
|
on_speech_recognized=None, on_volume_changed=None,
|
||||||
|
on_response=None):
|
||||||
|
self.language_code = language_code
|
||||||
|
self.device_model_id = device_model_id
|
||||||
|
self.device_id = device_id
|
||||||
|
self.conversation_stream = conversation_stream
|
||||||
|
self.display = display
|
||||||
|
|
||||||
|
# Opaque blob provided in AssistResponse that,
|
||||||
|
# when provided in a follow-up AssistRequest,
|
||||||
|
# gives the Assistant a context marker within the current state
|
||||||
|
# of the multi-Assist()-RPC "conversation".
|
||||||
|
# This value, along with MicrophoneMode, supports a more natural
|
||||||
|
# "conversation" with the Assistant.
|
||||||
|
self.conversation_state = None
|
||||||
|
# Force reset of first conversation.
|
||||||
|
self.is_new_conversation = True
|
||||||
|
|
||||||
|
# Create Google Assistant API gRPC client.
|
||||||
|
self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(
|
||||||
|
channel
|
||||||
|
)
|
||||||
|
self.deadline = deadline_sec
|
||||||
|
|
||||||
|
self.device_handler = device_handler
|
||||||
|
self.detected_speech = None
|
||||||
|
|
||||||
|
self.on_conversation_start = on_conversation_start
|
||||||
|
self.on_conversation_end = on_conversation_end
|
||||||
|
self.on_speech_recognized = on_speech_recognized
|
||||||
|
self.on_volume_changed = on_volume_changed
|
||||||
|
self.on_response = on_response
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, etype, e, traceback):
|
||||||
|
if e:
|
||||||
|
return False
|
||||||
|
self.conversation_stream.close()
|
||||||
|
|
||||||
|
def is_grpc_error_unavailable(e):
|
||||||
|
is_grpc_error = isinstance(e, grpc.RpcError)
|
||||||
|
if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE):
|
||||||
|
logging.error('grpc unavailable error: %s', e)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@retry(reraise=True, stop=stop_after_attempt(3),
|
||||||
|
retry=retry_if_exception(is_grpc_error_unavailable))
|
||||||
|
def assist(self):
|
||||||
|
"""Send a voice request to the Assistant and playback the response.
|
||||||
|
|
||||||
|
Returns: True if conversation should continue.
|
||||||
|
"""
|
||||||
|
continue_conversation = False
|
||||||
|
device_actions_futures = []
|
||||||
|
|
||||||
|
self.conversation_stream.start_recording()
|
||||||
|
if self.on_conversation_start:
|
||||||
|
self.on_conversation_start()
|
||||||
|
|
||||||
|
logging.info('Recording audio request.')
|
||||||
|
|
||||||
|
def iter_log_assist_requests():
|
||||||
|
for c in self.gen_assist_requests():
|
||||||
|
assistant_helpers.log_assist_request_without_audio(c)
|
||||||
|
yield c
|
||||||
|
logging.debug('Reached end of AssistRequest iteration.')
|
||||||
|
|
||||||
|
# This generator yields AssistResponse proto messages
|
||||||
|
# received from the gRPC Google Assistant API.
|
||||||
|
for resp in self.assistant.Assist(iter_log_assist_requests(),
|
||||||
|
self.deadline):
|
||||||
|
assistant_helpers.log_assist_response_without_audio(resp)
|
||||||
|
if resp.event_type == END_OF_UTTERANCE:
|
||||||
|
logging.info('End of audio request detected.')
|
||||||
|
logging.info('Stopping recording.')
|
||||||
|
self.conversation_stream.stop_recording()
|
||||||
|
|
||||||
|
if self.detected_speech and self.on_speech_recognized:
|
||||||
|
self.on_speech_recognized(self.detected_speech)
|
||||||
|
|
||||||
|
if resp.speech_results:
|
||||||
|
self.detected_speech = ' '.join(
|
||||||
|
r.transcript.strip() for r in resp.speech_results
|
||||||
|
if len(r).strip()).strip()
|
||||||
|
|
||||||
|
logging.info('Transcript of user request: "%s".', self.detected_speech)
|
||||||
|
if len(resp.audio_out.audio_data) > 0:
|
||||||
|
if not self.conversation_stream.playing:
|
||||||
|
self.conversation_stream.stop_recording()
|
||||||
|
self.conversation_stream.start_playback()
|
||||||
|
logging.info('Playing assistant response.')
|
||||||
|
self.conversation_stream.write(resp.audio_out.audio_data)
|
||||||
|
if resp.dialog_state_out.conversation_state:
|
||||||
|
conversation_state = resp.dialog_state_out.conversation_state
|
||||||
|
logging.debug('Updating conversation state.')
|
||||||
|
self.conversation_state = conversation_state
|
||||||
|
if resp.dialog_state_out.volume_percentage != 0:
|
||||||
|
volume_percentage = resp.dialog_state_out.volume_percentage
|
||||||
|
logging.info('Setting volume to %s%%', volume_percentage)
|
||||||
|
self.conversation_stream.volume_percentage = volume_percentage
|
||||||
|
|
||||||
|
if self.on_volume_changed:
|
||||||
|
self.on_volume_changed(volume_percentage)
|
||||||
|
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
|
||||||
|
continue_conversation = True
|
||||||
|
logging.info('Expecting follow-on query from user.')
|
||||||
|
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
|
||||||
|
continue_conversation = False
|
||||||
|
if resp.device_action.device_request_json:
|
||||||
|
device_request = json.loads(
|
||||||
|
resp.device_action.device_request_json
|
||||||
|
)
|
||||||
|
fs = self.device_handler(device_request)
|
||||||
|
if fs:
|
||||||
|
device_actions_futures.extend(fs)
|
||||||
|
if self.display and resp.screen_out.data:
|
||||||
|
system_browser = browser_helpers.system_browser
|
||||||
|
system_browser.display(resp.screen_out.data)
|
||||||
|
|
||||||
|
if resp.dialog_state_out.supplemental_display_text and self.on_response:
|
||||||
|
self.on_response(resp.dialog_state_out.supplemental_display_text)
|
||||||
|
|
||||||
|
if len(device_actions_futures):
|
||||||
|
logging.info('Waiting for device executions to complete.')
|
||||||
|
concurrent.futures.wait(device_actions_futures)
|
||||||
|
|
||||||
|
logging.info('Finished playing assistant response.')
|
||||||
|
self.conversation_stream.stop_playback()
|
||||||
|
|
||||||
|
if self.on_conversation_end:
|
||||||
|
self.on_conversation_end(continue_conversation)
|
||||||
|
|
||||||
|
return continue_conversation
|
||||||
|
|
||||||
|
def gen_assist_requests(self):
|
||||||
|
"""Yields: AssistRequest messages to send to the API."""
|
||||||
|
|
||||||
|
config = embedded_assistant_pb2.AssistConfig(
|
||||||
|
audio_in_config=embedded_assistant_pb2.AudioInConfig(
|
||||||
|
encoding='LINEAR16',
|
||||||
|
sample_rate_hertz=self.conversation_stream.sample_rate,
|
||||||
|
),
|
||||||
|
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
|
||||||
|
encoding='LINEAR16',
|
||||||
|
sample_rate_hertz=self.conversation_stream.sample_rate,
|
||||||
|
volume_percentage=self.conversation_stream.volume_percentage,
|
||||||
|
),
|
||||||
|
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
|
||||||
|
language_code=self.language_code,
|
||||||
|
conversation_state=self.conversation_state,
|
||||||
|
is_new_conversation=self.is_new_conversation,
|
||||||
|
),
|
||||||
|
device_config=embedded_assistant_pb2.DeviceConfig(
|
||||||
|
device_id=self.device_id,
|
||||||
|
device_model_id=self.device_model_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if self.display:
|
||||||
|
config.screen_out_config.screen_mode = PLAYING
|
||||||
|
# Continue current conversation with later requests.
|
||||||
|
self.is_new_conversation = False
|
||||||
|
# The first AssistRequest must contain the AssistConfig
|
||||||
|
# and no audio data.
|
||||||
|
yield embedded_assistant_pb2.AssistRequest(config=config)
|
||||||
|
for data in self.conversation_stream:
|
||||||
|
# Subsequent requests need audio data, but not config.
|
||||||
|
yield embedded_assistant_pb2.AssistRequest(audio_in=data)
|
||||||
|
|
||||||
|
|
@ -2,34 +2,242 @@
|
||||||
.. moduleauthor:: Fabio Manganiello <blacklight86@gmail.com>
|
.. moduleauthor:: Fabio Manganiello <blacklight86@gmail.com>
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from platypush.context import get_backend
|
import json
|
||||||
from platypush.plugins import Plugin, action
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
class AssistantGooglePushtotalkPlugin(Plugin):
|
import google.auth.transport.grpc
|
||||||
|
import google.auth.transport.requests
|
||||||
|
import google.oauth2.credentials
|
||||||
|
|
||||||
|
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
|
||||||
|
import googlesamples.assistant.grpc.device_helpers as device_helpers
|
||||||
|
|
||||||
|
from platypush.context import get_bus
|
||||||
|
from platypush.message.event.assistant import ConversationStartEvent, \
|
||||||
|
ConversationEndEvent, SpeechRecognizedEvent, VolumeChangedEvent, \
|
||||||
|
ResponseEvent
|
||||||
|
|
||||||
|
from platypush.plugins import action
|
||||||
|
from platypush.plugins.assistant import AssistantPlugin
|
||||||
|
from platypush.plugins.assistant.google.lib import SampleAssistant
|
||||||
|
|
||||||
|
class AssistantGooglePushtotalkPlugin(AssistantPlugin):
|
||||||
"""
|
"""
|
||||||
Plugin for the Google assistant pushtotalk API. It acts as a wrapper to
|
Plugin for the Google Assistant push-to-talk API.
|
||||||
programmatically control a
|
|
||||||
:mod:`platypush.backend.assistant.google.pushtotalk` backend.
|
Triggers:
|
||||||
|
|
||||||
|
* :class:`platypush.message.event.assistant.ConversationStartEvent` \
|
||||||
|
when a new conversation starts
|
||||||
|
* :class:`platypush.message.event.assistant.SpeechRecognizedEvent` \
|
||||||
|
when a new voice command is recognized
|
||||||
|
* :class:`platypush.message.event.assistant.ConversationEndEvent` \
|
||||||
|
when a new conversation ends
|
||||||
|
|
||||||
|
Requires:
|
||||||
|
|
||||||
|
* **tenacity** (``pip install tenacity``)
|
||||||
|
* **google-assistant-sdk** (``pip install google-assistant-sdk[samples]``)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
api_endpoint = 'embeddedassistant.googleapis.com'
|
||||||
super().__init__(*args, **kwargs)
|
audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE
|
||||||
|
audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH
|
||||||
|
audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE
|
||||||
|
audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE
|
||||||
|
audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE
|
||||||
|
grpc_deadline = 60 * 3 + 5
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
credentials_file=os.path.join(
|
||||||
|
os.path.expanduser('~'), '.config',
|
||||||
|
'google-oauthlib-tool', 'credentials.json'),
|
||||||
|
device_config=os.path.join(
|
||||||
|
os.path.expanduser('~'), '.config', 'googlesamples-assistant',
|
||||||
|
'device_config.json'),
|
||||||
|
language='en-US', **kwargs):
|
||||||
|
"""
|
||||||
|
:param credentials_file: Path to the Google OAuth credentials file \
|
||||||
|
(default: ~/.config/google-oauthlib-tool/credentials.json). \
|
||||||
|
See https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials \
|
||||||
|
for instructions to get your own credentials file.
|
||||||
|
:type credentials_file: str
|
||||||
|
|
||||||
|
:param device_config: Path to device_config.json. Register your device \
|
||||||
|
(see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device) \
|
||||||
|
and create a project, then run the pushtotalk.py script from \
|
||||||
|
googlesamples to create your device_config.json
|
||||||
|
:type device_config: str
|
||||||
|
|
||||||
|
:param language: Assistant language (default: en-US)
|
||||||
|
:type language: str
|
||||||
|
"""
|
||||||
|
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
|
self.language = language
|
||||||
|
self.credentials_file = credentials_file
|
||||||
|
self.device_config = device_config
|
||||||
|
self.assistant = None
|
||||||
|
self.interactions = []
|
||||||
|
|
||||||
|
with open(self.device_config) as f:
|
||||||
|
device = json.load(f)
|
||||||
|
self.device_id = device['id']
|
||||||
|
self.device_model_id = device['model_id']
|
||||||
|
|
||||||
|
# Load OAuth 2.0 credentials.
|
||||||
|
try:
|
||||||
|
with open(self.credentials_file, 'r') as f:
|
||||||
|
self.credentials = google.oauth2.credentials.Credentials(token=None,
|
||||||
|
**json.load(f))
|
||||||
|
self.http_request = google.auth.transport.requests.Request()
|
||||||
|
self.credentials.refresh(self.http_request)
|
||||||
|
except Exception as ex:
|
||||||
|
self.logger.error('Error loading credentials: %s', str(ex))
|
||||||
|
self.logger.error('Run google-oauthlib-tool to initialize '
|
||||||
|
'new OAuth 2.0 credentials.')
|
||||||
|
raise
|
||||||
|
|
||||||
|
self.grpc_channel = None
|
||||||
|
self.conversation_stream = None
|
||||||
|
self.device_handler = None
|
||||||
|
|
||||||
|
def _init_assistant(self):
|
||||||
|
self.interactions = []
|
||||||
|
|
||||||
|
# Create an authorized gRPC channel.
|
||||||
|
self.grpc_channel = google.auth.transport.grpc.secure_authorized_channel(
|
||||||
|
self.credentials, self.http_request, self.api_endpoint)
|
||||||
|
|
||||||
|
self.logger.info('Connecting to {}'.format(self.api_endpoint))
|
||||||
|
|
||||||
|
# Configure audio source and sink.
|
||||||
|
audio_device = None
|
||||||
|
audio_source = audio_device = (
|
||||||
|
audio_device or audio_helpers.SoundDeviceStream(
|
||||||
|
sample_rate=self.audio_sample_rate,
|
||||||
|
sample_width=self.audio_sample_width,
|
||||||
|
block_size=self.audio_block_size,
|
||||||
|
flush_size=self.audio_flush_size
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
audio_sink = audio_device = (
|
||||||
|
audio_device or audio_helpers.SoundDeviceStream(
|
||||||
|
sample_rate=self.audio_sample_rate,
|
||||||
|
sample_width=self.audio_sample_width,
|
||||||
|
block_size=self.audio_block_size,
|
||||||
|
flush_size=self.audio_flush_size
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create conversation stream with the given audio source and sink.
|
||||||
|
self.conversation_stream = audio_helpers.ConversationStream(
|
||||||
|
source=audio_source,
|
||||||
|
sink=audio_sink,
|
||||||
|
iter_size=self.audio_iter_size,
|
||||||
|
sample_width=self.audio_sample_width,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.device_handler = device_helpers.DeviceRequestHandler(self.device_id)
|
||||||
|
|
||||||
|
def on_conversation_start(self):
|
||||||
|
""" Conversation start handler """
|
||||||
|
def handler():
|
||||||
|
get_bus().post(ConversationStartEvent())
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
|
def on_conversation_end(self):
|
||||||
|
""" Conversation end handler """
|
||||||
|
def handler(with_follow_on_turn):
|
||||||
|
get_bus().post(ConversationEndEvent(with_follow_on_turn=with_follow_on_turn))
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
|
def on_speech_recognized(self):
|
||||||
|
""" Speech recognized handler """
|
||||||
|
def handler(phrase):
|
||||||
|
get_bus().post(SpeechRecognizedEvent(phrase=phrase))
|
||||||
|
self.interactions.append({'request': phrase})
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
|
def on_volume_changed(self):
|
||||||
|
""" Volume changed event """
|
||||||
|
def handler(volume):
|
||||||
|
get_bus().post(VolumeChangedEvent(volume=volume))
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
|
def on_response(self):
|
||||||
|
""" Response handler """
|
||||||
|
def handler(response):
|
||||||
|
get_bus().post(ResponseEvent(response_text=response))
|
||||||
|
|
||||||
|
if not self.interactions:
|
||||||
|
self.interactions.append({'response': response})
|
||||||
|
else:
|
||||||
|
self.interactions[-1]['response'] = response
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def start_conversation(self):
|
def start_conversation(self, language=None):
|
||||||
"""
|
"""
|
||||||
Programmatically start a conversation with the assistant
|
Start a conversation
|
||||||
|
|
||||||
|
:param language: Language code override (default: default configured
|
||||||
|
language)
|
||||||
|
:type language: str
|
||||||
|
|
||||||
|
:returns: A list of the interactions that happen within the conversation::
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"request": "request 1",
|
||||||
|
"response": "response 1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"request": "request 2",
|
||||||
|
"response": "response 2"
|
||||||
|
},
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
assistant = get_backend('assistant.google.pushtotalk')
|
|
||||||
assistant.start_conversation()
|
if not language:
|
||||||
|
language = self.language
|
||||||
|
|
||||||
|
self._init_assistant()
|
||||||
|
|
||||||
|
with SampleAssistant(language_code=language,
|
||||||
|
device_model_id=self.device_model_id,
|
||||||
|
device_id=self.device_id,
|
||||||
|
conversation_stream=self.conversation_stream,
|
||||||
|
display=None,
|
||||||
|
channel=self.grpc_channel,
|
||||||
|
deadline_sec=self.grpc_deadline,
|
||||||
|
device_handler=None,
|
||||||
|
on_conversation_start=self.on_conversation_start(),
|
||||||
|
on_conversation_end=self.on_conversation_end(),
|
||||||
|
on_volume_changed=self.on_volume_changed(),
|
||||||
|
on_response=self.on_response(),
|
||||||
|
on_speech_recognized=self.on_speech_recognized()) as self.assistant:
|
||||||
|
continue_conversation = True
|
||||||
|
|
||||||
|
while continue_conversation:
|
||||||
|
continue_conversation = self.assistant.assist()
|
||||||
|
|
||||||
|
return self.interactions
|
||||||
|
|
||||||
@action
|
@action
|
||||||
def stop_conversation(self):
|
def stop_conversation(self):
|
||||||
"""
|
""" Stop a conversation """
|
||||||
Programmatically stop a running conversation with the assistant
|
if self.assistant:
|
||||||
"""
|
self.conversation_stream.stop_playback()
|
||||||
assistant = get_backend('assistant.google.pushtotalk')
|
get_bus().post(ConversationEndEvent())
|
||||||
assistant.stop_conversation()
|
|
||||||
|
|
||||||
# vim:sw=4:ts=4:et:
|
# vim:sw=4:ts=4:et:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,9 @@ sqlalchemy
|
||||||
# Google APIs general layer support
|
# Google APIs general layer support
|
||||||
# google-api-python-client
|
# google-api-python-client
|
||||||
# oauth2client
|
# oauth2client
|
||||||
# google-cloud
|
|
||||||
|
# Google text-to-speech API support
|
||||||
|
# google-cloud-texttospeech
|
||||||
|
|
||||||
# Last.FM scrobbler plugin support
|
# Last.FM scrobbler plugin support
|
||||||
# pylast
|
# pylast
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue