platypush/platypush
platypush
/
platypush
2
2
Fork 4
Code Issues 44 Pull Requests Packages Projects 4 Releases Wiki Activity

Merge pull request 'New `assistant.google` integration' (#335) from 284/refactor-google-assistant-integration into master
continuous-integration/drone/push Build is passing Details 

Reviewed-on: #335
This commit is contained in:
Fabio Manganiello 2023-10-23 16:04:35 +02:00
parent debb4f6f36 a005958317
commit 6908a90d10
30 changed files with 1122 additions and 1311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/source/backends.rst
View File

@ -8,8 +8,6 @@ Backends
platypush/backend/adafruit.io.rst
platypush/backend/alarm.rst
platypush/backend/assistant.google.rst
platypush/backend/assistant.snowboy.rst
platypush/backend/button.flic.rst
platypush/backend/camera.pi.rst
platypush/backend/chat.telegram.rst

6
docs/source/platypush/backend/assistant.google.rst
View File

@ -1,6 +0,0 @@
``assistant.google``
======================================
.. automodule:: platypush.backend.assistant.google
:members:

6
docs/source/platypush/backend/assistant.snowboy.rst
View File

@ -1,6 +0,0 @@
``assistant.snowboy``
=======================================
.. automodule:: platypush.backend.assistant.snowboy
:members:

6
docs/source/platypush/plugins/assistant.echo.rst
View File

@ -1,6 +0,0 @@
``assistant.echo``
====================================
.. automodule:: platypush.plugins.assistant.echo
:members:

6
docs/source/platypush/plugins/assistant.google.pushtotalk.rst
View File

@ -1,6 +0,0 @@
``assistant.google.pushtotalk``
=================================================
.. automodule:: platypush.plugins.assistant.google.pushtotalk
:members:

2
docs/source/plugins.rst
View File

@ -10,9 +10,7 @@ Plugins
platypush/plugins/alarm.rst
platypush/plugins/application.rst
platypush/plugins/arduino.rst
platypush/plugins/assistant.echo.rst
platypush/plugins/assistant.google.rst
platypush/plugins/assistant.google.pushtotalk.rst
platypush/plugins/autoremote.rst
platypush/plugins/bluetooth.rst
platypush/plugins/calendar.rst

39
platypush/backend/assistant/__init__.py
View File

@ -1,39 +0,0 @@
from abc import ABC
import threading
from typing import Optional, Dict, Any, Tuple
from platypush.backend import Backend
from platypush.context import get_plugin
from platypush.plugins.tts import TtsPlugin
class AssistantBackend(Backend):
def __init__(self, tts_plugin: Optional[str] = None, tts_args: Optional[Dict[str, Any]] = None, **kwargs):
"""
Default assistant backend constructor.
:param tts_plugin: If set, and if the assistant returns the processed response as text, then the processed
response will be played through the selected text-to-speech plugin (can be e.g. "``tts``",
"``tts.google``" or any other implementation of :class:`platypush.plugins.tts.TtsPlugin`).
:param tts_args: Extra parameters to pass to the ``say`` method of the selected TTS plugin (e.g.
language, voice or gender).
"""
super().__init__(**kwargs)
self._detection_paused = threading.Event()
self.tts_plugin = tts_plugin
self.tts_args = tts_args or {}
def pause_detection(self):
self._detection_paused.set()
def resume_detection(self):
self._detection_paused.clear()
def is_detecting(self):
return not self._detection_paused.is_set()
def _get_tts_plugin(self) -> Tuple[Optional[TtsPlugin], Dict[str, Any]]:
return get_plugin(self.tts_plugin) if self.tts_plugin else None, self.tts_args
# vim:sw=4:ts=4:et:

191
platypush/backend/assistant/google/__init__.py
View File

@ -1,191 +0,0 @@
import json
import os
import time
from platypush.backend.assistant import AssistantBackend
from platypush.message.event.assistant import (
ConversationStartEvent,
ConversationEndEvent,
ConversationTimeoutEvent,
ResponseEvent,
NoResponseEvent,
SpeechRecognizedEvent,
AlarmStartedEvent,
AlarmEndEvent,
TimerStartedEvent,
TimerEndEvent,
AlertStartedEvent,
AlertEndEvent,
MicMutedEvent,
MicUnmutedEvent,
)
class AssistantGoogleBackend(AssistantBackend):
"""
Google Assistant backend.
It listens for voice commands and post conversation events on the bus.
**WARNING**: The Google Assistant library used by this backend has officially been deprecated:
https://developers.google.com/assistant/sdk/reference/library/python/. This backend still works on most of the
devices where I use it, but its correct functioning is not guaranteed as the assistant library is no longer
maintained.
"""
_default_credentials_file = os.path.join(
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
)
def __init__(
self,
credentials_file=_default_credentials_file,
device_model_id='Platypush',
**kwargs
):
"""
:param credentials_file: Path to the Google OAuth credentials file
(default: ~/.config/google-oauthlib-tool/credentials.json).
See
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
for instructions to get your own credentials file.
:type credentials_file: str
:param device_model_id: Device model ID to use for the assistant
(default: Platypush)
:type device_model_id: str
"""
super().__init__(**kwargs)
self.credentials_file = credentials_file
self.device_model_id = device_model_id
self.credentials = None
self.assistant = None
self._has_error = False
self._is_muted = False
self.logger.info('Initialized Google Assistant backend')
def _process_event(self, event):
from google.assistant.library.event import EventType, AlertType
self.logger.info('Received assistant event: {}'.format(event))
self._has_error = False
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
self.bus.post(ConversationStartEvent(assistant=self))
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
if not event.args.get('with_follow_on_turn'):
self.bus.post(ConversationEndEvent(assistant=self))
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
self.bus.post(ConversationTimeoutEvent(assistant=self))
elif event.type == EventType.ON_NO_RESPONSE:
self.bus.post(NoResponseEvent(assistant=self))
elif (
hasattr(EventType, 'ON_RENDER_RESPONSE')
and event.type == EventType.ON_RENDER_RESPONSE
):
self.bus.post(
ResponseEvent(assistant=self, response_text=event.args.get('text'))
)
tts, args = self._get_tts_plugin()
if tts and 'text' in event.args:
self.stop_conversation()
tts.say(text=event.args['text'], **args)
elif (
hasattr(EventType, 'ON_RESPONDING_STARTED')
and event.type == EventType.ON_RESPONDING_STARTED
and event.args.get('is_error_response', False) is True
):
self.logger.warning('Assistant response error')
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
phrase = event.args['text'].lower().strip()
self.logger.info('Speech recognized: {}'.format(phrase))
self.bus.post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
elif event.type == EventType.ON_ALERT_STARTED:
if event.args.get('alert_type') == AlertType.ALARM:
self.bus.post(AlarmStartedEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
self.bus.post(TimerStartedEvent(assistant=self))
else:
self.bus.post(AlertStartedEvent(assistant=self))
elif event.type == EventType.ON_ALERT_FINISHED:
if event.args.get('alert_type') == AlertType.ALARM:
self.bus.post(AlarmEndEvent(assistant=self))
elif event.args.get('alert_type') == AlertType.TIMER:
self.bus.post(TimerEndEvent(assistant=self))
else:
self.bus.post(AlertEndEvent(assistant=self))
elif event.type == EventType.ON_ASSISTANT_ERROR:
self._has_error = True
if event.args.get('is_fatal'):
self.logger.error('Fatal assistant error')
else:
self.logger.warning('Assistant error')
if event.type == EventType.ON_MUTED_CHANGED:
self._is_muted = event.args.get('is_muted')
event = MicMutedEvent() if self._is_muted else MicUnmutedEvent()
self.bus.post(event)
def start_conversation(self):
"""Starts a conversation."""
if self.assistant:
self.assistant.start_conversation()
def stop_conversation(self):
"""Stops an active conversation."""
if self.assistant:
self.assistant.stop_conversation()
def set_mic_mute(self, muted):
if not self.assistant:
self.logger.warning('Assistant not running')
return
self.assistant.set_mic_mute(muted)
def is_muted(self) -> bool:
return self._is_muted
def send_text_query(self, query):
if not self.assistant:
self.logger.warning('Assistant not running')
return
self.assistant.send_text_query(query)
def run(self):
import google.oauth2.credentials
from google.assistant.library import Assistant
super().run()
with open(self.credentials_file, 'r') as f:
self.credentials = google.oauth2.credentials.Credentials(
token=None, **json.load(f)
)
while not self.should_stop():
self._has_error = False
with Assistant(self.credentials, self.device_model_id) as assistant:
self.assistant = assistant
for event in assistant.start():
if not self.is_detecting():
self.logger.info(
'Assistant event received but detection is currently paused'
)
continue
self._process_event(event)
if self._has_error:
self.logger.info(
'Restarting the assistant after an unrecoverable error'
)
time.sleep(5)
break
# vim:sw=4:ts=4:et:

39
platypush/backend/assistant/google/manifest.yaml
View File

@ -1,39 +0,0 @@
manifest:
events:
platypush.message.event.assistant.AlarmEndEvent: when an alarm ends
platypush.message.event.assistant.AlarmStartedEvent: when an alarm starts
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
ends
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
starts
platypush.message.event.assistant.ConversationTimeoutEvent: when a conversation
times out
platypush.message.event.assistant.MicMutedEvent: when the microphone is muted.
platypush.message.event.assistant.MicUnmutedEvent: when the microphone is un-muted.
platypush.message.event.assistant.NoResponseEvent: when a conversation returned no
response
platypush.message.event.assistant.ResponseEvent: when the assistant is speaking
a response
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
is recognized
platypush.message.event.assistant.TimerEndEvent: when a timer ends
platypush.message.event.assistant.TimerStartedEvent: when a timer starts
install:
apk:
- py3-grpcio
- py3-google-auth
apt:
- python3-grpcio
- python3-google-auth
dnf:
- python-grpcio
- python-google-auth
pacman:
- python-grpcio
- python-google-auth
pip:
- google-assistant-library
- google-assistant-sdk[samples]
- google-auth
package: platypush.backend.assistant.google
type: backend

184
platypush/backend/assistant/snowboy/__init__.py
View File

@ -1,184 +0,0 @@
import os
import threading
from platypush.backend.assistant import AssistantBackend
from platypush.context import get_plugin
from platypush.message.event.assistant import HotwordDetectedEvent
class AssistantSnowboyBackend(AssistantBackend):
"""
Backend for detecting custom voice hotwords through Snowboy. The purpose of
this component is only to detect the hotword specified in your Snowboy voice
model. If you want to trigger proper assistant conversations or custom
speech recognition, you should create a hook in your configuration on
HotwordDetectedEvent to trigger the conversation on whichever assistant
plugin you're using (Google, Alexa...)
Manual installation for snowboy and its Python bindings if the installation via package fails::
$ [sudo] apt-get install libatlas-base-dev swig
$ [sudo] pip install pyaudio
$ git clone https://github.com/Kitt-AI/snowboy
$ cd snowboy/swig/Python3
$ make
$ cd ../..
$ python3 setup.py build
$ [sudo] python setup.py install
You will also need a voice model for the hotword detection. You can find
some under the ``resources/models`` directory of the Snowboy repository,
or train/download other models from https://snowboy.kitt.ai.
"""
def __init__(self, models, audio_gain=1.0, **kwargs):
"""
:param models: Map (name -> configuration) of voice models to be used by
the assistant. See https://snowboy.kitt.ai/ for training/downloading
models. Sample format::
ok_google: # Hotword model name
voice_model_file: /path/models/OK Google.pmdl # Voice model file location
sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5)
assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google
# push-to-talk assistant plugin (optional)
assistant_language: en-US # The assistant will conversate in English when this hotword is
detected (optional)
detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional)
ciao_google: # Hotword model name
voice_model_file: /path/models/Ciao Google.pmdl # Voice model file location
sensitivity: 0.5 # Model sensitivity, between 0 and 1 (default: 0.5)
assistant_plugin: assistant.google.pushtotalk # When the hotword is detected trigger the Google
# push-to-talk assistant plugin (optional)
assistant_language: it-IT # The assistant will conversate in Italian when this hotword is
# detected (optional)
detect_sound: /path/to/bell.wav # Sound file to be played when the hotword is detected (optional)
:type models: dict
:param audio_gain: Audio gain, between 0 and 1. Default: 1
:type audio_gain: float
"""
try:
import snowboydecoder
except ImportError:
import snowboy.snowboydecoder as snowboydecoder
super().__init__(**kwargs)
self.models = {}
self._init_models(models)
self.audio_gain = audio_gain
self.detector = snowboydecoder.HotwordDetector(
[model['voice_model_file'] for model in self.models.values()],
sensitivity=[model['sensitivity'] for model in self.models.values()],
audio_gain=self.audio_gain,
)
self.logger.info(
'Initialized Snowboy hotword detection with {} voice model configurations'.format(
len(self.models)
)
)
def _init_models(self, models):
if not models:
raise AttributeError('Please specify at least one voice model')
self.models = {}
for name, conf in models.items():
if name in self.models:
raise AttributeError('Duplicate model key {}'.format(name))
model_file = conf.get('voice_model_file')
detect_sound = conf.get('detect_sound')
if not model_file:
raise AttributeError(
'No voice_model_file specified for model {}'.format(name)
)
model_file = os.path.abspath(os.path.expanduser(model_file))
assistant_plugin_name = conf.get('assistant_plugin')
if detect_sound:
detect_sound = os.path.abspath(os.path.expanduser(detect_sound))
if not os.path.isfile(model_file):
raise FileNotFoundError(
'Voice model file {} does not exist or it not a regular file'.format(
model_file
)
)
self.models[name] = {
'voice_model_file': model_file,
'sensitivity': conf.get('sensitivity', 0.5),
'detect_sound': detect_sound,
'assistant_plugin': get_plugin(assistant_plugin_name)
if assistant_plugin_name
else None,
'assistant_language': conf.get('assistant_language'),
'tts_plugin': conf.get('tts_plugin'),
'tts_args': conf.get('tts_args', {}),
}
def hotword_detected(self, hotword):
"""
Callback called on hotword detection
"""
try:
import snowboydecoder
except ImportError:
import snowboy.snowboydecoder as snowboydecoder
def sound_thread(sound):
snowboydecoder.play_audio_file(sound)
def callback():
if not self.is_detecting():
self.logger.info(
'Hotword detected but assistant response currently paused'
)
return
self.bus.post(HotwordDetectedEvent(hotword=hotword))
model = self.models[hotword]
detect_sound = model.get('detect_sound')
assistant_plugin = model.get('assistant_plugin')
assistant_language = model.get('assistant_language')
tts_plugin = model.get('tts_plugin')
tts_args = model.get('tts_args')
if detect_sound:
threading.Thread(target=sound_thread, args=(detect_sound,)).start()
if assistant_plugin:
assistant_plugin.start_conversation(
language=assistant_language,
tts_plugin=tts_plugin,
tts_args=tts_args,
)
return callback
def on_stop(self):
super().on_stop()
if self.detector:
self.detector.terminate()
self.detector = None
def run(self):
super().run()
self.detector.start(
detected_callback=[
self.hotword_detected(hotword) for hotword in self.models.keys()
]
)
# vim:sw=4:ts=4:et:

9
platypush/backend/assistant/snowboy/manifest.yaml
View File

@ -1,9 +0,0 @@
manifest:
events:
platypush.message.event.assistant.HotwordDetectedEvent: whenever the hotword has
been detected
install:
pip:
- snowboy
package: platypush.backend.assistant.snowboy
type: backend

3
platypush/backend/http/webapp/src/App.vue
View File

@ -32,6 +32,7 @@ export default {
return {
config: {},
userAuthenticated: false,
connected: false,
pwaInstallEvent: null,
}
},
@ -97,6 +98,8 @@ export default {
mounted() {
bus.onNotification(this.onNotification)
bus.on('connect', () => this.connected = true)
bus.on('disconnect', () => this.connected = false)
},
}
</script>

3
platypush/backend/http/webapp/src/assets/icons.json
View File

@ -2,6 +2,9 @@
"arduino": {
"class": "fas fa-microchip"
},
"assistant.google": {
"class": "fas fa-microphone-lines"
},
"bluetooth": {
"class": "fab fa-bluetooth"
},

24
platypush/backend/http/webapp/src/components/Nav.vue
View File

@ -163,6 +163,7 @@ export default {
this.collapsed = this.collapsedDefault
bus.on('connect', this.setConnected(true))
bus.on('disconnect', this.setConnected(false))
this.setConnected(this.$root.connected)
},
}
</script>
@ -192,6 +193,11 @@ nav {
top: 0;
left: 0;
z-index: 5;
.icon.status {
top: 0.75em !important;
left: 2em;
}
}
}
@ -318,6 +324,10 @@ nav {
}
}
.icon.status {
width: 1em;
}
&.collapsed {
display: flex;
flex-direction: column;
@ -353,15 +363,19 @@ nav {
box-shadow: none;
background: $nav-toggler-collapsed-bg;
@media screen and (max-width: #{$tablet - 1px}) {
background: $nav-toggler-collapsed-mobile-bg;
color: $nav-toggler-collapsed-mobile-fg;
}
.icon.status {
top: 0.75em;
left: 2em;
}
@include until($tablet) {
background: $nav-toggler-collapsed-mobile-bg;
color: $nav-toggler-collapsed-mobile-fg;
.icon.status {
top: 0.75em !important;
}
}
}
.footer {

107
platypush/backend/http/webapp/src/components/elements/TextPrompt.vue Normal file
View File

@ -0,0 +1,107 @@
<template>
<Modal ref="modal" :title="title">
<form @submit.prevent="onConfirm">
<div class="dialog-content">
<slot />
<input type="text" ref="input" />
</div>
<div class="buttons">
<button type="submit" class="ok-btn" @click="onConfirm" @touch="onConfirm">
<i class="fas fa-check" /> &nbsp; {{ confirmText }}
</button>
<button type="button" class="cancel-btn" @click="close" @touch="close">
<i class="fas fa-xmark" /> &nbsp; {{ cancelText }}
</button>
</div>
</form>
</Modal>
</template>
<script>
import Modal from "@/components/Modal";
export default {
emits: ['input'],
components: {Modal},
props: {
title: {
type: String,
},
confirmText: {
type: String,
default: "OK",
},
cancelText: {
type: String,
default: "Cancel",
},
},
methods: {
onConfirm() {
this.$emit('input', this.$refs.input.value)
this.close()
},
show() {
this.$refs.modal.show()
},
close() {
this.$refs.modal.hide()
},
},
mounted() {
this.$nextTick(() => {
this.$refs.input.value = ""
this.$refs.input.focus()
})
},
}
</script>
<style lang="scss" scoped>
:deep(.modal) {
.dialog-content {
padding: 1em;
}
.body {
padding: 1.5em;
}
.buttons {
display: flex;
flex-direction: row;
justify-content: right;
margin-bottom: 1em;
border: 0;
border-radius: 0;
button {
margin-right: 1em;
padding: 0.5em 1em;
border: 1px solid $border-color-2;
border-radius: 1em;
&:hover {
background: $hover-bg;
}
}
}
}
form {
display: flex;
flex-direction: column !important;
.buttons {
flex-direction: row !important;
}
}
</style>

250
platypush/backend/http/webapp/src/components/panels/Entities/Assistant.vue Normal file
View File

@ -0,0 +1,250 @@
<template>
<div class="entity assistant-container">
<TextPrompt ref="prompt">
Enter a text query to send to the assistant.
</TextPrompt>
<div class="head" @click="onHeadClick">
<div class="col-1 icon entity-icon" ref="icon">
<EntityIcon
:entity="value"
:class="{active: value.conversation_running}"
:loading="loading"
:error="error" />
</div>
<div class="label">
<div class="name" ref="name" v-text="value.name" />
</div>
<div class="value-container">
<button @click.stop="collapsed = !collapsed">
<i class="fas"
:class="{'fa-angle-up': !collapsed, 'fa-angle-down': collapsed}" />
</button>
</div>
</div>
<div class="body" ref="body" v-if="!collapsed" @click.stop="prevent">
<div class="row" @click.stop="stopConversation" v-if="value.conversation_running">
<div class="icon">
<i class="fas fa-comment-slash" />
</div>
<div class="label">
<div class="name">Stop Conversation</div>
</div>
<div class="value">
<ToggleSwitch
@click.stop="stopConversation"
:value="false"
:disabled="loading" />
</div>
</div>
<div class="row" @click.stop="startConversation" v-else>
<div class="icon">
<i class="fas fa-comment" />
</div>
<div class="label">
<div class="name">Start Conversation</div>
</div>
<div class="value">
<ToggleSwitch
@click.stop="startConversation"
:value="false"
:disabled="loading" />
</div>
</div>
<div class="row" @click.stop="toggleMute">
<div class="icon">
<i class="fas fa-microphone-lines-slash" />
</div>
<div class="label">
<div class="name">Muted</div>
</div>
<div class="value">
<ToggleSwitch
@click.stop="toggleMute"
:value="value.is_muted"
:disabled="loading" />
</div>
</div>
<div class="row" @click.stop="showPrompt">
<div class="icon">
<i class="fas fa-keyboard" />
</div>
<div class="label">
<div class="name">Send query from text prompt</div>
</div>
<div class="value">
<ToggleSwitch
@click.stop="showPrompt"
:value="false"
:disabled="loading" />
</div>
</div>
</div>
</div>
</template>
<script>
import TextPrompt from "@/components/elements/TextPrompt"
import ToggleSwitch from "@/components/elements/ToggleSwitch"
import EntityIcon from "./EntityIcon"
import EntityMixin from "./EntityMixin"
export default {
name: 'Assistant',
mixins: [EntityMixin],
components: {
EntityIcon,
TextPrompt,
ToggleSwitch,
},
data() {
return {
collapsed: true,
showTextQueryPrompt: false,
modalId: 'assistant-text-prompt-modal',
}
},
methods: {
hidePrompt() {
document.body.querySelector(`#${this.modalId}`)?.remove()
},
showPrompt() {
const modalElement = this.$refs.prompt.$el
this.hidePrompt()
modalElement.id = this.modalId
modalElement.classList.remove('hidden')
const input = modalElement.querySelector('input[type="text"]')
const form = modalElement.querySelector('form')
if (form) {
form.addEventListener('submit', (event) => {
event.stopPropagation()
this.onTextPrompt(input?.value)
})
}
const cancelBtn = modalElement.querySelector('.cancel-btn')
if (cancelBtn) {
cancelBtn.onclick = (event) => {
this.hidePrompt()
event.stopPropagation()
}
}
modalElement.onclick = (event) => {
const modalContent = modalElement.querySelector('.modal')
if (modalContent?.contains(event.target)) {
event.stopPropagation()
return false
}
this.hidePrompt()
}
document.body.appendChild(modalElement)
this.$nextTick(() => {
modalElement.querySelector('input[type="text"]').focus()
})
},
onHeadClick(event) {
if (
this.$refs.name.contains(event.target) ||
this.$refs.icon.contains(event.target)
) {
// Propagate the event upwards and let it open the entity modal
return true
}
// Toggle the collapse state if the click is outside of the entity
// name/icon
this.collapsed = !this.collapsed
event.stopPropagation()
},
async toggleMute() {
await this.request('entities.execute', {
id: this.value.id,
action: 'toggle_mute',
})
},
async startConversation() {
await this.request('entities.execute', {
id: this.value.id,
action: 'start_conversation',
})
},
async stopConversation() {
await this.request('entities.execute', {
id: this.value.id,
action: 'stop_conversation',
})
},
async onTextPrompt(query) {
await this.request('entities.execute', {
id: this.value.id,
action: 'send_text_query',
query: query,
})
this.hidePrompt()
},
}
}
</script>
<style lang="scss" scoped>
@import "common";
$icon-size: 2em;
.assistant-container {
.body {
padding: 0;
}
.row {
margin: 0;
padding: 1em 0.5em;
display: flex;
&:hover {
background: $hover-bg;
}
&:not(:last-child) {
border-bottom: 1px solid $border-color-1;
}
.icon {
flex: 0 0 $icon-size;
display: flex;
align-items: center;
justify-content: center;
}
.label {
width: calc(100% - $icon-size);
}
}
:deep(.entity-icon) {
.active {
color: $selected-fg;
}
}
}
</style>

8
platypush/backend/http/webapp/src/components/panels/Entities/meta.json
View File

@ -1,4 +1,12 @@
{
"assistant": {
"name": "Assistant",
"name_plural": "Assistants",
"icon": {
"class": "fas fa-microphone-lines"
}
},
"battery": {
"name": "Battery",
"name_plural": "Batteries",

29
platypush/config/config.yaml
View File

@ -280,6 +280,35 @@ backend.http:
# - platypush/tests
###
###
# # Example configuration of a voice assistant.
# # Several voice assistant plugins and engines are available - Google
# # Assistant, Alexa, DeepSpeech, Picovoice etc.
# #
# # The Google Assistant is probably the most straightforward to configure and
# # the richest in terms of features provided out-of-the-box and speech
# # detection quality, while others may require further tinkering, may perform
# # worse than the Google model, and/or may run models on-device which could not
# # be within reach for some machines.
# #
# # Check the documentation of the `assistant.google` plugin for instructions on
# # how to get a credentials file that you can use with a custom assistant
# # installation.
# #
# # Note however that the Google Assistant plugin leverages the
# # `google-assistant-library`, which has been deprecated a while ago by Google
# # and it's unlikely to receive updates (although it still works and I'd still
# # expect it to work).
#
# assistant.google:
# # Path to your credentials file (by default it will look either under
# # ~/.config/google-oauthlib-tool/credentials.json or
# # <WORKDIR>/credentials/google/assistant.json
# # credentials_file: ~/credentials/assistant.json
# # If specified, then this sound will be played when a conversation starts
# # conversation_start_sound: ~/sounds/assistant-start.mp3
###
###
# # Example configuration of music.mpd plugin, a plugin to interact with MPD and
# # Mopidy music server instances. See

29
platypush/entities/assistants.py Normal file
View File

@ -0,0 +1,29 @@
from sqlalchemy import Column, Integer, ForeignKey, Boolean, String
from platypush.common.db import is_defined
from . import Entity
if not is_defined('assistant'):
class Assistant(Entity):
"""
Base class for voice assistant entities.
"""
__tablename__ = 'assistant'
id = Column(
Integer, ForeignKey(Entity.id, ondelete='CASCADE'), primary_key=True
)
last_query = Column(String)
last_response = Column(String)
conversation_running = Column(Boolean)
is_muted = Column(Boolean, default=False)
is_detecting = Column(Boolean, default=True)
__table_args__ = {'extend_existing': True}
__mapper_args__ = {
'polymorphic_identity': __tablename__,
}

78
platypush/entities/managers/assistants.py Normal file
View File

@ -0,0 +1,78 @@
from abc import ABC, abstractmethod
from . import EntityManager
class AssistantEntityManager(EntityManager, ABC):
"""
Base class for voice assistant integrations that support entity management.
"""
@abstractmethod
def start_conversation(self, *args, **kwargs):
"""
Programmatically starts a conversation.
"""
raise NotImplementedError()
@abstractmethod
def stop_conversation(self, *args, **kwargs):
"""
Programmatically stops a conversation.
"""
raise NotImplementedError()
@abstractmethod
def is_muted(self, *args, **kwargs) -> bool:
"""
:return: True if the microphone is muted, False otherwise.
"""
raise NotImplementedError()
@abstractmethod
def mute(self, *args, **kwargs):
"""
Mute the microphone.
"""
raise NotImplementedError()
@abstractmethod
def unmute(self, *args, **kwargs):
"""
Unmute the microphone.
"""
raise NotImplementedError()
def toggle_mute(self, *_, **__):
"""
Toggle the mute state of the microphone.
"""
return self.mute() if self.is_muted() else self.unmute()
@abstractmethod
def pause_detection(self, *args, **kwargs):
"""
Put the assistant on pause. No new conversation events will be triggered.
"""
raise NotImplementedError()
@abstractmethod
def resume_detection(self, *args, **kwargs):
"""
Resume the assistant hotword detection from a paused state.
"""
raise NotImplementedError()
@abstractmethod
def is_detecting(self, *args, **kwargs) -> bool:
"""
:return: True if the asistant is detecting, False otherwise.
"""
raise NotImplementedError()
@abstractmethod
def send_text_query(self, *args, **kwargs):
"""
Send a text query to the assistant.
"""
raise NotImplementedError()

15
platypush/message/event/assistant/__init__.py
View File

@ -1,8 +1,6 @@
import logging
import re
import sys
from platypush.context import get_backend, get_plugin
from platypush.message.event import Event
@ -11,18 +9,7 @@ class AssistantEvent(Event):
def __init__(self, *args, assistant=None, **kwargs):
super().__init__(*args, **kwargs)
self.logger = logging.getLogger('platypush:assistant')
if assistant:
self._assistant = assistant
else:
try:
self._assistant = get_backend('assistant.google')
if not self._assistant:
self._assistant = get_plugin('assistant.google.pushtotalk')
except Exception as e:
self.logger.debug('Could not initialize the assistant component: %s', e)
self._assistant = None
self._assistant = assistant
class ConversationStartEvent(AssistantEvent):

270
platypush/plugins/assistant/__init__.py
View File

@ -1,54 +1,286 @@
from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass
from enum import Enum
import os
from threading import Event
from typing import Any, Collection, Dict, Optional
from platypush.context import get_backend
from platypush.context import get_bus, get_plugin
from platypush.entities.assistants import Assistant
from platypush.entities.managers.assistants import AssistantEntityManager
from platypush.message.event.assistant import (
AssistantEvent,
ConversationStartEvent,
ConversationEndEvent,
ConversationTimeoutEvent,
ResponseEvent,
NoResponseEvent,
SpeechRecognizedEvent,
AlarmStartedEvent,
AlarmEndEvent,
TimerStartedEvent,
TimerEndEvent,
AlertStartedEvent,
AlertEndEvent,
MicMutedEvent,
MicUnmutedEvent,
)
from platypush.plugins import Plugin, action
from platypush.utils import get_plugin_name_by_class
class AssistantPlugin(ABC, Plugin):
@dataclass
class AlertType(Enum):
"""
Base class for assistant plugins
Enum representing the type of an alert.
"""
ALARM = 'alarm'
TIMER = 'timer'
ALERT = 'alert'
@dataclass
class AssistantState:
"""
Dataclass representing the state of an assistant.
"""
last_query: Optional[str] = None
last_response: Optional[str] = None
conversation_running: bool = False
is_muted: bool = False
is_detecting: bool = True
alert_state: Optional[str] = None
class AssistantPlugin(Plugin, AssistantEntityManager, ABC):
"""
Base class for assistant plugins.
"""
_entity_name = 'Assistant'
def __init__(
self,
*args,
tts_plugin: Optional[str] = None,
tts_plugin_args: Optional[Dict[str, Any]] = None,
conversation_start_sound: Optional[str] = None,
**kwargs
):
"""
:param tts_plugin: If set, the assistant will use this plugin (e.g.
``tts``, ``tts.google`` or ``tts.mimic3``) to render the responses,
instead of using the built-in assistant voice.
:param tts_plugin_args: Optional arguments to be passed to the TTS
``say`` action, if ``tts_plugin`` is set.
:param conversation_start_sound: If set, the assistant will play this
audio file when it detects a speech. The sound file will be played
on the default audio output device. If not set, the assistant won't
play any sound when it detects a speech.
"""
super().__init__(*args, **kwargs)
self.tts_plugin = tts_plugin
self.tts_plugin_args = tts_plugin_args or {}
if conversation_start_sound:
self._conversation_start_sound = os.path.abspath(
os.path.expanduser(conversation_start_sound)
)
self._detection_paused = Event()
self._conversation_running = Event()
self._is_muted = False
self._last_query: Optional[str] = None
self._last_response: Optional[str] = None
self._cur_alert_type: Optional[AlertType] = None
@property
def _state(self) -> AssistantState:
return AssistantState(
last_query=self._last_query,
last_response=self._last_response,
conversation_running=self._conversation_running.is_set(),
is_muted=self._is_muted,
is_detecting=not self._detection_paused.is_set(),
alert_state=self._cur_alert_type.value if self._cur_alert_type else None,
)
@abstractmethod
def start_conversation(self, *args, language=None, tts_plugin=None, tts_args=None, **kwargs):
def start_conversation(self, *_, **__):
"""
Start a conversation.
Programmatically starts a conversation.
"""
raise NotImplementedError
@abstractmethod
def stop_conversation(self, *args, **kwargs):
def stop_conversation(self, *_, **__):
"""
Stop a conversation.
Programmatically stops a conversation.
"""
raise NotImplementedError
def _get_assistant(self):
return get_backend('assistant.snowboy')
@action
def pause_detection(self):
def pause_detection(self, *_, **__):
"""
Put the assistant on pause. No new conversation events will be triggered.
"""
assistant = self._get_assistant()
assistant.pause_detection()
self._detection_paused.set()
@action
def resume_detection(self):
def resume_detection(self, *_, **__):
"""
Resume the assistant hotword detection from a paused state.
"""
assistant = self._get_assistant()
assistant.resume_detection()
self._detection_paused.clear()
@action
def is_detecting(self) -> bool:
def is_detecting(self, *_, **__) -> bool:
"""
:return: True if the asistant is detecting, False otherwise.
"""
assistant = self._get_assistant()
return assistant.is_detecting()
return not self._detection_paused.is_set()
@action
def is_muted(self, *_, **__) -> bool:
"""
:return: True if the microphone is muted, False otherwise.
"""
return self._is_muted
@action
def status(self, *_, **__):
"""
:return: The current assistant status:
.. code-block:: json
{
"last_query": "What time is it?",
"last_response": "It's 10:30 AM",
"conversation_running": true,
"is_muted": false,
"is_detecting": true
}
"""
self.publish_entities([self])
return asdict(self._state)
def _get_tts_plugin(self):
if not self.tts_plugin:
return None
return get_plugin(self.tts_plugin)
def _play_conversation_start_sound(self):
if not self._conversation_start_sound:
return
audio = get_plugin('sound')
if not audio:
self.logger.warning(
'Unable to play conversation start sound: sound plugin not found'
)
return
audio.play(self._conversation_start_sound)
def _send_event(self, event: AssistantEvent):
self.publish_entities([self])
get_bus().post(event)
def _on_conversation_start(self):
self._last_response = None
self._last_query = None
self._conversation_running.set()
self._send_event(ConversationStartEvent(assistant=self))
self._play_conversation_start_sound()
def _on_conversation_end(self):
self._conversation_running.clear()
self._send_event(ConversationEndEvent(assistant=self))
def _on_conversation_timeout(self):
self._last_response = None
self._last_query = None
self._conversation_running.clear()
self._send_event(ConversationTimeoutEvent(assistant=self))
def _on_no_response(self):
self._last_response = None
self._conversation_running.clear()
self._send_event(NoResponseEvent(assistant=self))
def _on_reponse_rendered(self, text: Optional[str]):
self._last_response = text
self._send_event(ResponseEvent(assistant=self, response_text=text))
tts = self._get_tts_plugin()
if tts and text:
self.stop_conversation()
tts.say(text=text, **self.tts_plugin_args)
def _on_speech_recognized(self, phrase: Optional[str]):
phrase = (phrase or '').lower().strip()
self._last_query = phrase
self._send_event(SpeechRecognizedEvent(assistant=self, phrase=phrase))
def _on_alarm_start(self):
self._cur_alert_type = AlertType.ALARM
self._send_event(AlarmStartedEvent(assistant=self))
def _on_alarm_end(self):
self._cur_alert_type = None
self._send_event(AlarmEndEvent(assistant=self))
def _on_timer_start(self):
self._cur_alert_type = AlertType.TIMER
self._send_event(TimerStartedEvent(assistant=self))
def _on_timer_end(self):
self._cur_alert_type = None
self._send_event(TimerEndEvent(assistant=self))
def _on_alert_start(self):
self._cur_alert_type = AlertType.ALERT
self._send_event(AlertStartedEvent(assistant=self))
def _on_alert_end(self):
self._cur_alert_type = None
self._send_event(AlertEndEvent(assistant=self))
def _on_mute(self):
self._is_muted = True
self._send_event(MicMutedEvent(assistant=self))
def _on_unmute(self):
self._is_muted = False
self._send_event(MicUnmutedEvent(assistant=self))
def _on_mute_changed(self, value: bool):
if value:
self._on_mute()
else:
self._on_unmute()
def transform_entities(self, entities: Collection['AssistantPlugin']):
return super().transform_entities(
[
Assistant(
external_id=get_plugin_name_by_class(type(dev)),
name=self._entity_name,
last_query=dev._state.last_query,
last_response=dev._state.last_response,
conversation_running=dev._state.conversation_running,
is_muted=dev._state.is_muted,
is_detecting=dev._state.is_detecting,
)
for dev in (entities or [])
]
)
# vim:sw=4:ts=4:et:

133
platypush/plugins/assistant/echo/__init__.py
View File

@ -1,133 +0,0 @@
import os
from typing import Optional
from platypush.context import get_bus
from platypush.plugins import action
from platypush.plugins.assistant import AssistantPlugin
from platypush.message.event.assistant import (
ConversationStartEvent,
ConversationEndEvent,
SpeechRecognizedEvent,
ResponseEvent,
)
class AssistantEchoPlugin(AssistantPlugin):
"""
Amazon Echo/Alexa assistant plugin.
In order to activate the Echo service on your device follow these steps:
1. Install avs (``pip install git+https://github.com/BlackLight/avs.git``)
2. Run the ``alexa-auth`` script. A local webservice will start on port 3000
3. If a browser instance doesn't open automatically then head to http://localhost:3000
4. Log in to your Amazon account
5. The required credentials will be stored to ~/.avs.json
"""
def __init__(
self,
avs_config_file: Optional[str] = None,
audio_device: str = 'default',
audio_player: str = 'default',
**kwargs
):
"""
:param avs_config_file: AVS credentials file - default: ~/.avs.json. If the file doesn't exist then
an instance of the AVS authentication service will be spawned. You can login through an Amazon
account either in the spawned browser window, if available, or by opening http://your-ip:3000
in the browser on another machine.
:param audio_device: Name of the input audio device (default: 'default')
:param audio_player: Player to be used for audio playback (default: 'default').
Supported values: 'mpv', 'mpg123', 'gstreamer'
"""
from avs.alexa import Alexa
from avs.config import DEFAULT_CONFIG_FILE
from avs.mic import Audio
super().__init__(**kwargs)
if not avs_config_file:
avs_config_file = DEFAULT_CONFIG_FILE
if not avs_config_file or not os.path.isfile(avs_config_file):
from avs.auth import auth
auth(None, avs_config_file)
self.logger.warning(
'Amazon Echo assistant credentials not configured. Open http://localhost:3000 '
+ 'to authenticate this client'
)
self.audio_device = audio_device
self.audio_player = audio_player
self.audio = Audio(device_name=audio_device)
self.alexa = Alexa(avs_config_file, audio_player=audio_player)
self._ready = False
self.alexa.state_listener.on_ready = self._on_ready()
self.alexa.state_listener.on_listening = self._on_listening()
self.alexa.state_listener.on_speaking = self._on_speaking()
self.alexa.state_listener.on_thinking = self._on_thinking()
self.alexa.state_listener.on_finished = self._on_finished()
self.alexa.state_listener.on_disconnected = self._on_disconnected()
self.audio.link(self.alexa)
self.alexa.start()
def _on_ready(self):
def _callback():
self._ready = True
return _callback
def _on_listening(self):
def _callback():
get_bus().post(ConversationStartEvent(assistant=self))
return _callback
def _on_speaking(self):
def _callback():
# AVS doesn't provide a way to access the response text
get_bus().post(ResponseEvent(assistant=self, response_text=''))
return _callback
def _on_finished(self):
def _callback():
get_bus().post(ConversationEndEvent(assistant=self))
return _callback
def _on_disconnected(self):
def _callback():
self._ready = False
return _callback
def _on_thinking(self):
def _callback():
# AVS doesn't provide a way to access the detected text
get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=''))
return _callback
@action
def start_conversation(self, **_):
if not self._ready:
raise RuntimeError('Echo assistant not ready')
self.audio.start()
self.alexa.listen()
@action
def stop_conversation(self):
self.audio.stop()
self._on_finished()()
# vim:sw=4:ts=4:et:

13
platypush/plugins/assistant/echo/manifest.yaml
View File

@ -1,13 +0,0 @@
manifest:
events:
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
ends
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
starts
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
is recognized
install:
pip:
- avs
package: platypush.plugins.assistant.echo
type: plugin

309
platypush/plugins/assistant/google/__init__.py
View File

@ -1,76 +1,313 @@
from platypush.backend.assistant.google import AssistantGoogleBackend
from platypush.context import get_backend
from platypush.plugins import action
import json
import os
from typing import Optional
from platypush.config import Config
from platypush.plugins import RunnablePlugin, action
from platypush.plugins.assistant import AssistantPlugin
class AssistantGooglePlugin(AssistantPlugin):
class AssistantGooglePlugin(AssistantPlugin, RunnablePlugin):
"""
Google assistant plugin.
It acts like a wrapper around the :mod:`platypush.backend.assistant.google`
backend to programmatically control the conversation status.
Google Assistant plugin.
This plugin allows you to run the Google Assistant _directly_ on your
device. It requires you to have an audio microphone and a speaker connected
to the device.
If you have multiple sound devices, you can specify which one(s) to use for
input and output through a ``~/.asoundrc`` configuration file like this:
.. code-block:: text
pcm.!default {
type asym
playback.pcm {
type plug
slave.pcm "hw:0,0"
}
capture.pcm {
type plug
slave.pcm "hw:1,0"
}
}
You can use ``aplay -l`` and ``arecord -l`` to respectively list the
detected audio output and input devices with their indices.
If you are using PulseAudio instead of bare ALSA, then you can:
1. Use the ``pavucontrol`` (GUI) tool to select the audio input and
output devices and volumes for the assistant.
2. Use a program like ``pamix`` (ncurses) or ``pamixer`` (CLI).
3. Run the ``pactl list sources`` and ``pactl list sinks`` commands to
respectively list the detected audio input and output devices. Take
note of their name, and specify which ones the assistant should use
by starting the application with the right ``PULSE_SOURCE`` and
``PULSE_SINK`` environment variables.
.. warning:: The Google Assistant library used by this backend has
been deprecated by Google:
https://developers.google.com/assistant/sdk/reference/library/python/.
This integration still works on all of my devices, but its future
functionality is not guaranteed - Google may decide to turn off the
API, the library may no longer be built against new architectures and
it's unlikely to be updated.
.. note:: Since the Google Assistant library hasn't been updated in several
years, some of its dependencies are quite old and may break more recent
Python installations. Please refer to the comments in the [manifest
file](https://git.platypush.tech/platypush/platypush/src/branch/master/platypush/plugins/assistant/google/manifest.yaml)
for more information on how to install the required dependencies, if
the automated ways fail.
"""
def __init__(self, **kwargs):
_entity_name = 'Google Assistant'
_default_credentials_files = (
os.path.join(Config.get_workdir(), 'credentials', 'google', 'assistant.json'),
os.path.join(
os.path.expanduser('~/.config'), 'google-oauthlib-tool', 'credentials.json'
),
)
def __init__(
self,
credentials_file: Optional[str] = None,
device_model_id: str = 'Platypush',
**kwargs,
):
"""
:param credentials_file: Path to the Google OAuth credentials file.
See
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
for instructions to get your own credentials file.
By default, it will search for the credentials file under:
* ``~/.config/google-oauthlib-tool/credentials.json``: default
location supported by the Google Assistant library
* ``<PLATYPUSH_WORKDIR>/credentials/google/assistant.json``:
recommended location, under the Platypush working directory.
:param device_model_id: The device model ID that identifies the device
where the assistant is running (default: Platypush). It can be a
custom string.
"""
super().__init__(**kwargs)
self._credentials_file = credentials_file
self.device_model_id = device_model_id
self.credentials = None
self._assistant = None
self.logger.info('Initialized Google Assistant plugin')
def _get_assistant(self) -> AssistantGoogleBackend:
backend = get_backend('assistant.google')
assert backend, 'The assistant.google backend is not configured.'
return backend
@property
def credentials_file(self) -> str:
if self._credentials_file:
return os.path.abspath(os.path.expanduser(self._credentials_file))
f = None
for default_file in self._default_credentials_files:
f = default_file
if os.path.isfile(default_file):
break
assert f, 'No credentials_file provided and no default file found'
return f
@property
def assistant(self):
if not self._assistant:
self.logger.warning('The assistant is not running')
return
return self._assistant
def _process_event(self, event):
from google.assistant.library.event import EventType, AlertType
self.logger.info('Received assistant event: %s', event)
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
self._on_conversation_start()
elif event.type == EventType.ON_CONVERSATION_TURN_FINISHED:
if not event.args.get('with_follow_on_turn'):
self._on_conversation_end()
elif event.type == EventType.ON_CONVERSATION_TURN_TIMEOUT:
self._on_conversation_timeout()
elif event.type == EventType.ON_NO_RESPONSE:
self._on_no_response()
elif (
hasattr(EventType, 'ON_RENDER_RESPONSE')
and event.type == EventType.ON_RENDER_RESPONSE
):
self._on_reponse_rendered(event.args.get('text'))
elif (
hasattr(EventType, 'ON_RESPONDING_STARTED')
and event.type == EventType.ON_RESPONDING_STARTED
and event.args.get('is_error_response') is True
):
self.logger.warning('Assistant response error: %s', json.dumps(event.args))
elif event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
self._on_speech_recognized(event.args.get('text'))
elif event.type == EventType.ON_ALERT_STARTED:
if event.args.get('alert_type') == AlertType.ALARM:
self._on_alarm_start()
elif event.args.get('alert_type') == AlertType.TIMER:
self._on_timer_start()
else:
self._on_alert_start()
elif event.type == EventType.ON_ALERT_FINISHED:
if event.args.get('alert_type') == AlertType.ALARM:
self._on_alarm_end()
elif event.args.get('alert_type') == AlertType.TIMER:
self._on_timer_end()
else:
self._on_alert_end()
elif event.type == EventType.ON_ASSISTANT_ERROR:
if event.args.get('is_fatal'):
raise RuntimeError(f'Fatal assistant error: {json.dumps(event.args)}')
self.logger.warning('Assistant error: %s', json.dumps(event.args))
elif event.type == EventType.ON_MUTED_CHANGED:
self._on_mute_changed(event.args.get('is_muted', False))
@action
def start_conversation(self):
def start_conversation(self, *_, **__):
"""
Programmatically start a conversation with the assistant
"""
assistant = self._get_assistant()
assistant.start_conversation()
if self.assistant:
self.assistant.start_conversation()
@action
def stop_conversation(self):
def stop_conversation(self, *_, **__):
"""
Programmatically stop a running conversation with the assistant
"""
assistant = self._get_assistant()
assistant.stop_conversation()
if self.assistant:
self.assistant.stop_conversation()
@action
def set_mic_mute(self, muted: bool = True):
def mute(self, *_, **__):
"""
Mute the microphone. Alias for :meth:`.set_mic_mute` with
``muted=True``.
"""
self.set_mic_mute(muted=True)
@action
def unmute(self, *_, **__):
"""
Unmute the microphone. Alias for :meth:`.set_mic_mute` with
``muted=False``.
"""
self.set_mic_mute(muted=False)
@action
def set_mic_mute(self, muted: bool):
"""
Programmatically mute/unmute the microphone.
:param muted: Set to True or False.
"""
assistant = self._get_assistant()
assistant.set_mic_mute(muted)
if self.assistant:
self.assistant.set_mic_mute(muted)
if muted:
self._on_mute()
else:
self._on_unmute()
@action
def toggle_mute(self, *_, **__):
"""
Toggle the mic mute state.
"""
self.set_mic_mute(muted=not self._is_muted)
@action
def toggle_mic_mute(self):
"""
Toggle the mic mute state.
Deprecated alias for :meth:`.toggle_mute`.
"""
assistant = self._get_assistant()
is_muted = assistant.is_muted()
self.set_mic_mute(muted=not is_muted)
return self.toggle_mute()
@action
def is_muted(self) -> bool:
"""
:return: True if the microphone is muted, False otherwise.
"""
assistant = self._get_assistant()
return assistant.is_muted()
@action
def send_text_query(self, query: str):
def send_text_query(self, *_, query: str, **__):
"""
Send a text query to the assistant.
This is equivalent to saying something to the assistant.
:param query: Query to be sent.
"""
assistant = self._get_assistant()
assistant.send_text_query(query)
if self.assistant:
self.assistant.send_text_query(query)
def main(self):
import google.oauth2.credentials
from google.assistant.library import Assistant
last_sleep = 0
while not self.should_stop():
try:
with open(self.credentials_file, 'r') as f:
self.credentials = google.oauth2.credentials.Credentials(
token=None, **json.load(f)
)
except Exception as e:
self.logger.error(
'Error while loading Google Assistant credentials: %s', e
)
self.logger.info(
'Please follow the instructions at '
'https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample'
'#generate_credentials to get your own credentials file'
)
self.logger.exception(e)
break
try:
with Assistant(
self.credentials, self.device_model_id
) as self._assistant:
self.publish_entities([self])
for event in self._assistant.start():
last_sleep = 0
if not self.is_detecting():
self.logger.info(
'Assistant event received but detection is currently paused'
)
continue
self._process_event(event)
except Exception as e:
self.logger.exception(e)
sleep_secs = min(60, max(5, last_sleep * 2))
self.logger.warning(
'Restarting the assistant in %d seconds after an unrecoverable error',
sleep_secs,
)
self.wait_stop(sleep_secs)
last_sleep = sleep_secs
continue
def stop(self):
try:
self.stop_conversation()
except RuntimeError:
pass
if self._assistant:
del self._assistant
self._assistant = None
super().stop()
# vim:sw=4:ts=4:et:

236
platypush/plugins/assistant/google/lib/__init__.py
View File

@ -1,236 +0,0 @@
"""Based on Google pushtotalk.py sample."""
import concurrent.futures
import json
import logging
import grpc
from google.assistant.embedded.v1alpha2 import (
embedded_assistant_pb2,
embedded_assistant_pb2_grpc
)
from tenacity import retry, stop_after_attempt, retry_if_exception
try:
from googlesamples.assistant.grpc import (
assistant_helpers,
audio_helpers,
browser_helpers,
device_helpers
)
except (SystemError, ImportError):
import assistant_helpers
import audio_helpers
import browser_helpers
import device_helpers
ASSISTANT_API_ENDPOINT = 'embeddedassistant.googleapis.com'
END_OF_UTTERANCE = embedded_assistant_pb2.AssistResponse.END_OF_UTTERANCE
DIALOG_FOLLOW_ON = embedded_assistant_pb2.DialogStateOut.DIALOG_FOLLOW_ON
CLOSE_MICROPHONE = embedded_assistant_pb2.DialogStateOut.CLOSE_MICROPHONE
PLAYING = embedded_assistant_pb2.ScreenOutConfig.PLAYING
DEFAULT_GRPC_DEADLINE = 60 * 3 + 5
class SampleAssistant(object):
"""Sample Assistant that supports conversations and device actions.
Args:
device_model_id: identifier of the device model.
device_id: identifier of the registered device instance.
conversation_stream(ConversationStream): audio stream
for recording query and playing back assistant answer.
channel: authorized gRPC channel for connection to the
Google Assistant API.
deadline_sec: gRPC deadline in seconds for Google Assistant API call.
device_handler: callback for device actions.
"""
def __init__(self, language_code, device_model_id, device_id,
conversation_stream, display,
channel, deadline_sec, device_handler, play_response=True,
on_conversation_start=None, on_conversation_end=None,
on_speech_recognized=None, on_volume_changed=None,
on_response=None):
self.language_code = language_code
self.device_model_id = device_model_id
self.device_id = device_id
self.conversation_stream = conversation_stream
self.display = display
self.play_response = play_response
# Opaque blob provided in AssistResponse that,
# when provided in a follow-up AssistRequest,
# gives the Assistant a context marker within the current state
# of the multi-Assist()-RPC "conversation".
# This value, along with MicrophoneMode, supports a more natural
# "conversation" with the Assistant.
self.conversation_state = None
# Force reset of first conversation.
self.is_new_conversation = True
# Create Google Assistant API gRPC client.
self.assistant = embedded_assistant_pb2_grpc.EmbeddedAssistantStub(channel)
self.deadline = deadline_sec
self.device_handler = device_handler
self.detected_speech = None
self.on_conversation_start = on_conversation_start
self.on_conversation_end = on_conversation_end
self.on_speech_recognized = on_speech_recognized
self.on_volume_changed = on_volume_changed
self.on_response = on_response
def __enter__(self):
return self
def __exit__(self, etype, e, traceback):
if e:
return False
self.conversation_stream.close()
@staticmethod
def is_grpc_error_unavailable(e):
is_grpc_error = isinstance(e, grpc.RpcError)
if is_grpc_error and (e.code() == grpc.StatusCode.UNAVAILABLE):
logging.error('grpc unavailable error: %s', e)
return True
return False
@retry(reraise=True, stop=stop_after_attempt(3),
retry=retry_if_exception(is_grpc_error_unavailable))
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
if self.on_conversation_start:
self.on_conversation_start()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(), self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if self.detected_speech and self.on_speech_recognized:
self.on_speech_recognized(self.detected_speech)
if resp.speech_results:
self.detected_speech = ' '.join(
r.transcript.strip() for r in resp.speech_results
if len(r.transcript.strip())).strip()
logging.info('Transcript of user request: "%s".', self.detected_speech)
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
if self.play_response:
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
if self.play_response and self.conversation_stream.playing:
self.conversation_stream.write(resp.audio_out.audio_data)
elif self.conversation_stream.playing:
self.conversation_stream.stop_playback()
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if self.on_volume_changed:
self.on_volume_changed(volume_percentage)
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if resp.dialog_state_out.supplemental_display_text and self.on_response:
self.on_response(resp.dialog_state_out.supplemental_display_text)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
if self.on_conversation_end:
self.on_conversation_end(continue_conversation)
return continue_conversation
def gen_assist_requests(self):
"""Yields: AssistRequest messages to send to the API."""
config = embedded_assistant_pb2.AssistConfig(
audio_in_config=embedded_assistant_pb2.AudioInConfig(
encoding='LINEAR16',
sample_rate_hertz=self.conversation_stream.sample_rate,
),
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=self.conversation_stream.sample_rate,
volume_percentage=self.conversation_stream.volume_percentage,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
)
)
if self.display:
config.screen_out_config.screen_mode = PLAYING
# Continue current conversation with later requests.
self.is_new_conversation = False
# The first AssistRequest must contain the AssistConfig
# and no audio data.
yield embedded_assistant_pb2.AssistRequest(config=config)
for data in self.conversation_stream:
# Subsequent requests need audio data, but not config.
yield embedded_assistant_pb2.AssistRequest(audio_in=data)

74
platypush/plugins/assistant/google/manifest.yaml
View File

@ -1,6 +1,74 @@
manifest:
events: {}
install:
pip: []
package: platypush.plugins.assistant.google
type: plugin
events:
- platypush.message.event.assistant.AlarmEndEvent
- platypush.message.event.assistant.AlarmStartedEvent
- platypush.message.event.assistant.ConversationEndEvent
- platypush.message.event.assistant.ConversationStartEvent
- platypush.message.event.assistant.ConversationTimeoutEvent
- platypush.message.event.assistant.MicMutedEvent
- platypush.message.event.assistant.MicUnmutedEvent
- platypush.message.event.assistant.NoResponseEvent
- platypush.message.event.assistant.ResponseEvent
- platypush.message.event.assistant.SpeechRecognizedEvent
- platypush.message.event.assistant.TimerEndEvent
- platypush.message.event.assistant.TimerStartedEvent
install:
apk:
- ffmpeg
- portaudio-dev
- py3-cachetools
- py3-grpcio
- py3-google-auth
- py3-numpy
- py3-pathlib2
- py3-tenacity
- py3-urllib3
apt:
- ffmpeg
- portaudio19-dev
- python3-cachetools
- python3-grpcio
- python3-google-auth
- python3-monotonic
- python3-tenacity
- python3-urllib3
dnf:
- ffmpeg
- portaudio-devel
- python-cachetools
- python-grpcio
- python-google-auth
- python-monotonic
- python-numpy
- python-tenacity
- python-urllib3
pacman:
- ffmpeg
- portaudio
- python-cachetools
- python-grpcio
- python-google-auth
- python-monotonic
- python-numpy
- python-sounddevice
- python-tenacity
- python-urllib3
pip:
- google-assistant-library
- google-assistant-sdk[samples]
- google-auth
- numpy
- sounddevice
after:
# Uninstall old versions of packages that break things on recent versions
# of Python, when the new versions work just fine
- yes | pip uninstall --break-system-packages enum34 click urllib3 requests google-auth
# Upgrade the dependencies (back) to the latest version.
# NOTE: Be careful when running this command on older distros that may
# not ship the latest versions of all the packages! This is a workaround
# caused by the fact that google-assistant-library pulls in some old
# breaking dependencies that need to be surgically removed.
- pip install -U --no-input --break-system-packages click urllib3 requests google-auth

330
platypush/plugins/assistant/google/pushtotalk/__init__.py
View File

@ -1,330 +0,0 @@
import json
import os
from typing import Optional, Dict, Any
from platypush.context import get_bus, get_plugin
from platypush.message.event.assistant import (
ConversationStartEvent,
ConversationEndEvent,
SpeechRecognizedEvent,
VolumeChangedEvent,
ResponseEvent,
)
from platypush.message.event.google import GoogleDeviceOnOffEvent
from platypush.plugins import action
from platypush.plugins.assistant import AssistantPlugin
class AssistantGooglePushtotalkPlugin(AssistantPlugin):
"""
Plugin for the Google Assistant push-to-talk API.
"""
api_endpoint = 'embeddedassistant.googleapis.com'
grpc_deadline = 60 * 3 + 5
device_handler = None
_default_credentials_file = os.path.join(
os.path.expanduser('~'),
'.config',
'google-oauthlib-tool',
'credentials.json',
)
_default_device_config = os.path.join(
os.path.expanduser('~'),
'.config',
'googlesamples-assistant',
'device_config.json',
)
def __init__(
self,
credentials_file=_default_credentials_file,
device_config=_default_device_config,
language='en-US',
play_response=True,
tts_plugin=None,
tts_args=None,
**kwargs
):
"""
:param credentials_file: Path to the Google OAuth credentials file
(default: ~/.config/google-oauthlib-tool/credentials.json).
See
https://developers.google.com/assistant/sdk/guides/library/python/embed/install-sample#generate_credentials
for instructions to get your own credentials file.
:type credentials_file: str
:param device_config: Path to device_config.json. Register your device
(see https://developers.google.com/assistant/sdk/guides/library/python/embed/register-device)
and create a project, then run the pushtotalk.py script from
googlesamples to create your device_config.json
:type device_config: str
:param language: Assistant language (default: en-US)
:type language: str
:param play_response: If True (default) then the plugin will play the assistant response upon processed
response. Otherwise nothing will be played - but you may want to handle the ``ResponseEvent`` manually.
:type play_response: bool
:param tts_plugin: Optional text-to-speech plugin to be used to process response text.
:type tts_plugin: str
:param tts_args: Optional arguments for the TTS plugin ``say`` method.
:type tts_args: dict
"""
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
super().__init__(**kwargs)
self.audio_sample_rate = audio_helpers.DEFAULT_AUDIO_SAMPLE_RATE
self.audio_sample_width = audio_helpers.DEFAULT_AUDIO_SAMPLE_WIDTH
self.audio_iter_size = audio_helpers.DEFAULT_AUDIO_ITER_SIZE
self.audio_block_size = audio_helpers.DEFAULT_AUDIO_DEVICE_BLOCK_SIZE
self.audio_flush_size = audio_helpers.DEFAULT_AUDIO_DEVICE_FLUSH_SIZE
self.language = language
self.credentials_file = credentials_file
self.device_config = device_config
self.play_response = play_response
self.tts_plugin = tts_plugin
self.tts_args = tts_args or {}
self.assistant = None
self.interactions = []
with open(self.device_config) as f:
device = json.load(f)
self.device_id = device['id']
self.device_model_id = device['model_id']
# Load OAuth 2.0 credentials.
try:
from google.oauth2.credentials import Credentials
from google.auth.transport.requests import Request
with open(self.credentials_file, 'r') as f:
self.credentials = Credentials(token=None, **json.load(f))
self.http_request = Request()
self.credentials.refresh(self.http_request)
except Exception as ex:
self.logger.error('Error loading credentials: %s', str(ex))
self.logger.error(
'Run google-oauthlib-tool to initialize ' 'new OAuth 2.0 credentials.'
)
raise
self.grpc_channel = None
self.conversation_stream = None
def _init_assistant(self):
import googlesamples.assistant.grpc.audio_helpers as audio_helpers
from google.auth.transport.grpc import secure_authorized_channel
self.interactions = []
# Create an authorized gRPC channel.
self.grpc_channel = secure_authorized_channel(
self.credentials, self.http_request, self.api_endpoint
)
self.logger.info('Connecting to {}'.format(self.api_endpoint))
# Configure audio source and sink.
audio_device = None
audio_source = audio_device = audio_device or audio_helpers.SoundDeviceStream(
sample_rate=self.audio_sample_rate,
sample_width=self.audio_sample_width,
block_size=self.audio_block_size,
flush_size=self.audio_flush_size,
)
audio_sink = audio_device or audio_helpers.SoundDeviceStream(
sample_rate=self.audio_sample_rate,
sample_width=self.audio_sample_width,
block_size=self.audio_block_size,
flush_size=self.audio_flush_size,
)
# Create conversation stream with the given audio source and sink.
self.conversation_stream = audio_helpers.ConversationStream(
source=audio_source,
sink=audio_sink,
iter_size=self.audio_iter_size,
sample_width=self.audio_sample_width,
)
self._install_device_handlers()
def on_conversation_start(self):
"""Conversation start handler"""
def handler():
get_bus().post(ConversationStartEvent(assistant=self))
return handler
def on_conversation_end(self):
"""Conversation end handler"""
def handler(with_follow_on_turn):
get_bus().post(
ConversationEndEvent(
assistant=self, with_follow_on_turn=with_follow_on_turn
)
)
return handler
def on_speech_recognized(self):
"""Speech recognized handler"""
def handler(phrase):
get_bus().post(SpeechRecognizedEvent(assistant=self, phrase=phrase))
self.interactions.append({'request': phrase})
return handler
def on_volume_changed(self):
"""Volume changed event"""
def handler(volume):
get_bus().post(VolumeChangedEvent(assistant=self, volume=volume))
return handler
def on_response(self):
"""Response handler"""
def handler(response):
get_bus().post(ResponseEvent(assistant=self, response_text=response))
if not self.interactions:
self.interactions.append({'response': response})
else:
self.interactions[-1]['response'] = response
if self.tts_plugin:
tts = get_plugin(self.tts_plugin)
tts.say(response, **self.tts_args)
return handler
@action
def start_conversation(
self,
*_,
language: Optional[str] = None,
tts_plugin: Optional[str] = None,
tts_args: Optional[Dict[str, Any]] = None,
**__
):
"""
Start a conversation
:param language: Language code override (default: default configured language).
:param tts_plugin: Optional text-to-speech plugin to be used for rendering text.
:param tts_args: Optional arguments for the TTS plugin say method.
:returns: A list of the interactions that happen within the conversation.
.. code-block:: json
[
{
"request": "request 1",
"response": "response 1"
},
{
"request": "request 2",
"response": "response 2"
}
]
"""
from platypush.plugins.assistant.google.lib import SampleAssistant
self.tts_plugin = tts_plugin
self.tts_args = tts_args
language = language or self.language
play_response = False if self.tts_plugin else self.play_response
self._init_assistant()
self.on_conversation_start()
with SampleAssistant(
language_code=language,
device_model_id=self.device_model_id,
device_id=self.device_id,
conversation_stream=self.conversation_stream,
display=None,
channel=self.grpc_channel,
deadline_sec=self.grpc_deadline,
play_response=play_response,
device_handler=self.device_handler,
on_conversation_start=self.on_conversation_start(),
on_conversation_end=self.on_conversation_end(),
on_volume_changed=self.on_volume_changed(),
on_response=self.on_response(),
on_speech_recognized=self.on_speech_recognized(),
) as self.assistant:
continue_conversation = True
while continue_conversation:
try:
continue_conversation = self.assistant.assist()
except Exception as e:
self.logger.warning(
'Unhandled assistant exception: {}'.format(str(e))
)
self.logger.exception(e)
self._init_assistant()
return self.interactions
@action
def stop_conversation(self):
if self.assistant:
self.assistant.play_response = False
if self.conversation_stream:
self.conversation_stream.stop_playback()
self.conversation_stream.stop_recording()
get_bus().post(ConversationEndEvent(assistant=self))
@action
def set_mic_mute(self, muted: bool = True):
"""
Programmatically mute/unmute the microphone.
:param muted: Set to True or False.
"""
if not self.conversation_stream:
self.logger.warning('The assistant is not running')
return
if muted:
self.conversation_stream.stop_recording()
else:
self.conversation_stream.start_recording()
def _install_device_handlers(self):
import googlesamples.assistant.grpc.device_helpers as device_helpers
self.device_handler = device_helpers.DeviceRequestHandler(self.device_id)
@self.device_handler.command('action.devices.commands.OnOff')
def handler(on): # type: ignore
get_bus().post(
GoogleDeviceOnOffEvent(
device_id=self.device_id,
device_model_id=self.device_model_id,
on=on,
)
)
# vim:sw=4:ts=4:et:

27
platypush/plugins/assistant/google/pushtotalk/manifest.yaml
View File

@ -1,27 +0,0 @@
manifest:
events:
platypush.message.event.assistant.ConversationEndEvent: when a new conversation
ends
platypush.message.event.assistant.ConversationStartEvent: when a new conversation
starts
platypush.message.event.assistant.SpeechRecognizedEvent: when a new voice command
is recognized
install:
apk:
- py3-tenacity
- py3-google-auth
apt:
- python3-tenacity
- python3-google-auth
dnf:
- python-tenacity
- python-google-auth
pacman:
- python-tenacity
- python-google-auth
pip:
- tenacity
- google-assistant-sdk
- google-auth
package: platypush.plugins.assistant.google.pushtotalk
type: plugin

5
setup.py
View File

@ -139,9 +139,6 @@ setup(
],
# Support for Last.FM scrobbler plugin
'lastfm': ['pylast'],
# Support for custom hotword detection
'hotword': ['snowboy'],
'snowboy': ['snowboy'],
# Support for real-time MIDI events
'midi': ['rtmidi'],
# Support for RaspberryPi GPIO
@ -193,8 +190,6 @@ setup(
'flic': [
'flic @ https://github.com/50ButtonsEach/fliclib-linux-hci/tarball/master'
],
# Support for Alexa/Echo plugin
'alexa': ['avs @ https://github.com/BlackLight/avs/tarball/master'],
# Support for Bluetooth devices
'bluetooth': [
'bleak',