Got microwakework working on the M5CoreS3 SE. Speaker output is garbage, microphone works pretty good. Might be helpful to anyone that already has one but wouldn’t recommend it at this point. Especially with the Seeed respeaker lite announcement and Nabu announcing that there voice assistant will have a XIOS chip in it like the respeaker lite. I had timers and text on the screen working but it kept locking up or stopped responding so I removed it. I also didn’t install ESP-ADF but the tensorflow/lite part still runs. Guess it’s not required? Maybe someone could clear that up for me. I saw the respeaker lite’s yaml didn’t have it either but I assumed that is because it’s using the XIOS chip for obvious reasons.
substitutions:
name: m5cores3
friendly_name: M5CoreS3
loading_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/loading_320_240.png
idle_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/idle_320_240.png
listening_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/listening_320_240.png
thinking_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/thinking_320_240.png
replying_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/replying_320_240.png
error_illustration_file: https://github.com/esphome/firmware/raw/main/voice-assistant/casita/error_320_240.png
loading_illustration_background_color: '000000'
idle_illustration_background_color: '000000'
listening_illustration_background_color: 'FFFFFF'
thinking_illustration_background_color: 'FFFFFF'
replying_illustration_background_color: 'FFFFFF'
error_illustration_background_color: '000000'
voice_assist_idle_phase_id: '1'
voice_assist_listening_phase_id: '2'
voice_assist_thinking_phase_id: '3'
voice_assist_replying_phase_id: '4'
voice_assist_not_ready_phase_id: '10'
voice_assist_error_phase_id: '11'
voice_assist_muted_phase_id: '12'
micro_wake_word_model: hey_jarvis
esphome:
name: m5cores3
friendly_name: M5CoreS3
platformio_options:
board_build.flash_mode: dio
#board_upload.maximum_size: 16777216
#board_build.f_cpu : 240000000L
libraries:
- m5stack/M5GFX@^0.1.11
- m5stack/M5Unified@^0.1.11
on_boot:
priority: 600
then:
- delay: 30s
- if:
condition:
- lambda: return id(init_in_progress);
then:
- lambda: id(init_in_progress) = false;
- script.execute: draw_display # Initialize the display
- delay: 1s # Wait a moment to ensure the display is ready
- switch.turn_on: switch_lcd_backlight # Turn on the backlight switch to set the initial state
esp32:
board: esp32-s3-devkitc-1
flash_size: 16MB
framework:
type: esp-idf
sdkconfig_options:
# need to set a s3 compatible board for the adf-sdk to compile
# board specific code is not used though
CONFIG_ESP32_S3_BOX_BOARD: "y"
CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240: "y"
CONFIG_ESP32S3_DATA_CACHE_64KB: "y"
CONFIG_ESP32S3_DATA_CACHE_LINE_64B: "y"
#CONFIG_LOG_DEFAULT_LEVEL_DEBUG: "y"
#CONFIG_LOG_DEFAULT_LEVEL: "4"
psram:
mode: quad
speed: 80MHz
external_components:
- source:
type: git
url: https://github.com/m5stack/M5CoreS3-Esphome
components: [ board_m5cores3, m5cores3_audio, m5cores3_display ]
refresh: 0s
#- source: github://pr#5230
# components: esp_adf
# refresh: 0s
#- source: github://jesserockz/esphome-components
# components: [file]
# refresh: 0s
text_sensor:
- platform: wifi_info
ip_address:
name: "${friendly_name} IP Address"
time:
- platform: homeassistant
id: homeassistant_time
# Enable logging
logger:
# Enable Home Assistant API
api:
encryption:
key: "APIKEY"
on_client_connected:
- script.execute: draw_display
on_client_disconnected:
- script.execute: draw_display
ota:
- platform: esphome
password: "OTAPW"
wifi:
ssid: !secret wifi_ssid
password: !secret wifi_password
# Enable fallback hotspot (captive portal) in case wifi connection fails
ap:
ssid: "M5Stack-Cores3 Fallback Hotspot"
password: "HOTSTPOTPW"
on_connect:
- script.execute: draw_display
# - delay: 5s # Gives time for improv results to be transmitted
on_disconnect:
- script.execute: draw_display
captive_portal:
#
#
# Globals
#
globals:
- id: init_in_progress
type: bool
restore_value: no
initial_value: 'true'
- id: voice_assistant_phase
type: int
restore_value: no
initial_value: ${voice_assist_not_ready_phase_id}
#
# Temp values for slider lcd brightnes
#
number:
- platform: template
name: "Display Brightness"
id: display_brightness
entity_category: config
optimistic: true
min_value: 0
max_value: 200
step: 1
unit_of_measurement: "%"
initial_value: 127
icon: "mdi:brightness-6"
set_action:
- lambda: |-
int brightness = (int)(x * 2.55);
M5.Display.setBrightness(brightness);
#
# Display
#
script:
- id: draw_display
then:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- if:
condition:
wifi.connected:
then:
- if:
condition:
api.connected:
then:
- lambda: |
switch(id(voice_assistant_phase)) {
case ${voice_assist_listening_phase_id}:
id(m5cores3_lcd).show_page(listening_page);
id(m5cores3_lcd).update();
break;
case ${voice_assist_thinking_phase_id}:
id(m5cores3_lcd).show_page(thinking_page);
id(m5cores3_lcd).update();
break;
case ${voice_assist_replying_phase_id}:
id(m5cores3_lcd).show_page(replying_page);
id(m5cores3_lcd).update();
break;
case ${voice_assist_error_phase_id}:
id(m5cores3_lcd).show_page(error_page);
id(m5cores3_lcd).update();
break;
case ${voice_assist_muted_phase_id}:
id(m5cores3_lcd).show_page(muted_page);
id(m5cores3_lcd).update();
break;
case ${voice_assist_not_ready_phase_id}:
id(m5cores3_lcd).show_page(no_ha_page);
id(m5cores3_lcd).update();
break;
default:
id(m5cores3_lcd).show_page(idle_page);
id(m5cores3_lcd).update();
}
else:
- display.page.show: no_ha_page
- component.update: m5cores3_lcd
else:
- display.page.show: no_wifi_page
- component.update: m5cores3_lcd
else:
- display.page.show: initializing_page
- component.update: m5cores3_lcd
image:
- file: ${error_illustration_file}
id: casita_error
resize: 320x240
type: RGB24
use_transparency: true
- file: ${idle_illustration_file}
id: casita_idle
resize: 320x240
type: RGB24
use_transparency: true
- file: ${listening_illustration_file}
id: casita_listening
resize: 320x240
type: RGB24
use_transparency: true
- file: ${thinking_illustration_file}
id: casita_thinking
resize: 320x240
type: RGB24
use_transparency: true
- file: ${replying_illustration_file}
id: casita_replying
resize: 320x240
type: RGB24
use_transparency: true
- file: ${loading_illustration_file}
id: casita_initializing
resize: 320x240
type: RGB24
use_transparency: true
- file: https://github.com/esphome/firmware/raw/main/voice-assistant/error_box_illustrations/error-no-wifi.png
id: error_no_wifi
resize: 320x240
type: RGB24
use_transparency: true
- file: https://github.com/esphome/firmware/raw/main/voice-assistant/error_box_illustrations/error-no-ha.png
id: error_no_ha
resize: 320x240
type: RGB24
use_transparency: true
color:
- id: idle_color
hex: ${idle_illustration_background_color}
- id: listening_color
hex: ${listening_illustration_background_color}
- id: thinking_color
hex: ${thinking_illustration_background_color}
- id: replying_color
hex: ${replying_illustration_background_color}
- id: loading_color
hex: ${loading_illustration_background_color}
- id: error_color
hex: ${error_illustration_background_color}
display:
- platform: m5cores3_display
model: ILI9342
dc_pin: 35
update_interval: never
id: m5cores3_lcd
pages:
- id: idle_page
lambda: |-
it.fill(id(idle_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_idle), ImageAlign::CENTER);
- id: listening_page
lambda: |-
it.fill(id(listening_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_listening), ImageAlign::CENTER);
- id: thinking_page
lambda: |-
it.fill(id(thinking_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_thinking), ImageAlign::CENTER);
- id: replying_page
lambda: |-
it.fill(id(replying_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_replying), ImageAlign::CENTER);
- id: error_page
lambda: |-
it.fill(id(error_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_error), ImageAlign::CENTER);
- id: no_ha_page
lambda: |-
it.image((it.get_width() / 2), (it.get_height() / 2), id(error_no_ha), ImageAlign::CENTER);
- id: no_wifi_page
lambda: |-
it.image((it.get_width() / 2), (it.get_height() / 2), id(error_no_wifi), ImageAlign::CENTER);
- id: initializing_page
lambda: |-
it.fill(id(loading_color));
it.image((it.get_width() / 2), (it.get_height() / 2), id(casita_initializing), ImageAlign::CENTER);
- id: muted_page
lambda: |-
it.fill(Color::BLACK);
#
# Audio
#
#esp_adf:
board_m5cores3:
m5cores3_audio:
id: m5cores3_audio_1
microphone:
- platform: m5cores3_audio
m5cores3_audio_id: m5cores3_audio_1
id: m5cores3_mic
adc_type: external
i2s_din_pin: 14
pdm: false
speaker:
- platform: m5cores3_audio
m5cores3_audio_id: m5cores3_audio_1
id: m5cores3_spk
dac_type: external
i2s_dout_pin: 13
mode: mono
#
# VA
#
micro_wake_word:
models: ${micro_wake_word_model}
on_wake_word_detected:
- voice_assistant.start:
wake_word: !lambda return wake_word;
voice_assistant:
id: va
microphone: m5cores3_mic
speaker: m5cores3_spk
use_wake_word: true
noise_suppression_level: 2
auto_gain: 31dBFS
volume_multiplier: 2.0
#vad_threshold: 3
on_listening:
- lambda: id(voice_assistant_phase) = ${voice_assist_listening_phase_id};
- script.execute: draw_display
on_stt_vad_end:
- lambda: id(voice_assistant_phase) = ${voice_assist_thinking_phase_id};
- script.execute: draw_display
on_tts_stream_start:
- lambda: id(voice_assistant_phase) = ${voice_assist_replying_phase_id};
- script.execute: draw_display
on_tts_stream_end:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
- script.execute: draw_display
on_error:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- lambda: id(voice_assistant_phase) = ${voice_assist_error_phase_id};
- script.execute: draw_display
- delay: 1s
- if:
condition:
switch.is_off: mute
then:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
else:
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- script.execute: draw_display
on_client_connected:
- if:
condition:
switch.is_off: mute
then:
- voice_assistant.start_continuous:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
else:
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- lambda: id(init_in_progress) = false;
- script.execute: draw_display
on_client_disconnected:
- lambda: id(voice_assistant_phase) = ${voice_assist_not_ready_phase_id};
- script.execute: draw_display
select:
- platform: template
entity_category: config
name: Wake word engine location
id: wake_word_engine_location
optimistic: true
restore_value: true
options:
- In Home Assistant
- On device
initial_option: On device
on_value:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- wait_until:
lambda: return id(voice_assistant_phase) == ${voice_assist_muted_phase_id} || id(voice_assistant_phase) == ${voice_assist_idle_phase_id};
- if:
condition:
lambda: return x == "In Home Assistant";
then:
- micro_wake_word.stop
- delay: 30ms
- if:
condition:
switch.is_off: mute
then:
- lambda: id(va).set_use_wake_word(true);
- voice_assistant.start_continuous:
- if:
condition:
lambda: return x == "On device";
then:
- lambda: id(va).set_use_wake_word(false);
- voice_assistant.stop
- delay: 30ms
- if:
condition:
switch.is_off: mute
then:
- micro_wake_word.start
switch:
- platform: template
name: "LCD Backlight"
id: switch_lcd_backlight
restore_mode: "ALWAYS_ON"
turn_on_action:
- lambda: |-
int brightness = id(display_brightness).state;
M5.Display.setBrightness(brightness);
id(switch_lcd_backlight).publish_state(true);
turn_off_action:
- lambda: |-
M5.Display.setBrightness(0);
id(switch_lcd_backlight).publish_state(false);
- platform: template
name: Mute
id: mute
optimistic: true
restore_mode: RESTORE_DEFAULT_OFF
entity_category: config
on_turn_off:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- lambda: id(va).set_use_wake_word(true);
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
- if:
condition:
not:
- voice_assistant.is_running
then:
- voice_assistant.start_continuous
- script.execute: draw_display
on_turn_on:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- voice_assistant.stop
- lambda: id(va).set_use_wake_word(false);
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- script.execute: draw_display
