|
|
@@ -0,0 +1,855 @@
|
|
|
+#include "application.h"
|
|
|
+#include "board.h"
|
|
|
+#include "display.h"
|
|
|
+#include "system_info.h"
|
|
|
+#include "ml307_ssl_transport.h"
|
|
|
+#include "audio_codec.h"
|
|
|
+#include "mqtt_protocol.h"
|
|
|
+#include "websocket_protocol.h"
|
|
|
+#include "font_awesome_symbols.h"
|
|
|
+#include "iot/thing_manager.h"
|
|
|
+#include "assets/lang_config.h"
|
|
|
+
|
|
|
+#include <cstring>
|
|
|
+#include <esp_log.h>
|
|
|
+#include <cJSON.h>
|
|
|
+#include <driver/gpio.h>
|
|
|
+#include <arpa/inet.h>
|
|
|
+#include <esp_app_desc.h>
|
|
|
+
|
|
|
+#define TAG "Application"
|
|
|
+
|
|
|
+
|
|
|
+static const char* const STATE_STRINGS[] = {
|
|
|
+ "unknown",
|
|
|
+ "starting",
|
|
|
+ "configuring",
|
|
|
+ "idle",
|
|
|
+ "connecting",
|
|
|
+ "listening",
|
|
|
+ "speaking",
|
|
|
+ "upgrading",
|
|
|
+ "activating",
|
|
|
+ "fatal_error",
|
|
|
+ "invalid_state"
|
|
|
+};
|
|
|
+
|
|
|
+Application::Application() {
|
|
|
+ event_group_ = xEventGroupCreate();
|
|
|
+ background_task_ = new BackgroundTask(4096 * 8);
|
|
|
+
|
|
|
+ esp_timer_create_args_t clock_timer_args = {
|
|
|
+ .callback = [](void* arg) {
|
|
|
+ Application* app = (Application*)arg;
|
|
|
+ app->OnClockTimer();
|
|
|
+ },
|
|
|
+ .arg = this,
|
|
|
+ .dispatch_method = ESP_TIMER_TASK,
|
|
|
+ .name = "clock_timer",
|
|
|
+ .skip_unhandled_events = true
|
|
|
+ };
|
|
|
+ esp_timer_create(&clock_timer_args, &clock_timer_handle_);
|
|
|
+}
|
|
|
+
|
|
|
+Application::~Application() {
|
|
|
+ if (clock_timer_handle_ != nullptr) {
|
|
|
+ esp_timer_stop(clock_timer_handle_);
|
|
|
+ esp_timer_delete(clock_timer_handle_);
|
|
|
+ }
|
|
|
+ if (background_task_ != nullptr) {
|
|
|
+ delete background_task_;
|
|
|
+ }
|
|
|
+ vEventGroupDelete(event_group_);
|
|
|
+}
|
|
|
+
|
|
|
+void Application::CheckNewVersion() {
|
|
|
+ auto& board = Board::GetInstance();
|
|
|
+ auto display = board.GetDisplay();
|
|
|
+ // Check if there is a new firmware version available
|
|
|
+ ota_.SetPostData(board.GetJson());
|
|
|
+
|
|
|
+ const int MAX_RETRY = 10;
|
|
|
+ int retry_count = 0;
|
|
|
+
|
|
|
+ while (true) {
|
|
|
+ if (!ota_.CheckVersion()) {
|
|
|
+ retry_count++;
|
|
|
+ if (retry_count >= MAX_RETRY) {
|
|
|
+ ESP_LOGE(TAG, "Too many retries, exit version check");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", 60, retry_count, MAX_RETRY);
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(60000));
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ retry_count = 0;
|
|
|
+
|
|
|
+ if (ota_.HasNewVersion()) {
|
|
|
+ Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "happy", Lang::Sounds::P3_UPGRADE);
|
|
|
+ // Wait for the chat state to be idle
|
|
|
+ do {
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(3000));
|
|
|
+ } while (GetDeviceState() != kDeviceStateIdle);
|
|
|
+
|
|
|
+ // Use main task to do the upgrade, not cancelable
|
|
|
+ Schedule([this, display]() {
|
|
|
+ SetDeviceState(kDeviceStateUpgrading);
|
|
|
+
|
|
|
+ display->SetIcon(FONT_AWESOME_DOWNLOAD);
|
|
|
+ std::string message = std::string(Lang::Strings::NEW_VERSION) + ota_.GetFirmwareVersion();
|
|
|
+ display->SetChatMessage("system", message.c_str());
|
|
|
+
|
|
|
+ auto& board = Board::GetInstance();
|
|
|
+ board.SetPowerSaveMode(false);
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ wake_word_detect_.StopDetection();
|
|
|
+#endif
|
|
|
+ // 预先关闭音频输出，避免升级过程有音频操作
|
|
|
+ auto codec = board.GetAudioCodec();
|
|
|
+ codec->EnableInput(false);
|
|
|
+ codec->EnableOutput(false);
|
|
|
+ {
|
|
|
+ std::lock_guard<std::mutex> lock(mutex_);
|
|
|
+ audio_decode_queue_.clear();
|
|
|
+ }
|
|
|
+ background_task_->WaitForCompletion();
|
|
|
+ delete background_task_;
|
|
|
+ background_task_ = nullptr;
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(1000));
|
|
|
+
|
|
|
+ ota_.StartUpgrade([display](int progress, size_t speed) {
|
|
|
+ char buffer[64];
|
|
|
+ snprintf(buffer, sizeof(buffer), "%d%% %zuKB/s", progress, speed / 1024);
|
|
|
+ display->SetChatMessage("system", buffer);
|
|
|
+ });
|
|
|
+
|
|
|
+ // If upgrade success, the device will reboot and never reach here
|
|
|
+ display->SetStatus(Lang::Strings::UPGRADE_FAILED);
|
|
|
+ ESP_LOGI(TAG, "Firmware upgrade failed...");
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(3000));
|
|
|
+ Reboot();
|
|
|
+ });
|
|
|
+
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // No new version, mark the current version as valid
|
|
|
+ ota_.MarkCurrentVersionValid();
|
|
|
+ std::string message = std::string(Lang::Strings::VERSION) + ota_.GetCurrentVersion();
|
|
|
+ display->ShowNotification(message.c_str());
|
|
|
+
|
|
|
+ if (ota_.HasActivationCode()) {
|
|
|
+ // Activation code is valid
|
|
|
+ SetDeviceState(kDeviceStateActivating);
|
|
|
+ ShowActivationCode();
|
|
|
+
|
|
|
+ // Check again in 60 seconds or until the device is idle
|
|
|
+ for (int i = 0; i < 60; ++i) {
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(1000));
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ display->SetChatMessage("system", "");
|
|
|
+ PlaySound(Lang::Sounds::P3_SUCCESS);
|
|
|
+ // Exit the loop if upgrade or idle
|
|
|
+ break;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::ShowActivationCode() {
|
|
|
+ auto& message = ota_.GetActivationMessage();
|
|
|
+ auto& code = ota_.GetActivationCode();
|
|
|
+
|
|
|
+ struct digit_sound {
|
|
|
+ char digit;
|
|
|
+ const std::string_view& sound;
|
|
|
+ };
|
|
|
+ static const std::array<digit_sound, 10> digit_sounds{{
|
|
|
+ digit_sound{'0', Lang::Sounds::P3_0},
|
|
|
+ digit_sound{'1', Lang::Sounds::P3_1},
|
|
|
+ digit_sound{'2', Lang::Sounds::P3_2},
|
|
|
+ digit_sound{'3', Lang::Sounds::P3_3},
|
|
|
+ digit_sound{'4', Lang::Sounds::P3_4},
|
|
|
+ digit_sound{'5', Lang::Sounds::P3_5},
|
|
|
+ digit_sound{'6', Lang::Sounds::P3_6},
|
|
|
+ digit_sound{'7', Lang::Sounds::P3_7},
|
|
|
+ digit_sound{'8', Lang::Sounds::P3_8},
|
|
|
+ digit_sound{'9', Lang::Sounds::P3_9}
|
|
|
+ }};
|
|
|
+
|
|
|
+ // This sentence uses 9KB of SRAM, so we need to wait for it to finish
|
|
|
+ Alert(Lang::Strings::ACTIVATION, message.c_str(), "happy", Lang::Sounds::P3_ACTIVATION);
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(1000));
|
|
|
+ background_task_->WaitForCompletion();
|
|
|
+
|
|
|
+ for (const auto& digit : code) {
|
|
|
+ auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
|
|
|
+ [digit](const digit_sound& ds) { return ds.digit == digit; });
|
|
|
+ if (it != digit_sounds.end()) {
|
|
|
+ PlaySound(it->sound);
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
|
|
|
+ ESP_LOGW(TAG, "Alert %s: %s [%s]", status, message, emotion);
|
|
|
+ auto display = Board::GetInstance().GetDisplay();
|
|
|
+ display->SetStatus(status);
|
|
|
+ display->SetEmotion(emotion);
|
|
|
+ display->SetChatMessage("system", message);
|
|
|
+ if (!sound.empty()) {
|
|
|
+ PlaySound(sound);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::DismissAlert() {
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ auto display = Board::GetInstance().GetDisplay();
|
|
|
+ display->SetStatus(Lang::Strings::STANDBY);
|
|
|
+ display->SetEmotion("neutral");
|
|
|
+ display->SetChatMessage("system", "");
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::PlaySound(const std::string_view& sound) {
|
|
|
+ auto codec = Board::GetInstance().GetAudioCodec();
|
|
|
+ codec->EnableOutput(true);
|
|
|
+ SetDecodeSampleRate(16000);
|
|
|
+ const char* data = sound.data();
|
|
|
+ size_t size = sound.size();
|
|
|
+ for (const char* p = data; p < data + size; ) {
|
|
|
+ auto p3 = (BinaryProtocol3*)p;
|
|
|
+ p += sizeof(BinaryProtocol3);
|
|
|
+
|
|
|
+ auto payload_size = ntohs(p3->payload_size);
|
|
|
+ std::vector<uint8_t> opus;
|
|
|
+ opus.resize(payload_size);
|
|
|
+ memcpy(opus.data(), p3->payload, payload_size);
|
|
|
+ p += payload_size;
|
|
|
+
|
|
|
+ std::lock_guard<std::mutex> lock(mutex_);
|
|
|
+ audio_decode_queue_.emplace_back(std::move(opus));
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::ToggleChatState() {
|
|
|
+ if (device_state_ == kDeviceStateActivating) {
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!protocol_) {
|
|
|
+ ESP_LOGE(TAG, "Protocol not initialized");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ Schedule([this]() {
|
|
|
+ SetDeviceState(kDeviceStateConnecting);
|
|
|
+ if (!protocol_->OpenAudioChannel()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ keep_listening_ = true;
|
|
|
+ protocol_->SendStartListening(kListeningModeAutoStop);
|
|
|
+ SetDeviceState(kDeviceStateListening);
|
|
|
+ });
|
|
|
+ } else if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ Schedule([this]() {
|
|
|
+ AbortSpeaking(kAbortReasonNone);
|
|
|
+ });
|
|
|
+ } else if (device_state_ == kDeviceStateListening) {
|
|
|
+ Schedule([this]() {
|
|
|
+ protocol_->CloseAudioChannel();
|
|
|
+ });
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::StartListening() {
|
|
|
+ if (device_state_ == kDeviceStateActivating) {
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!protocol_) {
|
|
|
+ ESP_LOGE(TAG, "Protocol not initialized");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ keep_listening_ = false;
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ Schedule([this]() {
|
|
|
+ if (!protocol_->IsAudioChannelOpened()) {
|
|
|
+ SetDeviceState(kDeviceStateConnecting);
|
|
|
+ if (!protocol_->OpenAudioChannel()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ protocol_->SendStartListening(kListeningModeManualStop);
|
|
|
+ SetDeviceState(kDeviceStateListening);
|
|
|
+ });
|
|
|
+ } else if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ Schedule([this]() {
|
|
|
+ AbortSpeaking(kAbortReasonNone);
|
|
|
+ protocol_->SendStartListening(kListeningModeManualStop);
|
|
|
+ SetDeviceState(kDeviceStateListening);
|
|
|
+ });
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::StopListening() {
|
|
|
+ Schedule([this]() {
|
|
|
+ if (device_state_ == kDeviceStateListening) {
|
|
|
+ protocol_->SendStopListening();
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ }
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+void Application::Start() {
|
|
|
+ auto& board = Board::GetInstance();
|
|
|
+ SetDeviceState(kDeviceStateStarting);
|
|
|
+
|
|
|
+ /* Setup the display */
|
|
|
+ auto display = board.GetDisplay();
|
|
|
+
|
|
|
+ /* Setup the audio codec */
|
|
|
+ auto codec = board.GetAudioCodec();
|
|
|
+ opus_decode_sample_rate_ = codec->output_sample_rate();
|
|
|
+ opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
|
|
|
+ opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
|
|
+ // For ML307 boards, we use complexity 5 to save bandwidth
|
|
|
+ // For other boards, we use complexity 3 to save CPU
|
|
|
+ if (board.GetBoardType() == "ml307") {
|
|
|
+ ESP_LOGI(TAG, "ML307 board detected, setting opus encoder complexity to 5");
|
|
|
+ opus_encoder_->SetComplexity(5);
|
|
|
+ } else {
|
|
|
+ ESP_LOGI(TAG, "WiFi board detected, setting opus encoder complexity to 3");
|
|
|
+ opus_encoder_->SetComplexity(3);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (codec->input_sample_rate() != 16000) {
|
|
|
+ input_resampler_.Configure(codec->input_sample_rate(), 16000);
|
|
|
+ reference_resampler_.Configure(codec->input_sample_rate(), 16000);
|
|
|
+ }
|
|
|
+ codec->OnInputReady([this, codec]() {
|
|
|
+ BaseType_t higher_priority_task_woken = pdFALSE;
|
|
|
+ xEventGroupSetBitsFromISR(event_group_, AUDIO_INPUT_READY_EVENT, &higher_priority_task_woken);
|
|
|
+ return higher_priority_task_woken == pdTRUE;
|
|
|
+ });
|
|
|
+ codec->OnOutputReady([this]() {
|
|
|
+ BaseType_t higher_priority_task_woken = pdFALSE;
|
|
|
+ xEventGroupSetBitsFromISR(event_group_, AUDIO_OUTPUT_READY_EVENT, &higher_priority_task_woken);
|
|
|
+ return higher_priority_task_woken == pdTRUE;
|
|
|
+ });
|
|
|
+ codec->Start();
|
|
|
+
|
|
|
+ /* Start the main loop */
|
|
|
+ xTaskCreate([](void* arg) {
|
|
|
+ Application* app = (Application*)arg;
|
|
|
+ app->MainLoop();
|
|
|
+ vTaskDelete(NULL);
|
|
|
+ }, "main_loop", 4096 * 2, this, 4, nullptr);
|
|
|
+
|
|
|
+ /* Wait for the network to be ready */
|
|
|
+ board.StartNetwork();
|
|
|
+
|
|
|
+ // Initialize the protocol
|
|
|
+ display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
|
|
|
+#ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
|
|
|
+ protocol_ = std::make_unique<WebsocketProtocol>();
|
|
|
+#else
|
|
|
+ protocol_ = std::make_unique<MqttProtocol>();
|
|
|
+#endif
|
|
|
+ protocol_->OnNetworkError([this](const std::string& message) {
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ Alert(Lang::Strings::ERROR, message.c_str(), "sad", Lang::Sounds::P3_EXCLAMATION);
|
|
|
+ });
|
|
|
+ protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
|
|
|
+ std::lock_guard<std::mutex> lock(mutex_);
|
|
|
+ if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ audio_decode_queue_.emplace_back(std::move(data));
|
|
|
+ }
|
|
|
+ });
|
|
|
+ protocol_->OnAudioChannelOpened([this, codec, &board]() {
|
|
|
+ board.SetPowerSaveMode(false);
|
|
|
+ if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
|
|
|
+ ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion",
|
|
|
+ protocol_->server_sample_rate(), codec->output_sample_rate());
|
|
|
+ }
|
|
|
+ SetDecodeSampleRate(protocol_->server_sample_rate());
|
|
|
+ auto& thing_manager = iot::ThingManager::GetInstance();
|
|
|
+ protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson());
|
|
|
+ std::string states;
|
|
|
+ if (thing_manager.GetStatesJson(states, false)) {
|
|
|
+ protocol_->SendIotStates(states);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ protocol_->OnAudioChannelClosed([this, &board]() {
|
|
|
+ board.SetPowerSaveMode(true);
|
|
|
+ Schedule([this]() {
|
|
|
+ auto display = Board::GetInstance().GetDisplay();
|
|
|
+ display->SetChatMessage("system", "");
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ });
|
|
|
+ });
|
|
|
+ protocol_->OnIncomingJson([this, display](const cJSON* root) {
|
|
|
+ // Parse JSON data
|
|
|
+ auto type = cJSON_GetObjectItem(root, "type");
|
|
|
+ if (strcmp(type->valuestring, "tts") == 0) {
|
|
|
+ auto state = cJSON_GetObjectItem(root, "state");
|
|
|
+ if (strcmp(state->valuestring, "start") == 0) {
|
|
|
+ Schedule([this]() {
|
|
|
+ aborted_ = false;
|
|
|
+ if (device_state_ == kDeviceStateIdle || device_state_ == kDeviceStateListening) {
|
|
|
+ SetDeviceState(kDeviceStateSpeaking);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ } else if (strcmp(state->valuestring, "stop") == 0) {
|
|
|
+ Schedule([this]() {
|
|
|
+ if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ background_task_->WaitForCompletion();
|
|
|
+ if (keep_listening_) {
|
|
|
+ protocol_->SendStartListening(kListeningModeAutoStop);
|
|
|
+ SetDeviceState(kDeviceStateListening);
|
|
|
+ } else {
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ });
|
|
|
+ } else if (strcmp(state->valuestring, "sentence_start") == 0) {
|
|
|
+ auto text = cJSON_GetObjectItem(root, "text");
|
|
|
+ if (text != NULL) {
|
|
|
+ ESP_LOGI(TAG, "<< %s", text->valuestring);
|
|
|
+ std::string message = text->valuestring;
|
|
|
+ // Schedule([this, display, message = std::string(text->valuestring)]() {
|
|
|
+ // display->SetChatMessage("assistant", message.c_str());
|
|
|
+ // });
|
|
|
+ // 简化去重：只对比上一次显示的消息
|
|
|
+ if (message != last_displayed_message_) {
|
|
|
+ Schedule([this, display, message]() {
|
|
|
+ display->SetChatMessage("assistant", message.c_str());
|
|
|
+ last_displayed_message_ = message; // 更新上一次显示的消息
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else if (strcmp(type->valuestring, "stt") == 0) {
|
|
|
+ auto text = cJSON_GetObjectItem(root, "text");
|
|
|
+ if (text != NULL) {
|
|
|
+ ESP_LOGI(TAG, ">> %s", text->valuestring);
|
|
|
+ Schedule([this, display, message = std::string(text->valuestring)]() {
|
|
|
+ display->SetChatMessage("user", message.c_str());
|
|
|
+ });
|
|
|
+ }
|
|
|
+ } else if (strcmp(type->valuestring, "llm") == 0) {
|
|
|
+ auto emotion = cJSON_GetObjectItem(root, "emotion");
|
|
|
+ if (emotion != NULL) {
|
|
|
+ Schedule([this, display, emotion_str = std::string(emotion->valuestring)]() {
|
|
|
+ display->SetEmotion(emotion_str.c_str());
|
|
|
+ });
|
|
|
+ }
|
|
|
+ } else if (strcmp(type->valuestring, "iot") == 0) {
|
|
|
+ auto commands = cJSON_GetObjectItem(root, "commands");
|
|
|
+ if (commands != NULL) {
|
|
|
+ auto& thing_manager = iot::ThingManager::GetInstance();
|
|
|
+ for (int i = 0; i < cJSON_GetArraySize(commands); ++i) {
|
|
|
+ auto command = cJSON_GetArrayItem(commands, i);
|
|
|
+ thing_manager.Invoke(command);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ });
|
|
|
+ protocol_->Start();
|
|
|
+
|
|
|
+ codec->SetOutputVolume(90);
|
|
|
+
|
|
|
+ // Check for new firmware version or get the MQTT broker address
|
|
|
+ ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
|
|
|
+ ota_.SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
|
|
|
+ ota_.SetHeader("Client-Id", board.GetUuid());
|
|
|
+ ota_.SetHeader("Accept-Language", Lang::CODE);
|
|
|
+ auto app_desc = esp_app_get_description();
|
|
|
+ ota_.SetHeader("User-Agent", std::string(BOARD_NAME "/") + app_desc->version);
|
|
|
+
|
|
|
+ xTaskCreate([](void* arg) {
|
|
|
+ Application* app = (Application*)arg;
|
|
|
+ app->CheckNewVersion();
|
|
|
+ vTaskDelete(NULL); //这个意思是任务完成之后自动删除自身
|
|
|
+ }, "check_new_version", 4096 * 2, this, 2, nullptr);
|
|
|
+
|
|
|
+#if CONFIG_USE_AUDIO_PROCESSOR
|
|
|
+ audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
|
|
|
+ audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
|
|
|
+ background_task_->Schedule([this, data = std::move(data)]() mutable {
|
|
|
+ opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
|
|
|
+ Schedule([this, opus = std::move(opus)]() {
|
|
|
+ protocol_->SendAudio(opus);
|
|
|
+ });
|
|
|
+ });
|
|
|
+ });
|
|
|
+ });
|
|
|
+ audio_processor_.OnVadStateChange([this](bool speaking) {
|
|
|
+ if (device_state_ == kDeviceStateListening) {
|
|
|
+ Schedule([this, speaking]() {
|
|
|
+ if (speaking) {
|
|
|
+ voice_detected_ = true;
|
|
|
+ } else {
|
|
|
+ voice_detected_ = false;
|
|
|
+ }
|
|
|
+ auto led = Board::GetInstance().GetLed();
|
|
|
+ led->OnStateChanged();
|
|
|
+ });
|
|
|
+ }
|
|
|
+ });
|
|
|
+#endif
|
|
|
+
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference());
|
|
|
+ wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) {
|
|
|
+ Schedule([this, &wake_word]() {
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ SetDeviceState(kDeviceStateConnecting);
|
|
|
+ wake_word_detect_.EncodeWakeWordData();
|
|
|
+
|
|
|
+ if (!protocol_->OpenAudioChannel()) {
|
|
|
+ wake_word_detect_.StartDetection();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ std::vector<uint8_t> opus;
|
|
|
+ // Encode and send the wake word data to the server
|
|
|
+ while (wake_word_detect_.GetWakeWordOpus(opus)) {
|
|
|
+ protocol_->SendAudio(opus);
|
|
|
+ }
|
|
|
+ // Set the chat state to wake word detected
|
|
|
+ protocol_->SendWakeWordDetected(wake_word);
|
|
|
+ ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
|
|
|
+ keep_listening_ = true;
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ } else if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ AbortSpeaking(kAbortReasonWakeWordDetected);
|
|
|
+ } else if (device_state_ == kDeviceStateActivating) {
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ });
|
|
|
+ wake_word_detect_.StartDetection();
|
|
|
+#endif
|
|
|
+
|
|
|
+ SetDeviceState(kDeviceStateIdle);
|
|
|
+ esp_timer_start_periodic(clock_timer_handle_, 1000000);
|
|
|
+}
|
|
|
+
|
|
|
+void Application::OnClockTimer() {
|
|
|
+ vTaskPrioritySet(NULL, 1);
|
|
|
+ clock_ticks_++;
|
|
|
+
|
|
|
+ // Print the debug info every 10 seconds
|
|
|
+ if (clock_ticks_ % 10 == 0) {
|
|
|
+ // SystemInfo::PrintTaskList();
|
|
|
+ // SystemInfo::PrintRealTimeStats(pdMS_TO_TICKS(1000));
|
|
|
+ int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
|
|
|
+ int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
|
|
|
+ ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);
|
|
|
+
|
|
|
+ // If we have synchronized server time, set the status to clock "HH:MM" if the device is idle
|
|
|
+ if (ota_.HasServerTime()) {
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ Schedule([this]() {
|
|
|
+ // Set status to clock "HH:MM"
|
|
|
+ time_t now = time(NULL);
|
|
|
+ char time_str[64];
|
|
|
+ strftime(time_str, sizeof(time_str), "%H:%M ", localtime(&now));
|
|
|
+ Board::GetInstance().GetDisplay()->SetStatus(time_str);
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::Schedule(std::function<void()> callback) {
|
|
|
+ {
|
|
|
+ std::lock_guard<std::mutex> lock(mutex_);
|
|
|
+ main_tasks_.push_back(std::move(callback));
|
|
|
+ }
|
|
|
+ xEventGroupSetBits(event_group_, SCHEDULE_EVENT);
|
|
|
+}
|
|
|
+
|
|
|
+// The Main Loop controls the chat state and websocket connection
|
|
|
+// If other tasks need to access the websocket or chat state,
|
|
|
+// they should use Schedule to call this function
|
|
|
+void Application::MainLoop() {
|
|
|
+ while (true) {
|
|
|
+ auto bits = xEventGroupWaitBits(event_group_,
|
|
|
+ SCHEDULE_EVENT | AUDIO_INPUT_READY_EVENT | AUDIO_OUTPUT_READY_EVENT,
|
|
|
+ pdTRUE, pdFALSE, portMAX_DELAY);
|
|
|
+
|
|
|
+ if (bits & AUDIO_INPUT_READY_EVENT) {
|
|
|
+ InputAudio();
|
|
|
+ }
|
|
|
+ if (bits & AUDIO_OUTPUT_READY_EVENT) {
|
|
|
+ OutputAudio();
|
|
|
+ }
|
|
|
+ if (bits & SCHEDULE_EVENT) {
|
|
|
+ std::unique_lock<std::mutex> lock(mutex_);
|
|
|
+ std::list<std::function<void()>> tasks = std::move(main_tasks_);
|
|
|
+ lock.unlock();
|
|
|
+ for (auto& task : tasks) {
|
|
|
+ task();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::ResetDecoder() {
|
|
|
+ std::lock_guard<std::mutex> lock(mutex_);
|
|
|
+ opus_decoder_->ResetState();
|
|
|
+ audio_decode_queue_.clear();
|
|
|
+ last_output_time_ = std::chrono::steady_clock::now();
|
|
|
+}
|
|
|
+
|
|
|
+void Application::OutputAudio() {
|
|
|
+ auto now = std::chrono::steady_clock::now();
|
|
|
+ auto codec = Board::GetInstance().GetAudioCodec();
|
|
|
+ const int max_silence_seconds = 10;
|
|
|
+
|
|
|
+ std::unique_lock<std::mutex> lock(mutex_);
|
|
|
+ if (audio_decode_queue_.empty()) {
|
|
|
+ // Disable the output if there is no audio data for a long time
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_output_time_).count();
|
|
|
+ if (duration > max_silence_seconds) {
|
|
|
+ codec->EnableOutput(false);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (device_state_ == kDeviceStateListening) {
|
|
|
+ audio_decode_queue_.clear();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ last_output_time_ = now;
|
|
|
+ auto opus = std::move(audio_decode_queue_.front());
|
|
|
+ audio_decode_queue_.pop_front();
|
|
|
+ lock.unlock();
|
|
|
+
|
|
|
+ background_task_->Schedule([this, codec, opus = std::move(opus)]() mutable {
|
|
|
+ if (aborted_) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ std::vector<int16_t> pcm;
|
|
|
+ if (!opus_decoder_->Decode(std::move(opus), pcm)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Resample if the sample rate is different
|
|
|
+ if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
|
|
|
+ int target_size = output_resampler_.GetOutputSamples(pcm.size());
|
|
|
+ std::vector<int16_t> resampled(target_size);
|
|
|
+ output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
|
|
|
+ pcm = std::move(resampled);
|
|
|
+ }
|
|
|
+
|
|
|
+ codec->OutputData(pcm);
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+void Application::InputAudio() {
|
|
|
+ auto codec = Board::GetInstance().GetAudioCodec();
|
|
|
+ std::vector<int16_t> data;
|
|
|
+ if (!codec->InputData(data)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (codec->input_sample_rate() != 16000) {
|
|
|
+ if (codec->input_channels() == 2) {
|
|
|
+ auto mic_channel = std::vector<int16_t>(data.size() / 2);
|
|
|
+ auto reference_channel = std::vector<int16_t>(data.size() / 2);
|
|
|
+ for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
|
|
|
+ mic_channel[i] = data[j];
|
|
|
+ reference_channel[i] = data[j + 1];
|
|
|
+ }
|
|
|
+ auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
|
|
|
+ auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
|
|
|
+ input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
|
|
|
+ reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
|
|
|
+ data.resize(resampled_mic.size() + resampled_reference.size());
|
|
|
+ for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
|
|
|
+ data[j] = resampled_mic[i];
|
|
|
+ data[j + 1] = resampled_reference[i];
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
|
|
|
+ input_resampler_.Process(data.data(), data.size(), resampled.data());
|
|
|
+ data = std::move(resampled);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ if (wake_word_detect_.IsDetectionRunning()) {
|
|
|
+ wake_word_detect_.Feed(data);
|
|
|
+ }
|
|
|
+#endif
|
|
|
+#if CONFIG_USE_AUDIO_PROCESSOR
|
|
|
+ if (audio_processor_.IsRunning()) {
|
|
|
+ audio_processor_.Input(data);
|
|
|
+ }
|
|
|
+#else
|
|
|
+ if (device_state_ == kDeviceStateListening) {
|
|
|
+ background_task_->Schedule([this, data = std::move(data)]() mutable {
|
|
|
+ opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
|
|
|
+ Schedule([this, opus = std::move(opus)]() {
|
|
|
+ protocol_->SendAudio(opus);
|
|
|
+ });
|
|
|
+ });
|
|
|
+ });
|
|
|
+ }
|
|
|
+#endif
|
|
|
+}
|
|
|
+
|
|
|
+void Application::AbortSpeaking(AbortReason reason) {
|
|
|
+ ESP_LOGI(TAG, "Abort speaking");
|
|
|
+ aborted_ = true;
|
|
|
+ protocol_->SendAbortSpeaking(reason);
|
|
|
+}
|
|
|
+
|
|
|
+void Application::SetDeviceState(DeviceState state) {
|
|
|
+ if (device_state_ == state) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ clock_ticks_ = 0;
|
|
|
+ auto previous_state = device_state_;
|
|
|
+ device_state_ = state;
|
|
|
+ ESP_LOGI(TAG, "STATE: %s", STATE_STRINGS[device_state_]);
|
|
|
+ // The state is changed, wait for all background tasks to finish
|
|
|
+ background_task_->WaitForCompletion();
|
|
|
+
|
|
|
+ auto& board = Board::GetInstance();
|
|
|
+ auto codec = board.GetAudioCodec();
|
|
|
+ auto display = board.GetDisplay();
|
|
|
+ auto led = board.GetLed();
|
|
|
+ led->OnStateChanged();
|
|
|
+ switch (state) {
|
|
|
+ case kDeviceStateUnknown:
|
|
|
+ case kDeviceStateIdle:
|
|
|
+ display->SetStatus(Lang::Strings::STANDBY);
|
|
|
+ display->SetEmotion("neutral");
|
|
|
+#if CONFIG_USE_AUDIO_PROCESSOR
|
|
|
+ audio_processor_.Stop();
|
|
|
+#endif
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ wake_word_detect_.StartDetection();
|
|
|
+#endif
|
|
|
+ break;
|
|
|
+ case kDeviceStateConnecting:
|
|
|
+ display->SetStatus(Lang::Strings::CONNECTING);
|
|
|
+ display->SetEmotion("neutral");
|
|
|
+ display->SetChatMessage("system", "");
|
|
|
+ break;
|
|
|
+ case kDeviceStateListening:
|
|
|
+ display->SetStatus(Lang::Strings::LISTENING);
|
|
|
+ display->SetEmotion("neutral");
|
|
|
+ ResetDecoder();
|
|
|
+ opus_encoder_->ResetState();
|
|
|
+#if CONFIG_USE_AUDIO_PROCESSOR
|
|
|
+ audio_processor_.Start();
|
|
|
+#endif
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ wake_word_detect_.StopDetection();
|
|
|
+#endif
|
|
|
+ UpdateIotStates();
|
|
|
+ if (previous_state == kDeviceStateSpeaking) {
|
|
|
+ // FIXME: Wait for the speaker to empty the buffer
|
|
|
+ vTaskDelay(pdMS_TO_TICKS(120));
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ case kDeviceStateSpeaking:
|
|
|
+ display->SetStatus(Lang::Strings::SPEAKING);
|
|
|
+ ResetDecoder();
|
|
|
+ codec->EnableOutput(true);
|
|
|
+#if CONFIG_USE_AUDIO_PROCESSOR
|
|
|
+ audio_processor_.Stop();
|
|
|
+#endif
|
|
|
+#if CONFIG_USE_WAKE_WORD_DETECT
|
|
|
+ wake_word_detect_.StartDetection();
|
|
|
+#endif
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ // Do nothing
|
|
|
+ break;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::SetDecodeSampleRate(int sample_rate) {
|
|
|
+ if (opus_decode_sample_rate_ == sample_rate) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ opus_decode_sample_rate_ = sample_rate;
|
|
|
+ opus_decoder_.reset();
|
|
|
+ opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
|
|
|
+
|
|
|
+ auto codec = Board::GetInstance().GetAudioCodec();
|
|
|
+ if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
|
|
|
+ ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decode_sample_rate_, codec->output_sample_rate());
|
|
|
+ output_resampler_.Configure(opus_decode_sample_rate_, codec->output_sample_rate());
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::UpdateIotStates() {
|
|
|
+ auto& thing_manager = iot::ThingManager::GetInstance();
|
|
|
+ std::string states;
|
|
|
+ if (thing_manager.GetStatesJson(states, true)) {
|
|
|
+ protocol_->SendIotStates(states);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void Application::Reboot() {
|
|
|
+ ESP_LOGI(TAG, "Rebooting...");
|
|
|
+ esp_restart();
|
|
|
+}
|
|
|
+
|
|
|
+void Application::WakeWordInvoke(const std::string& wake_word) {
|
|
|
+ if (device_state_ == kDeviceStateIdle) {
|
|
|
+ ToggleChatState();
|
|
|
+ Schedule([this, wake_word]() {
|
|
|
+ if (protocol_) {
|
|
|
+ protocol_->SendWakeWordDetected(wake_word);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ } else if (device_state_ == kDeviceStateSpeaking) {
|
|
|
+ Schedule([this]() {
|
|
|
+ AbortSpeaking(kAbortReasonNone);
|
|
|
+ });
|
|
|
+ } else if (device_state_ == kDeviceStateListening) {
|
|
|
+ Schedule([this]() {
|
|
|
+ if (protocol_) {
|
|
|
+ protocol_->CloseAudioChannel();
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+bool Application::CanEnterSleepMode() {
|
|
|
+ if (device_state_ != kDeviceStateIdle) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Now it is safe to enter sleep mode
|
|
|
+ return true;
|
|
|
+}
|
