update

CMakeLists.txt

@@ -92,11 +92,11 @@ zadd_executable_simple(
   src/service/main_control_service.cpp
   src/service/light_control_service.cpp
   src/service/report_service.cpp
-  src/service/voiceprocess/asr_service.cpp
   src/service/voiceprocess/beforeasr_voiceprocesser.cpp
   src/service/voiceprocess/beforewakeup_voiceprocesser.cpp
   src/service/voiceprocess/voiceprocess_service.cpp
   src/service/voiceprocess/wakeup_processer.cpp
+  src/service/voiceprocess/audio_recoder_service.cpp
   # aiui
 )
 

dep/iflytopcpp

@@ -1 +1 @@
-Subproject commit 59531ea26be163327f7cbc6bc31b9845521afff5
+Subproject commit e4c17891a53fda64a1beafa88ecc9f62119082c2

env/rootfs/etc/asound.conf

@@ -75,6 +75,18 @@ pcm.xa8326 {
 pcm.recorder {
     type plug
     slave {
+        pcm "recorder5ch"
+        rate 48000 
+        format S16_LE 
+        channels 5 
+    }
+    rate 16000
+    format S16_LE 
+    channels 5 
+}
+pcm.recorder5ch {
+    type plug
+    slave {
         pcm {
             type multi
             slaves.xa8326.pcm "hw:2,0"
@@ -89,31 +101,31 @@ pcm.recorder {
             bindings.1.channel 1
 
             bindings.2.slave pdm
-            bindings.2.channel 2
+            bindings.2.channel 5
 
             bindings.3.slave pdm
-            bindings.3.channel 3
+            bindings.3.channel 6
 
-            bindings.4.slave pdm
-            bindings.4.channel 4
+            bindings.4.slave xa8326
+            bindings.4.channel 0
     
-            bindings.5.slave pdm
-            bindings.5.channel 5
+            # bindings.5.slave pdm
+            # bindings.5.channel 5
 
-            bindings.6.slave pdm
-            bindings.6.channel 6
+            # bindings.6.slave pdm
+            # bindings.6.channel 6
 
-            bindings.7.slave pdm
-            bindings.7.channel 7
+            # bindings.7.slave pdm
+            # bindings.7.channel 7
 
-            bindings.8.slave xa8326
-            bindings.8.channel 0
+            # bindings.8.slave xa8326
+            # bindings.8.channel 0
 
-            bindings.9.slave xa8326
-            bindings.9.channel 1    
+            # bindings.9.slave xa8326
+            # bindings.9.channel 1    
         }
         rate 48000
         format S16_LE
-        channels 10
+        channels 5
     }
 }

sh/packet.sh

@@ -56,7 +56,6 @@ cd $PROJECT_PATH/env/ && cp --path -rf ./* $PACKET_DIR/
 ####################################################################
 # 下载唤醒词模型
 wget_and_unzip xiaomaoxiaomao ftp://zftpuser:zftpzwsd@192.168.1.2/zwsd/manufacturer_mycroft/hotworld/xiaomaoxiaomao/xiaomaoxiaomao_module_20220519.tar.gz
-mv $PACKET_DIR/xiaomaoxiaomao $PACKET_DIR/wakeupmodule
 # 下载唤醒词引擎
 wget_and_unzip precise 192.168.1.2:8021/manufacturer_mycroft/mycroft-precise-release/precise-all_0.3.0_aarch64.tar.gz
 # 下载唤醒词语音

src/main.cpp

@@ -1,11 +1,17 @@
+#include <curl/curl.h>
+
+#include <iostream>
+#include <string>
+
+#include "zlinuxcomponents/aiui_ws/aiui_service.hpp"
+#include "zlinuxcomponents/alsaplayer/AudioPlayerAlsaImpl.hpp"
+#include "zlinuxcomponents/alsaplayer/smart_soundbox_player.hpp"
 #include "zlinuxcomponents/zmainhelper.hpp"
 //
 #include "configs/config.hpp"
 #include "iflytopcpp/core/spdlogfactory/logger.hpp"
 #include "iflytopcpp/core/thread/thread.hpp"
-#include "spdlog/spdlog.h"
 #include "version.hpp"
-
 #include "zlinuxcomponents/rootfs_auto_update/rootfs_auto_update.hpp"
 #include "zservice_container/zservice_container.hpp"
 #include "zwebservice/zwebservice.hpp"
@@ -15,13 +21,13 @@
 #include "service/light_control_service.hpp"
 #include "service/main_control_service.hpp"
 #include "service/report_service.hpp"
+#include "service/voiceprocess/audio_recoder_service.hpp"
+#include "service/voiceprocess/beforeasr_voiceprocesser.hpp"
+#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp"
+#include "service/voiceprocess/voiceprocess_service.hpp"
+#include "service/voiceprocess/wakeup_processer.hpp"
 //
-#include "zlinuxcomponents/alsaplayer/AudioPlayerAlsaImpl.hpp"
-// #include "zlinuxcomponents/audio/audio_recoder.hpp"
-#include <curl/curl.h>
-#include <iostream>
-#include <string>
-//
+
 using namespace iflytop;
 using namespace core;
 using namespace std;
@@ -32,7 +38,7 @@ ZMAIN();
  * =======================================================Main======================================================== *
  ***********************************************************************************************************************/
 void Main::onSIGINT() { exit(0); }
-int Main::main(int argc, char *argv[]) {
+int  Main::main(int argc, char *argv[]) {
   string g_host_server_ip;
   string g_device_id;
 #if 0
@@ -53,9 +59,12 @@ int Main::main(int argc, char *argv[]) {
   logger->info("#  version:{}", VERSION);
   logger->info("#");
 
-  BUILD_AND_REG_SERRVICE(Config);
-  GET_SERVICE(Config)->initialize();
-  if (!g_device_id.empty()) GET_SERVICE(Config)->set_deviceId(g_device_id);
+  // BUILD_AND_REG_SERRVICE(Config);
+  logger->info("build {}.....", "Config");
+  shared_ptr<iflytop::Config> Config_val(new iflytop::Config());
+  ServiceContrainer::get().regService(Config_val);
+  GET_SERVICE(iflytop::Config)->initialize();
+  if (!g_device_id.empty()) GET_SERVICE(iflytop::Config)->set_deviceId(g_device_id);
 
   /**
    * @brief 比较rootfs目录中的文件和系统中对应路径的文件是否一致，如果不一致则替换
@@ -73,12 +82,9 @@ int Main::main(int argc, char *argv[]) {
     // system("reboot");
   }
 
-
-  // AudioPlayerAlsaImp audioPlayerAlsaImp("/dev/");
-
-  // AudioRecoder::get().initialize();
-  // shared_ptr<AudioRecoder> audioRecoder(new AudioRecoder());
-
+  /**
+   * @brief DeviceIOService初始化
+   */
 #if 0
   BUILD_AND_REG_SERRVICE(DeviceIOService);
 #else
@@ -86,6 +92,51 @@ int Main::main(int argc, char *argv[]) {
 #endif
   GET_SERVICE(DeviceIOService)->initialize();
 
+  /**
+   * @brief 音频处理相关组建初始化
+   */
+
+  // 音频采集
+  BUILD_AND_REG_SERRVICE(AudioRecoderService);
+  GET_SERVICE(AudioRecoderService)->initialize("recorder", 5, 16000, SND_PCM_FORMAT_S16_LE);
+
+  // BeforeWakeupVoiceProcesser
+  BUILD_AND_REG_SERRVICE(BeforeWakeupVoiceProcesser);
+  GET_SERVICE(BeforeWakeupVoiceProcesser)->initialize();
+
+  // BeforeAsrVoiceProcesser
+  BUILD_AND_REG_SERRVICE(BeforeasrVoiceProcesser);
+  GET_SERVICE(BeforeasrVoiceProcesser)->initialize();
+
+  // WakeupProcesser
+  BUILD_AND_REG_SERRVICE(WakeupProcesser);
+  GET_SERVICE(WakeupProcesser)->initialize("xiaomaoxiaomao/xiaomaoxiaomao.pb");
+
+  // AsrProcesser
+  const char *appid = "5938b7c7";                          // 应用ID，在AIUI开放平台创建并设置
+  const char *key   = "19c1f7becc78eedc7826b485aabe30de";  // 接口密钥，在AIUI开放平台查看
+  json        paramj;
+  paramj["result_level"] = "plain";
+  paramj["auth_id"]      = "ac30105366ea460f9ff08ddac0c4f71e";
+  paramj["data_type"]    = "text";
+  paramj["scene"]        = "main_box";
+  paramj["sample_rate"]  = "16000";
+  paramj["context"]      = R"({"sdk_support":["nlp","tts","vad","iat"]})";
+  BUILD_AND_REG_SERRVICE(AiuiService);
+  GET_SERVICE(AiuiService)->initialize(appid, key, paramj.dump());
+
+  // 播放器-SmartSoundBoxPlayer
+  BUILD_AND_REG_SERRVICE(SmartSoundBoxPlayer);
+  //
+  GET_SERVICE(SmartSoundBoxPlayer)->initialize(80);
+
+  // VoiceProcessService
+  BUILD_AND_REG_SERRVICE(VoiceProcessService);
+  GET_SERVICE(VoiceProcessService)->initialize();
+
+  /**
+   * @brief 其他服务
+   */
   BUILD_AND_REG_SERRVICE(ZWebService);
   GET_SERVICE(ZWebService)->initialize();
 

src/service/voiceprocess/audio_logging_service.cpp

@@ -0,0 +1,43 @@
+#include "audio_logging_service.hpp"
+
+#include "iflytopcpp/core/components/time_util.hpp"
+
+using namespace iflytop;
+using namespace core;
+void AudioLoggingService::initialize() {
+  m_originalVoice     = make_shared<AudioQueue>();
+  m_beforeWakeupVoice = make_shared<AudioQueue>();
+  m_beforeAsrVoice    = make_shared<AudioQueue>();
+
+  m_realtime_log_thread.reset(new Thread("m_realtime_log_thread", [this]() {
+    ThisThread thisThread;
+    while (!thisThread.getExitFlag()) {
+      thisThread.sleepForMs(33);
+    }
+  }));
+}
+
+void AudioLoggingService::loggerVoice(VoiceType voiceType, shared_ptr<AudioClip> audioClip) {
+  zsteady_tp now = zsteady_clock().now();
+
+  shared_ptr<AudioQueue> aq;
+  if (voiceType == kOriginalVoice) {
+    aq = m_originalVoice;
+  } else if (voiceType == kBeforeWakeupVoice) {
+    aq = m_beforeWakeupVoice;
+  } else if (voiceType == kBeforeAsrVoice) {
+    aq = m_beforeAsrVoice;
+  } else {
+    return;
+  }
+
+  aq->m_30sque.push_back(audioClip);
+  aq->m_rtque.push_back(audioClip);
+  if (wakeup) aq->m_wakque.push_back(audioClip);
+
+  aq->m_30sque.remove_if(
+      [&](shared_ptr<AudioClip> audioClip) { return zsteady_clock().dToMs(now - audioClip->getTp()) > (30 * 1000); });
+}
+
+void AudioLoggingService::triggerWakeupEvent() {}
+void AudioLoggingService::triggerEndWakeupEvent() {}

src/service/voiceprocess/audio_logging_service.hpp

@@ -0,0 +1,97 @@
+//
+// Created by zwsd
+//
+
+#pragma once
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <map>
+#include <memory>
+#include <set>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "iflytopcpp/core/spdlogfactory/logger.hpp"
+#include "iflytopcpp/core/thread/thread.hpp"
+#include "zlinuxcomponents/audio/audio_clip.hpp"
+#include "iflytopcpp/core/components/time_util.hpp"
+
+/**
+ * @brief
+ *
+ * service: AudioLoggingService
+ *
+ * 1. 存储每次唤醒后的音频，及其唤醒前20s的音频
+ * 2. 每隔1分钟录一帧,共录15分钟
+ * 3. 存储不同阶段的语音
+ *
+ */
+
+namespace iflytop {
+using namespace std;
+using namespace core;
+class AudioLoggingService : public enable_shared_from_this<AudioLoggingService> {
+  ENABLE_LOGGER(AudioLoggingService);
+
+ public:
+  enum VoiceType {
+    kOriginalVoice,
+    kBeforeWakeupVoice,
+    kBeforeAsrVoice,
+  };
+
+  /**
+   * @brief
+   * 存储一下文件:
+   *
+   * 每次唤醒存储一次语音，为了录全语音，会将唤醒前30s的语音也录制进来
+   * logVoice/bywakeup/
+   *       20230308183030_origi.wav
+   *       20230308183030_bfwak.wav
+   *       20230308183030_bfasr.wav
+   *
+   * 每隔1分钟录一次，一次录一分钟的语音，总共录15分钟，滚动覆盖.
+   * logVoice/bytime/
+   *       20230308183030_origi.wav
+   *       20230308183030_bfwak.wav
+   *       20230308183030_bfasr.wav
+   *
+   */
+  class AudioQueue {
+   public:
+    // 缓存30s语音的队列
+    list<shared_ptr<AudioClip>> m_30sque;
+    // 唤醒存储队列
+    list<shared_ptr<AudioClip>> m_wakque;
+    // 实时存储队列
+    list<shared_ptr<AudioClip>> m_rtque;
+  };
+
+  shared_ptr<AudioQueue> m_originalVoice;
+  shared_ptr<AudioQueue> m_beforeWakeupVoice;
+  shared_ptr<AudioQueue> m_beforeAsrVoice;
+
+  unique_ptr<Thread> m_realtime_log_thread;
+  bool               wakeup = false;
+
+  class OnceLog {
+   public:
+    zsteady_tp m_startTime;
+
+  };
+
+  shared_ptr<OnceLog> m_realTimeOnceLog;
+
+ public:
+  AudioLoggingService(){};
+
+  void loggerVoice(VoiceType voiceType, shared_ptr<AudioClip> audioClip);
+
+  void triggerWakeupEvent();
+  void triggerEndWakeupEvent();
+
+  void initialize();
+};
+}  // namespace iflytop

src/service/voiceprocess/audio_recoder_service.cpp

@@ -0,0 +1,12 @@
+#include "audio_recoder_service.hpp"
+
+using namespace std;
+using namespace iflytop;
+using namespace core;
+
+AudioRecoderService::AudioRecoderService() {}
+void AudioRecoderService::initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate,
+                                     snd_pcm_format_t format) {
+  m_audioRecoder = make_shared<AudioRecoder>();
+  m_audioRecoder->initialize(pcmName, channels, sample_rate, format);
+}

src/service/voiceprocess/audio_recoder_service.hpp

@@ -0,0 +1,46 @@
+//
+// Created by zwsd
+//
+
+#pragma once
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <map>
+#include <memory>
+#include <set>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "iflytopcpp/core/spdlogfactory/logger.hpp"
+#include "iflytopcpp/core/thread/thread.hpp"
+#include "zlinuxcomponents/audio/audio_recoder.hpp"
+
+/**
+ * @brief
+ *
+ * service: AudioRecoderService
+ *
+ * 监听事件:
+ * 依赖状态:
+ * 依赖服务:
+ * 作用:
+ *
+ */
+
+namespace iflytop {
+using namespace std;
+using namespace core;
+class AudioRecoderService : public enable_shared_from_this<AudioRecoderService> {
+  ENABLE_LOGGER(AudioRecoderService);
+
+ public:
+  shared_ptr<AudioRecoder> m_audioRecoder;
+
+ public:
+  AudioRecoderService();
+
+  void initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, snd_pcm_format_t format);
+};
+}  // namespace iflytop

src/service/voiceprocess/voiceprocess_service.cpp

@@ -1,9 +1,25 @@
 #include "voiceprocess_service.hpp"
 
 #include "iflytopcpp/core/spdlogfactory/logger.hpp"
+#include "zservice_container/zservice_container.hpp"
 
 using namespace iflytop;
 using namespace std;
 using namespace core;
 
-void VoiceProcessService::initialize() {}
+void VoiceProcessService::initialize() {
+  //
+    GET_TO_SERVICE(m_beforeWakeupVoiceProcesser);
+    GET_TO_SERVICE(m_beforeasrVoiceProcesser);
+    GET_TO_SERVICE(m_wakeupProcesser);
+    GET_TO_SERVICE(m_audioRecoderService);
+
+  m_audioRecoderService->m_audioRecoder->onRecordData.connect(
+      [this](shared_ptr<AudioClip> audioClip) { m_beforeWakeupVoiceProcesser->writeVoice(audioClip); });
+
+  m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect(
+      [this](shared_ptr<AudioClip> audioClip) { m_beforeasrVoiceProcesser->writeVoice(audioClip); });
+
+  //   m_wakeupProcesser->onWakeupSignal.connect(
+  //       [this](float wakeup_score) { logger->info("wakeup_score:{}", wakeup_score); });
+}

src/service/voiceprocess/voiceprocess_service.hpp

@@ -15,6 +15,10 @@
 
 #include "iflytopcpp/core/spdlogfactory/logger.hpp"
 #include "iflytopcpp/core/thread/thread.hpp"
+#include "service/voiceprocess/audio_recoder_service.hpp"
+#include "service/voiceprocess/beforeasr_voiceprocesser.hpp"
+#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp"
+#include "service/voiceprocess/wakeup_processer.hpp"
 
 /**
  * @brief
@@ -34,6 +38,11 @@ using namespace core;
 class VoiceProcessService : public enable_shared_from_this<VoiceProcessService> {
   ENABLE_LOGGER(VoiceProcessService);
 
+  shared_ptr<BeforeWakeupVoiceProcesser> m_beforeWakeupVoiceProcesser;
+  shared_ptr<BeforeasrVoiceProcesser>    m_beforeasrVoiceProcesser;
+  shared_ptr<WakeupProcesser>            m_wakeupProcesser;
+  shared_ptr<AudioRecoderService>        m_audioRecoderService;
+
  public:
   VoiceProcessService(){};
 

src/service/voiceprocess/wakeup_processer.cpp

@@ -7,9 +7,9 @@ using namespace core;
 // #define WAKEUP_MODULE_PATH "module/wakeup.pb"
 #define CHUNK_SIZE "2048"
 
-void WakeupProcesser::initialize(string wakeupmodule) {
+void WakeupProcesser::initialize(string wakeupmodulepath) {
   wakeupProcesser.reset(new MycroftPreciseWapper());
-  wakeupProcesser->initialize(ORDER_PATH, wakeupmodule.c_str(), CHUNK_SIZE);
+  wakeupProcesser->initialize(ORDER_PATH, wakeupmodulepath.c_str(), CHUNK_SIZE);
 }
 void WakeupProcesser::dumpwakeup_info(float source) {
   if (source > 0.9) {

src/service/voiceprocess/wakeup_processer.hpp

@@ -47,7 +47,7 @@ class WakeupProcesser : public enable_shared_from_this<WakeupProcesser> {
  public:
   WakeupProcesser(){};
 
-  void initialize(string wakeupmodule);
+  void initialize(string wakeupmodulepath);
 
   void processVoice(uint8_t* voice, size_t voiceLen);