update

dep/zlinuxcomponents

@@ -1 +1 @@
-Subproject commit f8ebf838a0aa7fdbe7915cc620875d4031c0b77a
+Subproject commit c771fa7af061fd5d672d37a703a536e27f2d6522

src/service/conversation_session.hpp

@@ -14,6 +14,7 @@
 #include <vector>
 
 #include "iflytopcpp/core/basic/nlohmann/json.hpp"
+#include "iflytopcpp/core/components/timeutils.hpp"
 #include "iflytopcpp/core/spdlogfactory/logger.hpp"
 #include "iflytopcpp/core/thread/thread.hpp"
 #include "iflytopcpp/core/utils/uuid/uuid.hpp"
@@ -40,15 +41,33 @@ class ConversationSession : public enable_shared_from_this<ConversationSession>
   string asrTTSLocalURL;
   json   nlpResult;
 
+  zsteady_tp buildtp = zsteady_clock().now();
+
+  static string gettimestamp() {
+    struct tm tm = {0};
+    time_t    t  = time(nullptr);
+    if (t == -1) {
+      return "";
+    }
+    struct tm* tmp = localtime_r(&t, &tm);
+    if (!tmp) {
+      return "";
+    }
+    return fmt::format("{:0>4}{:0>2}{:0>2}{:0>2}{:0>2}{:0>2}", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
+                       tm.tm_min, tm.tm_sec);
+  }
+
  public:
-  ConversationSession() { sessionId = UUID().toString(); };
+  ConversationSession() { sessionId = gettimestamp(); };
 
   string getSessionId() { return sessionId; };
 
   string getAsrTTSLocalURL() { return asrTTSLocalURL; };
   void   setAsrTTSLocalURL(string asrTTSLocalURL) { this->asrTTSLocalURL = asrTTSLocalURL; };
 
-  json &getNlpResult() { return nlpResult; };
+  json& getNlpResult() { return nlpResult; };
   void  setNlpResult(json nlpResult) { this->nlpResult = nlpResult; };
+
+  zsteady_tp getBuildtp() { return buildtp; };
 };
 }  // namespace iflytop

src/service/main_control_service.cpp

@@ -22,6 +22,9 @@ using namespace std;
 
 void MainControlService::initializeVoiceProcess() {
   m_beforeWakeupVoiceProcesser->setAmplifyDB(20);
+  m_beforeasrVoiceProcesser->setAmplifyDB(24);
+
+  logger->info("MainControlService::start.....");
 
   /**
    * @brief 声卡录音回调
@@ -33,6 +36,7 @@ void MainControlService::initializeVoiceProcess() {
       logger->error("onRecordData audioClip is null");
       return;
     }
+
     m_audioLoggingService->loggerMICVoice(audioClip);                             // 录音
     m_beforeWakeupVoiceProcesser->writeVoice(audioClip);                          // 交给唤醒词预处理逻辑
     if (m_conversationSession) m_beforeasrVoiceProcesser->writeVoice(audioClip);  // 交给asr预处理逻辑
@@ -71,14 +75,19 @@ void MainControlService::initializeVoiceProcess() {
       logger->error("onAfterProcessVoice audioClip is null");
       return;
     }
+    auto session = m_conversationSession;
+
+    if (!session) return;
 
-    if (!m_conversationSession) return;
+    if (zsteady_clock().elapsedTimeMs(session->getBuildtp()) < 1000) return;
+
+    m_audioLoggingService->loggerASRVoice(audioClip);
     m_aiuiService->aiuiWrite((const char*)audioClip->data(), audioClip->size());
   });
 
   m_aiuiService->onMessage.connect([&](json& rxjson) {
-    //
     lock_guard<recursive_mutex> lock(m_voiceprocessmutex);
+
     json msg = rxjson;
     m_workQueue->enQueue([this, msg]() {
       try {
@@ -91,6 +100,17 @@ void MainControlService::initializeVoiceProcess() {
 
   m_audioRecoderService->startRecord();
 }
+
+void MainControlService::triggerProcessConversationSession() {
+  m_smartSoundboxPlayer->playConversationTTS(m_conversationSession->getAsrTTSLocalURL(), nullptr);
+  json nlpResult = m_conversationSession->getNlpResult();
+  if (nlpResult["data"]["intent"]["shouldEndSession"]) {
+    logger->info("endSession");
+    endSession();
+    return;
+  }
+}
+
 void MainControlService::processasrResult_nlp(json& rxjson) {
   logger->info("rx nlp:{}", rxjson.dump());
   m_conversationSession->setNlpResult(rxjson);
@@ -107,6 +127,7 @@ void MainControlService::processasrResult_tts(json& rxjson) {
   if (isendFrame) {
     logger->info("rx tts end,url={}", ttsurl);
     m_conversationSession->setAsrTTSLocalURL(ttsurl);
+    triggerProcessConversationSession();
   }
 }
 
@@ -162,6 +183,8 @@ void MainControlService::constructSession() {
   }
 
   m_conversationSession = make_shared<ConversationSession>();
+  logger->info("constructSession:============  {}  ===========", m_conversationSession->getSessionId());
+  m_audioLoggingService->triggerWakeup(m_conversationSession->getSessionId());
   m_aiuiService->aiuiInit();
 
   if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
@@ -182,6 +205,7 @@ void MainControlService::endSession() {
     m_aiuiService->aiuiFinished();
     m_aiuiService->aiuiDestroy();
   }
+  m_audioLoggingService->endwakeup();
 }
 
 void MainControlService::initialize() {

src/service/main_control_service.hpp

@@ -66,6 +66,7 @@ class MainControlService : public enable_shared_from_this<MainControlService> {
   shared_ptr<SimpleTimer> m_endsessionTimer;
   shared_ptr<WorkQueue>   m_workQueue;                                //
   recursive_mutex         m_voiceprocessmutex;                        //
+  zsteady_tp              m_systemsetuptime = zsteady_clock().now();  //
 
   typedef enum {
     kzwebService,
@@ -92,5 +93,10 @@ class MainControlService : public enable_shared_from_this<MainControlService> {
   void processasrResult(json rxjson);
   void processasrResult_nlp(json& rxjson);
   void processasrResult_tts(json& rxjson);
+
+  /**
+   * @brief 处理本次对话，一般接收完全部的ASR结果后，调用此函数
+   */
+  void triggerProcessConversationSession();
 };
 }  // namespace iflytop

src/service/voiceprocess/audio_logging_service.cpp

@@ -48,7 +48,6 @@ void AudioLoggingService::loggerMICVoice(shared_ptr<AudioClip> audioClip) {
     cleanupLogVoiceByTime("./voice/rt/mic_voice_*.wav", RT_STORAGE_CLIP_NUMS);
     m_rtMicVoiceFile.reset(new WavRecorder("./voice/rt/mic_voice_" + gettimestamp() + ".wav"));
     m_rtMicVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
-    // m_rtMicVoiceFile->dumpheader(logger);
     m_rtMicVoiceFile->writeVoice(audioClip);
   } else {
     m_rtMicVoiceFile->writeVoice(audioClip);
@@ -57,33 +56,51 @@ void AudioLoggingService::loggerMICVoice(shared_ptr<AudioClip> audioClip) {
     }
   }
 
+  /**
+   * @brief 每次唤醒都记录下对应的原始语音
+   */
+  {
+    lock_guard<mutex> lock(m_mutex);
     if (m_wakeupState) {
-    if (!m_rtBfwakeupVoiceFile) {
+      if (!m_wakeupMicVoiceFile) {
         cleanupLogVoiceByTime("./voice/wakeup/mic_voice_*.wav", WAKEUP_STORAGE_CLIP_NUMS);
-      m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/wakeup/mic_voice_" + gettimestamp() + ".wav"));
-      // m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
-      m_rtBfwakeupVoiceFile->writeVoice(audioClip);
+        m_wakeupMicVoiceFile.reset(new WavRecorder("./voice/wakeup/mic_voice_" + m_wakeupsessionid + ".wav"));
+        m_wakeupMicVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
+        m_wakeupMicVoiceFile->writeVoice(audioClip);
       } else {
-      m_rtBfwakeupVoiceFile->writeVoice(audioClip);
+        m_wakeupMicVoiceFile->writeVoice(audioClip);
+      }
     }
   }
 }
 
-void AudioLoggingService::setWakeupState(bool state) {
-  if (!state) {
+void AudioLoggingService::triggerWakeup(string sessionid) {
+  lock_guard<mutex> lock(m_mutex);
+  m_wakeupsessionid = sessionid;
   m_wakeupState     = true;
-  } else {
+}
+void AudioLoggingService::endwakeup() {
+  lock_guard<mutex> lock(m_mutex);
   m_wakeupState = false;
-    m_rtBfwakeupVoiceFile.reset();
+
+  if (m_wakeupMicVoiceFile) {
+    m_wakeupMicVoiceFile.reset();
+  }
+  if (m_asrFile) {
+    m_asrFile.reset();
   }
 }
 
+/**
+ * @brief 记录送给唤醒词的语音
+ *
+ * @param audioClip
+ */
 void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr<AudioClip> audioClip) {
   /**
-   * @brief MIC的语音会存储在两个地方
+   * @brief
    *  1. ./voice/rt/beforewakeup*.wav
    */
-
   if (!m_rtBfwakeupVoiceFile) {
     cleanupLogVoiceByTime("./voice/rt/rtbfwakeup*.wav", RT_STORAGE_CLIP_NUMS);
     m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/rt/rtbfwakeup" + gettimestamp() + ".wav"));
@@ -96,4 +113,25 @@ void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr<AudioClip> audioCli
     }
   }
 }
-void AudioLoggingService::loggerASRVoice(shared_ptr<AudioClip> audioClip) {}
+
+/**
+ * @brief 记录送给ASR的语音
+ *
+ * @param audioClip
+ */
+void AudioLoggingService::loggerASRVoice(shared_ptr<AudioClip> audioClip) {
+  /**
+   * @brief
+   *   ./voice/wakeup/asr*.wav
+   */
+  lock_guard<mutex> lock(m_mutex);
+  if (!m_wakeupState) return;
+  if (!m_asrFile) {
+    cleanupLogVoiceByTime("./voice/wakeup/asr*.wav", WAKEUP_STORAGE_CLIP_NUMS);
+    m_asrFile.reset(new WavRecorder("./voice/wakeup/asr" + m_wakeupsessionid + ".wav"));
+    m_asrFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
+    m_asrFile->writeVoice(audioClip);
+  } else {
+    m_asrFile->writeVoice(audioClip);
+  }
+}

src/service/voiceprocess/audio_logging_service.hpp

@@ -86,8 +86,16 @@ class AudioLoggingService : public enable_shared_from_this<AudioLoggingService>
   };
 
   bool m_wakeupState = false;
-  unique_ptr<WavRecorder> m_rtMicVoiceFile;
-  unique_ptr<WavRecorder> m_rtBfwakeupVoiceFile;
+
+  unique_ptr<WavRecorder> m_rtMicVoiceFile;       // 实时记录语音-MIC语音
+  unique_ptr<WavRecorder> m_rtBfwakeupVoiceFile;  // 实时记录语音-送给唤醒词的语音
+
+  unique_ptr<WavRecorder> m_wakeupMicVoiceFile;  // 唤醒时记录的语音-MIC原始语音
+  unique_ptr<WavRecorder> m_asrFile;             // 唤醒时记录的语音-ASR识别语音
+
+  string m_wakeupsessionid;
+
+  mutex m_mutex;
 
  public:
   AudioLoggingService(){};
@@ -99,7 +107,9 @@ class AudioLoggingService : public enable_shared_from_this<AudioLoggingService>
   void loggerASRVoice(shared_ptr<AudioClip> audioClip);
 
   void setWakeupState(bool state);
-  void clearupWavRecorder();
+
+  void triggerWakeup(string sessionid);
+  void endwakeup();
 
  private:
   void cleanupLogVoiceByTime(string prefix, size_t maxnum);

src/service/voiceprocess/beforeasr_voiceprocesser.cpp

@@ -27,12 +27,25 @@ void BfAsrVProcesser::processVoice(shared_ptr<AudioClip> audioClip) {
   vector<uint8_t> voice;
   audioClip->getOneCHVoice(voice, 0);
 
+  ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE");
+
+  int16_t *p = (int16_t *)voice.data();
+  for (int i = 0; i < voice.size() / 2; i++) {
+    p[i] = p[i] * m_amplify;
+  }
+
   shared_ptr<AudioClip> afterProcessAudioClip =
       make_shared<AudioClip>((uint8_t *)voice.data(), voice.size(), 1, audioClip->getRate(), audioClip->getFormat());
 
   onAfterProcessVoice(afterProcessAudioClip);
 }
 
+void BfAsrVProcesser::setAmplify(float amplify) {
+  logger->info("BfAsrVProcesser::setAmplify amplify={}", amplify);
+  m_amplify = amplify;
+}
+void BfAsrVProcesser::setAmplifyDB(float amplifydb) { m_amplify = pow(10, amplifydb / 20); }
+
 void BfAsrVProcesser::writeVoice(shared_ptr<AudioClip> audioClip) {
   ZCHECK(audioClip != nullptr, "audioClip is null");
   ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE");

src/service/voiceprocess/beforeasr_voiceprocesser.hpp

@@ -23,8 +23,9 @@
  *
  * service:BfAsrVProcesser
  *
- * 作用: 波束增强，降噪，回声消除
- *
+ * 作用:
+ *    bosuzengqiang
+ *    简单的波束增强
  */
 
 namespace iflytop {
@@ -39,6 +40,8 @@ class BfAsrVProcesser : public enable_shared_from_this<BfAsrVProcesser> {
 
   unique_ptr<Thread> m_thread;
 
+  float m_amplify = 1.0;
+
  public:
   nod::signal<void(shared_ptr<AudioClip> audioClip)> onAfterProcessVoice;
 
@@ -46,6 +49,9 @@ class BfAsrVProcesser : public enable_shared_from_this<BfAsrVProcesser> {
 
   void initialize();
 
+  void setAmplify(float amplify);
+  void setAmplifyDB(float amplifydb);
+
   void writeVoice(shared_ptr<AudioClip> audioClip);
 
  private:

src/service/voiceprocess/wakeup_processer.cpp

@@ -3,7 +3,7 @@ using namespace std;
 using namespace iflytop;
 using namespace core;
 
-#define CHUNK_SIZE "1600"
+#define WAKEUP_MODULE_INIT_TIMEOUT_S 8
 
 void WakeupProcesser::initialize(string precise_engine, string wakeupmodulepath, string chunksize) {
   logger->info("initialize precise_engine:{}, wakeupmodulepath:{}, chunksize:{}", precise_engine, wakeupmodulepath,
@@ -11,6 +11,15 @@ void WakeupProcesser::initialize(string precise_engine, string wakeupmodulepath,
   wakeupProcesser.reset(new MycroftPreciseWapper());
   wakeupProcesser->initialize(precise_engine.c_str(), wakeupmodulepath.c_str(), chunksize.c_str());
   m_chucksize                          = atoi(chunksize.c_str());
+  uint16_t s_zerovoicebuf[m_chucksize] = {0};
+  wakeupProcesser->processVoice((uint8_t*)&s_zerovoicebuf[0], m_chucksize * 2);
+  wakeupProcesser->processVoice((uint8_t*)&s_zerovoicebuf[0], m_chucksize * 2);
+  for (size_t i = 0; i < WAKEUP_MODULE_INIT_TIMEOUT_S; i++) {
+    logger->info("wait for wakeup module init {}/{}", i, WAKEUP_MODULE_INIT_TIMEOUT_S);
+    sleep(1);
+  }
+
+
 }
 
 int WakeupProcesser::getChunkSize() { return m_chucksize; }

src/service/voiceprocess/wakeup_processer.hpp

@@ -40,7 +40,7 @@ class WakeupProcesser : public enable_shared_from_this<WakeupProcesser> {
   bool                               wakeupflag = false;
   tp_steady                          last_wakeup_timepoint;
   function<void(float wakeup_score)> m_cbfunc;
-  int                                m_chucksize = 0;
+  size_t                             m_chucksize = 0;
 
  public:
   nod::signal<void(float wakeup_score)> onWakeupSignal;