rk3588sdk/src/service/main_control_service.cpp

#include "main_control_service.hpp"

#include "iflytopcpp/core/components/stringutils.hpp"
#include "version.hpp"
using namespace iflytop;
using namespace core;
using namespace std;

/***********************************************************************************************************************
 * ======================================================宏定义======================================================= *
 ***********************************************************************************************************************/

#define SET_DEVICE_STATE(id, value, uint)                            \
  receipt["deviceState"][index]["id"]    = id;                       \
  receipt["deviceState"][index]["value"] = fmt::format("{}", value); \
  receipt["deviceState"][index]["unit"]  = uint;                     \
  index++;

/*********************************************************************************************************************
 * ======================================================代码======================================================= *
 *********************************************************************************************************************/

void MainControlService::initializeVoiceProcess() {
  m_beforeWakeupVoiceProcesser->setAmplifyDB(20);
  m_beforeasrVoiceProcesser->setAmplifyDB(24);

  logger->info("MainControlService::start.....");

  /**
   * @brief 声卡录音回调
   *          1路,交给唤醒词预处理算法
   *          1路,在唤醒的时候交给ASR预处理算法
   */
  m_audioRecoderService->onRecordData.connect([this](shared_ptr<AudioClip> audioClip) {
    if (!audioClip) {
      logger->error("onRecordData audioClip is null");
      return;
    }
    m_audioLoggingService->loggerMICVoice(audioClip);  // 录音
    m_commonVoiceProcesser->writeVoice(audioClip);     // 交给通用预处理逻辑
  });

  m_commonVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
    if (!audioClip) {
      logger->error("onAfterProcessVoice audioClip is null");
      return;
    }

    m_beforeWakeupVoiceProcesser->writeVoice(audioClip);                          // 交给唤醒词预处理逻辑
    if (m_conversationSession) m_beforeasrVoiceProcesser->writeVoice(audioClip);  // 交给asr预处理逻辑
  });

  /**
   * @brief 唤醒词预处理算法回调
   *          1路交给唤醒词算法
   */
  m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
    if (!audioClip) {
      logger->error("onAfterProcessVoice audioClip is null");
      return;
    }

    m_audioLoggingService->loggerBeforeWakeupVoice(audioClip);
    m_wakeupProcesser->processVoice(audioClip->data(), audioClip->size());
  });

  /**
   * @brief 唤醒词触发回调
   */
  m_wakeupProcesser->onWakeupSignal.connect([this](float wakeup_score) {
    logger->info("onWakeupSignal wakeup_score {}", wakeup_score);
    /**
     * @brief 触发唤醒，构造session
     */
    m_workQueue->enQueue([this]() { constructSession(); });
  });

  /**
   * @brief asr预处理算法回调
   */
  m_beforeasrVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
    if (!audioClip) {
      logger->error("onAfterProcessVoice audioClip is null");
      return;
    }
    auto session = m_conversationSession;

    if (!session) return;

    if (zsteady_clock().elapsedTimeMs(session->getBuildtp()) < 1000) return;

    m_audioLoggingService->loggerASRVoice(audioClip);
    m_aiuiService->aiuiWrite((const char*)audioClip->data(), audioClip->size());
  });

  m_aiuiService->onMessage.connect([&](json& rxjson) {
    lock_guard<recursive_mutex> lock(m_voiceprocessmutex);

    json msg = rxjson;
    m_workQueue->enQueue([this, msg]() {
      try {
        processasrResult(msg);
      } catch (const std::exception& e) {
        logger->error("processasrResult error:{}", e.what());
      }
    });
  });

  m_audioRecoderService->startRecord();
}

void MainControlService::triggerProcessConversationSession() {
  m_smartSoundboxPlayer->playConversationTTS(m_conversationSession->getAsrTTSLocalURL(), nullptr);
  json nlpResult = m_conversationSession->getNlpResult();
  if (nlpResult["data"]["intent"]["shouldEndSession"] == "true") {
    logger->info("endSession");
    if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
    endSession();
    return;
  }
}

void MainControlService::processasrResult_nlp(json& rxjson) {
  logger->info("rx nlp:{}", rxjson.dump());
  m_conversationSession->setNlpResult(rxjson);
  m_smartSoundboxPlayer->playConversationTTS(m_conversationSession->getAsrTTSLocalURL(), nullptr);
}
void MainControlService::processasrResult_tts(json& rxjson) {
  /**
   * @brief 接收从云端发过来的tts数据，并保存成文件
   */
  logger->debug("rx tts:frame {}", rxjson["data"]["json_args"]["frame_id"].get<int>());
  bool   isendFrame = false;
  string ttsurl;
  m_aiuiService->parseTTSContent(rxjson, isendFrame, ttsurl);
  if (isendFrame) {
    logger->info("rx tts end,url={}", ttsurl);
    m_conversationSession->setAsrTTSLocalURL(ttsurl);
    triggerProcessConversationSession();
  }
}

void MainControlService::processasrResult(json rxjson) {
  lock_guard<recursive_mutex> lock(m_voiceprocessmutex);

  string action = rxjson["action"];
  if (action == "started") {
    logger->info("processasr: rx started:{}", rxjson.dump());
  } else if (action == "error") {
    logger->info("processasr: rx error:{}", rxjson.dump());
  } else if (action == "vad") {
    logger->info("processasr: rx vad:{}", rxjson.dump());
  } else if (action == "result") {
    string sub = rxjson["data"]["sub"];
    if (sub == "nlp") {
      /**
       * @brief 解析nlp数据
       */
      processasrResult_nlp(rxjson);
    } else if (sub == "tts") {
      /**
       * @brief 解析tts数据
       */
      processasrResult_tts(rxjson);
    } else if (sub == "iat") {
      logger->info("rx iat:{}", rxjson.dump());
    } else if (sub == "vad") {
      logger->info("rx vad:{}", rxjson.dump());
    } else {
      logger->info("rx {}:{}", sub, rxjson.dump());
    }
  } else {
    logger->info("rx unkown:{}", rxjson.dump());
  }
}

void MainControlService::constructSession() {
  /**
   * @brief
   *  1. 触发唤醒词语
   *  2. 如果上一次session还没有结束,则结束上一次session
   *  3. 初始化asr
   *  4. 构建新的session
   *  5. 启动定时器,如果10s内无人应答，则结束session
   */

  lock_guard<recursive_mutex> lock(m_voiceprocessmutex);

  m_smartSoundboxPlayer->triggerWakeup();
  if (m_conversationSession) {
    if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
    endSession();
  }

  m_conversationSession = make_shared<ConversationSession>();
  logger->info("constructSession:============  {}  ===========", m_conversationSession->getSessionId());
  m_audioLoggingService->triggerWakeup(m_conversationSession->getSessionId());
  m_aiuiService->aiuiInit();

  if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();

  m_endsessionTimer->setTimeout(
      [this]() {
        logger->info("no hum voice detected, end session");
        endSession();
      },
      10000);
}

void MainControlService::endSession() {
  lock_guard<recursive_mutex> lock(m_voiceprocessmutex);

  if (m_conversationSession) {
    m_conversationSession = nullptr;
    m_aiuiService->aiuiFinished();
    m_aiuiService->aiuiDestroy();
  }
  m_audioLoggingService->endwakeup();
}

void MainControlService::initialize() {
  GET_TO_SERVICE(m_zwebService);
  GET_TO_SERVICE(m_deviceIOService);

  GET_TO_SERVICE(m_beforeWakeupVoiceProcesser);
  GET_TO_SERVICE(m_beforeasrVoiceProcesser);
  GET_TO_SERVICE(m_wakeupProcesser);
  GET_TO_SERVICE(m_audioRecoderService);
  GET_TO_SERVICE(m_audioLoggingService);
  GET_TO_SERVICE(m_smartSoundboxPlayer);
  GET_TO_SERVICE(m_aiuiService);
  GET_TO_SERVICE(m_commonVoiceProcesser);

  m_endsessionTimer.reset(new SimpleTimer("endSessionTimer"));
  m_workQueue.reset(new WorkQueue("mainControlServiceVoiceProcessCtrlWQ"));

  // 监听从webservice来的websocket消息
  m_zwebService->startWork([this](const json& command, json& receipt) {
    try {
      processReceiveMessage(kzwebService, command, receipt);
    } catch (const std::exception& e) {
      logger->error("process message fail {}", string(e.what()));
    } catch (...) {
      logger->error("process message fail {}", "catch unknown exception");
    }
  });

  /**
   * @brief 初始化语音处理流程
   */
  initializeVoiceProcess();
};
void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json& in, json& receipt) {
  logger->info("process receive message from {},{}", fromwhere2str(fromwhere), in.dump());

  if (in["command"] == "getVersion") {
    receipt["version"] = VERSION;
  }

  /*********************************************************************************************************************
   * ================================================日志动态配置接口================================================= *
   *********************************************************************************************************************/
  else if (in["command"] == "loggerSetLevel") {
    int    loggerLevel = in["loggerLevel"];
    string loggerName  = in["loggerName"];
    logger->info("loggerSetLevel {} {}", loggerName, loggerLevel);
    SpdLoggerFactory::Instance().createLogger(loggerName)->set_level((level::level_enum)loggerLevel);
  } else if (in["command"] == "loggerGetAllLoggers") {
    receipt["loggers"] = SpdLoggerFactory::Instance().loggerNames();
  }

  /*********************************************************************************************************************
   * ================================================硬件测试=============================================================*
   *********************************************************************************************************************/
  else if (in["command"] == "getDeviceState") {
#if 0
{
    "deviceState":[
        {
            "id":"wind_speed",
            "value":"12",
            "unit":"m/s"
        }
    ]
}
#endif
    int index = 0;

    DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
    SET_DEVICE_STATE("wind_speed", env.wind_speed, "m/s");
    SET_DEVICE_STATE("wind_direction", env.wind_direction, "m/s");
    SET_DEVICE_STATE("temperature", env.temperature, "℃");
    SET_DEVICE_STATE("humidity", env.humidity, "%");
    SET_DEVICE_STATE("noise", env.noise, "dB");
    SET_DEVICE_STATE("pm2_5", env.pm2_5, "ug/m³");
    SET_DEVICE_STATE("pm10", env.pm10, "ug/m³");
    SET_DEVICE_STATE("co2", env.co2, "ppm");
    SET_DEVICE_STATE("atmospheric_pressure", env.atmospheric_pressure, "hPa");
    SET_DEVICE_STATE("tvoc", env.tvoc, "ppm");
    SET_DEVICE_STATE("hcho", env.hcho, "mg/m³");
    SET_DEVICE_STATE("light_intensity", env.atmospheric_pressure, "lux");
    SET_DEVICE_STATE("InterTemperature", m_deviceIOService->getInterTemperature(), "℃");

    SET_DEVICE_STATE("fan1powerRate", m_deviceIOService->fanGetState(0), "%");
    SET_DEVICE_STATE("fan2powerRate", m_deviceIOService->fanGetState(1), "%");

    auto relaystate = m_deviceIOService->relayStateGet();
    SET_DEVICE_STATE("routerPower", relaystate.getState(DeviceIOService::kRouterPower), "-");
    SET_DEVICE_STATE("touchScreenPower", relaystate.getState(DeviceIOService::kTouchScreenPower), "-");
    SET_DEVICE_STATE("usbChargerPower", relaystate.getState(DeviceIOService::kUsbChargerPower), "-");
    SET_DEVICE_STATE("cameraPower", relaystate.getState(DeviceIOService::kCameraPower), "-");
    SET_DEVICE_STATE("lightPower", relaystate.getState(DeviceIOService::kLightPower), "-");

    auto inputdeviceState = m_deviceIOService->getinputState();
    SET_DEVICE_STATE("emergency", inputdeviceState.getState(DeviceIOService::kEmergency), "-");
    SET_DEVICE_STATE("waterImmersionSensor", inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor), "-");
    SET_DEVICE_STATE("humanProximitySensor", inputdeviceState.getState(DeviceIOService::kHumanProximitySensor), "-");
  } else if (in["command"] == "relayControl") {
    uint32_t type  = in["type"];
    bool     value = in["value"];
    m_deviceIOService->relayControl(type, value);
  } else if (in["command"] == "fanSetState") {
    int   id    = in["id"];
    float power = in["power"];
    m_deviceIOService->fanSetState(id, power);
  } else if (in["command"] == "relayStateGet") {
    auto relaystate                      = m_deviceIOService->relayStateGet();
    receipt["state"]["rawState"]         = StringUtils().bytet2Binary(relaystate.state, 32, false);
    receipt["state"]["routerPower"]      = relaystate.getState(DeviceIOService::kRouterPower);
    receipt["state"]["touchScreenPower"] = relaystate.getState(DeviceIOService::kTouchScreenPower);
    receipt["state"]["usbChargerPower"]  = relaystate.getState(DeviceIOService::kUsbChargerPower);
    receipt["state"]["cameraPower"]      = relaystate.getState(DeviceIOService::kCameraPower);
    receipt["state"]["lightPower"]       = relaystate.getState(DeviceIOService::kLightPower);
  } else if (in["command"] == "inputStateGet") {
    auto inputdeviceState = m_deviceIOService->getinputState();
    // logger->info("{}", inputdeviceState.state);
    receipt["state"]["rawState"]             = StringUtils().bytet2Binary(inputdeviceState.state, 32, false);
    receipt["state"]["emergency"]            = inputdeviceState.getState(DeviceIOService::kEmergency);
    receipt["state"]["waterImmersionSensor"] = inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor);
    receipt["state"]["humanProximitySensor"] = inputdeviceState.getState(DeviceIOService::kHumanProximitySensor);
  } else if (in["command"] == "idcardread") {
    bool   state = false;
    string info;
    m_deviceIOService->idcardread(state, info);
    receipt["state"] = state;
    receipt["info"]  = info;
  } else if (in["command"] == "getInterTemperature") {
    receipt["temperature"] = m_deviceIOService->getInterTemperature();
  } else if (in["command"] == "getEnvSensorState") {
    DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
    receipt["wind_speed"]                   = env.wind_speed;
    receipt["wind_direction"]               = env.wind_direction;
    receipt["temperature"]                  = env.temperature;
    receipt["humidity"]                     = env.humidity;
    receipt["noise"]                        = env.noise;
    receipt["pm2_5"]                        = env.pm2_5;
    receipt["pm10"]                         = env.pm10;
    receipt["co2"]                          = env.co2;
    receipt["atmospheric_pressure"]         = env.atmospheric_pressure;
    receipt["tvoc"]                         = env.tvoc;
    receipt["hcho"]                         = env.hcho;
    receipt["light_intensity"]              = env.light_intensity;
  } else if (in["command"] == "fanGetState") {
    // int id = in["id"];
    receipt["power"][0] = m_deviceIOService->fanGetState(0);
    receipt["power"][1] = m_deviceIOService->fanGetState(0);
  }
}