You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
368 lines
15 KiB
368 lines
15 KiB
#include "main_control_service.hpp"
|
|
|
|
#include "iflytopcpp/core/components/stringutils.hpp"
|
|
#include "version.hpp"
|
|
using namespace iflytop;
|
|
using namespace core;
|
|
using namespace std;
|
|
|
|
/***********************************************************************************************************************
|
|
* ======================================================宏定义======================================================= *
|
|
***********************************************************************************************************************/
|
|
|
|
#define SET_DEVICE_STATE(id, value, uint) \
|
|
receipt["deviceState"][index]["id"] = id; \
|
|
receipt["deviceState"][index]["value"] = fmt::format("{}", value); \
|
|
receipt["deviceState"][index]["unit"] = uint; \
|
|
index++;
|
|
|
|
/*********************************************************************************************************************
|
|
* ======================================================代码======================================================= *
|
|
*********************************************************************************************************************/
|
|
|
|
void MainControlService::initializeVoiceProcess() {
|
|
m_beforeWakeupVoiceProcesser->setAmplifyDB(20);
|
|
m_beforeasrVoiceProcesser->setAmplifyDB(24);
|
|
|
|
logger->info("MainControlService::start.....");
|
|
|
|
/**
|
|
* @brief 声卡录音回调
|
|
* 1路,交给唤醒词预处理算法
|
|
* 1路,在唤醒的时候交给ASR预处理算法
|
|
*/
|
|
m_audioRecoderService->onRecordData.connect([this](shared_ptr<AudioClip> audioClip) {
|
|
if (!audioClip) {
|
|
logger->error("onRecordData audioClip is null");
|
|
return;
|
|
}
|
|
m_audioLoggingService->loggerMICVoice(audioClip); // 录音
|
|
m_commonVoiceProcesser->writeVoice(audioClip); // 交给通用预处理逻辑
|
|
});
|
|
|
|
m_commonVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
|
|
if (!audioClip) {
|
|
logger->error("onAfterProcessVoice audioClip is null");
|
|
return;
|
|
}
|
|
|
|
m_beforeWakeupVoiceProcesser->writeVoice(audioClip); // 交给唤醒词预处理逻辑
|
|
if (m_conversationSession) m_beforeasrVoiceProcesser->writeVoice(audioClip); // 交给asr预处理逻辑
|
|
});
|
|
|
|
/**
|
|
* @brief 唤醒词预处理算法回调
|
|
* 1路交给唤醒词算法
|
|
*/
|
|
m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
|
|
if (!audioClip) {
|
|
logger->error("onAfterProcessVoice audioClip is null");
|
|
return;
|
|
}
|
|
|
|
m_audioLoggingService->loggerBeforeWakeupVoice(audioClip);
|
|
m_wakeupProcesser->processVoice(audioClip->data(), audioClip->size());
|
|
});
|
|
|
|
/**
|
|
* @brief 唤醒词触发回调
|
|
*/
|
|
m_wakeupProcesser->onWakeupSignal.connect([this](float wakeup_score) {
|
|
logger->info("onWakeupSignal wakeup_score {}", wakeup_score);
|
|
/**
|
|
* @brief 触发唤醒，构造session
|
|
*/
|
|
m_workQueue->enQueue([this]() { constructSession(); });
|
|
});
|
|
|
|
/**
|
|
* @brief asr预处理算法回调
|
|
*/
|
|
m_beforeasrVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
|
|
if (!audioClip) {
|
|
logger->error("onAfterProcessVoice audioClip is null");
|
|
return;
|
|
}
|
|
auto session = m_conversationSession;
|
|
|
|
if (!session) return;
|
|
|
|
if (zsteady_clock().elapsedTimeMs(session->getBuildtp()) < 1000) return;
|
|
|
|
m_audioLoggingService->loggerASRVoice(audioClip);
|
|
m_aiuiService->aiuiWrite((const char*)audioClip->data(), audioClip->size());
|
|
});
|
|
|
|
m_aiuiService->onMessage.connect([&](json& rxjson) {
|
|
lock_guard<recursive_mutex> lock(m_voiceprocessmutex);
|
|
|
|
json msg = rxjson;
|
|
m_workQueue->enQueue([this, msg]() {
|
|
try {
|
|
processasrResult(msg);
|
|
} catch (const std::exception& e) {
|
|
logger->error("processasrResult error:{}", e.what());
|
|
}
|
|
});
|
|
});
|
|
|
|
m_audioRecoderService->startRecord();
|
|
}
|
|
|
|
void MainControlService::triggerProcessConversationSession() {
|
|
m_smartSoundboxPlayer->playConversationTTS(m_conversationSession->getAsrTTSLocalURL(), nullptr);
|
|
json nlpResult = m_conversationSession->getNlpResult();
|
|
if (nlpResult["data"]["intent"]["shouldEndSession"] == "true") {
|
|
logger->info("endSession");
|
|
if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
|
|
endSession();
|
|
return;
|
|
}
|
|
}
|
|
|
|
void MainControlService::processasrResult_nlp(json& rxjson) {
|
|
logger->info("rx nlp:{}", rxjson.dump());
|
|
m_conversationSession->setNlpResult(rxjson);
|
|
m_smartSoundboxPlayer->playConversationTTS(m_conversationSession->getAsrTTSLocalURL(), nullptr);
|
|
}
|
|
void MainControlService::processasrResult_tts(json& rxjson) {
|
|
/**
|
|
* @brief 接收从云端发过来的tts数据，并保存成文件
|
|
*/
|
|
logger->debug("rx tts:frame {}", rxjson["data"]["json_args"]["frame_id"].get<int>());
|
|
bool isendFrame = false;
|
|
string ttsurl;
|
|
m_aiuiService->parseTTSContent(rxjson, isendFrame, ttsurl);
|
|
if (isendFrame) {
|
|
logger->info("rx tts end,url={}", ttsurl);
|
|
m_conversationSession->setAsrTTSLocalURL(ttsurl);
|
|
triggerProcessConversationSession();
|
|
}
|
|
}
|
|
|
|
void MainControlService::processasrResult(json rxjson) {
|
|
lock_guard<recursive_mutex> lock(m_voiceprocessmutex);
|
|
|
|
string action = rxjson["action"];
|
|
if (action == "started") {
|
|
logger->info("processasr: rx started:{}", rxjson.dump());
|
|
} else if (action == "error") {
|
|
logger->info("processasr: rx error:{}", rxjson.dump());
|
|
} else if (action == "vad") {
|
|
logger->info("processasr: rx vad:{}", rxjson.dump());
|
|
} else if (action == "result") {
|
|
string sub = rxjson["data"]["sub"];
|
|
if (sub == "nlp") {
|
|
/**
|
|
* @brief 解析nlp数据
|
|
*/
|
|
processasrResult_nlp(rxjson);
|
|
} else if (sub == "tts") {
|
|
/**
|
|
* @brief 解析tts数据
|
|
*/
|
|
processasrResult_tts(rxjson);
|
|
} else if (sub == "iat") {
|
|
logger->info("rx iat:{}", rxjson.dump());
|
|
} else if (sub == "vad") {
|
|
logger->info("rx vad:{}", rxjson.dump());
|
|
} else {
|
|
logger->info("rx {}:{}", sub, rxjson.dump());
|
|
}
|
|
} else {
|
|
logger->info("rx unkown:{}", rxjson.dump());
|
|
}
|
|
}
|
|
|
|
void MainControlService::constructSession() {
|
|
/**
|
|
* @brief
|
|
* 1. 触发唤醒词语
|
|
* 2. 如果上一次session还没有结束,则结束上一次session
|
|
* 3. 初始化asr
|
|
* 4. 构建新的session
|
|
* 5. 启动定时器,如果10s内无人应答，则结束session
|
|
*/
|
|
|
|
lock_guard<recursive_mutex> lock(m_voiceprocessmutex);
|
|
|
|
m_smartSoundboxPlayer->triggerWakeup();
|
|
if (m_conversationSession) {
|
|
if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
|
|
endSession();
|
|
}
|
|
|
|
m_conversationSession = make_shared<ConversationSession>();
|
|
logger->info("constructSession:============ {} ===========", m_conversationSession->getSessionId());
|
|
m_audioLoggingService->triggerWakeup(m_conversationSession->getSessionId());
|
|
m_aiuiService->aiuiInit();
|
|
|
|
if (m_endsessionTimer->isRunning()) m_endsessionTimer->stop();
|
|
|
|
m_endsessionTimer->setTimeout(
|
|
[this]() {
|
|
logger->info("no hum voice detected, end session");
|
|
endSession();
|
|
},
|
|
10000);
|
|
}
|
|
|
|
void MainControlService::endSession() {
|
|
lock_guard<recursive_mutex> lock(m_voiceprocessmutex);
|
|
|
|
if (m_conversationSession) {
|
|
m_conversationSession = nullptr;
|
|
m_aiuiService->aiuiFinished();
|
|
m_aiuiService->aiuiDestroy();
|
|
}
|
|
m_audioLoggingService->endwakeup();
|
|
}
|
|
|
|
void MainControlService::initialize() {
|
|
GET_TO_SERVICE(m_zwebService);
|
|
GET_TO_SERVICE(m_deviceIOService);
|
|
|
|
GET_TO_SERVICE(m_beforeWakeupVoiceProcesser);
|
|
GET_TO_SERVICE(m_beforeasrVoiceProcesser);
|
|
GET_TO_SERVICE(m_wakeupProcesser);
|
|
GET_TO_SERVICE(m_audioRecoderService);
|
|
GET_TO_SERVICE(m_audioLoggingService);
|
|
GET_TO_SERVICE(m_smartSoundboxPlayer);
|
|
GET_TO_SERVICE(m_aiuiService);
|
|
GET_TO_SERVICE(m_commonVoiceProcesser);
|
|
|
|
m_endsessionTimer.reset(new SimpleTimer("endSessionTimer"));
|
|
m_workQueue.reset(new WorkQueue("mainControlServiceVoiceProcessCtrlWQ"));
|
|
|
|
// 监听从webservice来的websocket消息
|
|
m_zwebService->startWork([this](const json& command, json& receipt) {
|
|
try {
|
|
processReceiveMessage(kzwebService, command, receipt);
|
|
} catch (const std::exception& e) {
|
|
logger->error("process message fail {}", string(e.what()));
|
|
} catch (...) {
|
|
logger->error("process message fail {}", "catch unknown exception");
|
|
}
|
|
});
|
|
|
|
/**
|
|
* @brief 初始化语音处理流程
|
|
*/
|
|
initializeVoiceProcess();
|
|
};
|
|
void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json& in, json& receipt) {
|
|
logger->info("process receive message from {},{}", fromwhere2str(fromwhere), in.dump());
|
|
|
|
if (in["command"] == "getVersion") {
|
|
receipt["version"] = VERSION;
|
|
}
|
|
|
|
/*********************************************************************************************************************
|
|
* ================================================日志动态配置接口================================================= *
|
|
*********************************************************************************************************************/
|
|
else if (in["command"] == "loggerSetLevel") {
|
|
int loggerLevel = in["loggerLevel"];
|
|
string loggerName = in["loggerName"];
|
|
logger->info("loggerSetLevel {} {}", loggerName, loggerLevel);
|
|
SpdLoggerFactory::Instance().createLogger(loggerName)->set_level((level::level_enum)loggerLevel);
|
|
} else if (in["command"] == "loggerGetAllLoggers") {
|
|
receipt["loggers"] = SpdLoggerFactory::Instance().loggerNames();
|
|
}
|
|
|
|
/*********************************************************************************************************************
|
|
* ================================================硬件测试=============================================================*
|
|
*********************************************************************************************************************/
|
|
else if (in["command"] == "getDeviceState") {
|
|
#if 0
|
|
{
|
|
"deviceState":[
|
|
{
|
|
"id":"wind_speed",
|
|
"value":"12",
|
|
"unit":"m/s"
|
|
}
|
|
]
|
|
}
|
|
#endif
|
|
int index = 0;
|
|
|
|
DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
|
|
SET_DEVICE_STATE("wind_speed", env.wind_speed, "m/s");
|
|
SET_DEVICE_STATE("wind_direction", env.wind_direction, "m/s");
|
|
SET_DEVICE_STATE("temperature", env.temperature, "℃");
|
|
SET_DEVICE_STATE("humidity", env.humidity, "%");
|
|
SET_DEVICE_STATE("noise", env.noise, "dB");
|
|
SET_DEVICE_STATE("pm2_5", env.pm2_5, "ug/m³");
|
|
SET_DEVICE_STATE("pm10", env.pm10, "ug/m³");
|
|
SET_DEVICE_STATE("co2", env.co2, "ppm");
|
|
SET_DEVICE_STATE("atmospheric_pressure", env.atmospheric_pressure, "hPa");
|
|
SET_DEVICE_STATE("tvoc", env.tvoc, "ppm");
|
|
SET_DEVICE_STATE("hcho", env.hcho, "mg/m³");
|
|
SET_DEVICE_STATE("light_intensity", env.atmospheric_pressure, "lux");
|
|
SET_DEVICE_STATE("InterTemperature", m_deviceIOService->getInterTemperature(), "℃");
|
|
|
|
SET_DEVICE_STATE("fan1powerRate", m_deviceIOService->fanGetState(0), "%");
|
|
SET_DEVICE_STATE("fan2powerRate", m_deviceIOService->fanGetState(1), "%");
|
|
|
|
auto relaystate = m_deviceIOService->relayStateGet();
|
|
SET_DEVICE_STATE("routerPower", relaystate.getState(DeviceIOService::kRouterPower), "-");
|
|
SET_DEVICE_STATE("touchScreenPower", relaystate.getState(DeviceIOService::kTouchScreenPower), "-");
|
|
SET_DEVICE_STATE("usbChargerPower", relaystate.getState(DeviceIOService::kUsbChargerPower), "-");
|
|
SET_DEVICE_STATE("cameraPower", relaystate.getState(DeviceIOService::kCameraPower), "-");
|
|
SET_DEVICE_STATE("lightPower", relaystate.getState(DeviceIOService::kLightPower), "-");
|
|
|
|
auto inputdeviceState = m_deviceIOService->getinputState();
|
|
SET_DEVICE_STATE("emergency", inputdeviceState.getState(DeviceIOService::kEmergency), "-");
|
|
SET_DEVICE_STATE("waterImmersionSensor", inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor), "-");
|
|
SET_DEVICE_STATE("humanProximitySensor", inputdeviceState.getState(DeviceIOService::kHumanProximitySensor), "-");
|
|
} else if (in["command"] == "relayControl") {
|
|
uint32_t type = in["type"];
|
|
bool value = in["value"];
|
|
m_deviceIOService->relayControl(type, value);
|
|
} else if (in["command"] == "fanSetState") {
|
|
int id = in["id"];
|
|
float power = in["power"];
|
|
m_deviceIOService->fanSetState(id, power);
|
|
} else if (in["command"] == "relayStateGet") {
|
|
auto relaystate = m_deviceIOService->relayStateGet();
|
|
receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false);
|
|
receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower);
|
|
receipt["state"]["touchScreenPower"] = relaystate.getState(DeviceIOService::kTouchScreenPower);
|
|
receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower);
|
|
receipt["state"]["cameraPower"] = relaystate.getState(DeviceIOService::kCameraPower);
|
|
receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower);
|
|
} else if (in["command"] == "inputStateGet") {
|
|
auto inputdeviceState = m_deviceIOService->getinputState();
|
|
// logger->info("{}", inputdeviceState.state);
|
|
receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false);
|
|
receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency);
|
|
receipt["state"]["waterImmersionSensor"] = inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor);
|
|
receipt["state"]["humanProximitySensor"] = inputdeviceState.getState(DeviceIOService::kHumanProximitySensor);
|
|
} else if (in["command"] == "idcardread") {
|
|
bool state = false;
|
|
string info;
|
|
m_deviceIOService->idcardread(state, info);
|
|
receipt["state"] = state;
|
|
receipt["info"] = info;
|
|
} else if (in["command"] == "getInterTemperature") {
|
|
receipt["temperature"] = m_deviceIOService->getInterTemperature();
|
|
} else if (in["command"] == "getEnvSensorState") {
|
|
DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
|
|
receipt["wind_speed"] = env.wind_speed;
|
|
receipt["wind_direction"] = env.wind_direction;
|
|
receipt["temperature"] = env.temperature;
|
|
receipt["humidity"] = env.humidity;
|
|
receipt["noise"] = env.noise;
|
|
receipt["pm2_5"] = env.pm2_5;
|
|
receipt["pm10"] = env.pm10;
|
|
receipt["co2"] = env.co2;
|
|
receipt["atmospheric_pressure"] = env.atmospheric_pressure;
|
|
receipt["tvoc"] = env.tvoc;
|
|
receipt["hcho"] = env.hcho;
|
|
receipt["light_intensity"] = env.light_intensity;
|
|
} else if (in["command"] == "fanGetState") {
|
|
// int id = in["id"];
|
|
receipt["power"][0] = m_deviceIOService->fanGetState(0);
|
|
receipt["power"][1] = m_deviceIOService->fanGetState(0);
|
|
}
|
|
}
|