From fb159606db9217b8f2f5270ee96544216626ce75 Mon Sep 17 00:00:00 2001 From: zhaohe Date: Wed, 15 Mar 2023 21:10:09 +0800 Subject: [PATCH] update --- .gitmodules | 5 +- CMakeLists.txt | 2 +- Linux测试指令说明.md | 3 + README.md | 2 +- buildtools | 1 + dep/zlinuxcomponents | 2 +- env/rootfs/etc/asound.conf | 20 +---- sh/packet.sh | 2 +- src/configs/config.hpp | 29 -------- src/configs/zconfig.hpp | 29 ++++++++ src/main.cpp | 82 +++++++++++++++----- src/service/device_io_service.hpp | 4 +- src/service/light_control_service.hpp | 6 +- src/service/main_control_service.cpp | 87 ++++++++++++++-------- src/service/main_control_service.hpp | 19 ++++- src/service/report_service.hpp | 5 +- src/service/voiceprocess/audio_logging_service.cpp | 26 ++++++- src/service/voiceprocess/audio_logging_service.hpp | 7 +- src/service/voiceprocess/audio_recoder_service.cpp | 4 + src/service/voiceprocess/audio_recoder_service.hpp | 4 +- .../voiceprocess/beforeasr_voiceprocesser.cpp | 22 +++--- .../voiceprocess/beforeasr_voiceprocesser.hpp | 8 +- .../voiceprocess/beforewakeup_voiceprocesser.cpp | 25 +++---- .../voiceprocess/beforewakeup_voiceprocesser.hpp | 8 +- src/service/voiceprocess/voiceprocess_service.cpp | 25 ------- src/service/voiceprocess/voiceprocess_service.hpp | 51 ------------- src/service/voiceprocess/wakeup_processer.cpp | 6 +- src/service/voiceprocess/wakeup_processer.hpp | 2 +- src/test_aiui_service.cpp | 2 +- 29 files changed, 257 insertions(+), 231 deletions(-) create mode 100644 Linux测试指令说明.md create mode 160000 buildtools delete mode 100644 src/configs/config.hpp create mode 100644 src/configs/zconfig.hpp delete mode 100644 src/service/voiceprocess/voiceprocess_service.cpp delete mode 100644 src/service/voiceprocess/voiceprocess_service.hpp diff --git a/.gitmodules b/.gitmodules index 7facb89..3d2a5f5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,4 +21,7 @@ url = ../../z2nd_lib/zclipp.git [submodule "dep/zlinuxcomponents"] path = dep/zlinuxcomponents - url = zwsd@192.168.1.3:z2nd_lib/zlinuxcomponents.git \ No newline at end of file + url = zwsd@192.168.1.3:z2nd_lib/zlinuxcomponents.git +[submodule "buildtools"] + path = buildtools + url = zwsd@192.168.1.3:z2nd_lib/iflytopbuildtools.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f1aad0..86bfb4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,9 +94,9 @@ zadd_executable_simple( src/service/report_service.cpp src/service/voiceprocess/beforeasr_voiceprocesser.cpp src/service/voiceprocess/beforewakeup_voiceprocesser.cpp - src/service/voiceprocess/voiceprocess_service.cpp src/service/voiceprocess/wakeup_processer.cpp src/service/voiceprocess/audio_recoder_service.cpp + src/service/voiceprocess/audio_logging_service.cpp # aiui ) diff --git a/Linux测试指令说明.md b/Linux测试指令说明.md new file mode 100644 index 0000000..9271720 --- /dev/null +++ b/Linux测试指令说明.md @@ -0,0 +1,3 @@ +录音 + +arecord -Drecorder -c5 -r16000 -fS16_LE test.wav \ No newline at end of file diff --git a/README.md b/README.md index a26d51d..cc90349 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ build/ 应用程序编译目录 ./sh/iflytool.sh --ip flash reboot # 编译并只部署应用 -./sh/iflytool.sh --ip flashapp +./sh/iflytool.sh --ip --flashapp "app.out" # 只部署webapp ./sh/iflytool.sh --ip flashwebapp diff --git a/buildtools b/buildtools new file mode 160000 index 0000000..2c5fa9e --- /dev/null +++ b/buildtools @@ -0,0 +1 @@ +Subproject commit 2c5fa9ee781af8c21d16ecaf50bfbab8ba0e137b diff --git a/dep/zlinuxcomponents b/dep/zlinuxcomponents index 99b2797..83404cd 160000 --- a/dep/zlinuxcomponents +++ b/dep/zlinuxcomponents @@ -1 +1 @@ -Subproject commit 99b27976a540084bb76a2d3cea03433dd8b8adba +Subproject commit 83404cdad908c9db64a161bef14dd1bbd2bcbf5d diff --git a/env/rootfs/etc/asound.conf b/env/rootfs/etc/asound.conf index 75329e6..a38dc85 100644 --- a/env/rootfs/etc/asound.conf +++ b/env/rootfs/etc/asound.conf @@ -35,7 +35,7 @@ pcm.default_playback { } min_dB -40.0 max_dB 0.0 - resolution 100 + resolution 256 } pcm.wakeup_playback { type softvol @@ -48,7 +48,7 @@ pcm.wakeup_playback { } min_dB -40.0 max_dB 0.0 - resolution 100 + resolution 256 } # 放音设备 pcm.xa8326 { @@ -75,18 +75,6 @@ pcm.xa8326 { pcm.recorder { type plug slave { - pcm "recorder5ch" - rate 48000 - format S16_LE - channels 5 - } - rate 16000 - format S16_LE - channels 5 -} -pcm.recorder5ch { - type plug - slave { pcm { type multi slaves.xa8326.pcm "hw:2,0" @@ -101,10 +89,10 @@ pcm.recorder5ch { bindings.1.channel 1 bindings.2.slave pdm - bindings.2.channel 5 + bindings.2.channel 2 bindings.3.slave pdm - bindings.3.channel 6 + bindings.3.channel 3 bindings.4.slave xa8326 bindings.4.channel 0 diff --git a/sh/packet.sh b/sh/packet.sh index 94b01d8..3467477 100755 --- a/sh/packet.sh +++ b/sh/packet.sh @@ -33,7 +33,7 @@ PACKET_DIR=$PROJECT_PATH/out/app # 删除打包目录,重新打包 rm -rf ${PACKET_DIR} -mkdir ${PACKET_DIR} +mkdir -p ${PACKET_DIR} ######################################################################################## ################################根据需要修改下面的内容即可################################## ######################################################################################## diff --git a/src/configs/config.hpp b/src/configs/config.hpp deleted file mode 100644 index 9854fd0..0000000 --- a/src/configs/config.hpp +++ /dev/null @@ -1,29 +0,0 @@ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "iflytopcpp/core/components/config_template/config_template.hpp" - -#define ConfigELEMENT_LIST(marco) \ - /**/ \ - marco(string /* */, deviceId, "") /**/ \ - marco(string /* */, lightControlMode, "auto") /*照明灯控制模式 auto/manual*/ \ - marco(int /* */, lightAutoOpenHour, 18) /**/ \ - marco(int /* */, lightAutoOpenMin, 0) /**/ \ - marco(int /* */, lightAutoCloseHour, 6) /**/ \ - marco(int /* */, lightAutoCloseMin, 0) /**/ - -configTemplateDEFILE_CONFIG_SERVICE2( // - Config, // - ConfigELEMENT_LIST, // - "./configs/config.json", {}); diff --git a/src/configs/zconfig.hpp b/src/configs/zconfig.hpp new file mode 100644 index 0000000..fd60fee --- /dev/null +++ b/src/configs/zconfig.hpp @@ -0,0 +1,29 @@ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytopcpp/core/components/config_template/config_template.hpp" + +#define ConfigELEMENT_LIST(marco) \ + /**/ \ + marco(string /* */, deviceId, "") /**/ \ + marco(string /* */, lightControlMode, "auto") /*照明灯控制模式 auto/manual*/ \ + marco(int /* */, lightAutoOpenHour, 18) /**/ \ + marco(int /* */, lightAutoOpenMin, 0) /**/ \ + marco(int /* */, lightAutoCloseHour, 6) /**/ \ + marco(int /* */, lightAutoCloseMin, 0) /**/ + +configTemplateDEFILE_CONFIG_SERVICE2( // + ZConfig, // + ConfigELEMENT_LIST, // + "./configs/config.json", {}); diff --git a/src/main.cpp b/src/main.cpp index f16abc5..1e2eee3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,7 +8,7 @@ #include "zlinuxcomponents/alsaplayer/smart_soundbox_player.hpp" #include "zlinuxcomponents/zmainhelper.hpp" // -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "version.hpp" @@ -21,10 +21,10 @@ #include "service/light_control_service.hpp" #include "service/main_control_service.hpp" #include "service/report_service.hpp" +#include "service/voiceprocess/audio_logging_service.hpp" #include "service/voiceprocess/audio_recoder_service.hpp" #include "service/voiceprocess/beforeasr_voiceprocesser.hpp" #include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" -#include "service/voiceprocess/voiceprocess_service.hpp" #include "service/voiceprocess/wakeup_processer.hpp" // @@ -32,13 +32,44 @@ using namespace iflytop; using namespace core; using namespace std; using namespace clipp; +class Main { + ENABLE_LOGGER(Main); + + private: + unique_ptr thread; + + private: + int main(int argc, char *argv[]); + + public: + Main(/* args */) {} + ~Main() {} + void run(int argc, char *argv[]) { + thread.reset(new Thread("main", [&]() { exit(main(argc, argv)); })); + while (true) sleep(1000); + } + void appSetup(int argc, char *argv[]); + void updateRootfs(); + void buildDeviceIOService(); + void buildVoiceProcessService(); + void onSIGINT(); +}; + +Main *g_main; +void onsignal(int signo) { g_main->onSIGINT(); } +int main(int argc, char *argv[]) { + Main main; + g_main = &main; + ::signal(SIGINT, onsignal); + main.run(argc, argv); +} -ZMAIN(); /*********************************************************************************************************************** * =======================================================Main======================================================== * ***********************************************************************************************************************/ void Main::onSIGINT() { exit(0); } -int Main::main(int argc, char *argv[]) { + +void Main::appSetup(int argc, char *argv[]) { string g_host_server_ip; string g_device_id; #if 0 @@ -61,11 +92,12 @@ int Main::main(int argc, char *argv[]) { // BUILD_AND_REG_SERRVICE(Config); logger->info("build {}.....", "Config"); - shared_ptr Config_val(new iflytop::Config()); - ServiceContrainer::get().regService(Config_val); - GET_SERVICE(iflytop::Config)->initialize(); - if (!g_device_id.empty()) GET_SERVICE(iflytop::Config)->set_deviceId(g_device_id); + BUILD_AND_REG_SERRVICE(ZConfig); + GET_SERVICE(ZConfig)->initialize(); + if (!g_device_id.empty()) GET_SERVICE(ZConfig)->set_deviceId(g_device_id); +} +void Main::updateRootfs() { /** * @brief 比较rootfs目录中的文件和系统中对应路径的文件是否一致，如果不一致则替换 * @@ -81,7 +113,9 @@ int Main::main(int argc, char *argv[]) { logger->warn("not auto reboot, please reboot manually."); // system("reboot"); } +} +void Main::buildDeviceIOService() { /** * @brief DeviceIOService初始化 */ @@ -91,26 +125,32 @@ int Main::main(int argc, char *argv[]) { BUILD_AND_REG_MOCK_SERRVICE(DeviceIOService, DeviceIOServiceMock); #endif GET_SERVICE(DeviceIOService)->initialize(); +} +void Main::buildVoiceProcessService() { /** * @brief 音频处理相关组建初始化 */ + // AudioLoggingService + BUILD_AND_REG_SERRVICE(AudioLoggingService); + GET_SERVICE(AudioLoggingService)->initialize(); + // 音频采集 BUILD_AND_REG_SERRVICE(AudioRecoderService); GET_SERVICE(AudioRecoderService)->initialize("recorder", 5, 16000, SND_PCM_FORMAT_S16_LE); - // BeforeWakeupVoiceProcesser - BUILD_AND_REG_SERRVICE(BeforeWakeupVoiceProcesser); - GET_SERVICE(BeforeWakeupVoiceProcesser)->initialize(); + // BfWakeupVProcesser + BUILD_AND_REG_SERRVICE(BfWakeupVProcesser); + GET_SERVICE(BfWakeupVProcesser)->initialize(); // BeforeAsrVoiceProcesser - BUILD_AND_REG_SERRVICE(BeforeasrVoiceProcesser); - GET_SERVICE(BeforeasrVoiceProcesser)->initialize(); + BUILD_AND_REG_SERRVICE(BfAsrVProcesser); + GET_SERVICE(BfAsrVProcesser)->initialize(); // WakeupProcesser BUILD_AND_REG_SERRVICE(WakeupProcesser); - GET_SERVICE(WakeupProcesser)->initialize("xiaomaoxiaomao/xiaomaoxiaomao.pb"); + GET_SERVICE(WakeupProcesser)->initialize("./precise/precise-engine", "xiaomaoxiaomao/xiaomaoxiaomao.pb"); // AsrProcesser const char *appid = "5938b7c7"; // 应用ID，在AIUI开放平台创建并设置 @@ -124,16 +164,18 @@ int Main::main(int argc, char *argv[]) { paramj["context"] = R"({"sdk_support":["nlp","tts","vad","iat"]})"; BUILD_AND_REG_SERRVICE(AiuiService); GET_SERVICE(AiuiService)->initialize(appid, key, paramj.dump()); +} + +int Main::main(int argc, char *argv[]) { + appSetup(argc, argv); + updateRootfs(); + buildDeviceIOService(); + buildVoiceProcessService(); + // while (true) sleep(1000); // 播放器-SmartSoundBoxPlayer BUILD_AND_REG_SERRVICE(SmartSoundBoxPlayer); - // GET_SERVICE(SmartSoundBoxPlayer)->initialize(80); - - // VoiceProcessService - BUILD_AND_REG_SERRVICE(VoiceProcessService); - GET_SERVICE(VoiceProcessService)->initialize(); - /** * @brief 其他服务 */ diff --git a/src/service/device_io_service.hpp b/src/service/device_io_service.hpp index c007865..2c460d8 100644 --- a/src/service/device_io_service.hpp +++ b/src/service/device_io_service.hpp @@ -13,10 +13,10 @@ #include #include -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" // -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/basic/nod/nod.hpp" #include "iflytopcpp/core/components/modbus/modbus.hpp" #include "iflytopcpp/core/thread/thread.hpp" diff --git a/src/service/light_control_service.hpp b/src/service/light_control_service.hpp index 9b0c008..93b3b7e 100644 --- a/src/service/light_control_service.hpp +++ b/src/service/light_control_service.hpp @@ -13,10 +13,10 @@ #include #include -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" // -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "service/device_io_service.hpp" #include "zservice_container/zservice_container.hpp" @@ -35,7 +35,7 @@ using namespace core; class LightControlService : public enable_shared_from_this { ENABLE_LOGGER(LightControlService); - shared_ptr config; + shared_ptr config; unique_ptr thread; shared_ptr deviceIoService; diff --git a/src/service/main_control_service.cpp b/src/service/main_control_service.cpp index e930115..d1d36d2 100644 --- a/src/service/main_control_service.cpp +++ b/src/service/main_control_service.cpp @@ -11,9 +11,9 @@ using namespace std; ***********************************************************************************************************************/ #define SET_DEVICE_STATE(id, value, uint) \ - receipt["deviceState"][index]["id"] = id; \ + receipt["deviceState"][index]["id"] = id; \ receipt["deviceState"][index]["value"] = fmt::format("{}", value); \ - receipt["deviceState"][index]["unit"] = uint; \ + receipt["deviceState"][index]["unit"] = uint; \ index++; /********************************************************************************************************************* @@ -24,6 +24,13 @@ void MainControlService::initialize() { GET_TO_SERVICE(m_zwebService); GET_TO_SERVICE(m_deviceIOService); + GET_TO_SERVICE(m_beforeWakeupVoiceProcesser); + GET_TO_SERVICE(m_beforeasrVoiceProcesser); + GET_TO_SERVICE(m_wakeupProcesser); + GET_TO_SERVICE(m_audioRecoderService); + GET_TO_SERVICE(m_audioLoggingService); + + // 监听从webservice来的websocket消息 m_zwebService->startWork([this](const json& command, json& receipt) { try { @@ -34,6 +41,27 @@ void MainControlService::initialize() { logger->error("process message fail {}", "catch unknown exception"); } }); + + m_audioRecoderService->onRecordData.connect([this](shared_ptr audioClip) { + if (!audioClip) { + logger->error("onRecordData audioClip is null"); + return; + } + logger->info("onRecordData {}", audioClip->size()); + m_beforeWakeupVoiceProcesser->writeVoice(audioClip); + m_audioLoggingService->loggerMICVoice(audioClip); + }); + + m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr audioClip) { + if (!audioClip) { + logger->error("onAfterProcessVoice audioClip is null"); + return; + } + m_audioLoggingService->loggerBeforeWakeupVoice(audioClip); + logger->info("onAfterProcessVoice {}", audioClip->size()); + }); + + m_audioRecoderService->startRecord(); }; void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json& in, json& receipt) { logger->info("process receive message from {},{}", fromwhere2str(fromwhere), in.dump()); @@ -46,8 +74,8 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json * ================================================日志动态配置接口================================================= * *********************************************************************************************************************/ else if (in["command"] == "loggerSetLevel") { - int loggerLevel = in["loggerLevel"]; - string loggerName = in["loggerName"]; + int loggerLevel = in["loggerLevel"]; + string loggerName = in["loggerName"]; logger->info("loggerSetLevel {} {}", loggerName, loggerLevel); SpdLoggerFactory::Instance().createLogger(loggerName)->set_level((level::level_enum)loggerLevel); } else if (in["command"] == "loggerGetAllLoggers") { @@ -70,6 +98,7 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json } #endif int index = 0; + DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState(); SET_DEVICE_STATE("wind_speed", env.wind_speed, "m/s"); SET_DEVICE_STATE("wind_direction", env.wind_direction, "m/s"); @@ -100,49 +129,49 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json SET_DEVICE_STATE("waterImmersionSensor", inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor), "-"); SET_DEVICE_STATE("humanProximitySensor", inputdeviceState.getState(DeviceIOService::kHumanProximitySensor), "-"); } else if (in["command"] == "relayControl") { - int type = in["type"]; + int type = in["type"]; bool value = in["value"]; m_deviceIOService->relayControl((DeviceIOService::relay_device_type_t)type, value); } else if (in["command"] == "fanSetState") { - int id = in["id"]; + int id = in["id"]; float power = in["power"]; m_deviceIOService->fanSetState(id, power); } else if (in["command"] == "relayStateGet") { - auto relaystate = m_deviceIOService->relayStateGet(); - receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false); - receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower); + auto relaystate = m_deviceIOService->relayStateGet(); + receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false); + receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower); receipt["state"]["touchScreenPower"] = relaystate.getState(DeviceIOService::kTouchScreenPower); - receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower); - receipt["state"]["cameraPower"] = relaystate.getState(DeviceIOService::kCameraPower); - receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower); + receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower); + receipt["state"]["cameraPower"] = relaystate.getState(DeviceIOService::kCameraPower); + receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower); } else if (in["command"] == "inputStateGet") { - auto inputdeviceState = m_deviceIOService->getinputState(); - receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false); - receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency); + auto inputdeviceState = m_deviceIOService->getinputState(); + receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false); + receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency); receipt["state"]["waterImmersionSensor"] = inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor); receipt["state"]["humanProximitySensor"] = inputdeviceState.getState(DeviceIOService::kHumanProximitySensor); } else if (in["command"] == "idcardread") { - bool state = false; + bool state = false; string info; m_deviceIOService->idcardread(state, info); receipt["state"] = state; - receipt["info"] = info; + receipt["info"] = info; } else if (in["command"] == "getInterTemperature") { receipt["temperature"] = m_deviceIOService->getInterTemperature(); } else if (in["command"] == "getEnvSensorState") { DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState(); - receipt["wind_speed"] = env.wind_speed; - receipt["wind_direction"] = env.wind_direction; - receipt["temperature"] = env.temperature; - receipt["humidity"] = env.humidity; - receipt["noise"] = env.noise; - receipt["pm2_5"] = env.pm2_5; - receipt["pm10"] = env.pm10; - receipt["co2"] = env.co2; - receipt["atmospheric_pressure"] = env.atmospheric_pressure; - receipt["tvoc"] = env.tvoc; - receipt["hcho"] = env.hcho; - receipt["light_intensity"] = env.light_intensity; + receipt["wind_speed"] = env.wind_speed; + receipt["wind_direction"] = env.wind_direction; + receipt["temperature"] = env.temperature; + receipt["humidity"] = env.humidity; + receipt["noise"] = env.noise; + receipt["pm2_5"] = env.pm2_5; + receipt["pm10"] = env.pm10; + receipt["co2"] = env.co2; + receipt["atmospheric_pressure"] = env.atmospheric_pressure; + receipt["tvoc"] = env.tvoc; + receipt["hcho"] = env.hcho; + receipt["light_intensity"] = env.light_intensity; } else if (in["command"] == "fanGetState") { // int id = in["id"]; receipt["power"][0] = m_deviceIOService->fanGetState(0); diff --git a/src/service/main_control_service.hpp b/src/service/main_control_service.hpp index 60b0d9b..b0a2211 100644 --- a/src/service/main_control_service.hpp +++ b/src/service/main_control_service.hpp @@ -13,13 +13,19 @@ #include #include -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" // -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "service/device_io_service.hpp" +#include "service/voiceprocess/audio_logging_service.hpp" +#include "service/voiceprocess/audio_recoder_service.hpp" +#include "service/voiceprocess/beforeasr_voiceprocesser.hpp" +#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" +#include "service/voiceprocess/wakeup_processer.hpp" #include "zservice_container/zservice_container.hpp" #include "zwebservice/zwebservice.hpp" + /** * @brief * @@ -38,8 +44,15 @@ using namespace core; class MainControlService : public enable_shared_from_this { ENABLE_LOGGER(MainControlService); - shared_ptr m_zwebService; + shared_ptr m_zwebService; shared_ptr m_deviceIOService; + + shared_ptr m_beforeWakeupVoiceProcesser; + shared_ptr m_beforeasrVoiceProcesser; + shared_ptr m_wakeupProcesser; + shared_ptr m_audioRecoderService; + shared_ptr m_audioLoggingService; + typedef enum { kzwebService, } fromwhere_t; diff --git a/src/service/report_service.hpp b/src/service/report_service.hpp index 55052f8..70815a0 100644 --- a/src/service/report_service.hpp +++ b/src/service/report_service.hpp @@ -13,10 +13,9 @@ #include #include -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" // -#include "configs/config.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "service/device_io_service.hpp" #include "zservice_container/zservice_container.hpp" @@ -39,7 +38,7 @@ using namespace core; class ReportService : public enable_shared_from_this { ENABLE_LOGGER(ReportService); - shared_ptr config; + shared_ptr config; unique_ptr thread; shared_ptr deviceIoService; diff --git a/src/service/voiceprocess/audio_logging_service.cpp b/src/service/voiceprocess/audio_logging_service.cpp index 8889091..c614c5b 100644 --- a/src/service/voiceprocess/audio_logging_service.cpp +++ b/src/service/voiceprocess/audio_logging_service.cpp @@ -32,6 +32,11 @@ void AudioLoggingService::cleanupLogVoiceByTime(string prefix, int maxnum) { } } +void AudioLoggingService::initialize() { + FileUtils().makeDirIfNoExist("./voice/rt/"); + FileUtils().makeDirIfNoExist("./voice/wakeup/"); +} + void AudioLoggingService::loggerMICVoice(shared_ptr audioClip) { /** * @brief MIC的语音会存储在两个地方 @@ -55,7 +60,7 @@ void AudioLoggingService::loggerMICVoice(shared_ptr audioClip) { if (!m_rtBfwakeupVoiceFile) { cleanupLogVoiceByTime("./voice/wakeup/mic_voice_*.wav", WAKEUP_STORAGE_CLIP_NUMS); m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/wakeup/mic_voice_" + gettimestamp() + ".wav")); - m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0); + // m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0); m_rtBfwakeupVoiceFile->writeVoice(audioClip); } else { m_rtBfwakeupVoiceFile->writeVoice(audioClip); @@ -72,5 +77,22 @@ void AudioLoggingService::setWakeupState(bool state) { } } -void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr audioClip) {} +void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr audioClip) { + /** + * @brief MIC的语音会存储在两个地方 + * 1. ./voice/rt/beforewakeup*.wav + */ + + if (!m_rtBfwakeupVoiceFile) { + cleanupLogVoiceByTime("./voice/rt/rtbfwakeup*.wav", RT_STORAGE_CLIP_NUMS); + m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/rt/rtbfwakeup" + gettimestamp() + ".wav")); + m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0); + m_rtBfwakeupVoiceFile->writeVoice(audioClip); + } else { + m_rtBfwakeupVoiceFile->writeVoice(audioClip); + if (m_rtBfwakeupVoiceFile->m_durationMS >= (RT_STORAGE_CLIP_DURATION_MS)) { + m_rtBfwakeupVoiceFile.reset(); + } + } +} void AudioLoggingService::loggerASRVoice(shared_ptr audioClip) {} \ No newline at end of file diff --git a/src/service/voiceprocess/audio_logging_service.hpp b/src/service/voiceprocess/audio_logging_service.hpp index d48bb4c..0402a19 100644 --- a/src/service/voiceprocess/audio_logging_service.hpp +++ b/src/service/voiceprocess/audio_logging_service.hpp @@ -13,6 +13,7 @@ #include #include +#include "iflytopcpp/core/components/audio/wavheader.hpp" #include "iflytopcpp/core/components/timeutils.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" @@ -61,8 +62,10 @@ class AudioLoggingService : public enable_shared_from_this WavRecorder(string filename) { m_file.open(filename, ios::binary); } void writeHeader(uint32_t sample_rate, uint16_t bits_per_sample, uint16_t num_channels, uint32_t num_samples) { WAVHeader wavHeader(sample_rate, bits_per_sample, num_channels, num_samples); + m_file.write((char*)wavHeader.data(), wavHeader.size()); } void writeVoice(shared_ptr audioClip) { + if (!audioClip) return; m_file.write((char*)audioClip->data(), audioClip->size()); m_durationMS += audioClip->getAudioDurationMs(); } @@ -75,6 +78,8 @@ class AudioLoggingService : public enable_shared_from_this public: AudioLoggingService(){}; + void initialize(); + void loggerMICVoice(shared_ptr audioClip); void loggerBeforeWakeupVoice(shared_ptr audioClip); void loggerASRVoice(shared_ptr audioClip); @@ -82,7 +87,5 @@ class AudioLoggingService : public enable_shared_from_this void setWakeupState(bool state); void cleanupLogVoiceByTime(string prefix, int maxnum); - - }; } // namespace iflytop diff --git a/src/service/voiceprocess/audio_recoder_service.cpp b/src/service/voiceprocess/audio_recoder_service.cpp index fb22a35..610af89 100644 --- a/src/service/voiceprocess/audio_recoder_service.cpp +++ b/src/service/voiceprocess/audio_recoder_service.cpp @@ -9,4 +9,8 @@ void AudioRecoderService::initialize(const char *pcmName, unsigned int channels, snd_pcm_format_t format) { m_audioRecoder = make_shared(); m_audioRecoder->initialize(pcmName, channels, sample_rate, format); + + m_audioRecoder->onRecordData.connect([this](shared_ptr audioclip) { onRecordData(audioclip); }); } + +void AudioRecoderService::startRecord() { m_audioRecoder->startRecord(); } diff --git a/src/service/voiceprocess/audio_recoder_service.hpp b/src/service/voiceprocess/audio_recoder_service.hpp index bd7f001..5498192 100644 --- a/src/service/voiceprocess/audio_recoder_service.hpp +++ b/src/service/voiceprocess/audio_recoder_service.hpp @@ -36,11 +36,13 @@ class AudioRecoderService : public enable_shared_from_this ENABLE_LOGGER(AudioRecoderService); public: - shared_ptr m_audioRecoder; + shared_ptr m_audioRecoder; + nod::signal audioclip)> onRecordData; public: AudioRecoderService(); void initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, snd_pcm_format_t format); + void startRecord(); }; } // namespace iflytop \ No newline at end of file diff --git a/src/service/voiceprocess/beforeasr_voiceprocesser.cpp b/src/service/voiceprocess/beforeasr_voiceprocesser.cpp index be17c59..6a5e182 100644 --- a/src/service/voiceprocess/beforeasr_voiceprocesser.cpp +++ b/src/service/voiceprocess/beforeasr_voiceprocesser.cpp @@ -4,10 +4,10 @@ using namespace std; using namespace iflytop; using namespace core; -void BeforeasrVoiceProcesser::initialize() { - logger->info("BeforeasrVoiceProcesser::initialize"); +void BfAsrVProcesser::initialize() { + logger->info("BfAsrVProcesser::initialize"); - m_thread.reset(new Thread("BeforeasrVoiceProcesser", [this]() { + m_thread.reset(new Thread("BfAsrVProcesser", [this]() { ThisThread thisThread; while (!thisThread.getExitFlag()) { shared_ptr audioClip; @@ -20,22 +20,20 @@ void BeforeasrVoiceProcesser::initialize() { } })); } -void BeforeasrVoiceProcesser::processVoice(shared_ptr audioClip) { - size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2; - size_t voiceLen = voiceNclips * 2; - uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2); - - for (size_t i = 0; i < voiceNclips; i++) { - voice[i] = ((uint16_t *)audioClip->getVoice())[i * 5]; +void BfAsrVProcesser::processVoice(shared_ptr audioClip) { + if (audioClip == nullptr) { + return; } + vector voice; + audioClip->getOneCHVoice(voice, 0); shared_ptr afterProcessAudioClip = - make_shared((uint8_t *)voice, voiceLen, 1, audioClip->getRate(), audioClip->getFormat()); + make_shared((uint8_t *)voice.data(), voice.size(), 1, audioClip->getRate(), audioClip->getFormat()); onAfterProcessVoice(afterProcessAudioClip); } -void BeforeasrVoiceProcesser::writeVoice(shared_ptr audioClip) { +void BfAsrVProcesser::writeVoice(shared_ptr audioClip) { ZCHECK(audioClip != nullptr, "audioClip is null"); ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE"); ZCHECK(audioClip->getCh() == 5, "audioClip ch is not 5"); // 4mic+1ref diff --git a/src/service/voiceprocess/beforeasr_voiceprocesser.hpp b/src/service/voiceprocess/beforeasr_voiceprocesser.hpp index 2c34c8d..6f0c9f5 100644 --- a/src/service/voiceprocess/beforeasr_voiceprocesser.hpp +++ b/src/service/voiceprocess/beforeasr_voiceprocesser.hpp @@ -21,7 +21,7 @@ /** * @brief * - * service:BeforeasrVoiceProcesser + * service:BfAsrVProcesser * * 作用: 波束增强，降噪，回声消除 * @@ -31,8 +31,8 @@ namespace iflytop { using namespace std; using namespace core; using namespace moodycamel; -class BeforeasrVoiceProcesser : public enable_shared_from_this { - ENABLE_LOGGER(BeforeasrVoiceProcesser); +class BfAsrVProcesser : public enable_shared_from_this { + ENABLE_LOGGER(BfAsrVProcesser); public: ConcurrentQueue> m_audioClips; @@ -42,7 +42,7 @@ class BeforeasrVoiceProcesser : public enable_shared_from_this audioClip)> onAfterProcessVoice; - BeforeasrVoiceProcesser(){}; + BfAsrVProcesser(){}; void initialize(); diff --git a/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp b/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp index 740d021..55e2a6a 100644 --- a/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp +++ b/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp @@ -4,10 +4,10 @@ using namespace std; using namespace iflytop; using namespace core; -void BeforeWakeupVoiceProcesser::initialize() { - logger->info("BeforeWakeupVoiceProcesser::initialize"); +void BfWakeupVProcesser::initialize() { + logger->info("BfWakeupVProcesser::initialize"); - m_thread.reset(new Thread("BeforeWakeupVoiceProcesser", [this]() { + m_thread.reset(new Thread("BfWakeupVProcesser", [this]() { ThisThread thisThread; while (!thisThread.getExitFlag()) { shared_ptr audioClip; @@ -20,23 +20,20 @@ void BeforeWakeupVoiceProcesser::initialize() { } })); } -void BeforeWakeupVoiceProcesser::processVoice(shared_ptr audioClip) { - - size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2; - size_t voiceLen = voiceNclips * 2; - uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2); - - for (size_t i = 0; i < voiceNclips; i++) { - voice[i] = ((uint16_t *)audioClip->getVoice())[i * 5]; +void BfWakeupVProcesser::processVoice(shared_ptr audioClip) { + if (audioClip == nullptr) { + return; } - + vector voice; + audioClip->getOneCHVoice(voice, 0); + shared_ptr afterProcessAudioClip = - make_shared((uint8_t *)voice, voiceLen, 1, audioClip->getRate(), audioClip->getFormat()); + make_shared((uint8_t *)voice.data(), voice.size(), 1, audioClip->getRate(), audioClip->getFormat()); onAfterProcessVoice(afterProcessAudioClip); } -void BeforeWakeupVoiceProcesser::writeVoice(shared_ptr audioClip) { +void BfWakeupVProcesser::writeVoice(shared_ptr audioClip) { ZCHECK(audioClip != nullptr, "audioClip is null"); ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE"); ZCHECK(audioClip->getCh() == 5, "audioClip ch is not 5"); // 4mic+1ref diff --git a/src/service/voiceprocess/beforewakeup_voiceprocesser.hpp b/src/service/voiceprocess/beforewakeup_voiceprocesser.hpp index e655bcb..9a4ece5 100644 --- a/src/service/voiceprocess/beforewakeup_voiceprocesser.hpp +++ b/src/service/voiceprocess/beforewakeup_voiceprocesser.hpp @@ -21,7 +21,7 @@ /** * @brief * - * service:BeforeWakeupVoiceProcesser + * service:BfWakeupVProcesser * * 作用: * bosuzengqiang @@ -32,8 +32,8 @@ namespace iflytop { using namespace std; using namespace core; using namespace moodycamel; -class BeforeWakeupVoiceProcesser : public enable_shared_from_this { - ENABLE_LOGGER(BeforeWakeupVoiceProcesser); +class BfWakeupVProcesser : public enable_shared_from_this { + ENABLE_LOGGER(BfWakeupVProcesser); public: ConcurrentQueue> m_audioClips; @@ -43,7 +43,7 @@ class BeforeWakeupVoiceProcesser : public enable_shared_from_this audioClip)> onAfterProcessVoice; - BeforeWakeupVoiceProcesser(){}; + BfWakeupVProcesser(){}; void initialize(); diff --git a/src/service/voiceprocess/voiceprocess_service.cpp b/src/service/voiceprocess/voiceprocess_service.cpp deleted file mode 100644 index 2002718..0000000 --- a/src/service/voiceprocess/voiceprocess_service.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "voiceprocess_service.hpp" - -#include "iflytopcpp/core/spdlogfactory/logger.hpp" -#include "zservice_container/zservice_container.hpp" - -using namespace iflytop; -using namespace std; -using namespace core; - -void VoiceProcessService::initialize() { - // - GET_TO_SERVICE(m_beforeWakeupVoiceProcesser); - GET_TO_SERVICE(m_beforeasrVoiceProcesser); - GET_TO_SERVICE(m_wakeupProcesser); - GET_TO_SERVICE(m_audioRecoderService); - - m_audioRecoderService->m_audioRecoder->onRecordData.connect( - [this](shared_ptr audioClip) { m_beforeWakeupVoiceProcesser->writeVoice(audioClip); }); - - m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect( - [this](shared_ptr audioClip) { m_beforeasrVoiceProcesser->writeVoice(audioClip); }); - - // m_wakeupProcesser->onWakeupSignal.connect( - // [this](float wakeup_score) { logger->info("wakeup_score:{}", wakeup_score); }); -} \ No newline at end of file diff --git a/src/service/voiceprocess/voiceprocess_service.hpp b/src/service/voiceprocess/voiceprocess_service.hpp deleted file mode 100644 index 90381cb..0000000 --- a/src/service/voiceprocess/voiceprocess_service.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// -// Created by zwsd -// - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "iflytopcpp/core/spdlogfactory/logger.hpp" -#include "iflytopcpp/core/thread/thread.hpp" -#include "service/voiceprocess/audio_recoder_service.hpp" -#include "service/voiceprocess/beforeasr_voiceprocesser.hpp" -#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" -#include "service/voiceprocess/wakeup_processer.hpp" - -/** - * @brief - * - * service: VoiceProcessService - * - * 监听事件: - * 依赖状态: - * 依赖服务: - * 作用: - * - */ - -namespace iflytop { -using namespace std; -using namespace core; -class VoiceProcessService : public enable_shared_from_this { - ENABLE_LOGGER(VoiceProcessService); - - shared_ptr m_beforeWakeupVoiceProcesser; - shared_ptr m_beforeasrVoiceProcesser; - shared_ptr m_wakeupProcesser; - shared_ptr m_audioRecoderService; - - public: - VoiceProcessService(){}; - - void initialize(); -}; -} // namespace iflytop \ No newline at end of file diff --git a/src/service/voiceprocess/wakeup_processer.cpp b/src/service/voiceprocess/wakeup_processer.cpp index e88f668..bb88cfd 100644 --- a/src/service/voiceprocess/wakeup_processer.cpp +++ b/src/service/voiceprocess/wakeup_processer.cpp @@ -3,13 +3,11 @@ using namespace std; using namespace iflytop; using namespace core; -#define ORDER_PATH "precise-engine/precise-engine" -// #define WAKEUP_MODULE_PATH "module/wakeup.pb" #define CHUNK_SIZE "2048" -void WakeupProcesser::initialize(string wakeupmodulepath) { +void WakeupProcesser::initialize(string precise_engine, string wakeupmodulepath) { wakeupProcesser.reset(new MycroftPreciseWapper()); - wakeupProcesser->initialize(ORDER_PATH, wakeupmodulepath.c_str(), CHUNK_SIZE); + wakeupProcesser->initialize(precise_engine.c_str(), wakeupmodulepath.c_str(), CHUNK_SIZE); } void WakeupProcesser::dumpwakeup_info(float source) { if (source > 0.9) { diff --git a/src/service/voiceprocess/wakeup_processer.hpp b/src/service/voiceprocess/wakeup_processer.hpp index 7ebf03a..64c918d 100644 --- a/src/service/voiceprocess/wakeup_processer.hpp +++ b/src/service/voiceprocess/wakeup_processer.hpp @@ -47,7 +47,7 @@ class WakeupProcesser : public enable_shared_from_this { public: WakeupProcesser(){}; - void initialize(string wakeupmodulepath); + void initialize(string precise_engine, string wakeupmodulepath); void processVoice(uint8_t* voice, size_t voiceLen); diff --git a/src/test_aiui_service.cpp b/src/test_aiui_service.cpp index 9d9ae0c..96da3c1 100644 --- a/src/test_aiui_service.cpp +++ b/src/test_aiui_service.cpp @@ -1,6 +1,6 @@ #include "zlinuxcomponents/zmainhelper.hpp" // -#include "configs/config.hpp" +#include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "spdlog/spdlog.h"