From e2868ebff7fee12f96a6307d3a47a116bc00ff38 Mon Sep 17 00:00:00 2001 From: zhaohe Date: Fri, 24 Mar 2023 21:28:15 +0800 Subject: [PATCH] =?UTF-8?q?=E9=9B=86=E6=88=90AEC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeListsAARCH64.cmake | 4 + dep/iflytopcpp | 2 +- dep/zlinuxcomponents | 2 +- src/configs/zconfig.hpp | 40 ++--- .../soundbox4mic_voice_processer.cpp | 184 +++++++++++++++++++++ .../soundbox4mic_voice_processer.hpp | 111 +++++++++++++ src/main.cpp | 42 +---- src/service/main_control_service.cpp | 11 +- src/service/main_control_service.hpp | 18 +- src/service/voiceprocess/common_voiceprocesser.cpp | 24 +++ src/service/voiceprocess/common_voiceprocesser.hpp | 50 ++++++ src/test/test_H8922S_gps_info_reader.cpp | 148 +++++++++++++++-- src/test/test_iflytop_voice.cpp | 162 ++++++++++++++++++ src/test/test_voice_process.cpp | 0 14 files changed, 716 insertions(+), 82 deletions(-) create mode 100644 src/iflytopvoicecpp/soundbox4mic_voice_processer.cpp create mode 100644 src/iflytopvoicecpp/soundbox4mic_voice_processer.hpp create mode 100644 src/service/voiceprocess/common_voiceprocesser.cpp create mode 100644 src/service/voiceprocess/common_voiceprocesser.hpp create mode 100644 src/test/test_iflytop_voice.cpp delete mode 100644 src/test/test_voice_process.cpp diff --git a/CMakeListsAARCH64.cmake b/CMakeListsAARCH64.cmake index 0c2abfa..9f9ff3a 100644 --- a/CMakeListsAARCH64.cmake +++ b/CMakeListsAARCH64.cmake @@ -14,6 +14,8 @@ zadd_executable_simple( src/service/voiceprocess/wakeup_processer.cpp src/service/voiceprocess/audio_recoder_service.cpp src/service/voiceprocess/audio_logging_service.cpp + src/service/voiceprocess/common_voiceprocesser.cpp + src/iflytopvoicecpp/soundbox4mic_voice_processer.cpp # aiui ) @@ -27,6 +29,8 @@ zadd_executable_simple( TARGET test_smart_soundbox_player.out SRC dep/zlinuxcomponents/alsaplayer/test_smart_soundbox_player.cpp) +zadd_executable_simple(TARGET test_iflytop_voice.out SRC + src/test/test_iflytop_voice.cpp) zadd_executable_simple( TARGET test_H8922S_gps_info_reader.out SRC src/test/test_H8922S_gps_info_reader.cpp diff --git a/dep/iflytopcpp b/dep/iflytopcpp index ff65e55..d43cf20 160000 --- a/dep/iflytopcpp +++ b/dep/iflytopcpp @@ -1 +1 @@ -Subproject commit ff65e5549835f4187d8ef6be14528ed72ae5f632 +Subproject commit d43cf20f0e6b6bd097b5c16793a1846ce5ba5da9 diff --git a/dep/zlinuxcomponents b/dep/zlinuxcomponents index 6fc40ee..7de088f 160000 --- a/dep/zlinuxcomponents +++ b/dep/zlinuxcomponents @@ -1 +1 @@ -Subproject commit 6fc40ee9db4094c2994c74aba684fe8577d2be2a +Subproject commit 7de088f8a8227e10279dcafaadaf79c3d46687a9 diff --git a/src/configs/zconfig.hpp b/src/configs/zconfig.hpp index 3e81b8f..8101837 100644 --- a/src/configs/zconfig.hpp +++ b/src/configs/zconfig.hpp @@ -14,26 +14,26 @@ #include "iflytopcpp/core/components/config_template/config_template.hpp" -#define ConfigELEMENT_LIST(marco) \ - marco(string /* */, deviceId, "") /*录音设备配置*/ \ - marco(string /* */, recoder_device_name, "recorder") /*录音设备配置*/ \ - marco(int /* */, recoder_ch_number, 5) /*录音设备配置*/ \ - marco(int /* */, recoder_sample_rate, 16000) /*录音设备配置*/ \ - marco(int /* */, recoder_period_time_ms, 50) /*录音设备配置*/ \ - marco(string /* */, wakeup_module_path, "xiaomaoxiaomao/xiaomaoxiaomao.pb") /*唤醒词配置*/ \ - marco(string /* */, wakeup_lib_module, "./precise/precise-engine") /*唤醒词配置*/ \ - marco(string /* */, wakeup_chunk_length, "1600") /*唤醒词配置,对应50ms*/ \ - marco(bool /* */, voice_logger_enable, true) /*是否记录语音配置*/ \ - marco(string /* */, aiui_appid, "5938b7c7") /*唤醒词配置,对应50ms*/ \ - marco(string /* */, aiui_appkey, "19c1f7becc78eedc7826b485aabe30de") /*唤醒词配置,对应50ms*/ \ - marco(string /* */, aiui_auth_id, "") /*唤醒词配置,对应50ms*/ \ - marco(string /* */, aiui_scene, "main_box") /*唤醒词配置,对应50ms*/ \ - marco(string /* */, device_io_uart_path, "/dev/ttyUSB1") /*唤醒词配置,对应50ms*/ \ - marco(int /* */, device_io_uart_baundrate, 115200) /*唤醒词配置,对应50ms*/ \ - marco(string /* */, lightControlMode, "auto") /*照明灯控制模式 auto/manual*/ \ - marco(int /* */, lightAutoOpenHour, 18) /**/ \ - marco(int /* */, lightAutoOpenMin, 0) /**/ \ - marco(int /* */, lightAutoCloseHour, 6) /**/ \ +#define ConfigELEMENT_LIST(marco) \ + marco(string /* */, deviceId, "") /*录音设备配置*/ \ + marco(string /* */, recoder_device_name, "recorder") /*录音设备配置*/ \ + marco(int /* */, recoder_ch_number, 5) /*录音设备配置*/ \ + marco(int /* */, recoder_sample_rate, 16000) /*录音设备配置*/ \ + marco(int /* */, recoder_period_time_ms, 80) /*录音设备配置*/ \ + marco(string /* */, wakeup_module_path, "xiaomaoxiaomao/xiaomaoxiaomao.pb") /*唤醒词配置*/ \ + marco(string /* */, wakeup_lib_module, "./precise/precise-engine") /*唤醒词配置*/ \ + marco(string /* */, wakeup_chunk_length, "1280") /*唤醒词配置,对应80ms*/ \ + marco(bool /* */, voice_logger_enable, true) /*是否记录语音配置*/ \ + marco(string /* */, aiui_appid, "5938b7c7") /*唤醒词配置,对应50ms*/ \ + marco(string /* */, aiui_appkey, "19c1f7becc78eedc7826b485aabe30de") /*唤醒词配置,对应50ms*/ \ + marco(string /* */, aiui_auth_id, "") /*唤醒词配置,对应50ms*/ \ + marco(string /* */, aiui_scene, "main_box") /*唤醒词配置,对应50ms*/ \ + marco(string /* */, device_io_uart_path, "/dev/ttyUSB1") /*唤醒词配置,对应50ms*/ \ + marco(int /* */, device_io_uart_baundrate, 115200) /*唤醒词配置,对应50ms*/ \ + marco(string /* */, lightControlMode, "auto") /*照明灯控制模式 auto/manual*/ \ + marco(int /* */, lightAutoOpenHour, 18) /**/ \ + marco(int /* */, lightAutoOpenMin, 0) /**/ \ + marco(int /* */, lightAutoCloseHour, 6) /**/ \ marco(int /* */, lightAutoCloseMin, 0) /**/ configTemplateDEFILE_CONFIG_SERVICE2( // diff --git a/src/iflytopvoicecpp/soundbox4mic_voice_processer.cpp b/src/iflytopvoicecpp/soundbox4mic_voice_processer.cpp new file mode 100644 index 0000000..d0cb10b --- /dev/null +++ b/src/iflytopvoicecpp/soundbox4mic_voice_processer.cpp @@ -0,0 +1,184 @@ +#include "soundbox4mic_voice_processer.hpp" + +using namespace iflytop; +using namespace core; + +#define MIC_NUM 4 +#define FRAME_LEN 128 // 必须是128,不支持其他 +float s_coord_data[18] = {-0.060, 0, 0, // + -0.020, 0, 0, // + +0.020, 0, 0, // + +0.060, 0, 0, // + 0, 0, 0, // + 0, 0, 0}; + +#if 0 +float *audio_aec_init(int mic_num); +int audio_aec_process(float *obj, short *mic_data, short *ref_data, short *aec_out, int *aec_state, int *aec_farend); +void audio_aec_uninit(float *obj); + +//====================doa================================================= +float *audio_doa_init(int mic_num, float *coord_data, float location_range1, float location_range2); +int audio_doa_process(float *obj, short *in_buff ,int aec_state, int aec_farend, float *doa1, float *doa2, float *doa3, int *vad_stat); +void audio_doa_uninit(float *audio_doa_obj); + +//====================gsc================================================= +float *audio_gsc_init(int mic_num, float *coord_data); +int audio_gsc_amb(float *obj, short *in_buff, int aec_state, int aec_farend, float dest_doa , float *location_obj, short *out_data, int *vad_stat); +int audio_gsc_fixed(float *obj, short *in_buff ,int aec_state, int aec_farend, float dest_doa, float int_doa, short *out_data,int *vad_stat); +void audio_gsc_uninit( float *audio_gsc_obj ); + +//====================ns================================================= +float *audio_ns_init(int mode); +int audio_ns_process(float *audio_ns_obj, short *in_data, short *out_data ,int aec_stat); +void audio_ns_uninit(float *audio_ns_obj); + +//====================agc================================================= +float *audio_agc_init(int frame_len, int mode, float arg_val); +int audio_agc_process(float *audio_agc_obj, short *in_data, short *out_data, int vad_stat, int aec_stat); +void audio_agc_uninit(float *audio_agc_obj); +#endif + +void SoundBox4MicVoiceProcesser::initialize() { + // + m_common_aec = audio_aec_init(4); + + m_wakeup_ns = audio_ns_init(1); + m_wakeuop_agc = audio_agc_init(128, 1, 24000.0); + + // | 40.00mm | 40.00mm | 40.00mm | + // 60 20 0 20 60 + + m_asr_gsc = audio_gsc_init(MIC_NUM, s_coord_data); + m_asr_agc = audio_agc_init(128, 1, 24000.0); + m_asr_ns = audio_ns_init(1); + + m_commonVoiceProcessThread.reset(new Thread("commonVoiceProcessThread", [this]() { + ThisThread thisThread; + while (!thisThread.getExitFlag()) { + shared_ptr audioClip; + if (m_commonVoiceProcessQ.try_dequeue(audioClip)) { + } + thisThread.sleepForMs(5); + } + })); +} + +void SoundBox4MicVoiceProcesser::commonVoiceProcess(shared_ptr audioClip) { + ZCHECK(audioClip != nullptr, "audioClip is null"); + ZCHECK(audioClip->getCh() == 5, "audioClip ch is not 5"); + ZCHECK(audioClip->getRate() == 16000, "audioClip sampleRate is not 16000"); + ZCHECK(audioClip->getBitsPerSample() == 16, "audioClip bitsPerSample is not 16"); + + // shared_ptr audioContext = make_shared(); + // audioClip->setContext("audioContext", audioContext); + + m_commonVoiceProcessQ.enqueue(audioClip); +} +// 处理语音送给ASR +void SoundBox4MicVoiceProcesser::bfAsrVoiceProcess(shared_ptr audioClip) { + shared_ptr audioContext = audioClip->getContext("audioContext"); + if (audioContext == nullptr) { + return; + } +} +// 通用处理语音 +void SoundBox4MicVoiceProcesser::bfWakeupVoiceProcess(shared_ptr audioClip) {} + +void SoundBox4MicVoiceProcesser::commonVoiceProcessThreadFunc(shared_ptr audioClip) { + ThisThread thisThread; + while (!thisThread.getExitFlag()) { + shared_ptr audioClip; + if (m_commonVoiceProcessQ.try_dequeue(audioClip)) { + if (audioClip) { + processCommonVoiceInter(audioClip); + } + } + thisThread.sleepForMs(5); + } +} +void SoundBox4MicVoiceProcesser::bfAsrVoiceProcessThreadFunc(shared_ptr audioClip) {} +void SoundBox4MicVoiceProcesser::bfWakeupVoiceProcessThreadFunc(shared_ptr audioClip) {} + +void SoundBox4MicVoiceProcesser::processCommonVoiceInter(shared_ptr audioClip) { + /** + * @brief 处理来自MIC的语音，主要是对语音进行AEC + */ + vector outputbuf; + + shared_ptr context = make_shared(); + + outputbuf.resize(audioClip->size()); + for (size_t i = 0; i < audioClip->size();) { + uint8_t* data = audioClip->data() + i; + uint8_t* output = outputbuf.data() + i; + size_t len = FRAME_LEN * 5; + AudioContext::OneFrameAECContext oneFrameContext; + + processCommonVoiceInter(oneFrameContext, data, output, len); + context->oneFrameContexts.push_back(oneFrameContext); + i += len; + } + + shared_ptr outputClip = make_shared(outputbuf.data(), outputbuf.size(), audioClip->getCh(), + audioClip->getRate(), audioClip->getFormat()); + outputClip->updateTp(audioClip->getTp(), audioClip->getHumanReadableTp()); + outputClip->setContext("audioContext", context); + + onAfterCommonVoiceProcess(outputClip); +} + +void SoundBox4MicVoiceProcesser::processBfAsrVoiceInter(shared_ptr audioClip) {} +void SoundBox4MicVoiceProcesser::processBfWakeupVoiceInter(shared_ptr audioClip) {} +void SoundBox4MicVoiceProcesser::processCommonVoiceInter(AudioContext::OneFrameAECContext& aec, uint8_t* data, + uint8_t* output, size_t len) { + /** + * @brief + * 输入语音 + * [1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref]... + * + * 1.将语音分解成 + * voice:[1,2,3,4][1,2,3,4][1,2,3,4][1,2,3,4][1,2,3,4][1,2,3,4]... + * ref :[ref][ref][ref][ref][ref][ref] + * + * 2.处理语音 + * + * 3.将处理的语音再合成语音 + * [1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref][1,2,3,4,ref].. + * + */ + + ZCHECK(len == FRAME_LEN * 5, "len is not FRAME_LEN * 5"); + + int eachframelen = 2 * 5; // 5mic*2byte + int frameNum = len / eachframelen; + + vector voice; + vector ref; + vector aecvoice; + vector outputvoice; + voice.resize(len / 5 * 4); + aecvoice.resize(len / 5 * 4); + ref.resize(len / 5); + outputvoice.resize(len); + + for (size_t i = 0; i < frameNum; frameNum++) { + memcpy(voice.data() + i * 4 * 2 /*4 channel 2byte*/, data + i * eachframelen, 4 * 2); + memcpy(ref.data() + i * 2 /*1 channel 2byte*/, data + i * eachframelen + 4 * 2, 2); + } + + int aec_state = 0; + int aec_farend = 0; + audio_aec_process(m_common_aec, voice.data(), ref.data(), aecvoice.data(), &aec.aec_state, &aec.aec_farend); + + for (size_t i = 0; i < frameNum; frameNum++) { + memcpy(outputvoice.data() + i * eachframelen, /* */ aecvoice.data() + i * 4 * 2, 4 * 2); + memcpy(outputvoice.data() + i * eachframelen + 4 * 2, ref.data() + i * 2 /* */, 2); + } + + memcpy(output, outputvoice.data(), outputvoice.size()); +} +void SoundBox4MicVoiceProcesser::processBfAsrVoiceInter(shared_ptr audioContext, uint8_t* data, + size_t len) {} +void SoundBox4MicVoiceProcesser::processBfWakeupVoiceInter(shared_ptr audioContext, uint8_t* data, + size_t len) {} \ No newline at end of file diff --git a/src/iflytopvoicecpp/soundbox4mic_voice_processer.hpp b/src/iflytopvoicecpp/soundbox4mic_voice_processer.hpp new file mode 100644 index 0000000..ba110f9 --- /dev/null +++ b/src/iflytopvoicecpp/soundbox4mic_voice_processer.hpp @@ -0,0 +1,111 @@ +// +// Created by zwsd +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytopcpp/core/basic/concurrentqueue/blockingconcurrentqueue.h" +#include "iflytopcpp/core/basic/concurrentqueue/concurrentqueue.h" +#include "iflytopcpp/core/basic/nod/nod.hpp" +#include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "iflytopcpp/core/thread/thread.hpp" +#include "zlinuxcomponents/audio/audio_clip.hpp" +// +#include "audio_process_api.h" + +/** + * @brief + * + * service: SoundBox4MicVoiceProcesser + * + * 监听事件: + * 依赖状态: + * 依赖服务: + * 作用: + * + */ + +namespace iflytop { +using namespace std; +using namespace core; +using namespace moodycamel; +class SoundBox4MicVoiceProcesser : public enable_shared_from_this { + ENABLE_LOGGER(SoundBox4MicVoiceProcesser); + + class AudioContext { + public: + struct OneFrameAECContext { + public: + int aec_state; + int aec_farend; + }; + vector oneFrameContexts; + }; + typedef float iflytop_voice_hander_t; + + iflytop_voice_hander_t* m_common_aec = nullptr; + + iflytop_voice_hander_t* m_wakeup_ns = nullptr; + iflytop_voice_hander_t* m_wakeuop_agc = nullptr; + + iflytop_voice_hander_t* m_asr_ns = nullptr; + iflytop_voice_hander_t* m_asr_agc = nullptr; + iflytop_voice_hander_t* m_asr_gsc = nullptr; + + ConcurrentQueue> m_commonVoiceProcessQ; + ConcurrentQueue> m_bfAsrVoiceProcessQ; + ConcurrentQueue> m_bfWakeupVoiceProcessQ; + + unique_ptr m_commonVoiceProcessThread; + unique_ptr m_bfAsrVoiceProcessThread; + unique_ptr m_bfWakeupVoiceProcessThread; + + public: + SoundBox4MicVoiceProcesser(){}; + void initialize(); + + // + /** + * @brief 处理MIC语音送给wakeup + * + * @param data + * 4+1路语音数据 + * + * + * @param len + */ + void commonVoiceProcess(shared_ptr audioClip); + // 处理语音送给ASR + void bfAsrVoiceProcess(shared_ptr audioClip); + // 通用处理语音 + void bfWakeupVoiceProcess(shared_ptr audioClip); + + nod::signal audioClip)> onAfterCommonVoiceProcess; + nod::signal audioClip)> onAfterBfAsrVoiceProcess; + nod::signal audioClip)> onAfterBfWakeupVoiceProcess; + + + private: + void commonVoiceProcessThreadFunc(shared_ptr audioClip); + void bfAsrVoiceProcessThreadFunc(shared_ptr audioClip); + void bfWakeupVoiceProcessThreadFunc(shared_ptr audioClip); + + void processCommonVoiceInter(shared_ptr audioClip); + void processBfAsrVoiceInter(shared_ptr audioClip); + void processBfWakeupVoiceInter(shared_ptr audioClip); + + void processCommonVoiceInter(AudioContext::OneFrameAECContext& aec, uint8_t* data, uint8_t* output, + size_t len); + void processBfAsrVoiceInter(shared_ptr audioContext, uint8_t* data, size_t len); + void processBfWakeupVoiceInter(shared_ptr audioContext, uint8_t* data, size_t len); +}; +} // namespace iflytop diff --git a/src/main.cpp b/src/main.cpp index 8fbf53c..2bb71fa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -16,7 +16,6 @@ #include "zservice_container/zservice_container.hpp" #include "zwebservice/zwebservice.hpp" // -#include "audio_process_api.h" #include "service/device_io_service.hpp" #include "service/device_io_service_mock.hpp" #include "service/light_control_service.hpp" @@ -26,6 +25,7 @@ #include "service/voiceprocess/audio_recoder_service.hpp" #include "service/voiceprocess/beforeasr_voiceprocesser.hpp" #include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" +#include "service/voiceprocess/common_voiceprocesser.hpp" #include "service/voiceprocess/wakeup_processer.hpp" // @@ -158,6 +158,9 @@ void Main::buildVoiceProcessService() { BUILD_AND_REG_SERRVICE(BfAsrVProcesser); GET_SERVICE(BfAsrVProcesser)->initialize(); + BUILD_AND_REG_SERRVICE(CommonVoiceProcesser); + GET_SERVICE(CommonVoiceProcesser)->initialize(5 /*总通道数*/, 4 /*参考通道通道INDEX*/); + // WakeupProcesser BUILD_AND_REG_SERRVICE(WakeupProcesser); GET_SERVICE(WakeupProcesser) @@ -215,42 +218,5 @@ int Main::main(int argc, char *argv[]) { GET_SERVICE(MainControlService)->initialize(); logger->info("system setup end."); - - audio_aec_init(0); - audio_aec_process(NULL, NULL, NULL, NULL, NULL, NULL); - // void audio_aec_uninit(float *obj); - // float *audio_doa_init(0, float *coord_data, float location_range1, float location_range2); - // int audio_doa_process(float *obj, short *in_buff, int aec_state, int aec_farend, float *doa1, float *doa2, - // float *doa3, int *vad_stat); - // void audio_doa_uninit(float *audio_doa_obj); - // float *audio_gsc_init(0, float *coord_data); - // int audio_gsc_amb(float *obj, short *in_buff, int aec_state, int aec_farend, float dest_doa, float - // *location_obj, - // short *out_data, int *vad_stat); - // int audio_gsc_fixed(float *obj, short *in_buff, int aec_state, int aec_farend, float dest_doa, float int_doa, - // short *out_data, int *vad_stat); - // void audio_gsc_uninit(float *audio_gsc_obj); - // float *audio_ns_init(int mode); - // int audio_ns_process(float *audio_ns_obj, short *in_data, short *out_data, int aec_stat); - // void audio_ns_uninit(float *audio_ns_obj); - // float *audio_agc_init(int frame_len, int mode, float arg_val); - // int audio_agc_process(float *audio_agc_obj, short *in_data, short *out_data, int vad_stat, int aec_stat); - // void audio_agc_uninit(float *audio_agc_obj); - - audio_aec_uninit(NULL); - audio_doa_init(NULL, NULL, NULL, NULL); - audio_doa_process(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); - audio_doa_uninit(NULL); - audio_gsc_init(NULL, NULL); - audio_gsc_amb(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); - audio_gsc_fixed(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); - audio_gsc_uninit(NULL); - audio_ns_init(NULL); - audio_ns_process(NULL, NULL, NULL, NULL); - audio_ns_uninit(NULL); - audio_agc_init(NULL, NULL, NULL); - audio_agc_process(NULL, NULL, NULL, NULL, NULL); - audio_agc_uninit(NULL); - while (true) sleep(1000); } diff --git a/src/service/main_control_service.cpp b/src/service/main_control_service.cpp index 7a8093d..48f8aee 100644 --- a/src/service/main_control_service.cpp +++ b/src/service/main_control_service.cpp @@ -36,8 +36,16 @@ void MainControlService::initializeVoiceProcess() { logger->error("onRecordData audioClip is null"); return; } + m_audioLoggingService->loggerMICVoice(audioClip); // 录音 + m_commonVoiceProcesser->writeVoice(audioClip); // 交给通用预处理逻辑 + }); + + m_commonVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr audioClip) { + if (!audioClip) { + logger->error("onAfterProcessVoice audioClip is null"); + return; + } - m_audioLoggingService->loggerMICVoice(audioClip); // 录音 m_beforeWakeupVoiceProcesser->writeVoice(audioClip); // 交给唤醒词预处理逻辑 if (m_conversationSession) m_beforeasrVoiceProcesser->writeVoice(audioClip); // 交给asr预处理逻辑 }); @@ -221,6 +229,7 @@ void MainControlService::initialize() { GET_TO_SERVICE(m_audioLoggingService); GET_TO_SERVICE(m_smartSoundboxPlayer); GET_TO_SERVICE(m_aiuiService); + GET_TO_SERVICE(m_commonVoiceProcesser); m_endsessionTimer.reset(new SimpleTimer("endSessionTimer")); m_workQueue.reset(new WorkQueue("mainControlServiceVoiceProcessCtrlWQ")); diff --git a/src/service/main_control_service.hpp b/src/service/main_control_service.hpp index 7eebb5a..5ad7054 100644 --- a/src/service/main_control_service.hpp +++ b/src/service/main_control_service.hpp @@ -22,6 +22,7 @@ #include "service/voiceprocess/audio_recoder_service.hpp" #include "service/voiceprocess/beforeasr_voiceprocesser.hpp" #include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" +#include "service/voiceprocess/common_voiceprocesser.hpp" #include "service/voiceprocess/wakeup_processer.hpp" #include "zservice_container/zservice_container.hpp" #include "zwebservice/zwebservice.hpp" @@ -54,14 +55,15 @@ class MainControlService : public enable_shared_from_this { shared_ptr m_zwebService; shared_ptr m_deviceIOService; - shared_ptr m_beforeWakeupVoiceProcesser; - shared_ptr m_beforeasrVoiceProcesser; - shared_ptr m_wakeupProcesser; - shared_ptr m_audioRecoderService; - shared_ptr m_audioLoggingService; - shared_ptr m_smartSoundboxPlayer; - shared_ptr m_conversationSession; - shared_ptr m_aiuiService; + shared_ptr m_beforeWakeupVoiceProcesser; + shared_ptr m_beforeasrVoiceProcesser; + shared_ptr m_wakeupProcesser; + shared_ptr m_audioRecoderService; + shared_ptr m_audioLoggingService; + shared_ptr m_smartSoundboxPlayer; + shared_ptr m_conversationSession; + shared_ptr m_aiuiService; + shared_ptr m_commonVoiceProcesser; shared_ptr m_endsessionTimer; shared_ptr m_workQueue; // diff --git a/src/service/voiceprocess/common_voiceprocesser.cpp b/src/service/voiceprocess/common_voiceprocesser.cpp new file mode 100644 index 0000000..08dea79 --- /dev/null +++ b/src/service/voiceprocess/common_voiceprocesser.cpp @@ -0,0 +1,24 @@ +#include "common_voiceprocesser.hpp" + +using namespace iflytop; +using namespace core; + +CommonVoiceProcesser::CommonVoiceProcesser() {} + +// CommonVoiceProcesser::~CommonVoiceProcesser() {} + +void CommonVoiceProcesser::initialize(int voicetotalchannel, int refchannel_index) { + m_voicetotalchannel = voicetotalchannel; + m_refchannel_index = refchannel_index; + + ZCHECK(m_voicetotalchannel == 5, "m_voicetotalchannel != 5"); + ZCHECK(m_refchannel_index == 4, "m_refchannel_index != 4"); +} + +void CommonVoiceProcesser::writeVoice(shared_ptr audioClip) { + ZCHECK(audioClip->getCh() == (size_t)m_voicetotalchannel, "audioClip->getCh() != m_voicetotalchannel"); + ZCHECK(audioClip->getCh() == 5, "audioClip->getCh() != 5"); + ZCHECK(m_refchannel_index == 4, "m_refchannel_index != 4"); + + onAfterProcessVoice(audioClip); +} \ No newline at end of file diff --git a/src/service/voiceprocess/common_voiceprocesser.hpp b/src/service/voiceprocess/common_voiceprocesser.hpp new file mode 100644 index 0000000..8d502e5 --- /dev/null +++ b/src/service/voiceprocess/common_voiceprocesser.hpp @@ -0,0 +1,50 @@ +// +// Created by zwsd +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytopcpp/core/basic/concurrentqueue/concurrentqueue.h" +#include "iflytopcpp/core/basic/nod/nod.hpp" +#include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "iflytopcpp/core/thread/thread.hpp" +#include "zlinuxcomponents/audio/audio_clip.hpp" +/** + * @brief + * + * service: CommonVoiceProcesser + * + * 监听事件: + * 依赖状态: + * 依赖服务: + * 作用: + * + */ + +namespace iflytop { +using namespace std; +using namespace core; +class CommonVoiceProcesser : public enable_shared_from_this { + ENABLE_LOGGER(CommonVoiceProcesser); + + int m_voicetotalchannel; + int m_refchannel_index; + + public: + CommonVoiceProcesser(); + void initialize(int voicetotalchannel, int refchannel_index); + + nod::signal audioClip)> onAfterProcessVoice; + + void writeVoice(shared_ptr audioClip); +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/test/test_H8922S_gps_info_reader.cpp b/src/test/test_H8922S_gps_info_reader.cpp index 28603b7..c8f48e6 100644 --- a/src/test/test_H8922S_gps_info_reader.cpp +++ b/src/test/test_H8922S_gps_info_reader.cpp @@ -3,16 +3,15 @@ #include "configs/zconfig.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" -#include "spdlog/spdlog.h" -#include "version.hpp" -#include "zlinuxcomponents/rootfs_auto_update/rootfs_auto_update.hpp" -#include "zservice_container/zservice_container.hpp" -// #include "service/device_io_service.hpp" #include "service/device_io_service_mock.hpp" #include "service/light_control_service.hpp" #include "service/report_service.hpp" +#include "spdlog/spdlog.h" +#include "version.hpp" #include "zlinuxcomponents/aiui_ws/aiui_service.hpp" +#include "zlinuxcomponents/rootfs_auto_update/rootfs_auto_update.hpp" +#include "zservice_container/zservice_container.hpp" // #include @@ -20,24 +19,147 @@ #include #include -#include "service/H8922S_gps_Info_reader.hpp" -#include "zlinuxcomponents/aiui_ws/aiui.h" +#include "audio_process_api.h" // using namespace iflytop; using namespace core; using namespace std; using namespace clipp; - +#define LEN 128 // Process Length Should Be 128 ,Do not Modify +#define MIC_CH 4 ZMAIN(); void Main::onSIGINT() { exit(0); } int Main::main(int argc, char *argv[]) { - H8922SGPSInfoReader h8922SGPSInfoReader; - h8922SGPSInfoReader.initialize(8922); - h8922SGPSInfoReader.onMessage.connect( - [&](H8922SGPSInfoReader::GPSINFO gpsinfo) { logger->info("gpsinfo: {}", gpsinfo.toString()); }); + FILE *mic_fd = NULL; + FILE *ref_fd = NULL; + FILE *out_fd1 = NULL; + FILE *out_fd2 = NULL; + + short aecout_buff[LEN * MIC_CH]; + short mic_buff[LEN * MIC_CH]; + short ref_buff[LEN]; + short asr_buf[LEN]; + + // 6mic + // float coord_data[18] = {0.04,0,0, 0.02,-0.0346,0, -0.02,-0.0346,0, -0.04,0,0, -0.02,0.0346,0, 0.02,0.0346,0}; + + // 4mic + float coord_data[18] = {0.038, 0, 0, 0, -0.038, 0, -0.038, 0, 0, 0, 0.038, 0, 0, 0, 0, 0, 0, 0}; + + // 2mic + // float coord_data[18] = {0.038,0,0, -0.038,0,0, 0,0,0, 0,0,0, 0,0,0, 0,0,0}; + + int aec_state = 1; + int aec_farend = 1; + int vad_state = 1; + float doa1, doa2, doa3; + + if (argc != 5) { + printf("usage:audio_test mic_file ref_file asr_file1 wkp_file2\n"); + return -1; + } + + mic_fd = fopen(argv[1], "rb"); + // mic_fd = fopen("zero.pcm","rb"); + if (mic_fd == NULL) { + printf("open mic file failed!\n"); + return -1; + } + + ref_fd = fopen(argv[2], "rb"); + // ref_fd = fopen("zero.pcm","rb"); + if (ref_fd == NULL) { + printf("open ref file failed!\n"); + fclose(mic_fd); + mic_fd = 0; + return -1; + } + + out_fd1 = fopen(argv[3], "wb"); + // out_fd1 = fopen("asr.pcm","wb"); + if (out_fd1 == NULL) { + printf("open output file1 failed!\n"); + fclose(mic_fd); + fclose(ref_fd); + mic_fd = 0; + ref_fd = 0; + return -1; + } + + out_fd2 = fopen(argv[4], "wb"); + // out_fd2 = fopen("wkp.pcm","wb"); + if (out_fd2 == NULL) { + printf("open output file2 failed!\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + mic_fd = 0; + ref_fd = 0; + out_fd1 = 0; + return -1; + } + + float *audio_aec_obj = audio_aec_init(MIC_CH); + if (audio_aec_obj == NULL) { + printf("Audio Aec Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_doa_obj = audio_doa_init(MIC_CH, coord_data, 0, 360); + if (audio_doa_obj == NULL) { + printf("Audio doa Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_gsc_obj = audio_gsc_init(MIC_CH, coord_data); + if (audio_gsc_obj == NULL) { + printf("Audio gsc Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_agc_obj = audio_agc_init(128, 1, 24000.0); + int frame = 0; while (1) { - sleep(1); + if (fread(mic_buff, sizeof(short), LEN * MIC_CH, mic_fd) <= 0) break; + if (fread(ref_buff, sizeof(short), LEN, ref_fd) <= 0) break; + + if (audio_aec_process(audio_aec_obj, mic_buff, ref_buff, aecout_buff, &aec_state, &aec_farend)) break; + + if (audio_doa_process(audio_doa_obj, aecout_buff, aec_state, aec_farend, &doa1, &doa2, &doa3, &vad_state)) break; + + // printf("doa:%d,%d,%d\n",(short)doa1, (short)doa2, (short)doa3); + + if (audio_gsc_fixed(audio_gsc_obj, aecout_buff, aec_state, aec_farend, 0.0, 180.0, asr_buf, &vad_state)) break; + + if (audio_agc_process(audio_agc_obj, asr_buf, asr_buf, vad_state, aec_state)) break; + // printf("frame:%d\n",frame++); + fwrite(asr_buf, sizeof(short), LEN, out_fd1); + fwrite(aecout_buff, sizeof(short), LEN * MIC_CH, out_fd2); } + + audio_aec_uninit(audio_aec_obj); + audio_doa_uninit(audio_doa_obj); + audio_gsc_uninit(audio_gsc_obj); + audio_agc_uninit(audio_agc_obj); + + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + + return 0; } diff --git a/src/test/test_iflytop_voice.cpp b/src/test/test_iflytop_voice.cpp new file mode 100644 index 0000000..471ad8c --- /dev/null +++ b/src/test/test_iflytop_voice.cpp @@ -0,0 +1,162 @@ +#include +#include + +#include "zlinuxcomponents/zmainhelper.hpp" +// +#include "configs/zconfig.hpp" +#include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "iflytopcpp/core/thread/thread.hpp" +#include "spdlog/spdlog.h" +#include "version.hpp" +#include "zlinuxcomponents/rootfs_auto_update/rootfs_auto_update.hpp" +#include "zservice_container/zservice_container.hpp" +// +#include "service/device_io_service.hpp" +#include "service/device_io_service_mock.hpp" +#include "service/light_control_service.hpp" +#include "service/report_service.hpp" +#include "zlinuxcomponents/aiui_ws/aiui_service.hpp" + +// + +#include "zlinuxcomponents/aiui_ws/aiui.h" +// +#include "audio_process_api.h" +ZMAIN(); + +#define LEN 128 // Process Length Should Be 128 ,Do not Modify +#define MIC_CH 4 +void Main::onSIGINT() { exit(0); } + +int Main::main(int argc, char *argv[]) { + FILE *mic_fd = NULL; + FILE *ref_fd = NULL; + FILE *out_fd1 = NULL; + FILE *out_fd2 = NULL; + + short aecout_buff[LEN * MIC_CH]; + short mic_buff[LEN * MIC_CH]; + short ref_buff[LEN]; + short asr_buf[LEN]; + + // 6mic + // float coord_data[18] = {0.04,0,0, 0.02,-0.0346,0, -0.02,-0.0346,0, -0.04,0,0, -0.02,0.0346,0, 0.02,0.0346,0}; + + // 4mic + float coord_data[18] = {0.038, 0, 0, 0, -0.038, 0, -0.038, 0, 0, 0, 0.038, 0, 0, 0, 0, 0, 0, 0}; + + // 2mic + // float coord_data[18] = {0.038,0,0, -0.038,0,0, 0,0,0, 0,0,0, 0,0,0, 0,0,0}; + + int aec_state = 1; + int aec_farend = 1; + int vad_state = 1; + float doa1, doa2, doa3; + + if (argc != 5) { + printf("usage:audio_test mic_file ref_file asr_file1 wkp_file2\n"); + return -1; + } + + mic_fd = fopen(argv[1], "rb"); + // mic_fd = fopen("zero.pcm","rb"); + if (mic_fd == NULL) { + printf("open mic file failed!\n"); + return -1; + } + + ref_fd = fopen(argv[2], "rb"); + // ref_fd = fopen("zero.pcm","rb"); + if (ref_fd == NULL) { + printf("open ref file failed!\n"); + fclose(mic_fd); + mic_fd = 0; + return -1; + } + + out_fd1 = fopen(argv[3], "wb"); + // out_fd1 = fopen("asr.pcm","wb"); + if (out_fd1 == NULL) { + printf("open output file1 failed!\n"); + fclose(mic_fd); + fclose(ref_fd); + mic_fd = 0; + ref_fd = 0; + return -1; + } + + out_fd2 = fopen(argv[4], "wb"); + // out_fd2 = fopen("wkp.pcm","wb"); + if (out_fd2 == NULL) { + printf("open output file2 failed!\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + mic_fd = 0; + ref_fd = 0; + out_fd1 = 0; + return -1; + } + + float *audio_aec_obj = audio_aec_init(MIC_CH); + if (audio_aec_obj == NULL) { + printf("Audio Aec Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_doa_obj = audio_doa_init(MIC_CH, coord_data, 0, 360); + if (audio_doa_obj == NULL) { + printf("Audio doa Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_gsc_obj = audio_gsc_init(MIC_CH, coord_data); + if (audio_gsc_obj == NULL) { + printf("Audio gsc Init Error\n"); + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + return -1; + } + + float *audio_agc_obj = audio_agc_init(128, 1, 24000.0); + int frame = 0; + while (1) { + if (fread(mic_buff, sizeof(short), LEN * MIC_CH, mic_fd) <= 0) break; + if (fread(ref_buff, sizeof(short), LEN, ref_fd) <= 0) break; + + if (audio_aec_process(audio_aec_obj, mic_buff, ref_buff, aecout_buff, &aec_state, &aec_farend)) break; + + if (audio_doa_process(audio_doa_obj, aecout_buff, aec_state, aec_farend, &doa1, &doa2, &doa3, &vad_state)) break; + + // printf("doa:%d,%d,%d\n",(short)doa1, (short)doa2, (short)doa3); + + if (audio_gsc_fixed(audio_gsc_obj, aecout_buff, aec_state, aec_farend, 0.0, 180.0, asr_buf, &vad_state)) break; + + if (audio_agc_process(audio_agc_obj, asr_buf, asr_buf, vad_state, aec_state)) break; + // printf("frame:%d\n",frame++); + fwrite(asr_buf, sizeof(short), LEN, out_fd1); + fwrite(aecout_buff, sizeof(short), LEN * MIC_CH, out_fd2); + } + + audio_aec_uninit(audio_aec_obj); + audio_doa_uninit(audio_doa_obj); + audio_gsc_uninit(audio_gsc_obj); + audio_agc_uninit(audio_agc_obj); + + fclose(mic_fd); + fclose(ref_fd); + fclose(out_fd1); + fclose(out_fd2); + + return 0; +} diff --git a/src/test/test_voice_process.cpp b/src/test/test_voice_process.cpp deleted file mode 100644 index e69de29..0000000