diff --git a/CMakeLists.txt b/CMakeLists.txt index a31c082..7f1aad0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,11 +92,11 @@ zadd_executable_simple( src/service/main_control_service.cpp src/service/light_control_service.cpp src/service/report_service.cpp - src/service/voiceprocess/asr_service.cpp src/service/voiceprocess/beforeasr_voiceprocesser.cpp src/service/voiceprocess/beforewakeup_voiceprocesser.cpp src/service/voiceprocess/voiceprocess_service.cpp src/service/voiceprocess/wakeup_processer.cpp + src/service/voiceprocess/audio_recoder_service.cpp # aiui ) diff --git a/dep/iflytopcpp b/dep/iflytopcpp index 59531ea..e4c1789 160000 --- a/dep/iflytopcpp +++ b/dep/iflytopcpp @@ -1 +1 @@ -Subproject commit 59531ea26be163327f7cbc6bc31b9845521afff5 +Subproject commit e4c17891a53fda64a1beafa88ecc9f62119082c2 diff --git a/env/rootfs/etc/asound.conf b/env/rootfs/etc/asound.conf index 09833b3..75329e6 100644 --- a/env/rootfs/etc/asound.conf +++ b/env/rootfs/etc/asound.conf @@ -75,6 +75,18 @@ pcm.xa8326 { pcm.recorder { type plug slave { + pcm "recorder5ch" + rate 48000 + format S16_LE + channels 5 + } + rate 16000 + format S16_LE + channels 5 +} +pcm.recorder5ch { + type plug + slave { pcm { type multi slaves.xa8326.pcm "hw:2,0" @@ -89,31 +101,31 @@ pcm.recorder { bindings.1.channel 1 bindings.2.slave pdm - bindings.2.channel 2 + bindings.2.channel 5 bindings.3.slave pdm - bindings.3.channel 3 + bindings.3.channel 6 - bindings.4.slave pdm - bindings.4.channel 4 + bindings.4.slave xa8326 + bindings.4.channel 0 - bindings.5.slave pdm - bindings.5.channel 5 + # bindings.5.slave pdm + # bindings.5.channel 5 - bindings.6.slave pdm - bindings.6.channel 6 + # bindings.6.slave pdm + # bindings.6.channel 6 - bindings.7.slave pdm - bindings.7.channel 7 + # bindings.7.slave pdm + # bindings.7.channel 7 - bindings.8.slave xa8326 - bindings.8.channel 0 + # bindings.8.slave xa8326 + # bindings.8.channel 0 - bindings.9.slave xa8326 - bindings.9.channel 1 + # bindings.9.slave xa8326 + # bindings.9.channel 1 } rate 48000 format S16_LE - channels 10 + channels 5 } } \ No newline at end of file diff --git a/env/wakeupvoice/zaina.mp3 b/env/wakeupvoice/zaina.mp3 new file mode 100644 index 0000000..171f90a Binary files /dev/null and b/env/wakeupvoice/zaina.mp3 differ diff --git a/sh/packet.sh b/sh/packet.sh index 74b68d8..94b01d8 100755 --- a/sh/packet.sh +++ b/sh/packet.sh @@ -56,7 +56,6 @@ cd $PROJECT_PATH/env/ && cp --path -rf ./* $PACKET_DIR/ #################################################################### # 下载唤醒词模型 wget_and_unzip xiaomaoxiaomao ftp://zftpuser:zftpzwsd@192.168.1.2/zwsd/manufacturer_mycroft/hotworld/xiaomaoxiaomao/xiaomaoxiaomao_module_20220519.tar.gz -mv $PACKET_DIR/xiaomaoxiaomao $PACKET_DIR/wakeupmodule # 下载唤醒词引擎 wget_and_unzip precise 192.168.1.2:8021/manufacturer_mycroft/mycroft-precise-release/precise-all_0.3.0_aarch64.tar.gz # 下载唤醒词语音 diff --git a/src/main.cpp b/src/main.cpp index f78cc73..f16abc5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,11 +1,17 @@ +#include + +#include +#include + +#include "zlinuxcomponents/aiui_ws/aiui_service.hpp" +#include "zlinuxcomponents/alsaplayer/AudioPlayerAlsaImpl.hpp" +#include "zlinuxcomponents/alsaplayer/smart_soundbox_player.hpp" #include "zlinuxcomponents/zmainhelper.hpp" // #include "configs/config.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" -#include "spdlog/spdlog.h" #include "version.hpp" - #include "zlinuxcomponents/rootfs_auto_update/rootfs_auto_update.hpp" #include "zservice_container/zservice_container.hpp" #include "zwebservice/zwebservice.hpp" @@ -15,13 +21,13 @@ #include "service/light_control_service.hpp" #include "service/main_control_service.hpp" #include "service/report_service.hpp" +#include "service/voiceprocess/audio_recoder_service.hpp" +#include "service/voiceprocess/beforeasr_voiceprocesser.hpp" +#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" +#include "service/voiceprocess/voiceprocess_service.hpp" +#include "service/voiceprocess/wakeup_processer.hpp" // -#include "zlinuxcomponents/alsaplayer/AudioPlayerAlsaImpl.hpp" -// #include "zlinuxcomponents/audio/audio_recoder.hpp" -#include -#include -#include -// + using namespace iflytop; using namespace core; using namespace std; @@ -32,7 +38,7 @@ ZMAIN(); * =======================================================Main======================================================== * ***********************************************************************************************************************/ void Main::onSIGINT() { exit(0); } -int Main::main(int argc, char *argv[]) { +int Main::main(int argc, char *argv[]) { string g_host_server_ip; string g_device_id; #if 0 @@ -53,9 +59,12 @@ int Main::main(int argc, char *argv[]) { logger->info("# version:{}", VERSION); logger->info("#"); - BUILD_AND_REG_SERRVICE(Config); - GET_SERVICE(Config)->initialize(); - if (!g_device_id.empty()) GET_SERVICE(Config)->set_deviceId(g_device_id); + // BUILD_AND_REG_SERRVICE(Config); + logger->info("build {}.....", "Config"); + shared_ptr Config_val(new iflytop::Config()); + ServiceContrainer::get().regService(Config_val); + GET_SERVICE(iflytop::Config)->initialize(); + if (!g_device_id.empty()) GET_SERVICE(iflytop::Config)->set_deviceId(g_device_id); /** * @brief 比较rootfs目录中的文件和系统中对应路径的文件是否一致，如果不一致则替换 @@ -73,12 +82,9 @@ int Main::main(int argc, char *argv[]) { // system("reboot"); } - - // AudioPlayerAlsaImp audioPlayerAlsaImp("/dev/"); - - // AudioRecoder::get().initialize(); - // shared_ptr audioRecoder(new AudioRecoder()); - + /** + * @brief DeviceIOService初始化 + */ #if 0 BUILD_AND_REG_SERRVICE(DeviceIOService); #else @@ -86,6 +92,51 @@ int Main::main(int argc, char *argv[]) { #endif GET_SERVICE(DeviceIOService)->initialize(); + /** + * @brief 音频处理相关组建初始化 + */ + + // 音频采集 + BUILD_AND_REG_SERRVICE(AudioRecoderService); + GET_SERVICE(AudioRecoderService)->initialize("recorder", 5, 16000, SND_PCM_FORMAT_S16_LE); + + // BeforeWakeupVoiceProcesser + BUILD_AND_REG_SERRVICE(BeforeWakeupVoiceProcesser); + GET_SERVICE(BeforeWakeupVoiceProcesser)->initialize(); + + // BeforeAsrVoiceProcesser + BUILD_AND_REG_SERRVICE(BeforeasrVoiceProcesser); + GET_SERVICE(BeforeasrVoiceProcesser)->initialize(); + + // WakeupProcesser + BUILD_AND_REG_SERRVICE(WakeupProcesser); + GET_SERVICE(WakeupProcesser)->initialize("xiaomaoxiaomao/xiaomaoxiaomao.pb"); + + // AsrProcesser + const char *appid = "5938b7c7"; // 应用ID，在AIUI开放平台创建并设置 + const char *key = "19c1f7becc78eedc7826b485aabe30de"; // 接口密钥，在AIUI开放平台查看 + json paramj; + paramj["result_level"] = "plain"; + paramj["auth_id"] = "ac30105366ea460f9ff08ddac0c4f71e"; + paramj["data_type"] = "text"; + paramj["scene"] = "main_box"; + paramj["sample_rate"] = "16000"; + paramj["context"] = R"({"sdk_support":["nlp","tts","vad","iat"]})"; + BUILD_AND_REG_SERRVICE(AiuiService); + GET_SERVICE(AiuiService)->initialize(appid, key, paramj.dump()); + + // 播放器-SmartSoundBoxPlayer + BUILD_AND_REG_SERRVICE(SmartSoundBoxPlayer); + // + GET_SERVICE(SmartSoundBoxPlayer)->initialize(80); + + // VoiceProcessService + BUILD_AND_REG_SERRVICE(VoiceProcessService); + GET_SERVICE(VoiceProcessService)->initialize(); + + /** + * @brief 其他服务 + */ BUILD_AND_REG_SERRVICE(ZWebService); GET_SERVICE(ZWebService)->initialize(); diff --git a/src/service/voiceprocess/audio_logging_service.cpp b/src/service/voiceprocess/audio_logging_service.cpp new file mode 100644 index 0000000..f4ca2e7 --- /dev/null +++ b/src/service/voiceprocess/audio_logging_service.cpp @@ -0,0 +1,43 @@ +#include "audio_logging_service.hpp" + +#include "iflytopcpp/core/components/time_util.hpp" + +using namespace iflytop; +using namespace core; +void AudioLoggingService::initialize() { + m_originalVoice = make_shared(); + m_beforeWakeupVoice = make_shared(); + m_beforeAsrVoice = make_shared(); + + m_realtime_log_thread.reset(new Thread("m_realtime_log_thread", [this]() { + ThisThread thisThread; + while (!thisThread.getExitFlag()) { + thisThread.sleepForMs(33); + } + })); +} + +void AudioLoggingService::loggerVoice(VoiceType voiceType, shared_ptr audioClip) { + zsteady_tp now = zsteady_clock().now(); + + shared_ptr aq; + if (voiceType == kOriginalVoice) { + aq = m_originalVoice; + } else if (voiceType == kBeforeWakeupVoice) { + aq = m_beforeWakeupVoice; + } else if (voiceType == kBeforeAsrVoice) { + aq = m_beforeAsrVoice; + } else { + return; + } + + aq->m_30sque.push_back(audioClip); + aq->m_rtque.push_back(audioClip); + if (wakeup) aq->m_wakque.push_back(audioClip); + + aq->m_30sque.remove_if( + [&](shared_ptr audioClip) { return zsteady_clock().dToMs(now - audioClip->getTp()) > (30 * 1000); }); +} + +void AudioLoggingService::triggerWakeupEvent() {} +void AudioLoggingService::triggerEndWakeupEvent() {} \ No newline at end of file diff --git a/src/service/voiceprocess/audio_logging_service.hpp b/src/service/voiceprocess/audio_logging_service.hpp new file mode 100644 index 0000000..f7e0161 --- /dev/null +++ b/src/service/voiceprocess/audio_logging_service.hpp @@ -0,0 +1,97 @@ +// +// Created by zwsd +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "iflytopcpp/core/thread/thread.hpp" +#include "zlinuxcomponents/audio/audio_clip.hpp" +#include "iflytopcpp/core/components/time_util.hpp" + +/** + * @brief + * + * service: AudioLoggingService + * + * 1. 存储每次唤醒后的音频，及其唤醒前20s的音频 + * 2. 每隔1分钟录一帧,共录15分钟 + * 3. 存储不同阶段的语音 + * + */ + +namespace iflytop { +using namespace std; +using namespace core; +class AudioLoggingService : public enable_shared_from_this { + ENABLE_LOGGER(AudioLoggingService); + + public: + enum VoiceType { + kOriginalVoice, + kBeforeWakeupVoice, + kBeforeAsrVoice, + }; + + /** + * @brief + * 存储一下文件: + * + * 每次唤醒存储一次语音，为了录全语音，会将唤醒前30s的语音也录制进来 + * logVoice/bywakeup/ + * 20230308183030_origi.wav + * 20230308183030_bfwak.wav + * 20230308183030_bfasr.wav + * + * 每隔1分钟录一次，一次录一分钟的语音，总共录15分钟，滚动覆盖. + * logVoice/bytime/ + * 20230308183030_origi.wav + * 20230308183030_bfwak.wav + * 20230308183030_bfasr.wav + * + */ + class AudioQueue { + public: + // 缓存30s语音的队列 + list> m_30sque; + // 唤醒存储队列 + list> m_wakque; + // 实时存储队列 + list> m_rtque; + }; + + shared_ptr m_originalVoice; + shared_ptr m_beforeWakeupVoice; + shared_ptr m_beforeAsrVoice; + + unique_ptr m_realtime_log_thread; + bool wakeup = false; + + class OnceLog { + public: + zsteady_tp m_startTime; + + }; + + shared_ptr m_realTimeOnceLog; + + public: + AudioLoggingService(){}; + + void loggerVoice(VoiceType voiceType, shared_ptr audioClip); + + void triggerWakeupEvent(); + void triggerEndWakeupEvent(); + + void initialize(); +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/service/voiceprocess/audio_recoder_service.cpp b/src/service/voiceprocess/audio_recoder_service.cpp new file mode 100644 index 0000000..fb22a35 --- /dev/null +++ b/src/service/voiceprocess/audio_recoder_service.cpp @@ -0,0 +1,12 @@ +#include "audio_recoder_service.hpp" + +using namespace std; +using namespace iflytop; +using namespace core; + +AudioRecoderService::AudioRecoderService() {} +void AudioRecoderService::initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, + snd_pcm_format_t format) { + m_audioRecoder = make_shared(); + m_audioRecoder->initialize(pcmName, channels, sample_rate, format); +} diff --git a/src/service/voiceprocess/audio_recoder_service.hpp b/src/service/voiceprocess/audio_recoder_service.hpp new file mode 100644 index 0000000..bd7f001 --- /dev/null +++ b/src/service/voiceprocess/audio_recoder_service.hpp @@ -0,0 +1,46 @@ +// +// Created by zwsd +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "iflytopcpp/core/thread/thread.hpp" +#include "zlinuxcomponents/audio/audio_recoder.hpp" + +/** + * @brief + * + * service: AudioRecoderService + * + * 监听事件: + * 依赖状态: + * 依赖服务: + * 作用: + * + */ + +namespace iflytop { +using namespace std; +using namespace core; +class AudioRecoderService : public enable_shared_from_this { + ENABLE_LOGGER(AudioRecoderService); + + public: + shared_ptr m_audioRecoder; + + public: + AudioRecoderService(); + + void initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, snd_pcm_format_t format); +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/service/voiceprocess/voiceprocess_service.cpp b/src/service/voiceprocess/voiceprocess_service.cpp index 15b0ce5..2002718 100644 --- a/src/service/voiceprocess/voiceprocess_service.cpp +++ b/src/service/voiceprocess/voiceprocess_service.cpp @@ -1,9 +1,25 @@ #include "voiceprocess_service.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" +#include "zservice_container/zservice_container.hpp" using namespace iflytop; using namespace std; using namespace core; -void VoiceProcessService::initialize() {} \ No newline at end of file +void VoiceProcessService::initialize() { + // + GET_TO_SERVICE(m_beforeWakeupVoiceProcesser); + GET_TO_SERVICE(m_beforeasrVoiceProcesser); + GET_TO_SERVICE(m_wakeupProcesser); + GET_TO_SERVICE(m_audioRecoderService); + + m_audioRecoderService->m_audioRecoder->onRecordData.connect( + [this](shared_ptr audioClip) { m_beforeWakeupVoiceProcesser->writeVoice(audioClip); }); + + m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect( + [this](shared_ptr audioClip) { m_beforeasrVoiceProcesser->writeVoice(audioClip); }); + + // m_wakeupProcesser->onWakeupSignal.connect( + // [this](float wakeup_score) { logger->info("wakeup_score:{}", wakeup_score); }); +} \ No newline at end of file diff --git a/src/service/voiceprocess/voiceprocess_service.hpp b/src/service/voiceprocess/voiceprocess_service.hpp index 3d3d2c6..90381cb 100644 --- a/src/service/voiceprocess/voiceprocess_service.hpp +++ b/src/service/voiceprocess/voiceprocess_service.hpp @@ -15,6 +15,10 @@ #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" +#include "service/voiceprocess/audio_recoder_service.hpp" +#include "service/voiceprocess/beforeasr_voiceprocesser.hpp" +#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp" +#include "service/voiceprocess/wakeup_processer.hpp" /** * @brief @@ -34,6 +38,11 @@ using namespace core; class VoiceProcessService : public enable_shared_from_this { ENABLE_LOGGER(VoiceProcessService); + shared_ptr m_beforeWakeupVoiceProcesser; + shared_ptr m_beforeasrVoiceProcesser; + shared_ptr m_wakeupProcesser; + shared_ptr m_audioRecoderService; + public: VoiceProcessService(){}; diff --git a/src/service/voiceprocess/wakeup_processer.cpp b/src/service/voiceprocess/wakeup_processer.cpp index 904c4be..e88f668 100644 --- a/src/service/voiceprocess/wakeup_processer.cpp +++ b/src/service/voiceprocess/wakeup_processer.cpp @@ -7,9 +7,9 @@ using namespace core; // #define WAKEUP_MODULE_PATH "module/wakeup.pb" #define CHUNK_SIZE "2048" -void WakeupProcesser::initialize(string wakeupmodule) { +void WakeupProcesser::initialize(string wakeupmodulepath) { wakeupProcesser.reset(new MycroftPreciseWapper()); - wakeupProcesser->initialize(ORDER_PATH, wakeupmodule.c_str(), CHUNK_SIZE); + wakeupProcesser->initialize(ORDER_PATH, wakeupmodulepath.c_str(), CHUNK_SIZE); } void WakeupProcesser::dumpwakeup_info(float source) { if (source > 0.9) { diff --git a/src/service/voiceprocess/wakeup_processer.hpp b/src/service/voiceprocess/wakeup_processer.hpp index 77f9862..fb60ab1 100644 --- a/src/service/voiceprocess/wakeup_processer.hpp +++ b/src/service/voiceprocess/wakeup_processer.hpp @@ -47,7 +47,7 @@ class WakeupProcesser : public enable_shared_from_this { public: WakeupProcesser(){}; - void initialize(string wakeupmodule); + void initialize(string wakeupmodulepath); void processVoice(uint8_t* voice, size_t voiceLen);