p_minhang_university_noise_detection
/
rk3588sdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

update

master
zhaohe 2 years ago
parent
ca6f79c199
commit
fb159606db
29 changed files with 257 additions and 231 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      .gitmodules
  2. 2
      CMakeLists.txt
  3. 3
      Linux测试指令说明.md
  4. 2
      README.md
  5. 1
      buildtools
  6. 2
      dep/zlinuxcomponents
  7. 20
      env/rootfs/etc/asound.conf
  8. 2
      sh/packet.sh
  9. 2
      src/configs/zconfig.hpp
  10. 82
      src/main.cpp
  11. 4
      src/service/device_io_service.hpp
  12. 6
      src/service/light_control_service.hpp
  13. 87
      src/service/main_control_service.cpp
  14. 19
      src/service/main_control_service.hpp
  15. 5
      src/service/report_service.hpp
  16. 26
      src/service/voiceprocess/audio_logging_service.cpp
  17. 7
      src/service/voiceprocess/audio_logging_service.hpp
  18. 4
      src/service/voiceprocess/audio_recoder_service.cpp
  19. 4
      src/service/voiceprocess/audio_recoder_service.hpp
  20. 22
      src/service/voiceprocess/beforeasr_voiceprocesser.cpp
  21. 8
      src/service/voiceprocess/beforeasr_voiceprocesser.hpp
  22. 23
      src/service/voiceprocess/beforewakeup_voiceprocesser.cpp
  23. 8
      src/service/voiceprocess/beforewakeup_voiceprocesser.hpp
  24. 25
      src/service/voiceprocess/voiceprocess_service.cpp
  25. 51
      src/service/voiceprocess/voiceprocess_service.hpp
  26. 6
      src/service/voiceprocess/wakeup_processer.cpp
  27. 2
      src/service/voiceprocess/wakeup_processer.hpp
  28. 2
      src/test_aiui_service.cpp

3
.gitmodules
View File

@ -22,3 +22,6 @@
[submodule "dep/zlinuxcomponents"]
path = dep/zlinuxcomponents
url = zwsd@192.168.1.3:z2nd_lib/zlinuxcomponents.git
[submodule "buildtools"]
path = buildtools
url = zwsd@192.168.1.3:z2nd_lib/iflytopbuildtools.git

2
CMakeLists.txt
View File

@ -94,9 +94,9 @@ zadd_executable_simple(
src/service/report_service.cpp
src/service/voiceprocess/beforeasr_voiceprocesser.cpp
src/service/voiceprocess/beforewakeup_voiceprocesser.cpp
src/service/voiceprocess/voiceprocess_service.cpp
src/service/voiceprocess/wakeup_processer.cpp
src/service/voiceprocess/audio_recoder_service.cpp
src/service/voiceprocess/audio_logging_service.cpp
# aiui
)

3
Linux测试指令说明.md
View File

@ -0,0 +1,3 @@
录音
arecord -Drecorder -c5 -r16000 -fS16_LE test.wav

2
README.md
View File

@ -26,7 +26,7 @@ build/ 应用程序编译目录
./sh/iflytool.sh --ip <ip> flash reboot
# 编译并只部署应用
./sh/iflytool.sh --ip <ip> flashapp
./sh/iflytool.sh --ip <ip> --flashapp "app.out"
# 只部署webapp
./sh/iflytool.sh --ip <ip> flashwebapp

1
buildtools

@ -0,0 +1 @@
Subproject commit 2c5fa9ee781af8c21d16ecaf50bfbab8ba0e137b

2
dep/zlinuxcomponents

@ -1 +1 @@
Subproject commit 99b27976a540084bb76a2d3cea03433dd8b8adba
Subproject commit 83404cdad908c9db64a161bef14dd1bbd2bcbf5d

20
env/rootfs/etc/asound.conf
View File

@ -35,7 +35,7 @@ pcm.default_playback {
}
min_dB -40.0
max_dB 0.0
resolution 100
resolution 256
}
pcm.wakeup_playback {
type softvol
@ -48,7 +48,7 @@ pcm.wakeup_playback {
}
min_dB -40.0
max_dB 0.0
resolution 100
resolution 256
}
# 放音设备
pcm.xa8326 {
@ -75,18 +75,6 @@ pcm.xa8326 {
pcm.recorder {
type plug
slave {
pcm "recorder5ch"
rate 48000
format S16_LE
channels 5
}
rate 16000
format S16_LE
channels 5
}
pcm.recorder5ch {
type plug
slave {
pcm {
type multi
slaves.xa8326.pcm "hw:2,0"
@ -101,10 +89,10 @@ pcm.recorder5ch {
bindings.1.channel 1
bindings.2.slave pdm
bindings.2.channel 5
bindings.2.channel 2
bindings.3.slave pdm
bindings.3.channel 6
bindings.3.channel 3
bindings.4.slave xa8326
bindings.4.channel 0

2
sh/packet.sh
View File

@ -33,7 +33,7 @@ PACKET_DIR=$PROJECT_PATH/out/app
# 删除打包目录,重新打包
rm -rf ${PACKET_DIR}
mkdir ${PACKET_DIR}
mkdir -p ${PACKET_DIR}
########################################################################################
################################根据需要修改下面的内容即可##################################
########################################################################################

2
src/configs/config.hpp → src/configs/zconfig.hpp
View File

@ -24,6 +24,6 @@
marco(int /* */, lightAutoCloseMin, 0) /**/
configTemplateDEFILE_CONFIG_SERVICE2( //
Config, //
ZConfig, //
ConfigELEMENT_LIST, //
"./configs/config.json", {});

82
src/main.cpp
View File

@ -8,7 +8,7 @@
#include "zlinuxcomponents/alsaplayer/smart_soundbox_player.hpp"
#include "zlinuxcomponents/zmainhelper.hpp"
//
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
#include "version.hpp"
@ -21,10 +21,10 @@
#include "service/light_control_service.hpp"
#include "service/main_control_service.hpp"
#include "service/report_service.hpp"
#include "service/voiceprocess/audio_logging_service.hpp"
#include "service/voiceprocess/audio_recoder_service.hpp"
#include "service/voiceprocess/beforeasr_voiceprocesser.hpp"
#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp"
#include "service/voiceprocess/voiceprocess_service.hpp"
#include "service/voiceprocess/wakeup_processer.hpp"
//
@ -32,13 +32,44 @@ using namespace iflytop;
using namespace core;
using namespace std;
using namespace clipp;
class Main {
ENABLE_LOGGER(Main);
private:
unique_ptr<Thread> thread;
private:
int main(int argc, char *argv[]);
public:
Main(/* args */) {}
~Main() {}
void run(int argc, char *argv[]) {
thread.reset(new Thread("main", [&]() { exit(main(argc, argv)); }));
while (true) sleep(1000);
}
void appSetup(int argc, char *argv[]);
void updateRootfs();
void buildDeviceIOService();
void buildVoiceProcessService();
void onSIGINT();
};
Main *g_main;
void onsignal(int signo) { g_main->onSIGINT(); }
int main(int argc, char *argv[]) {
Main main;
g_main = &main;
::signal(SIGINT, onsignal);
main.run(argc, argv);
}
ZMAIN();
/***********************************************************************************************************************
* =======================================================Main======================================================== *
***********************************************************************************************************************/
void Main::onSIGINT() { exit(0); }
int Main::main(int argc, char *argv[]) {
void Main::appSetup(int argc, char *argv[]) {
string g_host_server_ip;
string g_device_id;
#if 0
@ -61,11 +92,12 @@ int Main::main(int argc, char *argv[]) {
// BUILD_AND_REG_SERRVICE(Config);
logger->info("build {}.....", "Config");
shared_ptr<iflytop::Config> Config_val(new iflytop::Config());
ServiceContrainer::get().regService(Config_val);
GET_SERVICE(iflytop::Config)->initialize();
if (!g_device_id.empty()) GET_SERVICE(iflytop::Config)->set_deviceId(g_device_id);
BUILD_AND_REG_SERRVICE(ZConfig);
GET_SERVICE(ZConfig)->initialize();
if (!g_device_id.empty()) GET_SERVICE(ZConfig)->set_deviceId(g_device_id);
}
void Main::updateRootfs() {
/**
* @brief 比较rootfs目录中的文件和系统中对应路径的文件是否一致，如果不一致则替换
*
@ -81,7 +113,9 @@ int Main::main(int argc, char *argv[]) {
logger->warn("not auto reboot, please reboot manually.");
// system("reboot");
}
}
void Main::buildDeviceIOService() {
/**
* @brief DeviceIOService初始化
*/
@ -91,26 +125,32 @@ int Main::main(int argc, char *argv[]) {
BUILD_AND_REG_MOCK_SERRVICE(DeviceIOService, DeviceIOServiceMock);
#endif
GET_SERVICE(DeviceIOService)->initialize();
}
void Main::buildVoiceProcessService() {
/**
* @brief 音频处理相关组建初始化
*/
// AudioLoggingService
BUILD_AND_REG_SERRVICE(AudioLoggingService);
GET_SERVICE(AudioLoggingService)->initialize();
// 音频采集
BUILD_AND_REG_SERRVICE(AudioRecoderService);
GET_SERVICE(AudioRecoderService)->initialize("recorder", 5, 16000, SND_PCM_FORMAT_S16_LE);
// BeforeWakeupVoiceProcesser
BUILD_AND_REG_SERRVICE(BeforeWakeupVoiceProcesser);
GET_SERVICE(BeforeWakeupVoiceProcesser)->initialize();
// BfWakeupVProcesser
BUILD_AND_REG_SERRVICE(BfWakeupVProcesser);
GET_SERVICE(BfWakeupVProcesser)->initialize();
// BeforeAsrVoiceProcesser
BUILD_AND_REG_SERRVICE(BeforeasrVoiceProcesser);
GET_SERVICE(BeforeasrVoiceProcesser)->initialize();
BUILD_AND_REG_SERRVICE(BfAsrVProcesser);
GET_SERVICE(BfAsrVProcesser)->initialize();
// WakeupProcesser
BUILD_AND_REG_SERRVICE(WakeupProcesser);
GET_SERVICE(WakeupProcesser)->initialize("xiaomaoxiaomao/xiaomaoxiaomao.pb");
GET_SERVICE(WakeupProcesser)->initialize("./precise/precise-engine", "xiaomaoxiaomao/xiaomaoxiaomao.pb");
// AsrProcesser
const char *appid = "5938b7c7"; // 应用ID，在AIUI开放平台创建并设置
@ -124,16 +164,18 @@ int Main::main(int argc, char *argv[]) {
paramj["context"] = R"({"sdk_support":["nlp","tts","vad","iat"]})";
BUILD_AND_REG_SERRVICE(AiuiService);
GET_SERVICE(AiuiService)->initialize(appid, key, paramj.dump());
}
int Main::main(int argc, char *argv[]) {
appSetup(argc, argv);
updateRootfs();
buildDeviceIOService();
buildVoiceProcessService();
// while (true) sleep(1000);
// 播放器-SmartSoundBoxPlayer
BUILD_AND_REG_SERRVICE(SmartSoundBoxPlayer);
//
GET_SERVICE(SmartSoundBoxPlayer)->initialize(80);
// VoiceProcessService
BUILD_AND_REG_SERRVICE(VoiceProcessService);
GET_SERVICE(VoiceProcessService)->initialize();
/**
* @brief 其他服务
*/

4
src/service/device_io_service.hpp
View File

@ -13,10 +13,10 @@
#include <string>
#include <vector>
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
//
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/basic/nod/nod.hpp"
#include "iflytopcpp/core/components/modbus/modbus.hpp"
#include "iflytopcpp/core/thread/thread.hpp"

6
src/service/light_control_service.hpp
View File

@ -13,10 +13,10 @@
#include <string>
#include <vector>
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
//
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
#include "service/device_io_service.hpp"
#include "zservice_container/zservice_container.hpp"
@ -35,7 +35,7 @@ using namespace core;
class LightControlService : public enable_shared_from_this<LightControlService> {
ENABLE_LOGGER(LightControlService);
shared_ptr<Config> config;
shared_ptr<ZConfig> config;
unique_ptr<Thread> thread;
shared_ptr<DeviceIOService> deviceIoService;

87
src/service/main_control_service.cpp
View File

@ -11,9 +11,9 @@ using namespace std;
***********************************************************************************************************************/
#define SET_DEVICE_STATE(id, value, uint) \
receipt["deviceState"][index]["id"] = id; \
receipt["deviceState"][index]["id"] = id; \
receipt["deviceState"][index]["value"] = fmt::format("{}", value); \
receipt["deviceState"][index]["unit"] = uint; \
receipt["deviceState"][index]["unit"] = uint; \
index++;
/*********************************************************************************************************************
@ -24,6 +24,13 @@ void MainControlService::initialize() {
GET_TO_SERVICE(m_zwebService);
GET_TO_SERVICE(m_deviceIOService);
GET_TO_SERVICE(m_beforeWakeupVoiceProcesser);
GET_TO_SERVICE(m_beforeasrVoiceProcesser);
GET_TO_SERVICE(m_wakeupProcesser);
GET_TO_SERVICE(m_audioRecoderService);
GET_TO_SERVICE(m_audioLoggingService);
// 监听从webservice来的websocket消息
m_zwebService->startWork([this](const json& command, json& receipt) {
try {
@ -34,6 +41,27 @@ void MainControlService::initialize() {
logger->error("process message fail {}", "catch unknown exception");
}
});
m_audioRecoderService->onRecordData.connect([this](shared_ptr<AudioClip> audioClip) {
if (!audioClip) {
logger->error("onRecordData audioClip is null");
return;
}
logger->info("onRecordData {}", audioClip->size());
m_beforeWakeupVoiceProcesser->writeVoice(audioClip);
m_audioLoggingService->loggerMICVoice(audioClip);
});
m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect([this](shared_ptr<AudioClip> audioClip) {
if (!audioClip) {
logger->error("onAfterProcessVoice audioClip is null");
return;
}
m_audioLoggingService->loggerBeforeWakeupVoice(audioClip);
logger->info("onAfterProcessVoice {}", audioClip->size());
});
m_audioRecoderService->startRecord();
};
void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json& in, json& receipt) {
logger->info("process receive message from {},{}", fromwhere2str(fromwhere), in.dump());
@ -46,8 +74,8 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json
* ================================================日志动态配置接口================================================= *
*********************************************************************************************************************/
else if (in["command"] == "loggerSetLevel") {
int loggerLevel = in["loggerLevel"];
string loggerName = in["loggerName"];
int loggerLevel = in["loggerLevel"];
string loggerName = in["loggerName"];
logger->info("loggerSetLevel {} {}", loggerName, loggerLevel);
SpdLoggerFactory::Instance().createLogger(loggerName)->set_level((level::level_enum)loggerLevel);
} else if (in["command"] == "loggerGetAllLoggers") {
@ -70,6 +98,7 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json
}
#endif
int index = 0;
DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
SET_DEVICE_STATE("wind_speed", env.wind_speed, "m/s");
SET_DEVICE_STATE("wind_direction", env.wind_direction, "m/s");
@ -100,49 +129,49 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json
SET_DEVICE_STATE("waterImmersionSensor", inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor), "-");
SET_DEVICE_STATE("humanProximitySensor", inputdeviceState.getState(DeviceIOService::kHumanProximitySensor), "-");
} else if (in["command"] == "relayControl") {
int type = in["type"];
int type = in["type"];
bool value = in["value"];
m_deviceIOService->relayControl((DeviceIOService::relay_device_type_t)type, value);
} else if (in["command"] == "fanSetState") {
int id = in["id"];
int id = in["id"];
float power = in["power"];
m_deviceIOService->fanSetState(id, power);
} else if (in["command"] == "relayStateGet") {
auto relaystate = m_deviceIOService->relayStateGet();
receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false);
receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower);
auto relaystate = m_deviceIOService->relayStateGet();
receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false);
receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower);
receipt["state"]["touchScreenPower"] = relaystate.getState(DeviceIOService::kTouchScreenPower);
receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower);
receipt["state"]["cameraPower"] = relaystate.getState(DeviceIOService::kCameraPower);
receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower);
receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower);
receipt["state"]["cameraPower"] = relaystate.getState(DeviceIOService::kCameraPower);
receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower);
} else if (in["command"] == "inputStateGet") {
auto inputdeviceState = m_deviceIOService->getinputState();
receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false);
receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency);
auto inputdeviceState = m_deviceIOService->getinputState();
receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false);
receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency);
receipt["state"]["waterImmersionSensor"] = inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor);
receipt["state"]["humanProximitySensor"] = inputdeviceState.getState(DeviceIOService::kHumanProximitySensor);
} else if (in["command"] == "idcardread") {
bool state = false;
bool state = false;
string info;
m_deviceIOService->idcardread(state, info);
receipt["state"] = state;
receipt["info"] = info;
receipt["info"] = info;
} else if (in["command"] == "getInterTemperature") {
receipt["temperature"] = m_deviceIOService->getInterTemperature();
} else if (in["command"] == "getEnvSensorState") {
DeviceIOService::env_sensor_state_t env = m_deviceIOService->getEnvSensorState();
receipt["wind_speed"] = env.wind_speed;
receipt["wind_direction"] = env.wind_direction;
receipt["temperature"] = env.temperature;
receipt["humidity"] = env.humidity;
receipt["noise"] = env.noise;
receipt["pm2_5"] = env.pm2_5;
receipt["pm10"] = env.pm10;
receipt["co2"] = env.co2;
receipt["atmospheric_pressure"] = env.atmospheric_pressure;
receipt["tvoc"] = env.tvoc;
receipt["hcho"] = env.hcho;
receipt["light_intensity"] = env.light_intensity;
receipt["wind_speed"] = env.wind_speed;
receipt["wind_direction"] = env.wind_direction;
receipt["temperature"] = env.temperature;
receipt["humidity"] = env.humidity;
receipt["noise"] = env.noise;
receipt["pm2_5"] = env.pm2_5;
receipt["pm10"] = env.pm10;
receipt["co2"] = env.co2;
receipt["atmospheric_pressure"] = env.atmospheric_pressure;
receipt["tvoc"] = env.tvoc;
receipt["hcho"] = env.hcho;
receipt["light_intensity"] = env.light_intensity;
} else if (in["command"] == "fanGetState") {
// int id = in["id"];
receipt["power"][0] = m_deviceIOService->fanGetState(0);

19
src/service/main_control_service.hpp
View File

@ -13,13 +13,19 @@
#include <string>
#include <vector>
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
//
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "service/device_io_service.hpp"
#include "service/voiceprocess/audio_logging_service.hpp"
#include "service/voiceprocess/audio_recoder_service.hpp"
#include "service/voiceprocess/beforeasr_voiceprocesser.hpp"
#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp"
#include "service/voiceprocess/wakeup_processer.hpp"
#include "zservice_container/zservice_container.hpp"
#include "zwebservice/zwebservice.hpp"
/**
* @brief
*
@ -38,8 +44,15 @@ using namespace core;
class MainControlService : public enable_shared_from_this<MainControlService> {
ENABLE_LOGGER(MainControlService);
shared_ptr<ZWebService> m_zwebService;
shared_ptr<ZWebService> m_zwebService;
shared_ptr<DeviceIOService> m_deviceIOService;
shared_ptr<BfWakeupVProcesser> m_beforeWakeupVoiceProcesser;
shared_ptr<BfAsrVProcesser> m_beforeasrVoiceProcesser;
shared_ptr<WakeupProcesser> m_wakeupProcesser;
shared_ptr<AudioRecoderService> m_audioRecoderService;
shared_ptr<AudioLoggingService> m_audioLoggingService;
typedef enum {
kzwebService,
} fromwhere_t;

5
src/service/report_service.hpp
View File

@ -13,10 +13,9 @@
#include <string>
#include <vector>
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
//
#include "configs/config.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
#include "service/device_io_service.hpp"
#include "zservice_container/zservice_container.hpp"
@ -39,7 +38,7 @@ using namespace core;
class ReportService : public enable_shared_from_this<ReportService> {
ENABLE_LOGGER(ReportService);
shared_ptr<Config> config;
shared_ptr<ZConfig> config;
unique_ptr<Thread> thread;
shared_ptr<DeviceIOService> deviceIoService;

26
src/service/voiceprocess/audio_logging_service.cpp
View File

@ -32,6 +32,11 @@ void AudioLoggingService::cleanupLogVoiceByTime(string prefix, int maxnum) {
}
}
void AudioLoggingService::initialize() {
FileUtils().makeDirIfNoExist("./voice/rt/");
FileUtils().makeDirIfNoExist("./voice/wakeup/");
}
void AudioLoggingService::loggerMICVoice(shared_ptr<AudioClip> audioClip) {
/**
* @brief MIC的语音会存储在两个地方
@ -55,7 +60,7 @@ void AudioLoggingService::loggerMICVoice(shared_ptr<AudioClip> audioClip) {
if (!m_rtBfwakeupVoiceFile) {
cleanupLogVoiceByTime("./voice/wakeup/mic_voice_*.wav", WAKEUP_STORAGE_CLIP_NUMS);
m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/wakeup/mic_voice_" + gettimestamp() + ".wav"));
m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
// m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
m_rtBfwakeupVoiceFile->writeVoice(audioClip);
} else {
m_rtBfwakeupVoiceFile->writeVoice(audioClip);
@ -72,5 +77,22 @@ void AudioLoggingService::setWakeupState(bool state) {
}
}
void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr<AudioClip> audioClip) {}
void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr<AudioClip> audioClip) {
/**
* @brief MIC的语音会存储在两个地方
* 1. ./voice/rt/beforewakeup*.wav
*/
if (!m_rtBfwakeupVoiceFile) {
cleanupLogVoiceByTime("./voice/rt/rtbfwakeup*.wav", RT_STORAGE_CLIP_NUMS);
m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/rt/rtbfwakeup" + gettimestamp() + ".wav"));
m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0);
m_rtBfwakeupVoiceFile->writeVoice(audioClip);
} else {
m_rtBfwakeupVoiceFile->writeVoice(audioClip);
if (m_rtBfwakeupVoiceFile->m_durationMS >= (RT_STORAGE_CLIP_DURATION_MS)) {
m_rtBfwakeupVoiceFile.reset();
}
}
}
void AudioLoggingService::loggerASRVoice(shared_ptr<AudioClip> audioClip) {}

7
src/service/voiceprocess/audio_logging_service.hpp
View File

@ -13,6 +13,7 @@
#include <string>
#include <vector>
#include "iflytopcpp/core/components/audio/wavheader.hpp"
#include "iflytopcpp/core/components/timeutils.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
@ -61,8 +62,10 @@ class AudioLoggingService : public enable_shared_from_this<AudioLoggingService>
WavRecorder(string filename) { m_file.open(filename, ios::binary); }
void writeHeader(uint32_t sample_rate, uint16_t bits_per_sample, uint16_t num_channels, uint32_t num_samples) {
WAVHeader wavHeader(sample_rate, bits_per_sample, num_channels, num_samples);
m_file.write((char*)wavHeader.data(), wavHeader.size());
}
void writeVoice(shared_ptr<AudioClip> audioClip) {
if (!audioClip) return;
m_file.write((char*)audioClip->data(), audioClip->size());
m_durationMS += audioClip->getAudioDurationMs();
}
@ -75,6 +78,8 @@ class AudioLoggingService : public enable_shared_from_this<AudioLoggingService>
public:
AudioLoggingService(){};
void initialize();
void loggerMICVoice(shared_ptr<AudioClip> audioClip);
void loggerBeforeWakeupVoice(shared_ptr<AudioClip> audioClip);
void loggerASRVoice(shared_ptr<AudioClip> audioClip);
@ -82,7 +87,5 @@ class AudioLoggingService : public enable_shared_from_this<AudioLoggingService>
void setWakeupState(bool state);
void cleanupLogVoiceByTime(string prefix, int maxnum);
};
} // namespace iflytop

4
src/service/voiceprocess/audio_recoder_service.cpp
View File

@ -9,4 +9,8 @@ void AudioRecoderService::initialize(const char *pcmName, unsigned int channels,
snd_pcm_format_t format) {
m_audioRecoder = make_shared<AudioRecoder>();
m_audioRecoder->initialize(pcmName, channels, sample_rate, format);
m_audioRecoder->onRecordData.connect([this](shared_ptr<AudioClip> audioclip) { onRecordData(audioclip); });
}
void AudioRecoderService::startRecord() { m_audioRecoder->startRecord(); }

4
src/service/voiceprocess/audio_recoder_service.hpp
View File

@ -36,11 +36,13 @@ class AudioRecoderService : public enable_shared_from_this<AudioRecoderService>
ENABLE_LOGGER(AudioRecoderService);
public:
shared_ptr<AudioRecoder> m_audioRecoder;
shared_ptr<AudioRecoder> m_audioRecoder;
nod::signal<void(shared_ptr<AudioClip> audioclip)> onRecordData;
public:
AudioRecoderService();
void initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, snd_pcm_format_t format);
void startRecord();
};
} // namespace iflytop

22
src/service/voiceprocess/beforeasr_voiceprocesser.cpp
View File

@ -4,10 +4,10 @@ using namespace std;
using namespace iflytop;
using namespace core;
void BeforeasrVoiceProcesser::initialize() {
logger->info("BeforeasrVoiceProcesser::initialize");
void BfAsrVProcesser::initialize() {
logger->info("BfAsrVProcesser::initialize");
m_thread.reset(new Thread("BeforeasrVoiceProcesser", [this]() {
m_thread.reset(new Thread("BfAsrVProcesser", [this]() {
ThisThread thisThread;
while (!thisThread.getExitFlag()) {
shared_ptr<AudioClip> audioClip;
@ -20,22 +20,20 @@ void BeforeasrVoiceProcesser::initialize() {
}
}));
}
void BeforeasrVoiceProcesser::processVoice(shared_ptr<AudioClip> audioClip) {
size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2;
size_t voiceLen = voiceNclips * 2;
uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2);
for (size_t i = 0; i < voiceNclips; i++) {
voice[i] = ((uint16_t *)audioClip->getVoice())[i * 5];
void BfAsrVProcesser::processVoice(shared_ptr<AudioClip> audioClip) {
if (audioClip == nullptr) {
return;
}
vector<uint8_t> voice;
audioClip->getOneCHVoice(voice, 0);
shared_ptr<AudioClip> afterProcessAudioClip =
make_shared<AudioClip>((uint8_t *)voice, voiceLen, 1, audioClip->getRate(), audioClip->getFormat());
make_shared<AudioClip>((uint8_t *)voice.data(), voice.size(), 1, audioClip->getRate(), audioClip->getFormat());
onAfterProcessVoice(afterProcessAudioClip);
}
void BeforeasrVoiceProcesser::writeVoice(shared_ptr<AudioClip> audioClip) {
void BfAsrVProcesser::writeVoice(shared_ptr<AudioClip> audioClip) {
ZCHECK(audioClip != nullptr, "audioClip is null");
ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE");
ZCHECK(audioClip->getCh() == 5, "audioClip ch is not 5"); // 4mic+1ref

8
src/service/voiceprocess/beforeasr_voiceprocesser.hpp
View File

@ -21,7 +21,7 @@
/**
* @brief
*
* service:BeforeasrVoiceProcesser
* service:BfAsrVProcesser
*
* 作用: 波束增强，降噪，回声消除
*
@ -31,8 +31,8 @@ namespace iflytop {
using namespace std;
using namespace core;
using namespace moodycamel;
class BeforeasrVoiceProcesser : public enable_shared_from_this<BeforeasrVoiceProcesser> {
ENABLE_LOGGER(BeforeasrVoiceProcesser);
class BfAsrVProcesser : public enable_shared_from_this<BfAsrVProcesser> {
ENABLE_LOGGER(BfAsrVProcesser);
public:
ConcurrentQueue<shared_ptr<AudioClip>> m_audioClips;
@ -42,7 +42,7 @@ class BeforeasrVoiceProcesser : public enable_shared_from_this<BeforeasrVoicePro
public:
nod::signal<void(shared_ptr<AudioClip> audioClip)> onAfterProcessVoice;
BeforeasrVoiceProcesser(){};
BfAsrVProcesser(){};
void initialize();

23
src/service/voiceprocess/beforewakeup_voiceprocesser.cpp
View File

@ -4,10 +4,10 @@ using namespace std;
using namespace iflytop;
using namespace core;
void BeforeWakeupVoiceProcesser::initialize() {
logger->info("BeforeWakeupVoiceProcesser::initialize");
void BfWakeupVProcesser::initialize() {
logger->info("BfWakeupVProcesser::initialize");
m_thread.reset(new Thread("BeforeWakeupVoiceProcesser", [this]() {
m_thread.reset(new Thread("BfWakeupVProcesser", [this]() {
ThisThread thisThread;
while (!thisThread.getExitFlag()) {
shared_ptr<AudioClip> audioClip;
@ -20,23 +20,20 @@ void BeforeWakeupVoiceProcesser::initialize() {
}
}));
}
void BeforeWakeupVoiceProcesser::processVoice(shared_ptr<AudioClip> audioClip) {
size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2;
size_t voiceLen = voiceNclips * 2;
uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2);
for (size_t i = 0; i < voiceNclips; i++) {
voice[i] = ((uint16_t *)audioClip->getVoice())[i * 5];
void BfWakeupVProcesser::processVoice(shared_ptr<AudioClip> audioClip) {
if (audioClip == nullptr) {
return;
}
vector<uint8_t> voice;
audioClip->getOneCHVoice(voice, 0);
shared_ptr<AudioClip> afterProcessAudioClip =
make_shared<AudioClip>((uint8_t *)voice, voiceLen, 1, audioClip->getRate(), audioClip->getFormat());
make_shared<AudioClip>((uint8_t *)voice.data(), voice.size(), 1, audioClip->getRate(), audioClip->getFormat());
onAfterProcessVoice(afterProcessAudioClip);
}
void BeforeWakeupVoiceProcesser::writeVoice(shared_ptr<AudioClip> audioClip) {
void BfWakeupVProcesser::writeVoice(shared_ptr<AudioClip> audioClip) {
ZCHECK(audioClip != nullptr, "audioClip is null");
ZCHECK(audioClip->getFormat() == S16_LE, "audioClip format is not S16_LE");
ZCHECK(audioClip->getCh() == 5, "audioClip ch is not 5"); // 4mic+1ref

8
src/service/voiceprocess/beforewakeup_voiceprocesser.hpp
View File

@ -21,7 +21,7 @@
/**
* @brief
*
* service:BeforeWakeupVoiceProcesser
* service:BfWakeupVProcesser
*
* 作用:
* bosuzengqiang
@ -32,8 +32,8 @@ namespace iflytop {
using namespace std;
using namespace core;
using namespace moodycamel;
class BeforeWakeupVoiceProcesser : public enable_shared_from_this<BeforeWakeupVoiceProcesser> {
ENABLE_LOGGER(BeforeWakeupVoiceProcesser);
class BfWakeupVProcesser : public enable_shared_from_this<BfWakeupVProcesser> {
ENABLE_LOGGER(BfWakeupVProcesser);
public:
ConcurrentQueue<shared_ptr<AudioClip>> m_audioClips;
@ -43,7 +43,7 @@ class BeforeWakeupVoiceProcesser : public enable_shared_from_this<BeforeWakeupVo
public:
nod::signal<void(shared_ptr<AudioClip> audioClip)> onAfterProcessVoice;
BeforeWakeupVoiceProcesser(){};
BfWakeupVProcesser(){};
void initialize();

25
src/service/voiceprocess/voiceprocess_service.cpp
View File

@ -1,25 +0,0 @@
#include "voiceprocess_service.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
#include "zservice_container/zservice_container.hpp"
using namespace iflytop;
using namespace std;
using namespace core;
void VoiceProcessService::initialize() {
//
GET_TO_SERVICE(m_beforeWakeupVoiceProcesser);
GET_TO_SERVICE(m_beforeasrVoiceProcesser);
GET_TO_SERVICE(m_wakeupProcesser);
GET_TO_SERVICE(m_audioRecoderService);
m_audioRecoderService->m_audioRecoder->onRecordData.connect(
[this](shared_ptr<AudioClip> audioClip) { m_beforeWakeupVoiceProcesser->writeVoice(audioClip); });
m_beforeWakeupVoiceProcesser->onAfterProcessVoice.connect(
[this](shared_ptr<AudioClip> audioClip) { m_beforeasrVoiceProcesser->writeVoice(audioClip); });
// m_wakeupProcesser->onWakeupSignal.connect(
// [this](float wakeup_score) { logger->info("wakeup_score:{}", wakeup_score); });
}

51
src/service/voiceprocess/voiceprocess_service.hpp
View File

@ -1,51 +0,0 @@
//
// Created by zwsd
//
#pragma once
#include <fstream>
#include <iostream>
#include <list>
#include <map>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <vector>
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
#include "service/voiceprocess/audio_recoder_service.hpp"
#include "service/voiceprocess/beforeasr_voiceprocesser.hpp"
#include "service/voiceprocess/beforewakeup_voiceprocesser.hpp"
#include "service/voiceprocess/wakeup_processer.hpp"
/**
* @brief
*
* service: VoiceProcessService
*
* 监听事件:
* 依赖状态:
* 依赖服务:
* 作用:
*
*/
namespace iflytop {
using namespace std;
using namespace core;
class VoiceProcessService : public enable_shared_from_this<VoiceProcessService> {
ENABLE_LOGGER(VoiceProcessService);
shared_ptr<BeforeWakeupVoiceProcesser> m_beforeWakeupVoiceProcesser;
shared_ptr<BeforeasrVoiceProcesser> m_beforeasrVoiceProcesser;
shared_ptr<WakeupProcesser> m_wakeupProcesser;
shared_ptr<AudioRecoderService> m_audioRecoderService;
public:
VoiceProcessService(){};
void initialize();
};
} // namespace iflytop

6
src/service/voiceprocess/wakeup_processer.cpp
View File

@ -3,13 +3,11 @@ using namespace std;
using namespace iflytop;
using namespace core;
#define ORDER_PATH "precise-engine/precise-engine"
// #define WAKEUP_MODULE_PATH "module/wakeup.pb"
#define CHUNK_SIZE "2048"
void WakeupProcesser::initialize(string wakeupmodulepath) {
void WakeupProcesser::initialize(string precise_engine, string wakeupmodulepath) {
wakeupProcesser.reset(new MycroftPreciseWapper());
wakeupProcesser->initialize(ORDER_PATH, wakeupmodulepath.c_str(), CHUNK_SIZE);
wakeupProcesser->initialize(precise_engine.c_str(), wakeupmodulepath.c_str(), CHUNK_SIZE);
}
void WakeupProcesser::dumpwakeup_info(float source) {
if (source > 0.9) {

2
src/service/voiceprocess/wakeup_processer.hpp
View File

@ -47,7 +47,7 @@ class WakeupProcesser : public enable_shared_from_this<WakeupProcesser> {
public:
WakeupProcesser(){};
void initialize(string wakeupmodulepath);
void initialize(string precise_engine, string wakeupmodulepath);
void processVoice(uint8_t* voice, size_t voiceLen);

2
src/test_aiui_service.cpp
View File

@ -1,6 +1,6 @@
#include "zlinuxcomponents/zmainhelper.hpp"
//
#include "configs/config.hpp"
#include "configs/zconfig.hpp"
#include "iflytopcpp/core/spdlogfactory/logger.hpp"
#include "iflytopcpp/core/thread/thread.hpp"
#include "spdlog/spdlog.h"

Write Preview
Loading…
Cancel
Save