From ca6f79c1996be6b83b9b96cf9c03afeccaa35e3f Mon Sep 17 00:00:00 2001 From: zhaohe Date: Wed, 15 Mar 2023 17:58:56 +0800 Subject: [PATCH] temporary code submission --- dep/iflytopcpp | 2 +- dep/zlinuxcomponents | 2 +- dep/zwebservice | 2 +- src/service/device_io_service.cpp | 6 +- src/service/main_control_service.cpp | 6 +- src/service/voiceprocess/audio_logging_service.cpp | 93 +++++++++++++------- src/service/voiceprocess/audio_logging_service.hpp | 99 ++++++++++------------ .../voiceprocess/beforeasr_voiceprocesser.cpp | 2 +- .../voiceprocess/beforewakeup_voiceprocesser.cpp | 2 +- src/service/voiceprocess/wakeup_processer.hpp | 2 +- 10 files changed, 120 insertions(+), 96 deletions(-) diff --git a/dep/iflytopcpp b/dep/iflytopcpp index e4c1789..f5a2ccc 160000 --- a/dep/iflytopcpp +++ b/dep/iflytopcpp @@ -1 +1 @@ -Subproject commit e4c17891a53fda64a1beafa88ecc9f62119082c2 +Subproject commit f5a2ccc1a0f8dfabb7e162eca6d0245231658f0f diff --git a/dep/zlinuxcomponents b/dep/zlinuxcomponents index 4fe607a..99b2797 160000 --- a/dep/zlinuxcomponents +++ b/dep/zlinuxcomponents @@ -1 +1 @@ -Subproject commit 4fe607ac601efb4e3fb0215f16ea89296fcdae42 +Subproject commit 99b27976a540084bb76a2d3cea03433dd8b8adba diff --git a/dep/zwebservice b/dep/zwebservice index 89e0bfe..ef8c568 160000 --- a/dep/zwebservice +++ b/dep/zwebservice @@ -1 +1 @@ -Subproject commit 89e0bfea5db6bb713f49644d6e6d33900df9964c +Subproject commit ef8c5689f4ae8abf06b2a8009207e91ea72fdf7b diff --git a/src/service/device_io_service.cpp b/src/service/device_io_service.cpp index a0426e3..5c3576d 100644 --- a/src/service/device_io_service.cpp +++ b/src/service/device_io_service.cpp @@ -1,7 +1,7 @@ #include "device_io_service.hpp" -#include "iflytopcpp/core/components/string_util.hpp" -#include "iflytopcpp/core/components/time_util.hpp" +#include "iflytopcpp/core/components/stringutils.hpp" +#include "iflytopcpp/core/components/timeutils.hpp" #include "iflytopcpp/core/zexception/zexception.hpp" using namespace iflytop; using namespace core; @@ -293,7 +293,7 @@ void DeviceIOService::idcardread(bool& state, string& info) { { Modbus03Rx modbus03rx; EXEC_MODBUS(modbusMaster->modbus03(ksubboarddeviceid, 65, readreg, modbus03rx, kovertime)); - info = StringUtil().bytesToString(modbus03rx.getRegBegin(), datalen); + info = StringUtils().bytesToString(modbus03rx.getRegBegin(), datalen); } state = true; logger->debug("idcardread, state={},info={}", state, info); diff --git a/src/service/main_control_service.cpp b/src/service/main_control_service.cpp index f4143b7..e930115 100644 --- a/src/service/main_control_service.cpp +++ b/src/service/main_control_service.cpp @@ -1,6 +1,6 @@ #include "main_control_service.hpp" -#include "iflytopcpp/core/components/string_util.hpp" +#include "iflytopcpp/core/components/stringutils.hpp" #include "version.hpp" using namespace iflytop; using namespace core; @@ -109,7 +109,7 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json m_deviceIOService->fanSetState(id, power); } else if (in["command"] == "relayStateGet") { auto relaystate = m_deviceIOService->relayStateGet(); - receipt["state"]["rawState"] = StringUtil().bytet2Binary(relaystate.state, 32, false); + receipt["state"]["rawState"] = StringUtils().bytet2Binary(relaystate.state, 32, false); receipt["state"]["routerPower"] = relaystate.getState(DeviceIOService::kRouterPower); receipt["state"]["touchScreenPower"] = relaystate.getState(DeviceIOService::kTouchScreenPower); receipt["state"]["usbChargerPower"] = relaystate.getState(DeviceIOService::kUsbChargerPower); @@ -117,7 +117,7 @@ void MainControlService::processReceiveMessage(fromwhere_t fromwhere, const json receipt["state"]["lightPower"] = relaystate.getState(DeviceIOService::kLightPower); } else if (in["command"] == "inputStateGet") { auto inputdeviceState = m_deviceIOService->getinputState(); - receipt["state"]["rawState"] = StringUtil().bytet2Binary(inputdeviceState.state, 32, false); + receipt["state"]["rawState"] = StringUtils().bytet2Binary(inputdeviceState.state, 32, false); receipt["state"]["emergency"] = inputdeviceState.getState(DeviceIOService::kEmergency); receipt["state"]["waterImmersionSensor"] = inputdeviceState.getState(DeviceIOService::kWaterImmersionSensor); receipt["state"]["humanProximitySensor"] = inputdeviceState.getState(DeviceIOService::kHumanProximitySensor); diff --git a/src/service/voiceprocess/audio_logging_service.cpp b/src/service/voiceprocess/audio_logging_service.cpp index f4ca2e7..8889091 100644 --- a/src/service/voiceprocess/audio_logging_service.cpp +++ b/src/service/voiceprocess/audio_logging_service.cpp @@ -1,43 +1,76 @@ #include "audio_logging_service.hpp" -#include "iflytopcpp/core/components/time_util.hpp" - +#include "iflytopcpp/core/components/audio/wavheader.hpp" +#include "iflytopcpp/core/components/fileutils.hpp" +#include "iflytopcpp/core/components/timeutils.hpp" +#include "iflytopcpp/core/linuxcoreutils/linuxcoreutils.hpp" using namespace iflytop; using namespace core; -void AudioLoggingService::initialize() { - m_originalVoice = make_shared(); - m_beforeWakeupVoice = make_shared(); - m_beforeAsrVoice = make_shared(); - - m_realtime_log_thread.reset(new Thread("m_realtime_log_thread", [this]() { - ThisThread thisThread; - while (!thisThread.getExitFlag()) { - thisThread.sleepForMs(33); - } - })); + +static string gettimestamp() { + struct tm tm = {0}; + + time_t t = time(nullptr); + if (t == -1) { + return ""; + } + struct tm* tmp = localtime_r(&t, &tm); + if (!tmp) { + return ""; + } + return fmt::format("{:0>4}{:0>2}{:0>2}{:0>2}{:0>2}{:0>2}", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec); } +void AudioLoggingService::cleanupLogVoiceByTime(string prefix, int maxnum) { + vector files; + LinuxCoreUtils().ls(fmt::format("ls -1 -trd {}", prefix), files); -void AudioLoggingService::loggerVoice(VoiceType voiceType, shared_ptr audioClip) { - zsteady_tp now = zsteady_clock().now(); + if (files.size() > maxnum) { + for (int i = 0; i < files.size() - maxnum; i++) { + system(fmt::format("rm -f {}", files[i]).c_str()); + } + } +} - shared_ptr aq; - if (voiceType == kOriginalVoice) { - aq = m_originalVoice; - } else if (voiceType == kBeforeWakeupVoice) { - aq = m_beforeWakeupVoice; - } else if (voiceType == kBeforeAsrVoice) { - aq = m_beforeAsrVoice; +void AudioLoggingService::loggerMICVoice(shared_ptr audioClip) { + /** + * @brief MIC的语音会存储在两个地方 + * 1. ./voice/rt/mic_voice_*.wav + * 2. ./voice/wakeup/mic_voice_*.wav + * + */ + if (!m_rtMicVoiceFile) { + cleanupLogVoiceByTime("./voice/rt/mic_voice_*.wav", RT_STORAGE_CLIP_NUMS); + m_rtMicVoiceFile.reset(new WavRecorder("./voice/rt/mic_voice_" + gettimestamp() + ".wav")); + m_rtMicVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0); + m_rtMicVoiceFile->writeVoice(audioClip); } else { - return; + m_rtMicVoiceFile->writeVoice(audioClip); + if (m_rtMicVoiceFile->m_durationMS >= (RT_STORAGE_CLIP_DURATION_MS)) { + m_rtMicVoiceFile.reset(); + } } - aq->m_30sque.push_back(audioClip); - aq->m_rtque.push_back(audioClip); - if (wakeup) aq->m_wakque.push_back(audioClip); + if (m_wakeupState) { + if (!m_rtBfwakeupVoiceFile) { + cleanupLogVoiceByTime("./voice/wakeup/mic_voice_*.wav", WAKEUP_STORAGE_CLIP_NUMS); + m_rtBfwakeupVoiceFile.reset(new WavRecorder("./voice/wakeup/mic_voice_" + gettimestamp() + ".wav")); + m_rtBfwakeupVoiceFile->writeHeader(audioClip->getRate(), audioClip->getBitsPerSample(), audioClip->getCh(), 0); + m_rtBfwakeupVoiceFile->writeVoice(audioClip); + } else { + m_rtBfwakeupVoiceFile->writeVoice(audioClip); + } + } +} - aq->m_30sque.remove_if( - [&](shared_ptr audioClip) { return zsteady_clock().dToMs(now - audioClip->getTp()) > (30 * 1000); }); +void AudioLoggingService::setWakeupState(bool state) { + if (!state) { + m_wakeupState = true; + } else { + m_wakeupState = false; + m_rtBfwakeupVoiceFile.reset(); + } } -void AudioLoggingService::triggerWakeupEvent() {} -void AudioLoggingService::triggerEndWakeupEvent() {} \ No newline at end of file +void AudioLoggingService::loggerBeforeWakeupVoice(shared_ptr audioClip) {} +void AudioLoggingService::loggerASRVoice(shared_ptr audioClip) {} \ No newline at end of file diff --git a/src/service/voiceprocess/audio_logging_service.hpp b/src/service/voiceprocess/audio_logging_service.hpp index f7e0161..d48bb4c 100644 --- a/src/service/voiceprocess/audio_logging_service.hpp +++ b/src/service/voiceprocess/audio_logging_service.hpp @@ -13,85 +13,76 @@ #include #include +#include "iflytopcpp/core/components/timeutils.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "zlinuxcomponents/audio/audio_clip.hpp" -#include "iflytopcpp/core/components/time_util.hpp" /** * @brief * - * service: AudioLoggingService + * 1. 实时录制语音 + * 1.1 每隔5分钟录一帧,共录30分钟 + * 1.2 录制从MIC输入的语音和交给唤醒词前的语音。 + * + * 2. 每次唤醒录制语音 + * + * 1.1 录制MIC语音 + * 1.2 录制交给ASR的语音 + * 1.3 录制voiceprocess各个阶段的语音 + * + * 3. 存储目录结构 + * + * ./voice/rt/20200308183030_mic.wav + * ./voice/rt/20200308183030_bfwakeup.wav + * + * ./voice/bywakeup/20200308183030_mic.wav + * ./voice/bywakeup/20200308183030_asr.wav * - * 1. 存储每次唤醒后的音频，及其唤醒前20s的音频 - * 2. 每隔1分钟录一帧,共录15分钟 - * 3. 存储不同阶段的语音 * */ +#define RT_STORAGE_CLIP_DURATION_MS (1000 * 60 * 5) // 5分钟 +#define RT_STORAGE_CLIP_NUMS (6) // 6 + +#define WAKEUP_STORAGE_CLIP_NUMS (30) // 存储30次唤醒的语音 namespace iflytop { using namespace std; using namespace core; + class AudioLoggingService : public enable_shared_from_this { ENABLE_LOGGER(AudioLoggingService); - public: - enum VoiceType { - kOriginalVoice, - kBeforeWakeupVoice, - kBeforeAsrVoice, - }; - - /** - * @brief - * 存储一下文件: - * - * 每次唤醒存储一次语音，为了录全语音，会将唤醒前30s的语音也录制进来 - * logVoice/bywakeup/ - * 20230308183030_origi.wav - * 20230308183030_bfwak.wav - * 20230308183030_bfasr.wav - * - * 每隔1分钟录一次，一次录一分钟的语音，总共录15分钟，滚动覆盖. - * logVoice/bytime/ - * 20230308183030_origi.wav - * 20230308183030_bfwak.wav - * 20230308183030_bfasr.wav - * - */ - class AudioQueue { - public: - // 缓存30s语音的队列 - list> m_30sque; - // 唤醒存储队列 - list> m_wakque; - // 实时存储队列 - list> m_rtque; - }; - - shared_ptr m_originalVoice; - shared_ptr m_beforeWakeupVoice; - shared_ptr m_beforeAsrVoice; - - unique_ptr m_realtime_log_thread; - bool wakeup = false; - - class OnceLog { + class WavRecorder { public: - zsteady_tp m_startTime; + ofstream m_file; + uint32_t m_durationMS = 0; + WavRecorder(string filename) { m_file.open(filename, ios::binary); } + void writeHeader(uint32_t sample_rate, uint16_t bits_per_sample, uint16_t num_channels, uint32_t num_samples) { + WAVHeader wavHeader(sample_rate, bits_per_sample, num_channels, num_samples); + } + void writeVoice(shared_ptr audioClip) { + m_file.write((char*)audioClip->data(), audioClip->size()); + m_durationMS += audioClip->getAudioDurationMs(); + } }; - shared_ptr m_realTimeOnceLog; + bool m_wakeupState = false; + unique_ptr m_rtMicVoiceFile; + unique_ptr m_rtBfwakeupVoiceFile; public: AudioLoggingService(){}; - void loggerVoice(VoiceType voiceType, shared_ptr audioClip); + void loggerMICVoice(shared_ptr audioClip); + void loggerBeforeWakeupVoice(shared_ptr audioClip); + void loggerASRVoice(shared_ptr audioClip); + + void setWakeupState(bool state); + + void cleanupLogVoiceByTime(string prefix, int maxnum); - void triggerWakeupEvent(); - void triggerEndWakeupEvent(); - void initialize(); }; -} // namespace iflytop \ No newline at end of file +} // namespace iflytop diff --git a/src/service/voiceprocess/beforeasr_voiceprocesser.cpp b/src/service/voiceprocess/beforeasr_voiceprocesser.cpp index 545c662..be17c59 100644 --- a/src/service/voiceprocess/beforeasr_voiceprocesser.cpp +++ b/src/service/voiceprocess/beforeasr_voiceprocesser.cpp @@ -21,7 +21,7 @@ void BeforeasrVoiceProcesser::initialize() { })); } void BeforeasrVoiceProcesser::processVoice(shared_ptr audioClip) { - size_t voiceNclips = audioClip->getVoiceLen() / audioClip->getCh() / 2; + size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2; size_t voiceLen = voiceNclips * 2; uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2); diff --git a/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp b/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp index 94359a5..740d021 100644 --- a/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp +++ b/src/service/voiceprocess/beforewakeup_voiceprocesser.cpp @@ -22,7 +22,7 @@ void BeforeWakeupVoiceProcesser::initialize() { } void BeforeWakeupVoiceProcesser::processVoice(shared_ptr audioClip) { - size_t voiceNclips = audioClip->getVoiceLen() / audioClip->getCh() / 2; + size_t voiceNclips = audioClip->getVoiceBufSize() / audioClip->getCh() / 2; size_t voiceLen = voiceNclips * 2; uint16_t *voice = (uint16_t *)malloc(voiceNclips * 2); diff --git a/src/service/voiceprocess/wakeup_processer.hpp b/src/service/voiceprocess/wakeup_processer.hpp index fb60ab1..7ebf03a 100644 --- a/src/service/voiceprocess/wakeup_processer.hpp +++ b/src/service/voiceprocess/wakeup_processer.hpp @@ -14,7 +14,7 @@ #include #include "iflytopcpp/core/basic/nod/nod.hpp" -#include "iflytopcpp/core/components/time_util.hpp" +#include "iflytopcpp/core/components/timeutils.hpp" #include "iflytopcpp/core/spdlogfactory/logger.hpp" #include "iflytopcpp/core/thread/thread.hpp" #include "zlinuxcomponents/mycroft_precise/mycroft_precise_wapper.hpp"