From 83a2f88390474baf7b6fe96ffb19a1ed1a8d516b Mon Sep 17 00:00:00 2001 From: zhaohe Date: Sun, 10 Sep 2023 16:52:56 +0800 Subject: [PATCH] add audio --- src/iflytop/components/audio/audio.hpp | 8 ++ src/iflytop/components/audio/audio_clip.cpp | 86 +++++++++++++ src/iflytop/components/audio/audio_clip.hpp | 86 +++++++++++++ src/iflytop/components/audio/audio_dumper.cpp | 4 + src/iflytop/components/audio/audio_dumper.hpp | 41 ++++++ src/iflytop/components/audio/audio_format.hpp | 53 ++++++++ src/iflytop/components/audio/audio_recoder.cpp | 172 +++++++++++++++++++++++++ src/iflytop/components/audio/audio_recoder.hpp | 71 ++++++++++ src/iflytop/core/core.hpp | 1 + 9 files changed, 522 insertions(+) create mode 100644 src/iflytop/components/audio/audio.hpp create mode 100644 src/iflytop/components/audio/audio_clip.cpp create mode 100644 src/iflytop/components/audio/audio_clip.hpp create mode 100644 src/iflytop/components/audio/audio_dumper.cpp create mode 100644 src/iflytop/components/audio/audio_dumper.hpp create mode 100644 src/iflytop/components/audio/audio_format.hpp create mode 100644 src/iflytop/components/audio/audio_recoder.cpp create mode 100644 src/iflytop/components/audio/audio_recoder.hpp diff --git a/src/iflytop/components/audio/audio.hpp b/src/iflytop/components/audio/audio.hpp new file mode 100644 index 0000000..469d54f --- /dev/null +++ b/src/iflytop/components/audio/audio.hpp @@ -0,0 +1,8 @@ +#pragma once +#include "audio_clip.hpp" +// +#include "audio_dumper.hpp" +// +#include "audio_format.hpp" +// +#include "audio_recoder.hpp" \ No newline at end of file diff --git a/src/iflytop/components/audio/audio_clip.cpp b/src/iflytop/components/audio/audio_clip.cpp new file mode 100644 index 0000000..026614a --- /dev/null +++ b/src/iflytop/components/audio/audio_clip.cpp @@ -0,0 +1,86 @@ +// +// Created by iflytop +// + +#include "audio_clip.hpp" + +using namespace std; +using namespace iflytop; +using namespace core; + +/** + * @brief + * + * service: zvoice + */ + +AudioClip::AudioClip(uint8_t *voice, size_t voicebufsize, int ch, int rate, AudioFormat format) { + m_ch = ch; + m_rate = rate; + m_tp = tu_steady().now(); + m_human_readable_tp = tu_sys().now(); + m_voice = (uint8_t *)malloc(voicebufsize); + m_voicebufsize = voicebufsize; + m_format = format; + m_bits_per_sample = AudioFormatToBytes(m_format) * 8; + m_number_of_samples = m_voicebufsize / (AudioFormatToBytes(m_format) * ch); + + ZCHECK(m_voice != nullptr, "malloc failed"); + memcpy(m_voice, voice, voicebufsize); + + m_audio_duration_ms = (m_voicebufsize * 1000) / (m_ch * m_rate * (AudioFormatToBytes(m_format))); +} + +size_t AudioClip::getAudioDurationMs() { return m_audio_duration_ms; } + +size_t AudioClip::getNumberOfSamples() { return m_number_of_samples; } + +uint32_t AudioClip::getCh() { return m_ch; } + +uint32_t AudioClip::getRate() { return m_rate; } + +uint8_t *AudioClip::getVoice() { return m_voice; } + +AudioFormat AudioClip::getFormat() { return m_format; } + +size_t AudioClip::getVoiceBufSize() { return m_voicebufsize; } + +tp_sys AudioClip::getHumanReadableTp() { return m_human_readable_tp; } + +uint32_t AudioClip::getBitsPerSample() { return m_bits_per_sample; } + +tp_steady AudioClip::getTp() { return m_tp; } + +uint8_t *AudioClip::data() { return m_voice; } + +size_t AudioClip::size() { return m_voicebufsize; } + +void AudioClip::updateTp(tp_steady tp, tp_sys humreadtp) { + m_tp = tp; + m_human_readable_tp = humreadtp; +} + +string AudioClip::toString() { + time_t tt = system_clock::to_time_t(m_human_readable_tp); + tm tm = *localtime(&tt); + char buf[64] = {0}; + strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tm); + + string str = fmt::format("audio:{} c-{} r-{} f-{} {}:{}", buf, m_ch, m_rate, AudioFormatToStr(m_format), + (void *)m_voice, m_voicebufsize); + return str; +} + +void AudioClip::getOneCHVoice(vector &voice, size_t ch) { + if (ch >= m_ch) { + return; + } + + size_t one_ch_size = m_voicebufsize / m_ch; + voice.resize(one_ch_size); + + size_t oneframebytes = AudioFormatToBytes(m_format); + for (size_t i = 0; i < one_ch_size / oneframebytes; i++) { + memcpy(&voice[i * oneframebytes], &m_voice[i * m_ch * oneframebytes + ch * oneframebytes], oneframebytes); + } +} diff --git a/src/iflytop/components/audio/audio_clip.hpp b/src/iflytop/components/audio/audio_clip.hpp new file mode 100644 index 0000000..5e9ad7e --- /dev/null +++ b/src/iflytop/components/audio/audio_clip.hpp @@ -0,0 +1,86 @@ +// +// Created by iflytop +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytop/core/core.hpp" +#include "audio_format.hpp" + +/** + * @brief + * + * service: zvoice + */ + +namespace iflytop { +using namespace std; +using namespace iflytop; +using namespace core; +using namespace chrono; +class AudioClip : public enable_shared_from_this { + ENABLE_LOGGER(AudioClip); + uint8_t *m_voice = nullptr; // 声音数据 + size_t m_voicebufsize = 0; // 声音数据长度 + uint32_t m_ch = 0; // 声音通道数 + uint32_t m_rate = 0; // 声音采样率 + AudioFormat m_format = S16_LE; // 声音格式 + uint32_t m_bits_per_sample = 0; // 每个采样点的大小 + size_t m_audio_duration_ms = 0; // 音频时长 + uint32_t m_number_of_samples = 0; // 采样点数 + + tp_sys m_human_readable_tp = {}; // 人类可读的时间戳，可能由于时间服务器的原因，会发生突变 + tp_steady m_tp = {}; // 系统时间戳 + + mutex m_mutex; + map m_context; + + public: + AudioClip(uint8_t *voice, size_t m_voicebufsize, int ch, int rate, AudioFormat format); + + uint32_t getCh(); + uint32_t getRate(); + uint8_t *getVoice(); + AudioFormat getFormat(); + size_t getVoiceBufSize(); + size_t getNumberOfSamples(); + tp_sys getHumanReadableTp(); + tp_steady getTp(); + uint32_t getBitsPerSample(); + + void getOneCHVoice(vector &voice, size_t ch); + + uint8_t *data(); + size_t size(); + + size_t getAudioDurationMs(); + + string toString(); + void updateTp(tp_steady tp, tp_sys humreadtp); + + template + void setContext(string key, shared_ptr object) { + lock_guard lock(m_mutex); + m_context[key] = Any(object); + } + template + shared_ptr getContext(string key) { + lock_guard lock(m_mutex); + auto result = m_context.find(key); + if (result == m_context.end()) { + return nullptr; + } + return result->second.Get>(); + }; +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/iflytop/components/audio/audio_dumper.cpp b/src/iflytop/components/audio/audio_dumper.cpp new file mode 100644 index 0000000..f6f155d --- /dev/null +++ b/src/iflytop/components/audio/audio_dumper.cpp @@ -0,0 +1,4 @@ +#include "audio_dumper.hpp" + +using namespace iflytop; +using namespace core; diff --git a/src/iflytop/components/audio/audio_dumper.hpp b/src/iflytop/components/audio/audio_dumper.hpp new file mode 100644 index 0000000..2ddab00 --- /dev/null +++ b/src/iflytop/components/audio/audio_dumper.hpp @@ -0,0 +1,41 @@ +// +// Created by zwsd +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytop/core/core.hpp" + + +/** + * @brief + * + * service: AudioDumper + * + * 监听事件: + * 依赖状态: + * 依赖服务: + * 作用: + * + */ + +namespace iflytop { +using namespace std; +using namespace core; +class AudioDumper { + ENABLE_LOGGER(AudioDumper); + + public: + AudioDumper(); + void initialize(); +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/iflytop/components/audio/audio_format.hpp b/src/iflytop/components/audio/audio_format.hpp new file mode 100644 index 0000000..f4e5132 --- /dev/null +++ b/src/iflytop/components/audio/audio_format.hpp @@ -0,0 +1,53 @@ +// +// Created by iflytop +// + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytop/core/core.hpp" + + +namespace iflytop { + +enum AudioFormat { + S16_LE = 0, // 16位有符号整数，小端模式 + S16_BE, // 16位有符号整数，大端模式 + S32_LE, // 32位有符号整数，小端模式 + S32_BE, // 32位有符号整数，大端模式 +}; + +static inline int AudioFormatToBytes(AudioFormat audioformat) { + switch (audioformat) { + case S16_LE: + case S16_BE: + return 2; + case S32_LE: + case S32_BE: + return 4; + } + return 0; +} + +static inline std::string AudioFormatToStr(AudioFormat audioformat) { + switch (audioformat) { + case S16_LE: + return "S16_LE"; + case S16_BE: + return "S16_BE"; + case S32_LE: + return "S32_LE"; + case S32_BE: + return "S32_BE"; + } + return "unknown"; +} +} // namespace iflytop \ No newline at end of file diff --git a/src/iflytop/components/audio/audio_recoder.cpp b/src/iflytop/components/audio/audio_recoder.cpp new file mode 100644 index 0000000..f51e5fe --- /dev/null +++ b/src/iflytop/components/audio/audio_recoder.cpp @@ -0,0 +1,172 @@ +#include "audio_recoder.hpp" + +// +using namespace std; +using namespace iflytop; +using namespace core; + +AudioFormat AudioRecoder::snd_pcm_format_t2AudioFormat(snd_pcm_format_t format) { + switch (format) { + case SND_PCM_FORMAT_S16_LE: + return AudioFormat::S16_LE; + case SND_PCM_FORMAT_S16_BE: + return AudioFormat::S16_BE; + case SND_PCM_FORMAT_S32_LE: + return AudioFormat::S32_LE; + case SND_PCM_FORMAT_S32_BE: + return AudioFormat::S32_BE; + default: + logger->error("snd_pcm_format_t2AudioFormat fail,unsupport format:{}", format); + exit(1); + } +} +int AudioRecoder::getBitsPerSample(snd_pcm_format_t format) { + switch (format) { + case SND_PCM_FORMAT_S16_LE: + case SND_PCM_FORMAT_S16_BE: + return 16; + case SND_PCM_FORMAT_S32_LE: + case SND_PCM_FORMAT_S32_BE: + return 32; + default: + logger->error("getBitsPerSample fail,unsupport format:{}", format); + exit(1); + } +} + +#define CALL(exptr, errorinfo) \ + { \ + ret = exptr; \ + if (ret < 0) { \ + logger->error("call {},failed: {},{}", #exptr, snd_strerror(ret), errorinfo); \ + goto err; \ + } \ + } + +bool AudioRecoder::initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, + snd_pcm_format_t format, + int rec_period_time_ms) { // + logger->info("AudioRecoder::initialize pcmName:{},channels:{},sample_rate:{},format:{},rec_period_time_ms:{}", + pcmName, channels, sample_rate, format, rec_period_time_ms); + + /** + * @brief 打开设备 + */ + int ret = 0; + snd_pcm_hw_params_t *params = NULL; + snd_pcm_sw_params_t *swparams = NULL; + m_sample_rate = sample_rate; + m_channels = channels; + m_format = format; + + unsigned int alsa_rec_period_time = rec_period_time_ms * 1000; + unsigned int alsa_rec_buffer_time = rec_period_time_ms * 1000 * 4; + size_t sz = 0; + + m_pcmName = pcmName; + CALL(snd_pcm_open(&m_pcmhandle, m_pcmName, SND_PCM_STREAM_CAPTURE, 0), "open pcm failed"); + + /** + * @brief 设置参数 + */ + + snd_pcm_hw_params_alloca(¶ms); + if (!params) { + logger->error("alloc params failed"); + goto err; + } + CALL(snd_pcm_hw_params_any(m_pcmhandle, params), "Broken configuration for this PCM"); + CALL(snd_pcm_hw_params_set_access(m_pcmhandle, params, SND_PCM_ACCESS_RW_INTERLEAVED), "Access type not available"); + // set audio format + CALL(snd_pcm_hw_params_set_format(m_pcmhandle, params, m_format), "Sample format non available"); + // set channel + CALL(snd_pcm_hw_params_set_channels(m_pcmhandle, params, m_channels), "Channels count non available"); + // set rate + CALL(snd_pcm_hw_params_set_rate_near(m_pcmhandle, params, &m_sample_rate, 0), "Set rate failed"); + // set rec period time + CALL(snd_pcm_hw_params_set_period_time_near(m_pcmhandle, params, &alsa_rec_period_time, 0), "set period time fail"); + // set buffer time + CALL(snd_pcm_hw_params_set_buffer_time_near(m_pcmhandle, params, &alsa_rec_buffer_time, 0), "set buffer time failed"); + + CALL(snd_pcm_hw_params_get_period_size(params, &m_period_frames, 0), "get period size failed"); + CALL(snd_pcm_hw_params_get_buffer_size(params, &m_buffer_frames), "get buffer size failed"); + + /** + * @brief 设置参数到内核中 + */ + CALL(snd_pcm_hw_params(m_pcmhandle, params), "set params failed"); + // + snd_pcm_sw_params_alloca(&swparams); + if (!swparams) { + logger->error("alloc sw params failed"); + goto err; + } + CALL(snd_pcm_sw_params_current(m_pcmhandle, swparams), "get current sw para fail"); + CALL(snd_pcm_sw_params_set_avail_min(m_pcmhandle, swparams, m_period_frames), "set avail min failed"); + CALL(snd_pcm_sw_params(m_pcmhandle, swparams), "unable to install sw params"); + /** + * @brief 准备语音接收Buffer + */ + + sz = (m_period_frames * getBitsPerSample(m_format) / 8) * m_channels; + m_recordbuf = (uint8_t *)malloc(sz); + if (m_recordbuf == NULL) { + logger->error("malloc record buffer failed"); + goto err; + } + m_recordbuf_size = sz; + + // snd_pcm_hw_params_free(params); + // snd_pcm_sw_params_free(swparams); + return true; + +err: + if (m_pcmhandle) snd_pcm_close(m_pcmhandle); + // if (params) snd_pcm_hw_params_free(params); + // if (swparams) snd_pcm_sw_params_free(swparams); + if (m_recordbuf) { + free(m_recordbuf); + m_recordbuf = nullptr; + } + return false; +} + +void AudioRecoder::onAudio(uint8_t *voice, int size) { + shared_ptr audioClip; + audioClip.reset(new AudioClip(voice, // voice + size, // voice len + m_channels, // ch + m_sample_rate, // rate + snd_pcm_format_t2AudioFormat(m_format) // format + )); + + onRecordData(audioClip); +} + +void AudioRecoder::startRecord() { + m_recordThread.reset(new Thread("AudioRecoder", [this]() { + ThisThread thisThread; + int err = 0; + if ((err = snd_pcm_prepare(m_pcmhandle)) < 0) { + logger->error("cannot prepare audio interface for use ({})", snd_strerror(err)); + exit(1); + } + + while (!thisThread.getExitFlag()) { + int readframes = 0; + // int bytes = 0; + + readframes = snd_pcm_readi(m_pcmhandle, m_recordbuf, m_period_frames); + if (readframes == -EINTR || readframes == -EPIPE || readframes == -ESTRPIPE) { + snd_pcm_recover(m_pcmhandle, readframes, 1); + continue; + } else if (readframes < 0) { + logger->error("read from audio interface failed ({})", snd_strerror(readframes)); + exit(1); + } + + int readbytes = readframes * getBitsPerSample(m_format) / 8 * m_channels; + onAudio(m_recordbuf, readbytes); + } + })); +} \ No newline at end of file diff --git a/src/iflytop/components/audio/audio_recoder.hpp b/src/iflytop/components/audio/audio_recoder.hpp new file mode 100644 index 0000000..09dac6b --- /dev/null +++ b/src/iflytop/components/audio/audio_recoder.hpp @@ -0,0 +1,71 @@ +// +// Created by zwsd +// + +#pragma once +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iflytop/core/core.hpp" +#include "audio_clip.hpp" + + +/** + * @brief + * + * service: AudioRecoder + * + * 监听事件: + * 依赖状态: + * 依赖服务: + * 作用: + * + */ + +namespace iflytop { +using namespace std; +using namespace iflytop; +using namespace core; + +class AudioRecoder : public enable_shared_from_this { + ENABLE_LOGGER(AudioRecoder); + + snd_pcm_t *m_pcmhandle = nullptr; + const char *m_pcmName = nullptr; + snd_pcm_stream_t stream; + int mode; + + unsigned int m_channels = 0; + unsigned int m_sample_rate = 0; + // int m_sample_real_rate = 0; + snd_pcm_format_t m_format = SND_PCM_FORMAT_S16_LE; + + size_t m_period_frames = 0; + size_t m_buffer_frames = 0; + + uint8_t *m_recordbuf = nullptr; + size_t m_recordbuf_size = 0; + + unique_ptr m_recordThread; + + public: + bool initialize(const char *pcmName, unsigned int channels, unsigned int sample_rate, snd_pcm_format_t format, + int rec_period_time_ms); + void startRecord(); + nod::signal audioclip)> onRecordData; + + private: + AudioFormat snd_pcm_format_t2AudioFormat(snd_pcm_format_t format); + int getBitsPerSample(snd_pcm_format_t format); + void onAudio(uint8_t *voice, int size); +}; +} // namespace iflytop \ No newline at end of file diff --git a/src/iflytop/core/core.hpp b/src/iflytop/core/core.hpp index 696d384..d7b18fd 100644 --- a/src/iflytop/core/core.hpp +++ b/src/iflytop/core/core.hpp @@ -19,5 +19,6 @@ #include "iflytop/core/error/error_code.hpp" #include "iflytop/core/spdlogfactory/logger.hpp" #include "iflytop/core/thread/thread.hpp" +#include "iflytop/core/basic/any.hpp" #define ZARRAYSIZE(a) (sizeof(a) / sizeof(a[0])) \ No newline at end of file