Skip to content

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
singalsu wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff89574
Audio: MFCC: Add Voice Activity Detection based on Mel spectrum
singalsu May 12, 2026
f1a72af
Audio: MFCC: Add Python script for speech to text with Whisper model
singalsu May 19, 2026
8586762
Audio: MFCC: Add VAD switch control notification to user space
singalsu May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/audio/mfcc/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
add_subdirectory(llext ${PROJECT_BINARY_DIR}/mfcc_llext)
add_dependencies(app mfcc)
else()
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c)
if(CONFIG_IPC_MAJOR_4)
add_local_sources(sof mfcc_ipc4.c)
endif()
Comment thread
singalsu marked this conversation as resolved.
endif()
34 changes: 10 additions & 24 deletions src/audio/mfcc/mfcc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,36 +97,13 @@ static int mfcc_free(struct processing_module *mod)
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");
ipc_msg_free(cd->msg);
mod_data_blob_handler_free(mod, cd->model_handler);
mfcc_free_buffers(mod);
mod_free(mod, cd);
return 0;
}

static int mfcc_get_config(struct processing_module *mod,
uint32_t config_id, uint32_t *data_offset_size,
uint8_t *fragment, size_t fragment_size)
{
struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
}

static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
enum module_cfg_fragment_position pos, uint32_t data_offset_size,
const uint8_t *fragment, size_t fragment_size, uint8_t *response,
size_t response_size)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
}

static int mfcc_process(struct processing_module *mod,
struct input_stream_buffer *input_buffers, int num_input_buffers,
Expand Down Expand Up @@ -198,6 +175,15 @@ static int mfcc_prepare(struct processing_module *mod,
goto err;
}

/* Initialize VAD switch control notification if enabled */
if (cd->config && cd->config->update_controls) {
ret = mfcc_ipc_notification_init(mod);
if (ret < 0)
goto err;

cd->vad_prev = false;
}

return 0;

err:
Expand Down
88 changes: 57 additions & 31 deletions src/audio/mfcc/mfcc_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,17 @@
#include <stddef.h>
#include <stdint.h>

#include <sof/audio/mfcc/mfcc_vad.h>

LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);

/*
* The main processing function for MFCC
*/

static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *cd)
static int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd)
{
const struct comp_dev *dev = mod->dev;
struct sof_mfcc_config *config = cd->config;
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &state->buf;
Expand Down Expand Up @@ -169,6 +172,28 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *

cc_count += state->dct.num_out;
}

/* To start count from 0 */
state->header.frame_number = cd->vad.frame_count;

/* Run VAD on the mel log spectrum (available in both modes) */
if (config->enable_vad)
mfcc_vad_update(&cd->vad, state->mel_log_32);
Comment on lines +180 to +181

/* Populate data header for this output frame */
state->header.energy = cd->vad.energy;
state->header.noise_energy = cd->vad.noise_energy;
state->header.vad_flag = cd->vad.is_speech ? 1 : 0;
Comment thread
singalsu marked this conversation as resolved.

/* Send notification when VAD state changes */
if (config->update_controls) {
bool vad_now = cd->vad.is_speech;

if (vad_now != cd->vad_prev) {
mfcc_send_vad_notification(mod, vad_now ? 1 : 0);
cd->vad_prev = vad_now;
}
}
}

return cc_count;
Expand Down Expand Up @@ -267,9 +292,8 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int16_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 2;
const int num_header_s16 = sizeof(state->header) / sizeof(int16_t);
int num_ceps;
int sink_samples;
int to_copy;
Expand All @@ -278,27 +302,29 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s16(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT: Mel auditory filter and DCT. */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data and mark magic pending */
/* If new output produced, set up pointer into scratch data and mark header pending */
if (num_ceps > 0) {
if (state->mel_only)
if (state->mel_only) {
state->out_data_ptr = state->mel_spectra->data;
else
} else {
state->out_data_ptr = state->cepstral_coef->data;
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_magic, (int16_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s16) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_header_s16,
(int16_t *)&state->header);
sink_samples -= num_header_s16;
state->header_pending = false;
}

/* Write cepstral/mel data from scratch buffer */
Expand Down Expand Up @@ -363,9 +389,8 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -376,7 +401,7 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s24(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -391,17 +416,18 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down Expand Up @@ -443,9 +469,8 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -455,7 +480,7 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s32(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -466,17 +491,18 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down
140 changes: 140 additions & 0 deletions src/audio/mfcc/mfcc_ipc4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// SPDX-License-Identifier: BSD-3-Clause
//
// Copyright(c) 2026 Intel Corporation.
//
// Author: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

/**
* \file mfcc_ipc4.c
* \brief IPC4-specific functions for MFCC component.
*
* Provides VAD switch control notification to user space via the
* IPC4 module notification mechanism.
*/

#include <sof/audio/mfcc/mfcc_comp.h>
#include <sof/audio/module_adapter/module/generic.h>
#include <sof/audio/component.h>
#include <sof/ipc/msg.h>
#include <sof/trace/trace.h>
#include <ipc4/base-config.h>
#include <ipc4/header.h>
#include <ipc4/module.h>
#include <ipc4/notification.h>
#include <rtos/string.h>
#include <errno.h>
#include <stdint.h>

LOG_MODULE_DECLARE(mfcc, CONFIG_SOF_LOG_LEVEL);

/**
* \brief Initialize IPC notification message for VAD switch control.
*
* Allocates and configures the IPC message used to send VAD state
* change notifications to user space via a switch control.
*/
int mfcc_ipc_notification_init(struct processing_module *mod)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct ipc_msg msg_proto;
struct comp_dev *dev = mod->dev;
struct comp_ipc_config *ipc_config = &dev->ipc_config;
union ipc4_notification_header *primary =
(union ipc4_notification_header *)&msg_proto.header;
struct sof_ipc4_notify_module_data *msg_module_data;
struct sof_ipc4_control_msg_payload *msg_payload;

memset_s(&msg_proto, sizeof(msg_proto), 0, sizeof(msg_proto));
primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION;
primary->r.type = SOF_IPC4_GLB_NOTIFICATION;
primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST;
primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG;
cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension,
sizeof(struct sof_ipc4_notify_module_data) +
sizeof(struct sof_ipc4_control_msg_payload) +
sizeof(struct sof_ipc4_ctrl_value_chan));
if (!cd->msg) {
comp_err(dev, "Failed to initialize VAD notification");
return -ENOMEM;
}

msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
msg_module_data->instance_id = IPC4_INST_ID(ipc_config->id);
msg_module_data->module_id = IPC4_MOD_ID(ipc_config->id);
msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL |
SOF_IPC4_SWITCH_CONTROL_PARAM_ID;
msg_module_data->event_data_size = sizeof(struct sof_ipc4_control_msg_payload) +
sizeof(struct sof_ipc4_ctrl_value_chan);

msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
msg_payload->id = MFCC_CTRL_INDEX_VAD;
msg_payload->num_elems = 1;
msg_payload->chanv[0].channel = 0;

comp_dbg(dev, "VAD notification init: instance_id = 0x%08x, module_id = 0x%08x",
msg_module_data->instance_id, msg_module_data->module_id);
return 0;
}

/**
* \brief Send VAD switch control notification to user space.
* \param mod Processing module.
* \param val VAD value: 1 = speech, 0 = silence.
*/
void mfcc_send_vad_notification(struct processing_module *mod, uint32_t val)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct sof_ipc4_notify_module_data *msg_module_data;
struct sof_ipc4_control_msg_payload *msg_payload;

if (!cd->msg)
return;

msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
msg_payload->chanv[0].value = val;
ipc_msg_send(cd->msg, NULL, false);
}

int mfcc_get_config(struct processing_module *mod,
uint32_t config_id, uint32_t *data_offset_size,
uint8_t *fragment, size_t fragment_size)
{
struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct sof_ipc4_control_msg_payload *ctl;

comp_info(mod->dev, "entry");

switch (config_id) {
case SOF_IPC4_SWITCH_CONTROL_PARAM_ID:
ctl = (struct sof_ipc4_control_msg_payload *)fragment;
if (ctl->id == MFCC_CTRL_INDEX_VAD && ctl->num_elems == 1) {
ctl->chanv[0].value = cd->vad_prev ? 1 : 0;
*data_offset_size = sizeof(*ctl) + sizeof(ctl->chanv[0]);
return 0;
}
return -EINVAL;
default:
return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
}
}

int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
enum module_cfg_fragment_position pos, uint32_t data_offset_size,
const uint8_t *fragment, size_t fragment_size, uint8_t *response,
size_t response_size)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

switch (config_id) {
case SOF_IPC4_SWITCH_CONTROL_PARAM_ID:
/* VAD switch is read-only, ignore set requests */
return 0;
default:
return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
}
}
Loading
Loading