本篇文章介紹 使用Edge Impulse 語音機器學習，在Raspberry Pi Pico(RP2040) MCU上執行 Voice recognition，用來控制一輛遙控小車。
使用元件：
一. 控制端：
- Raspberry Pi Pico
- INMP441 I2S microphone
- SD card module
- nRF24L01
二、受控端：
- 向前進(forward)
- 往後退(backward)
- 停在原地(stop)
- 方向左轉(turn left)
- 方向右轉(turn right)
- 速度加快(speed up)
- 速度減慢(slow down)
Edge Impulse 本專案使用參數：
- Create Impulse
- MFCC
- Classifier
- Performance calibration
機器學習訓練過程請參閱成果影片的動態說明。
成果影片：
程式碼：
控制端：
有關nRF24L01, pico_audio_recorder, storage_driver等library使用起參閱前一篇文章：MCU(RP2040) voice recognition/voice commands using TinyML(Edge Impulse)
- pico_tinyML_voice.cpp
#include <stdlib.h>
#include <stdio.h>
#include <sttring.h>
#include "pico/stdlib.h"
#include "bsp/board.h"
#include "tusb.h"
#include "pico_storage.h"
#include "hardware/rtc.h"
#include "pico_audio_recorder.h"
#include "hardware/dma.h"
#include "nRF24L01.h"
#include "edge-impulse-sdk/classifier/ei_run_classifier.h"
#include "edge-impulse-sdk/classifier/ei_classifier_smooth.h"
extern bool new_read_data;
extern int16_t *fw_ptr;
#if DATA_ACQUISITION
#else
int raw_feature_get_data(size_t offset, size_t length, float *out_ptr) {
numpy::int16_to_float(fw_ptr+offset, out_ptr, length);
return 0;
}
#endif
//=== nRF24L01 irq
char const *addr[5] = {"0node", "1node", "2node","3node","4node"};
#define TX_PIN 17
void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) {
static uint32_t ack_payload=0;
uint8_t ack_buff[15];
switch(event_type) {
case EVENT_RX_DR:
data[width]='\0';
printf("RECV== pipe:%d, width:%d, %s\n", datapipe, width, data);
break;
case EVENT_TX_DS:
gpio_put(TX_PIN, !gpio_get(TX_PIN));
//printf("event_data_sent:%d\n", datapipe);
break;
case EVENT_MAX_RT:
printf("event_max_rt:%d, \n", datapipe);
break;
}
}
/*------------- MAIN -------------*/
int main(void)
{
stdio_init_all();
#if DATA_ACQUISITION
rtc_init();
datetime_t t = {
.year=2024, .month=02, .day=12,
.dotw=6,
.hour=10, .min=32, .sec=50
};
if (!rtc_set_datetime(&t)) printf("set rtc error\n");
storage_driver_init();
board_init();
// init device stack on configured roothub port
tud_init(BOARD_TUD_RHPORT);
#else
gpio_init(TX_PIN);
gpio_set_dir(TX_PIN,true);
// == nRF24L01 init
nRF24_spi_default_init(8, 9, irq_callback);
nRF24_config_mode(TRANSMITTER);
nRF24_enable_feature(FEATURE_EN_DPL, true);
nRF24_set_TX_addr((uint8_t*)addr[0], 5);
nRF24_set_RX_addr(0, (uint8_t*)addr[0], 5);
nRF24_enable_data_pipe_dynamic_payload_length(0, true);
//== edge impulse
run_classifier_init();
EI_IMPULSE_ERROR res;
ei_impulse_result_t result = {nullptr};
signal_t features_signal;
features_signal.total_length = EI_CLASSIFIER_SLICE_SIZE;
features_signal.get_data = &raw_feature_get_data;
ei_classifier_smooth_t smooth;
ei_classifier_smooth_init(&smooth, 4, 3, 0.9, 0.3);
#endif
inmp441_pio_init(INMP441_pio, INMP441_SM, INMP441_SD, INMP441_SCK, INMP441_SAMPLE_RATE);
uint8_t f_idx=0;
while (1)
{
#if DATA_ACQUISITION
if (get_recorder_state() == STATE_START_RECORDING) {
inmp441_starting_recording_to_file_wav();
}
tud_task(); // tinyusb device task
#else
if (new_read_data) {
new_read_data = false;
//absolute_time_t t1 = get_absolute_time();
res = run_classifier_continuous(&features_signal, &result, false,true );
if (res != EI_IMPULSE_OK) {
ei_printf("ERR: Failed to run classifier (%d)\n", res);
}
//display_results(&result); printf("\n"); // only for debug
ei_classifier_smooth_update(&smooth, &result);
for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
//result.classification[i].value = run_moving_average_filter(&classifier_maf[ix], result.classification[i].value);
if (result.classification[i].value > 0.9f)
{
// for debug
//ei_printf(" %s: ", ei_classifier_inferencing_categories[i]);
//ei_printf("%.5f\r\n", result.classification[i].value);
// for running
nRF24_write_payload((uint8_t*)ei_classifier_inferencing_categories[i], strlen(ei_classifier_inferencing_categories[i]));
if (strcmp(ei_classifier_inferencing_categories[i], "noise")!=0)
run_classifier_init();
break;
}
}
//absolute_time_t t2 = get_absolute_time();
//printf("time:%lld\n", absolute_time_diff_us(t1,t2));
//printf("\n");
}
#endif
tight_loop_contents();
}
return 0;
}
受控端(小車)
- pico_tinyML_voice_RC_CAR.c
#include <stdio.h>
#include "pico/stdlib.h"
#include "nRF24L01.h"
#include "string.h"
#include "hardware/pwm.h"
#include "ws2812.h"
#define CAR_PIN1A 16
#define CAR_PIN1B 17
#define CAR_PIN2A 18
#define CAR_PIN2B 19
#define SPEED_MAX 5
uint slice1a, slice1b, slice2a, slice2b;
uint chan_1a,chan_1b,chan_2a,chan_2b;
uint8_t addr[5][5] = {"0node", "1node", "2node","3node","4node"};
enum {
NOISE=0,
FORWARD,
BACKWARD,
STOP,
TURN_LEFT,
TURN_RIGHT,
SPEED_UP,
SLOW_DOWN,
ACTION_MAX,
};
int8_t speed=0;
uint16_t pwm_top_count[SPEED_MAX+1] = {0, 4000,8000,12000,16000,20000};
uint32_t ws2812_color[SPEED_MAX+1] = {0x0, 0xFF0000, 0xFF0000, 0x8FFF00, 0x8FFF00, 0xFF00};
uint8_t voice_state = 0;
uint8_t current_direction=STOP;
#define NUM_PIXELS 5
void play_ws2812() {
for (int j=0; j < speed; j++) {
put_grb_pixel(ws2812_color[j+1]);
}
for (int j=speed; j < SPEED_MAX; j++) {
put_grb_pixel(ws2812_color[0]);
}
//sleep_ms(100);
//printf("speed:%d\n", speed);
}
//nRF24L01 irq callback
void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) {
static uint32_t ack_payload=0;
uint8_t ack_buff[15];
switch(event_type) {
case EVENT_RX_DR:
data[width]='\0';
if (strcmp(data, "noise")==0) voice_state=NOISE;
if (strcmp(data, "forward")==0) voice_state=FORWARD;
if (strcmp(data, "backward")==0) voice_state=BACKWARD;
if (strcmp(data, "stop")==0) voice_state=STOP;
if (strcmp(data, "turn_left")==0) voice_state=TURN_LEFT;
if (strcmp(data, "turn_right")==0) voice_state=TURN_RIGHT;
if (strcmp(data, "speed_up")==0) voice_state=SPEED_UP;
if (strcmp(data, "slow_down")==0) voice_state=SLOW_DOWN;
printf("RECV== stat:%d, width:%d, %s\n", voice_state, width, data);
break;
case EVENT_TX_DS:
printf("event_data_sent:%d\n", datapipe);
break;
case EVENT_MAX_RT:
printf("EVENT_MAX_RT:%d\n", datapipe);
break;
}
}
void voice_action(uint8_t stat) {
switch(stat) {
case FORWARD:
pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]);
pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]);
pwm_set_chan_level(slice1b, chan_1b, 0);
pwm_set_chan_level(slice2b, chan_2b, 0);
current_direction = FORWARD;
break;
case BACKWARD:
pwm_set_chan_level(slice1a, chan_1a, 0);
pwm_set_chan_level(slice2a, chan_2a, 0);
pwm_set_chan_level(slice1b, chan_1b, pwm_top_count[speed]);
pwm_set_chan_level(slice2b, chan_2b, pwm_top_count[speed]);
current_direction = BACKWARD;
break;
case STOP:
pwm_set_chan_level(slice1a, chan_1a, 0);
pwm_set_chan_level(slice2a, chan_2a, 0);
pwm_set_chan_level(slice1b, chan_1b, 0);
pwm_set_chan_level(slice2b, chan_2b, 0);
current_direction=STOP;
break;
case TURN_LEFT:
pwm_set_chan_level(slice1a, chan_1a, 0);
pwm_set_chan_level(slice2a, chan_2a, 0);
pwm_set_chan_level(slice1b, chan_1b, 0);
pwm_set_chan_level(slice2b, chan_2b, 0);
pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]);
sleep_ms(500);
break;
case TURN_RIGHT:
pwm_set_chan_level(slice1a, chan_1a, 0);
pwm_set_chan_level(slice2a, chan_2a, 0);
pwm_set_chan_level(slice1b, chan_1b, 0);
pwm_set_chan_level(slice2b, chan_2b, 0);
pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]);
sleep_ms(500);
break;
case SPEED_UP:
speed++;
if (speed > SPEED_MAX) speed=SPEED_MAX;
play_ws2812();
break;
case SLOW_DOWN:
speed--;
if (speed < 0) speed=0;
play_ws2812();
break;
default:
break;
}
}
int main()
{
stdio_init_all();
ws2812_init(pio0, 0, 20);
gpio_set_function(CAR_PIN1A, GPIO_FUNC_PWM);
gpio_set_function(CAR_PIN1B, GPIO_FUNC_PWM);
gpio_set_function(CAR_PIN2A, GPIO_FUNC_PWM);
gpio_set_function(CAR_PIN2B, GPIO_FUNC_PWM);
slice1a = pwm_gpio_to_slice_num(CAR_PIN1A);
slice1b = pwm_gpio_to_slice_num(CAR_PIN1B);
slice2a = pwm_gpio_to_slice_num(CAR_PIN2A);
slice2b = pwm_gpio_to_slice_num(CAR_PIN2B);
chan_1a = pwm_gpio_to_channel(CAR_PIN1A);
chan_1b = pwm_gpio_to_channel(CAR_PIN1B);
chan_2a = pwm_gpio_to_channel(CAR_PIN2A);
chan_2b = pwm_gpio_to_channel(CAR_PIN2B);
pwm_config c = pwm_get_default_config();
pwm_config_set_clkdiv(&c, 125); // 20ms period, steps 20000, clkdiv = 125
pwm_config_set_wrap(&c, 20000);
pwm_config_set_phase_correct(&c, false);
pwm_init(slice1a, &c, true);
pwm_init(slice1b, &c, true);
pwm_init(slice2a, &c, true);
pwm_init(slice2b, &c, true);
uint32_t count=0;
uint8_t status;
nRF24_spi_default_init(8, 9, irq_callback);
nRF24_config_mode(RECEIVER);
nRF24_enable_feature(FEATURE_EN_DPL, true);
nRF24_set_RX_addr(0, addr[0], 5);
nRF24_enable_data_pipe_dynamic_payload_length(0, true);
while(1) {
if (voice_state > 0 && voice_state < ACTION_MAX) {
voice_action(voice_state);
voice_action(current_direction);
voice_state = NOISE;
}
tight_loop_contents();
}
return 0;
}
- CMakeLists.txt
# Generated Cmake Pico project file
cmake_minimum_required(VERSION 3.13)
set(CMAKE_C_STANDARD 11)
set(CMAKE_CXX_STANDARD 17)
# Initialise pico_sdk from installed location
# (note this can come from environment, CMake cache etc)
set(PICO_SDK_PATH "/home/duser/pico/pico-sdk")
set(PICO_BOARD pico CACHE STRING "Board type")
# Pull in Raspberry Pi Pico SDK (must be before project)
include(pico_sdk_import.cmake)
if (PICO_SDK_VERSION_STRING VERSION_LESS "1.4.0")
message(FATAL_ERROR "Raspberry Pi Pico SDK version 1.4.0 (or later) required. Your version is ${PICO_SDK_VERSION_STRING}")
endif()
project(pico_tinyML_voice_RC_CAR C CXX ASM)
# Initialise the Raspberry Pi Pico SDK
pico_sdk_init()
# Add executable. Default name is the project name, version 0.1
add_executable(pico_tinyML_voice_RC_CAR pico_tinyML_voice_RC_CAR.c )
pico_set_program_name(pico_tinyML_voice_RC_CAR "pico_guesture_ws2812")
pico_set_program_version(pico_tinyML_voice_RC_CAR "0.1")
pico_enable_stdio_uart(pico_tinyML_voice_RC_CAR 0)
pico_enable_stdio_usb(pico_tinyML_voice_RC_CAR 1)
# Add the standard library to the build
target_link_libraries(pico_tinyML_voice_RC_CAR
pico_stdlib
hardware_pwm)
# Add the standard include files to the build
target_include_directories(pico_tinyML_voice_RC_CAR PRIVATE
${CMAKE_CURRENT_LIST_DIR}
${CMAKE_CURRENT_LIST_DIR}/.. # for our common lwipopts or any other standard includes, if required
)
# Add any user requested libraries
add_subdirectory(nRF24L01)
target_link_libraries(pico_tinyML_voice_RC_CAR
nRF24L01_drv
)
add_subdirectory(ws2812)
target_link_libraries(pico_tinyML_voice_RC_CAR
ws2812
)
pico_add_extra_outputs(pico_tinyML_voice_RC_CAR)
ws2812:
- ws2812.c
#include "ws2812.h"
#include "stdlib.h"
#include "string.h"
#define NUM_PIXELS 12
#define green 0x1f0000
#define red 0x001f00
#define blue 0x00001f
static inline void ws2812_program_init(PIO pio, uint sm, uint pin, float freq) {
pio_gpio_init(pio, pin);
pio_sm_set_consecutive_pindirs(pio, sm, pin, 1, true);
uint offset = pio_add_program(pio, &ws2812_program);
pio_sm_config c = ws2812_program_get_default_config(offset);
sm_config_set_sideset_pins(&c, pin);
sm_config_set_out_shift(&c, false, true, 24);
sm_config_set_fifo_join(&c, PIO_FIFO_JOIN_TX);
int cycles_per_bit = ws2812_T1 + ws2812_T2 *2;
float div = clock_get_hz(clk_sys) / (freq * cycles_per_bit);
sm_config_set_clkdiv(&c, div);
pio_sm_init(pio, sm, offset, &c);
pio_sm_set_enabled(pio, sm, true);
}
void put_grb_pixel(uint32_t pixel_grb) {
pio_sm_put_blocking(pio0, 0, pixel_grb << 8u);
}
static inline uint32_t urgb_u32(uint8_t r, uint8_t g, uint8_t b) {
return
((uint32_t) (r) << 8) |
((uint32_t) (g) << 16) |
(uint32_t) (b);
}
void clear_pixel() {
for (int i = 0; i < NUM_PIXELS; i++) put_grb_pixel(0x000000);
}
void ws2812_init(PIO pio, uint sm, uint ws2812_pin) {
ws2812_program_init(pio, sm, ws2812_pin, 800000);
}
- ws2812.h
#ifndef _WS2812_H_
#define _WS2812_H_
#include <stdio.h>
#include "pico/stdlib.h"
#include "hardware/pio.h"
#include "ws2812.pio.h"
#include "hardware/clocks.h"
void ws2812_init(PIO pio, uint sm, uint ws2812_pin);
void put_grb_pixel(uint32_t pixel_grb);
#endif
- ws2812.pio
;
; Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
;
; SPDX-License-Identifier: BSD-3-Clause
;
.program ws2812
.side_set 1
.define public T1 2
.define public T2 2
.lang_opt python sideset_init = pico.PIO.OUT_HIGH
.lang_opt python out_init = pico.PIO.OUT_HIGH
.lang_opt python out_shiftdir = 1
.wrap_target
bitloop:
out x, 1 side 0 [T1 - 1] ; Side-set still takes place when instruction stalls
jmp !x do_zero side 1 [T2 - 1] ; Branch on the bit we shifted out. Positive pulse
do_one:
jmp bitloop side 1 [T2 - 1] ; Continue driving high, for a long pulse
do_zero:
nop side 0 [T2 - 1] ; Or drive low, for a short pulse
.wrap
% c-sdk {
%}
- CMakeLists.txt(ws2812)
add_library(ws2812 INTERFACE)
pico_generate_pio_header(ws2812 ${CMAKE_CURRENT_LIST_DIR}/ws2812.pio)
target_sources(ws2812 INTERFACE
${CMAKE_CURRENT_LIST_DIR}/ws2812.c
)
target_include_directories(ws2812 INTERFACE
${CMAKE_CURRENT_LIST_DIR}
)
target_link_libraries(ws2812 INTERFACE
hardware_pio
)
