本篇文章介紹 使用Edge Impulse 語音機器學習，在Raspberry Pi Pico(RP2040) MCU上執行 Voice recognition，用來控制一輛遙控小車。
使用元件：
一. 控制端：
- Raspberry Pi Pico
- INMP441 I2S microphone
- SD card module
- nRF24L01
二、受控端：
- 向前進(forward)
- 往後退(backward)
- 停在原地(stop)
- 方向左轉(turn left)
- 方向右轉(turn right)
- 速度加快(speed up)
- 速度減慢(slow down)
Edge Impulse 本專案使用參數：
- Create Impulse
- MFCC
- Classifier
- Performance calibration
機器學習訓練過程請參閱成果影片的動態說明。
成果影片：
程式碼：
控制端：
有關nRF24L01, pico_audio_recorder, storage_driver等library使用起參閱前一篇文章：MCU(RP2040) voice recognition/voice commands using TinyML(Edge Impulse)
- pico_tinyML_voice.cpp
#include <stdlib.h> #include <stdio.h> #include <sttring.h> #include "pico/stdlib.h" #include "bsp/board.h" #include "tusb.h" #include "pico_storage.h" #include "hardware/rtc.h" #include "pico_audio_recorder.h" #include "hardware/dma.h" #include "nRF24L01.h" #include "edge-impulse-sdk/classifier/ei_run_classifier.h" #include "edge-impulse-sdk/classifier/ei_classifier_smooth.h" extern bool new_read_data; extern int16_t *fw_ptr; #if DATA_ACQUISITION #else int raw_feature_get_data(size_t offset, size_t length, float *out_ptr) { numpy::int16_to_float(fw_ptr+offset, out_ptr, length); return 0; } #endif //=== nRF24L01 irq char const *addr[5] = {"0node", "1node", "2node","3node","4node"}; #define TX_PIN 17 void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) { static uint32_t ack_payload=0; uint8_t ack_buff[15]; switch(event_type) { case EVENT_RX_DR: data[width]='\0'; printf("RECV== pipe:%d, width:%d, %s\n", datapipe, width, data); break; case EVENT_TX_DS: gpio_put(TX_PIN, !gpio_get(TX_PIN)); //printf("event_data_sent:%d\n", datapipe); break; case EVENT_MAX_RT: printf("event_max_rt:%d, \n", datapipe); break; } } /*------------- MAIN -------------*/ int main(void) { stdio_init_all(); #if DATA_ACQUISITION rtc_init(); datetime_t t = { .year=2024, .month=02, .day=12, .dotw=6, .hour=10, .min=32, .sec=50 }; if (!rtc_set_datetime(&t)) printf("set rtc error\n"); storage_driver_init(); board_init(); // init device stack on configured roothub port tud_init(BOARD_TUD_RHPORT); #else gpio_init(TX_PIN); gpio_set_dir(TX_PIN,true); // == nRF24L01 init nRF24_spi_default_init(8, 9, irq_callback); nRF24_config_mode(TRANSMITTER); nRF24_enable_feature(FEATURE_EN_DPL, true); nRF24_set_TX_addr((uint8_t*)addr[0], 5); nRF24_set_RX_addr(0, (uint8_t*)addr[0], 5); nRF24_enable_data_pipe_dynamic_payload_length(0, true); //== edge impulse run_classifier_init(); EI_IMPULSE_ERROR res; ei_impulse_result_t result = {nullptr}; signal_t features_signal; features_signal.total_length = EI_CLASSIFIER_SLICE_SIZE; features_signal.get_data = &raw_feature_get_data; ei_classifier_smooth_t smooth; ei_classifier_smooth_init(&smooth, 4, 3, 0.9, 0.3); #endif inmp441_pio_init(INMP441_pio, INMP441_SM, INMP441_SD, INMP441_SCK, INMP441_SAMPLE_RATE); uint8_t f_idx=0; while (1) { #if DATA_ACQUISITION if (get_recorder_state() == STATE_START_RECORDING) { inmp441_starting_recording_to_file_wav(); } tud_task(); // tinyusb device task #else if (new_read_data) { new_read_data = false; //absolute_time_t t1 = get_absolute_time(); res = run_classifier_continuous(&features_signal, &result, false,true ); if (res != EI_IMPULSE_OK) { ei_printf("ERR: Failed to run classifier (%d)\n", res); } //display_results(&result); printf("\n"); // only for debug ei_classifier_smooth_update(&smooth, &result); for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) { //result.classification[i].value = run_moving_average_filter(&classifier_maf[ix], result.classification[i].value); if (result.classification[i].value > 0.9f) { // for debug //ei_printf(" %s: ", ei_classifier_inferencing_categories[i]); //ei_printf("%.5f\r\n", result.classification[i].value); // for running nRF24_write_payload((uint8_t*)ei_classifier_inferencing_categories[i], strlen(ei_classifier_inferencing_categories[i])); if (strcmp(ei_classifier_inferencing_categories[i], "noise")!=0) run_classifier_init(); break; } } //absolute_time_t t2 = get_absolute_time(); //printf("time:%lld\n", absolute_time_diff_us(t1,t2)); //printf("\n"); } #endif tight_loop_contents(); } return 0; }
受控端(小車)
- pico_tinyML_voice_RC_CAR.c
#include <stdio.h> #include "pico/stdlib.h" #include "nRF24L01.h" #include "string.h" #include "hardware/pwm.h" #include "ws2812.h" #define CAR_PIN1A 16 #define CAR_PIN1B 17 #define CAR_PIN2A 18 #define CAR_PIN2B 19 #define SPEED_MAX 5 uint slice1a, slice1b, slice2a, slice2b; uint chan_1a,chan_1b,chan_2a,chan_2b; uint8_t addr[5][5] = {"0node", "1node", "2node","3node","4node"}; enum { NOISE=0, FORWARD, BACKWARD, STOP, TURN_LEFT, TURN_RIGHT, SPEED_UP, SLOW_DOWN, ACTION_MAX, }; int8_t speed=0; uint16_t pwm_top_count[SPEED_MAX+1] = {0, 4000,8000,12000,16000,20000}; uint32_t ws2812_color[SPEED_MAX+1] = {0x0, 0xFF0000, 0xFF0000, 0x8FFF00, 0x8FFF00, 0xFF00}; uint8_t voice_state = 0; uint8_t current_direction=STOP; #define NUM_PIXELS 5 void play_ws2812() { for (int j=0; j < speed; j++) { put_grb_pixel(ws2812_color[j+1]); } for (int j=speed; j < SPEED_MAX; j++) { put_grb_pixel(ws2812_color[0]); } //sleep_ms(100); //printf("speed:%d\n", speed); } //nRF24L01 irq callback void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) { static uint32_t ack_payload=0; uint8_t ack_buff[15]; switch(event_type) { case EVENT_RX_DR: data[width]='\0'; if (strcmp(data, "noise")==0) voice_state=NOISE; if (strcmp(data, "forward")==0) voice_state=FORWARD; if (strcmp(data, "backward")==0) voice_state=BACKWARD; if (strcmp(data, "stop")==0) voice_state=STOP; if (strcmp(data, "turn_left")==0) voice_state=TURN_LEFT; if (strcmp(data, "turn_right")==0) voice_state=TURN_RIGHT; if (strcmp(data, "speed_up")==0) voice_state=SPEED_UP; if (strcmp(data, "slow_down")==0) voice_state=SLOW_DOWN; printf("RECV== stat:%d, width:%d, %s\n", voice_state, width, data); break; case EVENT_TX_DS: printf("event_data_sent:%d\n", datapipe); break; case EVENT_MAX_RT: printf("EVENT_MAX_RT:%d\n", datapipe); break; } } void voice_action(uint8_t stat) { switch(stat) { case FORWARD: pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]); pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]); pwm_set_chan_level(slice1b, chan_1b, 0); pwm_set_chan_level(slice2b, chan_2b, 0); current_direction = FORWARD; break; case BACKWARD: pwm_set_chan_level(slice1a, chan_1a, 0); pwm_set_chan_level(slice2a, chan_2a, 0); pwm_set_chan_level(slice1b, chan_1b, pwm_top_count[speed]); pwm_set_chan_level(slice2b, chan_2b, pwm_top_count[speed]); current_direction = BACKWARD; break; case STOP: pwm_set_chan_level(slice1a, chan_1a, 0); pwm_set_chan_level(slice2a, chan_2a, 0); pwm_set_chan_level(slice1b, chan_1b, 0); pwm_set_chan_level(slice2b, chan_2b, 0); current_direction=STOP; break; case TURN_LEFT: pwm_set_chan_level(slice1a, chan_1a, 0); pwm_set_chan_level(slice2a, chan_2a, 0); pwm_set_chan_level(slice1b, chan_1b, 0); pwm_set_chan_level(slice2b, chan_2b, 0); pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]); sleep_ms(500); break; case TURN_RIGHT: pwm_set_chan_level(slice1a, chan_1a, 0); pwm_set_chan_level(slice2a, chan_2a, 0); pwm_set_chan_level(slice1b, chan_1b, 0); pwm_set_chan_level(slice2b, chan_2b, 0); pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]); sleep_ms(500); break; case SPEED_UP: speed++; if (speed > SPEED_MAX) speed=SPEED_MAX; play_ws2812(); break; case SLOW_DOWN: speed--; if (speed < 0) speed=0; play_ws2812(); break; default: break; } } int main() { stdio_init_all(); ws2812_init(pio0, 0, 20); gpio_set_function(CAR_PIN1A, GPIO_FUNC_PWM); gpio_set_function(CAR_PIN1B, GPIO_FUNC_PWM); gpio_set_function(CAR_PIN2A, GPIO_FUNC_PWM); gpio_set_function(CAR_PIN2B, GPIO_FUNC_PWM); slice1a = pwm_gpio_to_slice_num(CAR_PIN1A); slice1b = pwm_gpio_to_slice_num(CAR_PIN1B); slice2a = pwm_gpio_to_slice_num(CAR_PIN2A); slice2b = pwm_gpio_to_slice_num(CAR_PIN2B); chan_1a = pwm_gpio_to_channel(CAR_PIN1A); chan_1b = pwm_gpio_to_channel(CAR_PIN1B); chan_2a = pwm_gpio_to_channel(CAR_PIN2A); chan_2b = pwm_gpio_to_channel(CAR_PIN2B); pwm_config c = pwm_get_default_config(); pwm_config_set_clkdiv(&c, 125); // 20ms period, steps 20000, clkdiv = 125 pwm_config_set_wrap(&c, 20000); pwm_config_set_phase_correct(&c, false); pwm_init(slice1a, &c, true); pwm_init(slice1b, &c, true); pwm_init(slice2a, &c, true); pwm_init(slice2b, &c, true); uint32_t count=0; uint8_t status; nRF24_spi_default_init(8, 9, irq_callback); nRF24_config_mode(RECEIVER); nRF24_enable_feature(FEATURE_EN_DPL, true); nRF24_set_RX_addr(0, addr[0], 5); nRF24_enable_data_pipe_dynamic_payload_length(0, true); while(1) { if (voice_state > 0 && voice_state < ACTION_MAX) { voice_action(voice_state); voice_action(current_direction); voice_state = NOISE; } tight_loop_contents(); } return 0; }
- CMakeLists.txt
# Generated Cmake Pico project file cmake_minimum_required(VERSION 3.13) set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) # Initialise pico_sdk from installed location # (note this can come from environment, CMake cache etc) set(PICO_SDK_PATH "/home/duser/pico/pico-sdk") set(PICO_BOARD pico CACHE STRING "Board type") # Pull in Raspberry Pi Pico SDK (must be before project) include(pico_sdk_import.cmake) if (PICO_SDK_VERSION_STRING VERSION_LESS "1.4.0") message(FATAL_ERROR "Raspberry Pi Pico SDK version 1.4.0 (or later) required. Your version is ${PICO_SDK_VERSION_STRING}") endif() project(pico_tinyML_voice_RC_CAR C CXX ASM) # Initialise the Raspberry Pi Pico SDK pico_sdk_init() # Add executable. Default name is the project name, version 0.1 add_executable(pico_tinyML_voice_RC_CAR pico_tinyML_voice_RC_CAR.c ) pico_set_program_name(pico_tinyML_voice_RC_CAR "pico_guesture_ws2812") pico_set_program_version(pico_tinyML_voice_RC_CAR "0.1") pico_enable_stdio_uart(pico_tinyML_voice_RC_CAR 0) pico_enable_stdio_usb(pico_tinyML_voice_RC_CAR 1) # Add the standard library to the build target_link_libraries(pico_tinyML_voice_RC_CAR pico_stdlib hardware_pwm) # Add the standard include files to the build target_include_directories(pico_tinyML_voice_RC_CAR PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/.. # for our common lwipopts or any other standard includes, if required ) # Add any user requested libraries add_subdirectory(nRF24L01) target_link_libraries(pico_tinyML_voice_RC_CAR nRF24L01_drv ) add_subdirectory(ws2812) target_link_libraries(pico_tinyML_voice_RC_CAR ws2812 ) pico_add_extra_outputs(pico_tinyML_voice_RC_CAR)
ws2812:
- ws2812.c
#include "ws2812.h" #include "stdlib.h" #include "string.h" #define NUM_PIXELS 12 #define green 0x1f0000 #define red 0x001f00 #define blue 0x00001f static inline void ws2812_program_init(PIO pio, uint sm, uint pin, float freq) { pio_gpio_init(pio, pin); pio_sm_set_consecutive_pindirs(pio, sm, pin, 1, true); uint offset = pio_add_program(pio, &ws2812_program); pio_sm_config c = ws2812_program_get_default_config(offset); sm_config_set_sideset_pins(&c, pin); sm_config_set_out_shift(&c, false, true, 24); sm_config_set_fifo_join(&c, PIO_FIFO_JOIN_TX); int cycles_per_bit = ws2812_T1 + ws2812_T2 *2; float div = clock_get_hz(clk_sys) / (freq * cycles_per_bit); sm_config_set_clkdiv(&c, div); pio_sm_init(pio, sm, offset, &c); pio_sm_set_enabled(pio, sm, true); } void put_grb_pixel(uint32_t pixel_grb) { pio_sm_put_blocking(pio0, 0, pixel_grb << 8u); } static inline uint32_t urgb_u32(uint8_t r, uint8_t g, uint8_t b) { return ((uint32_t) (r) << 8) | ((uint32_t) (g) << 16) | (uint32_t) (b); } void clear_pixel() { for (int i = 0; i < NUM_PIXELS; i++) put_grb_pixel(0x000000); } void ws2812_init(PIO pio, uint sm, uint ws2812_pin) { ws2812_program_init(pio, sm, ws2812_pin, 800000); }
- ws2812.h
#ifndef _WS2812_H_ #define _WS2812_H_ #include <stdio.h> #include "pico/stdlib.h" #include "hardware/pio.h" #include "ws2812.pio.h" #include "hardware/clocks.h" void ws2812_init(PIO pio, uint sm, uint ws2812_pin); void put_grb_pixel(uint32_t pixel_grb); #endif
- ws2812.pio
; ; Copyright (c) 2020 Raspberry Pi (Trading) Ltd. ; ; SPDX-License-Identifier: BSD-3-Clause ; .program ws2812 .side_set 1 .define public T1 2 .define public T2 2 .lang_opt python sideset_init = pico.PIO.OUT_HIGH .lang_opt python out_init = pico.PIO.OUT_HIGH .lang_opt python out_shiftdir = 1 .wrap_target bitloop: out x, 1 side 0 [T1 - 1] ; Side-set still takes place when instruction stalls jmp !x do_zero side 1 [T2 - 1] ; Branch on the bit we shifted out. Positive pulse do_one: jmp bitloop side 1 [T2 - 1] ; Continue driving high, for a long pulse do_zero: nop side 0 [T2 - 1] ; Or drive low, for a short pulse .wrap % c-sdk { %}
- CMakeLists.txt(ws2812)
add_library(ws2812 INTERFACE) pico_generate_pio_header(ws2812 ${CMAKE_CURRENT_LIST_DIR}/ws2812.pio) target_sources(ws2812 INTERFACE ${CMAKE_CURRENT_LIST_DIR}/ws2812.c ) target_include_directories(ws2812 INTERFACE ${CMAKE_CURRENT_LIST_DIR} ) target_link_libraries(ws2812 INTERFACE hardware_pio )
