TinyML(Edge Impulse) : Voice remote control car || RP2040 || Cortex-M0+

 本篇文章介紹 使用Edge Impulse 語音機器學習，在Raspberry Pi Pico(RP2040) MCU上執行 Voice recognition，用來控制一輛遙控小車。

使用元件：

一. 控制端：

1. Raspberry Pi Pico 
2. INMP441 I2S microphone
3. SD card module
4. nRF24L01

二、受控端：

1. Raspberry Pi Pico
2. TT馬達+65mm 車輪 X 4
3. L298N 直流馬達驅動板
4. nRF24L01

本專案使用7個語音來控制小車的動作：

1. 向前進(forward)
2. 往後退(backward)
3. 停在原地(stop)
4. 方向左轉(turn left)
5. 方向右轉(turn right)
6. 速度加快(speed up)
7. 速度減慢(slow down)

Edge Impulse 本專案使用參數：

• Create Impulse

• MFCC

• Classifier

• Performance calibration

機器學習訓練過程請參閱成果影片的動態說明。

成果影片：

程式碼：

 控制端：

有關nRF24L01, pico_audio_recorder, storage_driver等library使用起參閱前一篇文章：MCU(RP2040) voice recognition/voice commands using TinyML(Edge Impulse)

• pico_tinyML_voice.cpp

#include <stdlib.h>
#include <stdio.h>
#include <sttring.h>
#include "pico/stdlib.h"
#include "bsp/board.h"
#include "tusb.h"

#include "pico_storage.h"
#include "hardware/rtc.h"
#include "pico_audio_recorder.h"
#include "hardware/dma.h"
#include "nRF24L01.h"

#include "edge-impulse-sdk/classifier/ei_run_classifier.h"
#include "edge-impulse-sdk/classifier/ei_classifier_smooth.h"
extern bool new_read_data;
extern int16_t *fw_ptr;

#if DATA_ACQUISITION
#else
int raw_feature_get_data(size_t offset, size_t length, float *out_ptr) {
    numpy::int16_to_float(fw_ptr+offset, out_ptr, length);
    return 0;
}
#endif

//=== nRF24L01 irq
char const *addr[5] = {"0node", "1node", "2node","3node","4node"};
#define TX_PIN 17

void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) {
    static uint32_t ack_payload=0;
    uint8_t ack_buff[15];
      switch(event_type) {
        case EVENT_RX_DR:
            data[width]='\0';
            printf("RECV== pipe:%d, width:%d, %s\n", datapipe, width, data);
        break;
        case EVENT_TX_DS:
            gpio_put(TX_PIN, !gpio_get(TX_PIN));
            //printf("event_data_sent:%d\n", datapipe);
        break;
        case EVENT_MAX_RT:
        printf("event_max_rt:%d, \n", datapipe);
        break;
    }

}

/*------------- MAIN -------------*/
int main(void)
{
  stdio_init_all();
#if DATA_ACQUISITION 
      rtc_init();
      datetime_t t = {
        .year=2024, .month=02, .day=12, 
        .dotw=6,
        .hour=10, .min=32, .sec=50
      };
     
      if (!rtc_set_datetime(&t)) printf("set rtc error\n");
        
      storage_driver_init();
      
      board_init();
      // init device stack on configured roothub port
      tud_init(BOARD_TUD_RHPORT);

#else 
    gpio_init(TX_PIN);
    gpio_set_dir(TX_PIN,true);
    // == nRF24L01 init
    nRF24_spi_default_init(8, 9, irq_callback);
    nRF24_config_mode(TRANSMITTER);
    nRF24_enable_feature(FEATURE_EN_DPL, true);
    nRF24_set_TX_addr((uint8_t*)addr[0], 5);
    nRF24_set_RX_addr(0, (uint8_t*)addr[0], 5);
    nRF24_enable_data_pipe_dynamic_payload_length(0, true);
    //== edge impulse
      run_classifier_init();
    
      EI_IMPULSE_ERROR res;
      ei_impulse_result_t result = {nullptr};
      signal_t features_signal;
      features_signal.total_length = EI_CLASSIFIER_SLICE_SIZE;
      features_signal.get_data = &raw_feature_get_data;

      ei_classifier_smooth_t smooth;
      ei_classifier_smooth_init(&smooth, 4, 3, 0.9, 0.3);
#endif

  inmp441_pio_init(INMP441_pio, INMP441_SM, INMP441_SD, INMP441_SCK, INMP441_SAMPLE_RATE);

  uint8_t f_idx=0;

  while (1)
  {
#if DATA_ACQUISITION
      if (get_recorder_state() == STATE_START_RECORDING) {
        inmp441_starting_recording_to_file_wav();
      }
         
      tud_task(); // tinyusb device task
#else

    if (new_read_data) {
          new_read_data = false;
//absolute_time_t t1 = get_absolute_time();
          res = run_classifier_continuous(&features_signal, &result, false,true );

          if (res != EI_IMPULSE_OK) {
                  ei_printf("ERR: Failed to run classifier (%d)\n", res);
          }
         
        //display_results(&result); printf("\n"); // only for debug
          ei_classifier_smooth_update(&smooth, &result);
          for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
            //result.classification[i].value = run_moving_average_filter(&classifier_maf[ix], result.classification[i].value);
            if (result.classification[i].value > 0.9f) 
            { 
              // for debug
              //ei_printf("  %s: ", ei_classifier_inferencing_categories[i]);
              //ei_printf("%.5f\r\n", result.classification[i].value);
              // for running
              nRF24_write_payload((uint8_t*)ei_classifier_inferencing_categories[i], strlen(ei_classifier_inferencing_categories[i]));
              if (strcmp(ei_classifier_inferencing_categories[i], "noise")!=0)
                run_classifier_init();
              break;
            } 
          }

        //absolute_time_t t2 = get_absolute_time();
        //printf("time:%lld\n", absolute_time_diff_us(t1,t2));
        //printf("\n");
   
    }
   
#endif
      tight_loop_contents();
  }

  return 0;
}

受控端(小車)

• pico_tinyML_voice_RC_CAR.c

#include <stdio.h>
#include "pico/stdlib.h"
#include "nRF24L01.h"
#include "string.h"
#include "hardware/pwm.h"
#include "ws2812.h"

#define CAR_PIN1A   16
#define CAR_PIN1B   17
#define CAR_PIN2A   18
#define CAR_PIN2B   19
#define SPEED_MAX   5
uint slice1a, slice1b, slice2a, slice2b;
uint chan_1a,chan_1b,chan_2a,chan_2b;

uint8_t addr[5][5] = {"0node", "1node", "2node","3node","4node"};

enum {
    NOISE=0,
    FORWARD,
    BACKWARD,
    STOP,
    TURN_LEFT,
    TURN_RIGHT,
    SPEED_UP,
    SLOW_DOWN,
    ACTION_MAX,
};

int8_t speed=0;
uint16_t pwm_top_count[SPEED_MAX+1] = {0, 4000,8000,12000,16000,20000};
uint32_t ws2812_color[SPEED_MAX+1] = {0x0, 0xFF0000, 0xFF0000, 0x8FFF00, 0x8FFF00, 0xFF00};
uint8_t voice_state = 0;
uint8_t current_direction=STOP;

#define NUM_PIXELS 5
void play_ws2812() {
    for (int j=0; j < speed; j++) { 
        put_grb_pixel(ws2812_color[j+1]);
    }
    for (int j=speed; j < SPEED_MAX; j++) { 
        put_grb_pixel(ws2812_color[0]);
    }
    //sleep_ms(100);
    //printf("speed:%d\n", speed);
}
//nRF24L01 irq callback
void irq_callback(uint8_t event_type, uint8_t datapipe, uint8_t* data, uint8_t width) {
    static uint32_t ack_payload=0;
    uint8_t ack_buff[15];
      switch(event_type) {
        case EVENT_RX_DR:
            data[width]='\0';
            
            if (strcmp(data, "noise")==0) voice_state=NOISE;
            if (strcmp(data, "forward")==0) voice_state=FORWARD;
            if (strcmp(data, "backward")==0) voice_state=BACKWARD;
            if (strcmp(data, "stop")==0) voice_state=STOP;
            if (strcmp(data, "turn_left")==0) voice_state=TURN_LEFT;
            if (strcmp(data, "turn_right")==0) voice_state=TURN_RIGHT;
            if (strcmp(data, "speed_up")==0) voice_state=SPEED_UP;
            if (strcmp(data, "slow_down")==0) voice_state=SLOW_DOWN;

            printf("RECV== stat:%d, width:%d, %s\n", voice_state, width, data);
        break;
        case EVENT_TX_DS:
            printf("event_data_sent:%d\n", datapipe);
        break;
        case EVENT_MAX_RT:
            printf("EVENT_MAX_RT:%d\n", datapipe);
        break;
    }
}


void voice_action(uint8_t stat) {
    switch(stat) {
        case FORWARD:
            pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]);
            pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]);
            pwm_set_chan_level(slice1b, chan_1b, 0);
            pwm_set_chan_level(slice2b, chan_2b, 0);
            current_direction = FORWARD;
        break;
        case BACKWARD:
            pwm_set_chan_level(slice1a, chan_1a, 0);
            pwm_set_chan_level(slice2a, chan_2a, 0);
            pwm_set_chan_level(slice1b, chan_1b, pwm_top_count[speed]);
            pwm_set_chan_level(slice2b, chan_2b, pwm_top_count[speed]);
            current_direction = BACKWARD;
        break;
        case STOP:
            pwm_set_chan_level(slice1a, chan_1a, 0);
            pwm_set_chan_level(slice2a, chan_2a, 0);
            pwm_set_chan_level(slice1b, chan_1b, 0);
            pwm_set_chan_level(slice2b, chan_2b, 0);
            current_direction=STOP;
        break;
        case TURN_LEFT:
            pwm_set_chan_level(slice1a, chan_1a, 0);         
            pwm_set_chan_level(slice2a, chan_2a, 0);
            pwm_set_chan_level(slice1b, chan_1b, 0);
            pwm_set_chan_level(slice2b, chan_2b, 0);
            pwm_set_chan_level(slice1a, chan_1a, pwm_top_count[speed]);
            sleep_ms(500);
        break;
        case TURN_RIGHT:
            pwm_set_chan_level(slice1a, chan_1a, 0);
            pwm_set_chan_level(slice2a, chan_2a, 0);
            pwm_set_chan_level(slice1b, chan_1b, 0);
            pwm_set_chan_level(slice2b, chan_2b, 0);
            pwm_set_chan_level(slice2a, chan_2a, pwm_top_count[speed]);
            sleep_ms(500);
        break;
        case SPEED_UP:
           speed++;
           if (speed > SPEED_MAX) speed=SPEED_MAX;   
           play_ws2812();      
        break;
        case SLOW_DOWN:
           speed--;
           if (speed < 0) speed=0; 
           play_ws2812();
        break;
        default:
        break;
    }
}

int main()
{
    
    stdio_init_all(); 

    ws2812_init(pio0, 0, 20);

    gpio_set_function(CAR_PIN1A, GPIO_FUNC_PWM);
    gpio_set_function(CAR_PIN1B, GPIO_FUNC_PWM);
    gpio_set_function(CAR_PIN2A, GPIO_FUNC_PWM);
    gpio_set_function(CAR_PIN2B, GPIO_FUNC_PWM);

    slice1a = pwm_gpio_to_slice_num(CAR_PIN1A);
    slice1b = pwm_gpio_to_slice_num(CAR_PIN1B);
    slice2a = pwm_gpio_to_slice_num(CAR_PIN2A);
    slice2b = pwm_gpio_to_slice_num(CAR_PIN2B);

    chan_1a = pwm_gpio_to_channel(CAR_PIN1A);
    chan_1b = pwm_gpio_to_channel(CAR_PIN1B);
    chan_2a = pwm_gpio_to_channel(CAR_PIN2A);
    chan_2b = pwm_gpio_to_channel(CAR_PIN2B);

    pwm_config c = pwm_get_default_config();
    pwm_config_set_clkdiv(&c, 125);  // 20ms period, steps 20000, clkdiv = 125 
    pwm_config_set_wrap(&c, 20000);
    pwm_config_set_phase_correct(&c, false);
    pwm_init(slice1a, &c, true);
    pwm_init(slice1b, &c, true);
    pwm_init(slice2a, &c, true);
    pwm_init(slice2b, &c, true);
    
    uint32_t count=0;
    uint8_t status;

    nRF24_spi_default_init(8, 9, irq_callback);
    nRF24_config_mode(RECEIVER);
    nRF24_enable_feature(FEATURE_EN_DPL, true);
    nRF24_set_RX_addr(0, addr[0], 5);
    nRF24_enable_data_pipe_dynamic_payload_length(0, true);

   
    while(1) {
        if (voice_state > 0 && voice_state < ACTION_MAX) {
            voice_action(voice_state);
            voice_action(current_direction);
            voice_state = NOISE;
        }
       
        tight_loop_contents();
    }
    
    return 0;
}

• CMakeLists.txt

# Generated Cmake Pico project file

cmake_minimum_required(VERSION 3.13)

set(CMAKE_C_STANDARD 11)
set(CMAKE_CXX_STANDARD 17)

# Initialise pico_sdk from installed location
# (note this can come from environment, CMake cache etc)
set(PICO_SDK_PATH "/home/duser/pico/pico-sdk")

set(PICO_BOARD pico CACHE STRING "Board type")

# Pull in Raspberry Pi Pico SDK (must be before project)
include(pico_sdk_import.cmake)

if (PICO_SDK_VERSION_STRING VERSION_LESS "1.4.0")
  message(FATAL_ERROR "Raspberry Pi Pico SDK version 1.4.0 (or later) required. Your version is ${PICO_SDK_VERSION_STRING}")
endif()

project(pico_tinyML_voice_RC_CAR C CXX ASM)

# Initialise the Raspberry Pi Pico SDK
pico_sdk_init()

# Add executable. Default name is the project name, version 0.1

add_executable(pico_tinyML_voice_RC_CAR pico_tinyML_voice_RC_CAR.c )

pico_set_program_name(pico_tinyML_voice_RC_CAR "pico_guesture_ws2812")
pico_set_program_version(pico_tinyML_voice_RC_CAR "0.1")

pico_enable_stdio_uart(pico_tinyML_voice_RC_CAR 0)
pico_enable_stdio_usb(pico_tinyML_voice_RC_CAR 1)

# Add the standard library to the build
target_link_libraries(pico_tinyML_voice_RC_CAR
        pico_stdlib
        hardware_pwm)

# Add the standard include files to the build
target_include_directories(pico_tinyML_voice_RC_CAR PRIVATE
  ${CMAKE_CURRENT_LIST_DIR}
  ${CMAKE_CURRENT_LIST_DIR}/.. # for our common lwipopts or any other standard includes, if required
)

# Add any user requested libraries
add_subdirectory(nRF24L01)
target_link_libraries(pico_tinyML_voice_RC_CAR 
        nRF24L01_drv
)

add_subdirectory(ws2812)
target_link_libraries(pico_tinyML_voice_RC_CAR 
      ws2812
)

pico_add_extra_outputs(pico_tinyML_voice_RC_CAR)

ws2812:

• ws2812.c

#include "ws2812.h"
#include "stdlib.h"
#include "string.h"

#define NUM_PIXELS 12


#define green 0x1f0000
#define red 0x001f00
#define blue 0x00001f

static inline void ws2812_program_init(PIO pio, uint sm, uint pin, float freq) {
    pio_gpio_init(pio, pin);
    pio_sm_set_consecutive_pindirs(pio, sm, pin, 1, true);

    uint offset = pio_add_program(pio, &ws2812_program);

    pio_sm_config c = ws2812_program_get_default_config(offset);
    sm_config_set_sideset_pins(&c, pin);
    sm_config_set_out_shift(&c, false, true, 24);
    sm_config_set_fifo_join(&c, PIO_FIFO_JOIN_TX);

    int cycles_per_bit = ws2812_T1 + ws2812_T2 *2;
    float div = clock_get_hz(clk_sys) / (freq * cycles_per_bit);
    sm_config_set_clkdiv(&c, div);

    pio_sm_init(pio, sm, offset, &c);
    pio_sm_set_enabled(pio, sm, true);
}

void put_grb_pixel(uint32_t pixel_grb) {
    pio_sm_put_blocking(pio0, 0, pixel_grb << 8u);
}

static inline uint32_t urgb_u32(uint8_t r, uint8_t g, uint8_t b) {
    return
            ((uint32_t) (r) << 8) |
            ((uint32_t) (g) << 16) |
            (uint32_t) (b);
}

void clear_pixel() {
    for (int i = 0; i < NUM_PIXELS; i++) put_grb_pixel(0x000000);
}

void ws2812_init(PIO pio, uint sm, uint ws2812_pin) {

    ws2812_program_init(pio, sm,  ws2812_pin, 800000);
}

• ws2812.h

#ifndef _WS2812_H_
#define _WS2812_H_
#include <stdio.h>
#include "pico/stdlib.h"
#include "hardware/pio.h"
#include "ws2812.pio.h"
#include "hardware/clocks.h"



void ws2812_init(PIO pio, uint sm, uint ws2812_pin);
void put_grb_pixel(uint32_t pixel_grb);

#endif 

• ws2812.pio

;
; Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
;
; SPDX-License-Identifier: BSD-3-Clause
;

.program ws2812
.side_set 1

.define public T1 2
.define public T2 2


.lang_opt python sideset_init = pico.PIO.OUT_HIGH
.lang_opt python out_init     = pico.PIO.OUT_HIGH
.lang_opt python out_shiftdir = 1

.wrap_target
bitloop:
    out x, 1       side 0 [T1 - 1] ; Side-set still takes place when instruction stalls
    jmp !x do_zero side 1 [T2 - 1] ; Branch on the bit we shifted out. Positive pulse
do_one:
    jmp  bitloop   side 1 [T2 - 1] ; Continue driving high, for a long pulse
do_zero:
    nop            side 0 [T2 - 1] ; Or drive low, for a short pulse
.wrap

% c-sdk {

%}

• CMakeLists.txt(ws2812)

add_library(ws2812 INTERFACE)
pico_generate_pio_header(ws2812 ${CMAKE_CURRENT_LIST_DIR}/ws2812.pio)
target_sources(ws2812 INTERFACE
    ${CMAKE_CURRENT_LIST_DIR}/ws2812.c
)

target_include_directories(ws2812 INTERFACE
    ${CMAKE_CURRENT_LIST_DIR}
)

target_link_libraries(ws2812 INTERFACE
        hardware_pio
)