ESP-IDF: ESP32 I2S介面語音錄音機(ESP32 I2S Audio Recorder using ESP-ADF)

 本實驗製作一個I2S介面錄音設備，使用元件如下:

1. ESP32-WROOM-32(30pins) development board
2. INMP441 microphone(I2S)
3. MAX98357A 
4. 3W4Ω Speaker
5. 2.4" TFT (8-bit parallel port with touch)
6. SPI SD Card

實驗內容：

1. 使用touch screen控制錄放音樂存放在SD上。
2. 錄音時動畫顯示
3. 可調整錄音或撥放音量
4. 可捲動選擇撥放的錄製清單

軟體開發環境: VSCode 使用ESP-IDF & ESP-ADF extension; 不使用ESP32 audio development board。

本實驗幾乎用完所有GPIO pins，在不使用ESP32 audio development board下，利用ESP-ADF環境開發audio application，實驗內容不另外define custom board，在menuconfig Audio board選用ESP32-Lyart-Mini，再自行重新另訂GPIO pins。

一、GPIO pins:

I2S:

SPI SD Card

TFT Display:

二、Record audio pipeline

INMP441使用left channel，接線如下:

相對應的參數設定,

I2S 參數:由I2S_STREAM_CFG_DEFAULT()修改:

i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();

i2s_cfg.task_core = APP_CPU_NUM;

i2s_cfg.type = AUDIO_STREAM_READER;

i2s_cfg.i2s_config.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;

i2s_cfg.use_alc = true;

i2s_cfg.volume=-63;

i2s_stream_reader = i2s_stream_init(&i2s_cfg);

I2S pins:

i2s_pin_config_t rp = {

        .bck_io_num = 5,

        .ws_io_num = 25,

        .data_out_num = -1,

        .data_in_num = 39

    };

    i2s_set_pin(I2S_NUM_0, &rp);

三、Play audio pipeline

I2S參數使用default即可，因為audio_element_setinfo 函數填入

i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();

    i2s_cfg.type = AUDIO_STREAM_WRITER;

    i2s_cfg.use_alc = true;

    i2s_cfg.volume=0;

i2s_stream_writer = i2s_stream_init(&i2s_cfg);

audio_element_getinfo(audio_decoder, &music_info);

audio_element_setinfo(i2s_stream_writer, &music_info);

I2S pins:

i2s_pin_config_t wp = {

        .bck_io_num = 22,

        .ws_io_num = 25,

        .data_out_num = 26,

        .data_in_num = -1

    };

 i2s_set_pin(I2S_NUM_0, &wp);

SPI SD Card:

設定VFS base_path:

#define SDSPI_MOUNT "/sdcard"

SPI pin 設定

sdmmc_host_t sdspi_host = SDSPI_HOST_DEFAULT();

sdspi_device_config_t sdspi_device = SDSPI_DEVICE_CONFIG_DEFAULT();

spi_bus_config_t bus_config = {

    .miso_io_num = GPIO_NUM_19,

    .mosi_io_num = GPIO_NUM_23,

    .sclk_io_num = GPIO_NUM_18

};

sdspi_device.host_id = VSPI_HOST;

spi_dma_chan_t dma_chan = SPI_DMA_CH1;

esp_vfs_fat_mount_config_t mount_config = {

     .format_if_mount_failed=true,

     .max_files = 5,

     .allocation_unit_size=16*512   //8192

 };

 ret = spi_bus_initialize(sdspi_device.host_id, &bus_config, dma_chan);

 if (ret != ESP_OK) {

           return;

 }

 ret = esp_vfs_fat_sdspi_mount(SDSPI_MOUNT, &sdspi_host, &sdspi_device, &mount_config, &sdspi_card);

 if(ret != ESP_OK) {

      return;

}

相關SD FAT filesystem操作可參閱千一篇文章(

EPS32 ESP-IDF開發環境儲存設備與檔案系統實驗(一) -- SDMMC 與SDSPI )

四、8-bit parallel TFT(with touch)  

ILI3941 ESP32 Driver使用https://github.com/nopnop2002/esp-idf-parallel-tft， 相關的檔案為ili3941, lcd_com, lcd_lib, fontx,與font檔案，TFT程式碼放在components/tft_library下，font資料夾下放font檔案與相關的UI button JPEG檔案。font資料夾會先預載到flash 的spiffs partition下。Kconfig.projbuild是相關tft menuconfig設定，放在`main資料夾下。相關位置如下圖

Flash partition tables:

五、成果展示:

六、main程式碼:

#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "sdkconfig.h"
#include "audio_element.h"
#include "audio_pipeline.h"
#include "audio_event_iface.h"
#include "audio_common.h"
//#include "board.h"
#include "esp_peripherals.h"
#include "periph_sdcard.h"
#include "fatfs_stream.h"
#include "i2s_stream.h"
#include "wav_encoder.h"
#include "wav_decoder.h"
#include "lcd_com.h"
#include "lcd_lib.h"
#include "fontx.h"
#include "decode_jpeg.h"
#include "string.h"
#include "driver/i2s.h"

#if CONFIG_INTERFACE_I2S
#define INTERFACE INTERFACE_I2S
#elif CONFIG_INTERFACE_GPIO
#define INTERFACE INTERFACE_GPIO
#elif CONFIG_INTERFACE_REG
#define INTERFACE INTERFACE_REG
#endif

#include "ili9341.h"
#define DRIVER "ILI9340"
#define INIT_FUNCTION(a, b, c, d, e) ili9341_lcdInit(a, b, c, d, e)

#define RECORD_STATE	(1)
#define PLAY_STATE		(2)
#define PAUSE_STATE		(3)
#define STOP_STATE		(0)
#define VOLUME_MIN		(-50)
#define VOLUME_MAX		(50)
#define MAX_INDEX 		(1024)
int  volume=0;
static char **fltable=NULL;
static TFT_t dev;

static int16_t tindex=0, bindex=0, totalindex=0, lastnumber=0;
static int16_t selectindex=-1;
static char selectFile[11];

uint8_t gStatus=STOP_STATE; 
static TaskHandle_t hPlayWave, hRecordingTask;
static TaskHandle_t hPlayTask;
audio_element_handle_t fatfs_stream_reader, i2s_stream_writer, audio_decoder;
audio_element_handle_t fatfs_stream_writer, i2s_stream_reader, audio_encoder;

FontxFile fx16G[2];
FontxFile fx24G[2];
FontxFile fx32G[2];
FontxFile fx16M[2];
FontxFile fx24M[2];
FontxFile fx32M[2];


#define RECORD_TIME_SECONDS (20)

//static const char *TAG = "Audio Recoder";

#include "esp_vfs_fat.h"
#include "esp_vfs.h"
#include "driver/sdspi_host.h"
#include "esp_spiffs.h"

static sdmmc_card_t *sdspi_card;

#define SDSPI_MOUNT "/sdcard"

static audio_pipeline_handle_t rec_pipeline;
static audio_pipeline_handle_t play_pipeline;


void drawJPEG(TFT_t * dev, char * file, int px, int py, int width, int height) {
	lcdSetFontDirection(dev, 0);
	int _width = width;
	if (width > 240) _width = 240;
	int _height = height;
	if (height > 320) _height = 320;

	pixel_jpeg **pixels;
	uint16_t imageWidth;
	uint16_t imageHeight;
	esp_err_t err = decode_jpeg(&pixels, file, _width, _height, &imageWidth, &imageHeight);
	if (err == ESP_OK) {
		uint16_t jpegWidth = width;
		uint16_t offsetX = px;
		if (width > imageWidth) {
			jpegWidth = imageWidth;
		}
		uint16_t jpegHeight = height;
		uint16_t offsetY = py;
		if (height > imageHeight) {
			jpegHeight = imageHeight;
		}
		uint16_t *colors = (uint16_t*)malloc(sizeof(uint16_t) * jpegWidth);


		for(int y = 0; y < jpegHeight; y++){
			for(int x = 0;x < jpegWidth; x++){
				colors[x] = pixels[y][x];
				}
			lcdDrawMultiPixels(dev, offsetX, y+offsetY, jpegWidth, colors);
		}

		free(colors);
		release_image(&pixels, _width, _height);

	}
}


int button[7][2] = {{15,275},{60,275},{105,275},{150,275},{195,275},{202,10},{202,220}};

bool getTouchPos(TFT_t * dev, int *posx, int *posy) {
	
	float _xd = dev->_max_xp - dev->_min_xp;
	float _yd = dev->_max_yp - dev->_min_yp;
	
	float _xs = dev->_max_xc - dev->_min_xc;
	float _ys = dev->_max_yc - dev->_min_yc;
	//ESP_LOGD(TAG, "_xs=%f _ys=%f", _xs, _ys);

	int _xpos = 0;
	int _ypos = 0;

	int _xp;
	int _yp;
	if (touch_getxy(dev, &_xp, &_yp)) { 
		if (dev->_max_xp > dev->_min_xp) {
			if (_xp < dev->_min_xp && _xp > dev->_max_xp) return false;
		} else {
			if (_xp < dev->_max_xp && _xp > dev->_min_xp) return false;
		}
		if (dev->_max_yp > dev->_min_yp) {
			if (_yp < dev->_min_yp && _yp > dev->_max_yp) return false;
		} else {
			if (_yp < dev->_max_yp && _yp > dev->_min_yp) return false;
		}
		// Convert from position to coordinate
		_xpos = ( (float)(_xp - dev->_min_xp) / _xd * _xs ) + dev->_min_xc;
		_ypos = ( (float)(_yp - dev->_min_yp) / _yd * _ys ) + dev->_min_yc;
		*posx = _xpos;
		*posy = _ypos;
		return true;
	} else {
		return false;
	}
}

void playWave(void* param) {
	char *file[5] = {"/spiffs/w0.jpg", "/spiffs/w1.jpg", "/spiffs/w2.jpg", "/spiffs/w3.jpg", "/spiffs/w4.jpg"};
	int i = 0;
	lcdDrawFillRect(&dev,6,6,199,259,BLACK);
	while (1) {
		drawJPEG(&dev, file[i], 15, 80, 180, 100);
		vTaskDelay(500/portTICK_PERIOD_MS);
		i = (i+1) % 5;
	}
}

void drawBackground(TFT_t *dev) {
	lcdFillScreen(dev, BLACK);
	lcdDrawRect(dev, 5,5,200, 260,  CYAN);
	lcdDrawRect(dev, 201,5,239, 260,  CYAN);
	drawJPEG(dev, "/spiffs/mic.jpg", 15, 275, 36,36);
	drawJPEG(dev, "/spiffs/play.jpg", 60, 275, 36,36);
	drawJPEG(dev, "/spiffs/stop.jpg", 105, 275, 36,36);
	drawJPEG(dev, "/spiffs/volume_up.jpg", 150, 275, 36,36);
	drawJPEG(dev, "/spiffs/volume_down.jpg", 195, 275, 36,36);
	drawJPEG(dev, "/spiffs/up.jpg", 202, 10, 36,36);
	drawJPEG(dev, "/spiffs/down.jpg", 202, 220, 36,36);

	fltable = malloc(MAX_INDEX*sizeof(char*));
	char tf[12];
	lastnumber = 0;
	totalindex=-1;
	DIR* dir = opendir("/sdcard/");
	assert(dir != NULL);
	while (true) {
		struct dirent*pe = readdir(dir);
		if (!pe) break;
		if ((pe->d_name)[0] == 'v' && strlen(pe->d_name) >=6){
			totalindex++;
			fltable[totalindex] = malloc(strlen(pe->d_name)*sizeof(char));
			strcpy(fltable[totalindex], pe->d_name);
			strncpy(tf, pe->d_name+1,5);
			int i = atoi(tf);
			if (lastnumber < i) lastnumber = i;
		}
	}
	closedir(dir);	

	int items;
	if (totalindex < 9) items = totalindex+1; else items = 10;
	tindex=0;
	for (int i = 0; i < items; i++) {
		bindex=i;
		lcdDrawString(dev, fx24G, 15, 10+(i+1)*24, (uint8_t*)fltable[i], WHITE);
	}
}
void TFT_Touch_init(bool touchEnable) {
	esp_vfs_spiffs_conf_t conf = {
		.base_path = "/spiffs",
		.partition_label = NULL,
		.max_files = 10,
		.format_if_mount_failed =true
	};
	esp_err_t ret = esp_vfs_spiffs_register(&conf);

	if (ret != ESP_OK) {
		if (ret == ESP_FAIL) {
			//ESP_LOGE(TAG, "Failed to mount or format filesystem");
		} else if (ret == ESP_ERR_NOT_FOUND) {
			//ESP_LOGE(TAG, "Failed to find SPIFFS partition");
		} else {
			//ESP_LOGE(TAG, "Failed to initialize SPIFFS (%s)",esp_err_to_name(ret));
		}
		return;
	}

	size_t total = 0, used = 0;
	ret = esp_spiffs_info(NULL, &total,&used);
	if (ret != ESP_OK) {
		//ESP_LOGE(TAG,"Failed to get SPIFFS partition information (%s)",esp_err_to_name(ret));
	} else {
		//ESP_LOGI(TAG,"Partition size: total: %d, used: %d", total, used);
	}

	// set font file
	InitFontx(fx16G,"/spiffs/ILGH16XB.FNT",""); // 8x16Dot Gothic
	InitFontx(fx24G,"/spiffs/ILGH24XB.FNT",""); // 12x24Dot Gothic
	InitFontx(fx32G,"/spiffs/ILGH32XB.FNT",""); // 16x32Dot Gothic
	
	InitFontx(fx16M,"/spiffs/ILMH16XB.FNT",""); // 8x16Dot Mincyo
	InitFontx(fx24M,"/spiffs/ILMH24XB.FNT",""); // 12x24Dot Mincyo
	InitFontx(fx32M,"/spiffs/ILMH32XB.FNT",""); // 16x32Dot Mincyo
	//TFT_t dev;
	lcd_interface_cfg(&dev, INTERFACE);

	INIT_FUNCTION(&dev, CONFIG_WIDTH, CONFIG_HEIGHT, CONFIG_OFFSETX, CONFIG_OFFSETY);

	if (touchEnable) {
		int gpio_xp = dev._d6;
		int gpio_xm = dev._rs;
		int gpio_yp = dev._wr;
		int gpio_ym = dev._d7;
		touch_interface_cfg(&dev, CONFIG_ADC_CHANNEL_YP, CONFIG_ADC_CHANNEL_XM, gpio_xp, gpio_xm, gpio_yp, gpio_ym);
	}
}
void TouchCalibration(TFT_t * dev, int width, int height) {
	if (dev->_calibration == false) return;
	lcdFillScreen(dev, BLACK);
	// get font width & height
	uint8_t buffer[FontxGlyphBufSize];
	uint8_t fontWidth;
	uint8_t fontHeight;
	GetFontx(fx24G, 0, buffer, &fontWidth, &fontHeight);
	//ESP_LOGD(__FUNCTION__,"fontWidth=%d fontHeight=%d",fontWidth,fontHeight);

	uint8_t ascii[24];
	int xpos = 0;
	int ypos = 0;

	// Calibration
	lcdFillScreen(dev, BLACK);
	dev->_min_xc = 15;
	dev->_min_yc = 15;
	lcdDrawFillCircle(dev, dev->_min_xc, dev->_min_yc, 10, CYAN);
	strcpy((char *)ascii, "Calibration");
	ypos = ((height - fontHeight) / 2) - 1;
	xpos = (width - (strlen((char *)ascii) * fontWidth)) / 2;
	lcdSetFontDirection(dev, DIRECTION0);
	lcdDrawString(dev, fx24G, xpos, ypos, ascii, WHITE);
	ypos = ypos + fontHeight;
	int _xpos = xpos;
	
	for(int i=0;i<10;i++) {
		lcdDrawFillCircle(dev, _xpos, ypos, fontWidth/2, RED);
		_xpos = _xpos + fontWidth + 5;
	}

	int32_t xp = 0;
	int32_t yp = 0;
	int counter = 0;
	while(1) {
		vTaskDelay(1);
		int _xp;
		int _yp;
		if (touch_getxy(dev, &_xp, &_yp) == false) continue;
		xp += _xp;
		yp += _yp;
		//ESP_LOGI(TAG, "counter=%d _xp=%d _yp=%d xp=%d yp=%d", counter, _xp, _yp, xp, yp);
		counter++;
		if (counter == 100) break;
		if ((counter % 10) == 0) {
			lcdDrawFillCircle(dev, xpos, ypos, fontWidth/2, GREEN);
			xpos = xpos + fontWidth + 5;
		}
	} // end while
	dev->_min_xp = xp/100;
	dev->_min_yp = yp/100;
	//ESP_LOGI(TAG, "_min_xp=%d _min_yp=%d", dev->_min_xp, dev->_min_yp);

	lcdFillScreen(dev, BLACK);
	dev->_max_xc = width-10;
	dev->_max_yc = height-10;
	lcdDrawFillCircle(dev, dev->_max_xc, dev->_max_yc, 10, CYAN);
	ypos = ((height - fontHeight) / 2) - 1;
	xpos = (width - (strlen((char *)ascii) * fontWidth)) / 2;
	lcdDrawString(dev, fx24G, xpos, ypos, ascii, WHITE);
	ypos = ypos + fontHeight;
	_xpos = xpos;
	for(int i=0;i<10;i++) {
		lcdDrawFillCircle(dev, _xpos, ypos, fontWidth/2, RED);
		_xpos = _xpos + fontWidth + 5;
	}

	xp = 0;
	yp = 0;
	counter=0;
	while(1) {
		vTaskDelay(1);
		int _xp;
		int _yp;
		if (touch_getxy(dev, &_xp, &_yp) == false) continue;
		if (_xp < dev->_min_xp+100 || _yp < dev->_min_yp+100) continue;
		xp += _xp;
		yp += _yp;
		counter++;
		if (counter == 100) break;
		if ((counter % 10) == 0) {
			lcdDrawFillCircle(dev, xpos, ypos, fontWidth/2, GREEN);
			xpos = xpos + fontWidth + 5;
		}
	} // end while
	
	dev->_max_xp = xp/100;
	dev->_max_yp = yp/100;
	dev->_calibration = false;
}


void vTaskRecord(void *param) {
    esp_err_t ret;
    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    rec_pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(rec_pipeline);

    fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT();
	fatfs_cfg.task_core=APP_CPU_NUM;
    fatfs_cfg.type = AUDIO_STREAM_WRITER;
    fatfs_stream_writer = fatfs_stream_init(&fatfs_cfg);

    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();
	i2s_cfg.task_core = APP_CPU_NUM;
    i2s_cfg.type = AUDIO_STREAM_READER;
    i2s_cfg.i2s_config.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;
    i2s_cfg.use_alc = true;
    i2s_cfg.volume=-63;
	i2s_stream_reader = i2s_stream_init(&i2s_cfg);	

    i2s_pin_config_t rp = {
        .bck_io_num = 5, 
        .ws_io_num = 25,
        .data_out_num = -1,
        .data_in_num = 39 //35
    };
    i2s_set_pin(I2S_NUM_0, &rp);

    wav_encoder_cfg_t wav_cfg = DEFAULT_WAV_ENCODER_CONFIG();
	wav_cfg.task_core = APP_CPU_NUM;
    audio_encoder = wav_encoder_init(&wav_cfg);
   
    audio_pipeline_register(rec_pipeline, i2s_stream_reader, "i2s");
    audio_pipeline_register(rec_pipeline, audio_encoder, "wav");
    audio_pipeline_register(rec_pipeline, fatfs_stream_writer, "file");

    const char *link_tag[3] = {"i2s", "wav", "file"};
    audio_pipeline_link(rec_pipeline, &link_tag[0], 3);

	audio_element_info_t music_info = {0};
    audio_element_getinfo(i2s_stream_reader, &music_info);
    audio_element_setinfo(fatfs_stream_writer, &music_info);

    char filename[100];
    sprintf(filename, "%s/%s", SDSPI_MOUNT, (char*) param);
    audio_element_set_uri(fatfs_stream_writer, filename);
    
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

    audio_pipeline_set_listener(rec_pipeline, evt);

    i2s_alc_volume_set(i2s_stream_reader, -63);
	
    audio_pipeline_run(rec_pipeline);
    
	vTaskDelay(310/portTICK_PERIOD_MS);
	i2s_alc_volume_set(i2s_stream_reader, 20);
	int second_recorded = 0;
    while (1) {
        audio_event_iface_msg_t msg;
		ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
		if (ret != ESP_OK) {
            continue;
        }
		/*
        if (audio_event_iface_listen(evt, &msg, 1000/portTICK_PERIOD_MS) != ESP_OK) {
            second_recorded ++;
            if (second_recorded >= RECORD_TIME_SECONDS) {
               audio_element_set_ringbuf_done(i2s_stream_reader);
            }
			continue;
        }
		*/

        /* Stop when the last pipeline element (fatfs_stream_writer in this case) receives stop event */
        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && msg.source == (void *) fatfs_stream_writer
            && msg.cmd == AEL_MSG_CMD_REPORT_STATUS
            && (((int)msg.data == AEL_STATUS_STATE_STOPPED) || ((int)msg.data == AEL_STATUS_STATE_FINISHED)
                || ((int)msg.data == AEL_STATUS_ERROR_OPEN))) {
            break;
        }
    }
    
	audio_pipeline_stop(rec_pipeline);
    audio_pipeline_wait_for_stop(rec_pipeline);
    audio_pipeline_terminate(rec_pipeline);

    audio_pipeline_unregister(rec_pipeline, audio_encoder);
    audio_pipeline_unregister(rec_pipeline, i2s_stream_reader);
    audio_pipeline_unregister(rec_pipeline, fatfs_stream_writer);


    /* Terminal the pipeline before removing the listener */
    audio_pipeline_remove_listener(rec_pipeline);

    /* Make sure audio_pipeline_remove_listener & audio_event_iface_remove_listener are called before destroying event_iface */
    audio_event_iface_destroy(evt);

    /* Release all resources */
    audio_pipeline_deinit(rec_pipeline);
    audio_element_deinit(fatfs_stream_writer);
    audio_element_deinit(i2s_stream_reader);
    audio_element_deinit(audio_encoder);

    vTaskDelete(NULL);

}

void vTaskPlay(void *param) {
    
	audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    play_pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(play_pipeline);

    //ESP_LOGI(TAG, "[Playing] Create fatfs stream to read data from sdcard");
    fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT();
    fatfs_cfg.type = AUDIO_STREAM_READER;
    fatfs_stream_reader = fatfs_stream_init(&fatfs_cfg);

    //ESP_LOGI(TAG, "[Playing]  Create i2s stream to write audio data to MAX98357A");
    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();
    i2s_cfg.type = AUDIO_STREAM_WRITER;
    i2s_cfg.use_alc = true;
    i2s_cfg.volume=0;
	i2s_stream_writer = i2s_stream_init(&i2s_cfg);

    i2s_pin_config_t wp = {
        .bck_io_num = 22,
        .ws_io_num = 25,
        .data_out_num = 26, 
        .data_in_num = -1
    };
    i2s_set_pin(I2S_NUM_0, &wp);

    wav_decoder_cfg_t wav_cfg = DEFAULT_WAV_DECODER_CONFIG();
    audio_decoder = wav_decoder_init(&wav_cfg);
    
	audio_pipeline_register(play_pipeline, fatfs_stream_reader, "file");
    audio_pipeline_register(play_pipeline, audio_decoder, "wav");
    audio_pipeline_register(play_pipeline, i2s_stream_writer, "i2s");

    const char *link_tag[3] = {"file", "wav", "i2s"};
    audio_pipeline_link(play_pipeline, &link_tag[0], 3);
    
	char filename[100];
    sprintf(filename, "%s/%s", SDSPI_MOUNT, (char*) param);
    printf("%s\n", filename);
    audio_element_set_uri(fatfs_stream_reader, filename);
    
	audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

	audio_pipeline_set_listener(play_pipeline, evt);

    audio_pipeline_run(play_pipeline);

   
    while (1) {
        audio_event_iface_msg_t msg;
         esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
        if (ret != ESP_OK) {
            continue;
        }
		
        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT
            && msg.source == (void *) audio_decoder
            && msg.cmd == AEL_MSG_CMD_REPORT_MUSIC_INFO) {
                audio_element_info_t music_info = {0};
            audio_element_getinfo(audio_decoder, &music_info);
            audio_element_setinfo(i2s_stream_writer, &music_info);
            i2s_stream_set_clk(i2s_stream_writer, music_info.sample_rates, music_info.bits, music_info.channels);
            continue;
        }
	
        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT
            && (msg.source == (void *) i2s_stream_writer || msg.source == (void *) fatfs_stream_reader)
            && msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
			(((int)msg.data == AEL_STATUS_STATE_STOPPED) || ((int)msg.data == AEL_STATUS_STATE_FINISHED))) {            
            break;
        }
    }
    ////ESP_LOGI(TAG, "[Playing]  Stop audio_pipeline");
    audio_pipeline_stop(play_pipeline);
    audio_pipeline_wait_for_stop(play_pipeline);
    audio_pipeline_terminate(play_pipeline);

	audio_pipeline_unregister(play_pipeline, fatfs_stream_reader);
    audio_pipeline_unregister(play_pipeline, audio_decoder);
    audio_pipeline_unregister(play_pipeline, i2s_stream_writer);
   
    /* Terminal the pipeline before removing the listener */
    audio_pipeline_remove_listener(play_pipeline);

    /* Make sure audio_pipeline_remove_listener & audio_event_iface_remove_listener are called before destroying event_iface */
    audio_event_iface_destroy(evt);

    /* Release all resources */
    audio_pipeline_deinit(play_pipeline);
    audio_element_deinit(fatfs_stream_reader);
    audio_element_deinit(i2s_stream_writer);
    audio_element_deinit(audio_decoder);
	gStatus = PAUSE_STATE;
	drawJPEG(&dev, "/spiffs/play.jpg", button[1][0], button[1][1], 36,36);

	vTaskDelete(NULL);

}

void app_main(void)
{
    esp_err_t ret;
	audio_element_state_t m_el_state;
	TFT_Touch_init(true);
	TouchCalibration(&dev, CONFIG_WIDTH, CONFIG_HEIGHT);
	lcdDrawFillRect(&dev, 1,1,239,319, BLACK);

    
// mount sdspi sd card
    sdmmc_host_t sdspi_host = SDSPI_HOST_DEFAULT();
    sdspi_device_config_t sdspi_device = SDSPI_DEVICE_CONFIG_DEFAULT();
    spi_bus_config_t bus_config = {
        .miso_io_num = GPIO_NUM_19,
        .mosi_io_num = GPIO_NUM_23,
        .sclk_io_num = GPIO_NUM_18
    };
    sdspi_device.host_id = VSPI_HOST;
    spi_dma_chan_t dma_chan = SPI_DMA_CH1;
    esp_vfs_fat_mount_config_t mount_config = {
        .format_if_mount_failed=true,
        .max_files = 5,
        .allocation_unit_size=16*512   //8192 
    };
    ret = spi_bus_initialize(sdspi_device.host_id, &bus_config, dma_chan);
    if (ret != ESP_OK) {
		lcdDrawString(&dev, fx16M, 10,50, (uint8_t*)"SPI BUS initialize error", WHITE);
		vTaskDelay(2000/portTICK_PERIOD_MS);
        return;
    }
    ret = esp_vfs_fat_sdspi_mount(SDSPI_MOUNT, &sdspi_host, &sdspi_device, &mount_config, &sdspi_card);
    if(ret != ESP_OK) {
		lcdDrawString(&dev, fx16M, 10,50, (uint8_t*)"SDSPI sdcard mount error", WHITE);
		vTaskDelay(2000/portTICK_PERIOD_MS);
        return;
    }
	
	lcdDrawString(&dev, fx16M, 10,50, (uint8_t*)"SDSPI sdcard mounted", WHITE);
	vTaskDelay(1000/portTICK_PERIOD_MS);
	drawBackground(&dev);

	int posx, posy;
	int selItem=-1;
	char fileName[12];

	while(1) {
		vTaskDelay(100/portTICK_PERIOD_MS);
		if (getTouchPos(&dev, &posx, &posy)) {
			selItem = -1;
			for (int i = 0 ; i < 7; i++) {
				if (posx >= button[i][0]-5 && posx <= button[i][0]+41 && 
					posy >= button[i][1]-5 && posy <= button[i][1]+41) { //+=5
					selItem=i;
					break;
				}
			}
			switch(selItem) {
				case 0:
				if (gStatus != RECORD_STATE) {
					if (gStatus != STOP_STATE) {
						// stop play
						m_el_state = audio_element_get_state(i2s_stream_writer);
						if (m_el_state == AEL_STATE_RUNNING || m_el_state == AEL_STATE_PAUSED){
							audio_pipeline_stop(play_pipeline);
							audio_pipeline_wait_for_stop(play_pipeline);
							drawJPEG(&dev, "/spiffs/play.jpg", button[1][0], button[1][1], 36,36);
						}
					
					}
					gStatus = RECORD_STATE;
					lastnumber++;
					sprintf(fileName, "v%05d.wav", lastnumber);
					//lcdDrawFillRect(&dev, 6,6,199,259, BLACK);
					//lcdDrawString(&dev, fx32G, 15, 100, (uint8_t*)"Recording...",WHITE);
					xTaskCreatePinnedToCore(playWave, "play Wave", 1024*3, NULL, 3, &hPlayWave, PRO_CPU_NUM);
					xTaskCreatePinnedToCore(vTaskRecord, "record task", 1024*4, fileName, 3, &hRecordingTask, APP_CPU_NUM);
					selectindex=-1;
				}
				break;
				case 1:
				if (selectindex != -1 && gStatus == STOP_STATE) {
							xTaskCreatePinnedToCore(vTaskPlay, "play task", 1024*4, selectFile, 3, &hPlayTask, APP_CPU_NUM);
							drawJPEG(&dev, "/spiffs/pause.jpg", button[1][0], button[1][1], 36,36);
							gStatus = PLAY_STATE;
				} else {
					
					m_el_state = audio_element_get_state(i2s_stream_writer);
					if (m_el_state == AEL_STATE_PAUSED){
						audio_pipeline_resume(play_pipeline);
						gStatus = PLAY_STATE;
						drawJPEG(&dev, "/spiffs/pause.jpg", button[1][0], button[1][1], 36,36);
					} else if (m_el_state == AEL_STATE_RUNNING){
						audio_pipeline_pause(play_pipeline);
						gStatus = PAUSE_STATE;
						drawJPEG(&dev, "/spiffs/play.jpg", button[1][0], button[1][1], 36,36);
					} 
					
				}
				break;
				case 2:
					if (gStatus == PAUSE_STATE || gStatus == PLAY_STATE) {
						drawJPEG(&dev, "/spiffs/play.jpg", button[1][0], button[1][1], 36,36);
						//m_el_handle = audio_pipeline_get_el_by_tag(hPlayTask, "is2");
						//audio_element_set_ringbuf_done(fatfs_stream_reader);
						audio_pipeline_stop(play_pipeline);
						audio_pipeline_wait_for_stop(play_pipeline);
					} 
					if (gStatus == RECORD_STATE && totalindex < MAX_INDEX-1) {
						totalindex++;
						audio_pipeline_stop(rec_pipeline);
						audio_pipeline_wait_for_stop(rec_pipeline);
						vTaskDelete(hPlayWave);
						fltable[totalindex] = malloc(11*sizeof(char));
						sprintf(fileName,"v%05d.wav", lastnumber);
						strcpy(fltable[totalindex], fileName);
						bindex=totalindex;
						if (bindex > 9) tindex = bindex-9; else tindex=0;
						selectindex = -1;
						lcdDrawFillRect(&dev, 6,6,199,259, BLACK);
						for (int i = tindex; i <= bindex; i++) {
							lcdDrawString(&dev, fx24G, 15, 10+(i-tindex+1)*24, (uint8_t*)fltable[i], WHITE);
						}
					}
					gStatus = STOP_STATE;
				break;
				case 3:
					if (gStatus == PLAY_STATE) {
						ret = i2s_alc_volume_get(i2s_stream_writer, &volume);
						if (ret == ESP_OK) {
							if (volume < VOLUME_MAX) {
								volume = volume+2;
								i2s_alc_volume_set(i2s_stream_writer, volume);
							}
						}
					}
					if (gStatus == RECORD_STATE) {
						ret = i2s_alc_volume_get(i2s_stream_reader, &volume);
						if (ret == ESP_OK) {
							if (volume < VOLUME_MAX) {
								volume = volume+2;
								i2s_alc_volume_set(i2s_stream_reader, volume);
							}
						}
					}
				break;
				case 4:
				if (gStatus == PLAY_STATE) {
						ret = i2s_alc_volume_get(i2s_stream_writer, &volume);
						if (ret == ESP_OK) {
							if (volume > VOLUME_MIN) {
								volume = volume-2;
								i2s_alc_volume_set(i2s_stream_writer, volume);
							}
						}
					}
					if (gStatus == RECORD_STATE) {
						ret = i2s_alc_volume_get(i2s_stream_reader, &volume);
						if (ret == ESP_OK) {
							if (volume > VOLUME_MIN) {
								volume = volume-2;
								i2s_alc_volume_set(i2s_stream_reader, volume);
							}
						}
					}
				break;
				case 5:
					if (tindex == 0 || bindex - tindex < 9) break;
					tindex--;
					selectindex=-1;
					lcdDrawFillRect(&dev, 6,6,199,259, BLACK);
					for (int i = tindex; i < tindex+10; i++) {
						bindex=i;
						lcdDrawString(&dev, fx24G, 15, 10+(i-tindex+1)*24, (uint8_t*)fltable[i], WHITE);
					}
				break;
				case 6:
					if (bindex == totalindex || bindex - tindex < 9) break;
					tindex++;
					selectindex=-1;
					lcdDrawFillRect(&dev, 6,6,199,259, BLACK);
					for (int i = tindex; i < tindex+10; i++) {
						bindex=i;
						lcdDrawString(&dev, fx24G, 15, 10+(i-tindex+1)*24, (uint8_t*)fltable[i], WHITE);
					}
				break;
				default:
					if (posx > 5 && posx < 200 && posy > 5 && posy < 260) {
						if (selectindex != -1) {
							lcdDrawFillRect(&dev, 15, 10+(selectindex-tindex)*24, 199,10+(selectindex-tindex+1)*24, BLACK);
							lcdDrawString(&dev, fx24G, 15, 10+(selectindex-tindex+1)*24, (uint8_t*)fltable[selectindex], WHITE);
						}
						selectindex = (posy-5)/24+tindex;
						if (selectindex <= totalindex && selectindex-tindex < 10) {
							strcpy(selectFile, fltable[selectindex]);
							lcdDrawFillRect(&dev, 15, 10+(selectindex-tindex)*24, 199,10+(selectindex-tindex+1)*24, BLUE);
							lcdDrawString(&dev, fx24G, 15, 10+(selectindex-tindex+1)*24, (uint8_t*)fltable[selectindex], WHITE);
						} else {
							selectindex = -1;
						}
						
					}
				break;
			}
		}

	}
    esp_vfs_fat_sdcard_unmount(SDSPI_MOUNT, sdspi_card);   
}
	

無線智能音箱實驗--使用Node-RED透過MQTT控制(Smart Wireless Speaker Experiment--Controlling by Node-RED via MQTT)

     本實驗製作一個WiFi音箱，使用著介面使用Node-RED Dashboard，透過MQTT protocol控制分布在不同地點的音箱，來播放音樂、網路收音機、Text to Speech廣播。可分別針對個別或全部音箱即時或排程播送。

    本實驗可應用於像校園廣播系統，可排程上下課鐘聲、或活動音樂分區播放、透過Node-RED dashboard介面輸入文字即可廣播語音。

一、實驗使用元件:

1. ESP32 NodeMCU-32S開發版
2. MAX98357A模組
3. RGB LED
4. NPN 電晶體
5. 3W4Ω喇叭
6. 200Ω電阻

二、開發軟體環境:

1. ESP-ADF(ESP-IDF)
2. VSCode
3. Node_RED
4. Mosquitto MQTT broker
5. SQLite

三、功能說明:

1. 即時顯示每個喇叭狀態(斷線、已連線或播放中)
2. Dashboard 能自動動態增加連線喇叭數量。
3. 設備端LED顯示狀態(LED紅燈為斷線、綠燈正常連線中)
4. 可針對各別或全部喇叭操作
5. 可即時播放音樂、網路收音機或TTS語音。
6. 可排程(月、日、時、分、星期)播放音樂、網路收音機或TTS語音。
7. MQTT使用TLS連線與帳密安全保護
8. 能使用HTTPS(secure HTTP) URI

四、實驗過程:

    1.定義MQTT TOPIC and message flows:

TOPIC分成兩大類，一為Node-RED送出控制Wifi Speaker動作，與Wifi Speaker回報狀態(STATE)給Node-RED，分別為/WIFI_SPEAKER/DeviceID/CMD 與 /WIFI_SPEAKER/DeviceID/STATE，DeviceID為每個WiFi Speaker  MAC Address 的後3bytes，用於區別對個別的音箱做控制與狀態顯示。詳細定義如下圖:

• CMD topic :
  TOPIC:/WIFI_SPEAKER/deviceID/CMD, 
  PAYLOAD: {"action":"play", "uri":"https://...../..mp3"}
  PAYLOAD: {"action":"stop"}
  PAYLOAD: {"action":"pause"}
  PAYLOAD: {"action":"resume"}
  PAYLOAD: {"action":"volume", "value":number}
• STATE topic:
  PAYLOAD: {"ID":"deviceID", "NAME":"deviceName", "TYPE":"NETWORK","STATE":"OFFLINE"}
  PAYLOAD: {"ID":"deviceID", "NAME":"deviceName", "TYPE":"NETWORK","STATE":"CONNECT"}
  PAYLOAD: {"ID":"deviceID",  "TYPE":"MUSIC_PLAY","STATE":"play"}
  PAYLOAD: {"ID":"deviceID",  "TYPE":"MUSIC_PLAY","STATE":"pause"}
  PAYLOAD: {"ID":"deviceID",  "TYPE":"MUSIC_PLAY","STATE":"resume"}
  PAYLOAD: {"ID":"deviceID",  "TYPE":"MUSIC_PLAY","STATE":"stop"}
  PAYLOAD: {"ID":"deviceID",  "TYPE":"MUSIC_PLAY","STATE":"FINISH"}
  
  
  兩個端點Node-RED與device(Wifi Speaker)對於接收到MQTT Topic要實行的端點動作如下圖所示:

2.ESP-ADF:

ESP-ADF (Espressif Audio Development Framework)，是ESP-IDF的擴充組件，用於開發Audio Application。在使用VSCode下，按下F1即可安裝ESP-ADF擴充組件，如下圖所示

有關ESP-ADF詳細文件說明位址:https://docs.espressif.com/projects/esp-adf/en/latest/。

本實驗以播放MP3格式的聲音為主，加入其他audio element codec即可播放其他格式audio，便於開發。

主要架構以audio pipeline透過ringbuffer串聯input audio element, process(codec) audio element與 output audio element，如下圖所示(取自官方文件)

本實驗I2S Output to DAC，不使用ESP32內建8bit DAC，使用MAX98357A 包含DAC與Ampiler。

3.TTS(Text To Speech)播放：

本實驗使用Google Translate TTS把Text轉成MP3語音於無線音箱播放。可使用兩種方式實行。

1. 安裝Node-RED google-translate-tts套件，將text轉成MP3 audio file存到server local file，再將local audio file的uri透過MQTT protocol送到WiFi Speaker播放，如下flow所示:
   
   
2. 安裝Node-RED node-red-contrib-google-tts套件，將欲播放的TEXT當成uri參數，透過node-red-contrib-google-tts轉成google translate URI，再透過MQTT protocol傳URI給WiFi Speaker播放。flow如下圖所示:
   
   

   
   
3. URI參數: a. tl=:為指定語言(en:英文, zh-TW中文)。 b. q=:為欲播放的語音文字。搜尋網路上有關google translate TTS的探討，URI亦可簡化類似:https://translate.google.com/translate_tts?ie=UTF-8&client=tw-ob&tl=en&ttsspeed=1&q=TEXT。

4.硬體線路:

• PIN_NUM_23連接npn電晶體與RGB LED用來指示Wifi連線MQTT是否正常，紅燈為斷線，綠燈為正常連線。
• PIN_NUM_5  I2S SCLK, PIN_NUM_25 I2S LRCK, PIN_NUM_26 I2S DSDIN

詳細線路圖如下所示：

4.Node-RED flows:

UI部分，本實驗主要使用Node-RED dashboard template(angular)來完成。

五、實驗成果展示

六、原始碼

parm_define.h

#define PRO_CPU         0
#define APP_CPU         1
#define SSID            "your-ssid"
#define PWD             "your-pwd"
#define MQTT_HOST       "your_mqtt_broker" 
#define MQTT_PORT       8883
#define MQTT_CLIENT_USER  "your_mqtt_user"
#define MQTT_CLIENT_PASSWORD  "your_mqtt_pwd"
#define DEVICE_NAME         "define_your_device_name"

static const char cert[] = "-----BEGIN CERTIFICATE-----\n"
"................................................................\n"
".....................\n"
"-----END CERTIFICATE-----";

static const uint8_t client_cert[]="-----BEGIN CERTIFICATE-----\n"
"................................................................\n"
"..................\n"
"-----END CERTIFICATE-----";

static const uint8_t client_key[] = "-----BEGIN RSA PRIVATE KEY-----\n"
"................................................................\n"
".....................\n"
"-----END RSA PRIVATE KEY-----";

wifi_speaker.c

#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "freertos/event_groups.h"

#include "nvs_flash.h"
#include "audio_element.h"
#include "audio_pipeline.h"
#include "audio_event_iface.h"
#include "audio_common.h"
#include "http_stream.h"
#include "i2s_stream.h"
#include "mp3_decoder.h"
#include "driver/i2c.h"

#include "esp_log.h"
#include "esp_audio.h"
#include "esp_wifi.h"
#include "esp_peripherals.h"
#include "esp_netif.h"
#include "periph_wifi.h"
#include "esp_tls.h"
#include "driver/gpio.h"

#include "cJSON.h"
#include "mqtt_client.h"

#include "param_define.h" //wifi and mqtt connection parameters

static audio_event_iface_handle_t evt={0};
static audio_pipeline_handle_t pipeline;
static audio_element_handle_t http_stream_reader, i2s_stream_writer, mp3_decoder;
static esp_mqtt_client_handle_t hMQTTClient;
static char sID[7];
static char cmd_topic[50]={0};
static char state_topic[50]={0};

static esp_err_t mqtt_event_handler_cb(esp_mqtt_event_handle_t event)
{
    esp_mqtt_client_handle_t client = event->client;
    char msg[100]={0};
    char topic[100] = {0};
    switch (event->event_id) {
        case MQTT_EVENT_CONNECTED:
            sprintf(msg, "{\"ID\":\"%s\",\"NAME\":\"%s\",\"TYPE\":\"NETWORK\",\"STATE\":\"CONNECTED\"}", sID, DEVICE_NAME);
            esp_mqtt_client_publish(client, state_topic, msg, 0, 1, 0);
            esp_mqtt_client_subscribe(client, cmd_topic, 1);
            gpio_set_level(GPIO_NUM_23, 0);
            break;
        case MQTT_EVENT_DISCONNECTED:
            gpio_set_level(GPIO_NUM_23, 1);
            esp_mqtt_client_reconnect(client);
            break;
        case MQTT_EVENT_SUBSCRIBED:
            break;
        case MQTT_EVENT_UNSUBSCRIBED:
            break;
        case MQTT_EVENT_PUBLISHED:
            break;
        case MQTT_EVENT_DATA:
            strncpy(topic, event->topic, event->topic_len);
            if (strcmp(topic, cmd_topic) == 0) {
                cJSON *root = cJSON_Parse(event->data);
                char *action=cJSON_GetObjectItem(root, "action")->valuestring;
                if (strcmp(action,"play")==0) {
                    char *uri = cJSON_GetObjectItem(root, "uri")->valuestring;
                    int http_state = audio_element_get_state(http_stream_reader);
                    if (http_state == AEL_STATE_PAUSED) {
                        audio_pipeline_resume(pipeline);
                        while((http_state = audio_element_get_state(http_stream_reader)) != AEL_STATE_RUNNING) {
                            vTaskDelay(10/portTICK_PERIOD_MS);
                        }
                    }
                    audio_pipeline_stop(pipeline);
                    audio_pipeline_wait_for_stop(pipeline);
                    
                    audio_pipeline_reset_elements(pipeline);
                    audio_pipeline_reset_ringbuffer(pipeline);
                    audio_element_set_uri(http_stream_reader,uri);
                    
                    audio_pipeline_run(pipeline);
                    
                    sprintf(msg, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"play\"}", sID);
                    esp_mqtt_client_publish(client,state_topic,msg,0,1,0);
                }
                if (strcmp(action,"stop")==0) {
                    audio_pipeline_stop(pipeline);
                    audio_pipeline_wait_for_stop(pipeline);
                    sprintf(msg, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"stop\"}", sID);
                    esp_mqtt_client_publish(client,state_topic,msg,0,1,0);
                }
                if (strcmp(action,"pause")==0) {
                    audio_pipeline_pause(pipeline);
                    sprintf(msg, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"pause\"}", sID);
                    esp_mqtt_client_publish(client,state_topic,msg,0,1,0);
                }
                if (strcmp(action,"resume")==0) {
                    audio_pipeline_resume(pipeline);
                    sprintf(msg, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"resume\"}", sID);
                    esp_mqtt_client_publish(client,state_topic,msg,0,1,0);
                }
                if (strcmp(action,"volume")==0) {
                    int volume = cJSON_GetObjectItem(root, "value")->valueint;
                    i2s_alc_volume_set(i2s_stream_writer, volume);
                    sprintf(msg, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"VOLUME\",\"volume\":%d}", sID, volume);
                    esp_mqtt_client_publish(client,state_topic,msg,0,1,0);
                }
            }
            break;
        case MQTT_EVENT_ERROR:
            if (event->error_handle->error_type == MQTT_ERROR_TYPE_TCP_TRANSPORT) {
            }
            break;
        default:
            break;
    }
    return ESP_OK;
}

static void mqtt_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data) {
    mqtt_event_handler_cb(event_data);
}

static void mqtt_client_start(void)
{
    char msg[100];
    sprintf(msg, "{\"ID\":\"%s\",\"NAME\":\"%s\",\"TYPE\":\"NETWORK\",\"STATE\":\"OFFLINE\"}", sID, DEVICE_NAME);
    esp_mqtt_client_config_t mqtt_cfg = {
        .host = MQTT_HOST, 
        .port = MQTT_PORT,
        //.uri = MQTT_URI,
        .username = MQTT_CLIENT_USER,
        .password = MQTT_CLIENT_PASSWORD,
        .lwt_topic = state_topic,
        .lwt_msg = msg,
        .keepalive=30,
        .cert_pem=(const char *)cert,
        .cert_len=0,
        .client_cert_pem=(const char *)client_cert,
        .client_cert_len=0,
        .client_key_pem=(const char *)client_key,
        .client_key_len=0,
        .transport = MQTT_TRANSPORT_OVER_SSL
        
    };
    hMQTTClient = esp_mqtt_client_init(&mqtt_cfg);
    esp_mqtt_client_register_event(hMQTTClient, ESP_EVENT_ANY_ID, mqtt_event_handler, hMQTTClient);
    esp_mqtt_client_start(hMQTTClient);
}

void wifi_speaker_audio_pipeline_init(void) {
  
    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    //pipeline_cfg.rb_size=40960;
    pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(pipeline);

    // audio element http_stream
    http_stream_cfg_t http_cfg = HTTP_STREAM_CFG_DEFAULT();
    http_cfg.type = AUDIO_STREAM_READER;
     http_cfg.task_core = APP_CPU;
    http_stream_reader = http_stream_init(&http_cfg);

    //audio element i2s_stream
    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();
    i2s_cfg.task_core = APP_CPU;
    i2s_cfg.use_alc=true;
    i2s_cfg.type = AUDIO_STREAM_WRITER;
    i2s_stream_writer = i2s_stream_init(&i2s_cfg);
   
    // audio element mp3_decoder
    mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG();
    mp3_cfg.task_core = APP_CPU;
    mp3_decoder = mp3_decoder_init(&mp3_cfg);

    // register audio element in the order http->mp3->i2s(input -> process -> output)
    audio_pipeline_register(pipeline, http_stream_reader, "http");
    audio_pipeline_register(pipeline, mp3_decoder,        "mp3");
    audio_pipeline_register(pipeline, i2s_stream_writer,  "i2s");
    
    // Link it together http_stream-->mp3_decoder-->i2s_stream-->[MAX98357A]");
    const char *link_tag[3] = {"http", "mp3", "i2s"};
     audio_pipeline_link(pipeline, &link_tag[0], 3);
    
}

void app_main(void)
{
    //led turn off
    gpio_pad_select_gpio(GPIO_NUM_23);
    gpio_set_direction(GPIO_NUM_23,GPIO_MODE_OUTPUT);
    gpio_set_level(GPIO_NUM_23, 1);
    esp_err_t err = nvs_flash_init();
    if (err == ESP_ERR_NVS_NO_FREE_PAGES) {
        // NVS partition was truncated and needs to be erased
        // Retry nvs_flash_init
        ESP_ERROR_CHECK(nvs_flash_erase());
        err = nvs_flash_init();
    }
    
    ESP_ERROR_CHECK(esp_netif_init());

    //esp_log_level_set("*", ESP_LOG_WARN);
    //esp_log_level_set(TAG, ESP_LOG_DEBUG);

    wifi_speaker_audio_pipeline_init();


    esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
    esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);
    periph_wifi_cfg_t wifi_cfg = {
        .ssid = SSID,
        .password = PWD,
        
    };
    esp_periph_handle_t wifi_handle = periph_wifi_init(&wifi_cfg);
    esp_periph_start(set, wifi_handle);
    esp_err_t cr = periph_wifi_wait_for_connected(wifi_handle, 120000/portTICK_PERIOD_MS);
    if (cr == ESP_FAIL) {
        return;
    }

    // register audio event
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    evt = audio_event_iface_init(&evt_cfg);
    // Listening event from all elements of pipeline
    audio_pipeline_set_listener(pipeline, evt);
    // Listening event from peripherals
    audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt);
    uint8_t tmac[6];
    esp_efuse_mac_get_default(tmac);
    sprintf(sID, "%02x%02x%02x", tmac[3],tmac[4],tmac[5]);
    sprintf(cmd_topic, "/WIFI_SPEAKER/%s/CMD", sID);
    sprintf(state_topic, "/WIFI_SPEAKER/%s/STATE", sID);
    
    // default volume level  (range -64 ~ 64)
    i2s_alc_volume_set(i2s_stream_writer, -20);
    
    mqtt_client_start();

    while(1) {
        audio_event_iface_msg_t msg;
        esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
        if (ret != ESP_OK) {
            continue;
        }

        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT
            && msg.source == (void *) mp3_decoder
            && msg.cmd == AEL_MSG_CMD_REPORT_MUSIC_INFO) {
            audio_element_info_t music_info = {0};
            audio_element_getinfo(mp3_decoder, &music_info);
            audio_element_setinfo(i2s_stream_writer, &music_info);
            i2s_stream_set_clk(i2s_stream_writer, music_info.sample_rates, music_info.bits, music_info.channels);
            continue;
        }
        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT
            && msg.source == (void *) i2s_stream_writer
            && msg.cmd == AEL_MSG_CMD_REPORT_STATUS && (int)msg.data == AEL_STATUS_STATE_FINISHED) {
                char mqttdata[100];
                sprintf(mqttdata, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"FINISH\"}", sID);
                esp_mqtt_client_publish(hMQTTClient,state_topic,mqttdata,0,1,0);
            
            continue;
        }
        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT
            && msg.source == (void *) http_stream_reader
            && msg.cmd == AEL_MSG_CMD_REPORT_STATUS && (int)msg.data == AEL_STATUS_ERROR_OPEN) {
                char mqttdata[100];
                sprintf(mqttdata, "{\"ID\":\"%s\",\"TYPE\":\"MUSIC_PLAY\",\"STATE\":\"openerror\"}", sID);
                esp_mqtt_client_publish(hMQTTClient,state_topic,mqttdata,0,1,0);
            
            continue;
        }
    }
}