diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 51bbaec..bae4143 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -9,7 +9,7 @@ jobs: build: strategy: matrix: - target: [esp32s3, linux] + board: [esp-box, esp32s3_korvo_2, esp32s3_others, linux] runs-on: ubuntu-latest @@ -24,5 +24,5 @@ jobs: docker run -v $PWD:/project -w /project -u 0 \ -e HOME=/tmp -e WIFI_SSID=A -e WIFI_PASSWORD=B -e OPENAI_API_KEY=X \ espressif/idf:latest \ - /bin/bash -c 'idf.py --preview set-target ${{ matrix.target }} && idf.py build' + /bin/bash -c 'idf.py @boards/${{ matrix.board }}.cfg build' shell: bash diff --git a/.gitignore b/.gitignore index 38c3a5f..5cb82be 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build sdkconfig sdkconfig.old managed_components +dependencies.lock diff --git a/.gitmodules b/.gitmodules index 087f53a..c2a629b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,6 @@ -[submodule "components/srtp"] - path = components/srtp - url = https://git@github.com/sepfy/esp_ports -[submodule "deps/libpeer"] - path = deps/libpeer - url = https://github.com/sean-der/libpeer -[submodule "components/esp-libopus"] - path = components/esp-libopus - url = https://github.com/XasWorks/esp-libopus.git [submodule "components/esp-protocols"] path = components/esp-protocols url = https://github.com/espressif/esp-protocols.git +[submodule "components/peer/libpeer"] + path = components/peer/libpeer + url = https://github.com/sean-der/libpeer diff --git a/CMakeLists.txt b/CMakeLists.txt index 84ad50c..9ecc003 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,16 @@ endif() add_compile_definitions(OPENAI_API_KEY="$ENV{OPENAI_API_KEY}") add_compile_definitions(OPENAI_REALTIMEAPI="https://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview-2024-12-17") -set(COMPONENTS src) -set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus") +# Select board configuration based on -DBUILD_BOARD +if(NOT DEFINED BUILD_BOARD) + message(FATAL_ERROR "BUILD_BOARD CMake variable is not set. Use idf.py @boards/BOARD.cfg reconfigure") +endif() + +set(ENV{BUILD_BOARD} ${BUILD_BOARD}) +message(STATUS "Building for board: ${BUILD_BOARD}") + +set(COMPONENTS src ${BUILD_BOARD}) +set(EXTRA_COMPONENT_DIRS "src") if(IDF_TARGET STREQUAL linux) add_compile_definitions(LINUX_BUILD=1) diff --git a/README.md b/README.md index e0faf87..49cb824 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,10 @@ Set your Wifi SSID + Password as env variables * `export OPENAI_API_KEY=bing` Build -* `idf.py build` +* `idf.py @boards/esp-box.cfg build` +* `idf.py @boards/esp32s3_korvo_2.cfg build` +* `idf.py @boards/esp32s3_others.cfg build` +* `idf.py @boards/linux.cfg build` If you built for `esp32s3` run the following to flash to the device * `sudo -E idf.py flash` diff --git a/boards/esp-box.cfg b/boards/esp-box.cfg new file mode 100644 index 0000000..23fb5c6 --- /dev/null +++ b/boards/esp-box.cfg @@ -0,0 +1 @@ +-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp-box;sdkconfig.defaults" -DBUILD_BOARD="esp-box" -DIDF_TARGET=esp32s3 diff --git a/boards/esp32s3_korvo_2.cfg b/boards/esp32s3_korvo_2.cfg new file mode 100644 index 0000000..97b9e57 --- /dev/null +++ b/boards/esp32s3_korvo_2.cfg @@ -0,0 +1 @@ +-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp32s3_korvo_2;sdkconfig.defaults" -DBUILD_BOARD="esp32s3_korvo_2" -DIDF_TARGET=esp32s3 diff --git a/boards/esp32s3_others.cfg b/boards/esp32s3_others.cfg new file mode 100644 index 0000000..79b2261 --- /dev/null +++ b/boards/esp32s3_others.cfg @@ -0,0 +1 @@ +-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp32s3_others;sdkconfig.defaults" -DBUILD_BOARD="others" -DIDF_TARGET=esp32s3 diff --git a/boards/linux.cfg b/boards/linux.cfg new file mode 100644 index 0000000..277ade7 --- /dev/null +++ b/boards/linux.cfg @@ -0,0 +1 @@ +-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.linux;sdkconfig.defaults" -DBUILD_BOARD="linux" -DIDF_TARGET=linux diff --git a/components/esp-libopus b/components/esp-libopus deleted file mode 160000 index 260b16c..0000000 --- a/components/esp-libopus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 260b16cc540285e84220c709de1bb2796ea7ec41 diff --git a/components/peer/CMakeLists.txt b/components/peer/CMakeLists.txt index d965443..0443837 100644 --- a/components/peer/CMakeLists.txt +++ b/components/peer/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PEER_PROJECT_PATH "../../deps/libpeer") +set(PEER_PROJECT_PATH "./libpeer") file(GLOB CODES "${PEER_PROJECT_PATH}/src/*.c") idf_component_register( @@ -8,9 +8,9 @@ idf_component_register( ) # Disable KeepAlives -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h INPUT_CONTENT) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/libpeer/src/config.h INPUT_CONTENT) string(REPLACE "#define KEEPALIVE_CONNCHECK 10000" "#define KEEPALIVE_CONNCHECK 0" MODIFIED_CONTENT ${INPUT_CONTENT}) -file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h ${MODIFIED_CONTENT}) +file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/libpeer/src/config.h ${MODIFIED_CONTENT}) if(NOT IDF_TARGET STREQUAL linux) add_definitions("-DESP32 -DCONFIG_USE_LWIP=1 -DCONFIG_AUDIO_BUFFER_SIZE=8096 -DCONFIG_DATA_BUFFER_SIZE=102400 -D__BYTE_ORDER=__LITTLE_ENDIAN") diff --git a/deps/libpeer b/components/peer/libpeer similarity index 100% rename from deps/libpeer rename to components/peer/libpeer diff --git a/components/srtp b/components/srtp deleted file mode 160000 index f39a4a2..0000000 --- a/components/srtp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f39a4a2c70a4b7d09f9f852a58b5ea39b9c1182c diff --git a/dependencies.lock b/dependencies.lock deleted file mode 100644 index 9b916ea..0000000 --- a/dependencies.lock +++ /dev/null @@ -1,10 +0,0 @@ -dependencies: - idf: - source: - type: idf - version: 5.5.0 -direct_dependencies: -- idf -manifest_hash: 655e4ae2c4a00dc0e9b6d66aa2a909e40e81c57604a11f1553343408aeddfb41 -target: esp32s3 -version: 2.0.0 diff --git a/partitions.csv b/partitions.csv index 03d5d13..483809a 100644 --- a/partitions.csv +++ b/partitions.csv @@ -2,5 +2,5 @@ # Name, Type, SubType, Offset, Size, Flags nvs, data, nvs, 0x9000, 0x6000, phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 0x180000, +factory, app, factory, 0x10000, 0x2M, diff --git a/sdkconfig.defaults.esp-box b/sdkconfig.defaults.esp-box new file mode 100644 index 0000000..2570200 --- /dev/null +++ b/sdkconfig.defaults.esp-box @@ -0,0 +1,37 @@ +# ESP Event Loop on Linux +CONFIG_ESP_EVENT_POST_FROM_ISR=n +CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n + +# Disable TLS verification +# Production needs to include specific cert chain you care about +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y + +# Enable DTLS-SRTP +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y + +# Defaults to partitions.csv +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +# Set highest CPU Freq +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y + +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# Disable Watchdog +# CONFIG_ESP_INT_WDT is not set +# CONFIG_ESP_TASK_WDT_EN is not set + +# Enable Compiler Optimization +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y + +CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64 +CONFIG_LWIP_MAX_UDP_PCBS=1024 +CONFIG_LWIP_UDP_RECVMBOX_SIZE=64 + +# CONFIG_ESP_AUDIO_SIMPLE_PLAYER_RESAMPLE_EN is not set +# CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE is not set diff --git a/sdkconfig.defaults.esp32s3_korvo_2 b/sdkconfig.defaults.esp32s3_korvo_2 new file mode 100644 index 0000000..2570200 --- /dev/null +++ b/sdkconfig.defaults.esp32s3_korvo_2 @@ -0,0 +1,37 @@ +# ESP Event Loop on Linux +CONFIG_ESP_EVENT_POST_FROM_ISR=n +CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n + +# Disable TLS verification +# Production needs to include specific cert chain you care about +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y + +# Enable DTLS-SRTP +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y + +# Defaults to partitions.csv +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +# Set highest CPU Freq +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y + +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# Disable Watchdog +# CONFIG_ESP_INT_WDT is not set +# CONFIG_ESP_TASK_WDT_EN is not set + +# Enable Compiler Optimization +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y + +CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64 +CONFIG_LWIP_MAX_UDP_PCBS=1024 +CONFIG_LWIP_UDP_RECVMBOX_SIZE=64 + +# CONFIG_ESP_AUDIO_SIMPLE_PLAYER_RESAMPLE_EN is not set +# CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE is not set diff --git a/sdkconfig.defaults.esp32s3_others b/sdkconfig.defaults.esp32s3_others new file mode 100644 index 0000000..ec5362b --- /dev/null +++ b/sdkconfig.defaults.esp32s3_others @@ -0,0 +1,32 @@ +# ESP Event Loop on Linux +CONFIG_ESP_EVENT_POST_FROM_ISR=n +CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n + +# Disable TLS verification +# Production needs to include specific cert chain you care about +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y + +# Enable DTLS-SRTP +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y + +# libpeer requires large stack allocations +CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 + +# Defaults to partitions.csv +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +# Set highest CPU Freq +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y + +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y + +# Disable Watchdog +# CONFIG_ESP_INT_WDT is not set +# CONFIG_ESP_TASK_WDT_EN is not set + +# Enable Compiler Optimization +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y diff --git a/sdkconfig.defaults.linux b/sdkconfig.defaults.linux new file mode 100644 index 0000000..16712df --- /dev/null +++ b/sdkconfig.defaults.linux @@ -0,0 +1,31 @@ +# ESP Event Loop on Linux +CONFIG_ESP_EVENT_POST_FROM_ISR=n +CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n + +# Disable TLS verification +# Production needs to include specific cert chain you care about +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y + +# Enable DTLS-SRTP +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y + +# libpeer requires large stack allocations +CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 + +# Defaults to partitions.csv +CONFIG_PARTITION_TABLE_CUSTOM=y + +# Set highest CPU Freq +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y + +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y + +# Disable Watchdog +# CONFIG_ESP_INT_WDT is not set +# CONFIG_ESP_TASK_WDT_EN is not set + +# Enable Compiler Optimization +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index df7f328..0eb2537 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,20 +3,34 @@ set(COMMON_SRC "webrtc.cpp" "main.cpp" "http.cpp") if(IDF_TARGET STREQUAL linux) idf_component_register( SRCS ${COMMON_SRC} - REQUIRES peer esp-libopus esp_http_client) + REQUIRES peer esp_http_client) else() idf_component_register( - SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp" - REQUIRES driver esp_wifi nvs_flash peer esp_psram esp-libopus esp_http_client) + SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp" "media_gmf.cpp" + REQUIRES driver esp_wifi nvs_flash peer esp_psram esp_timer esp_http_client) endif() idf_component_get_property(lib peer COMPONENT_LIB) target_compile_options(${lib} PRIVATE -Wno-error=restrict) target_compile_options(${lib} PRIVATE -Wno-error=stringop-truncation) -idf_component_get_property(lib srtp COMPONENT_LIB) +idf_component_get_property(lib sepfy__srtp COMPONENT_LIB) target_compile_options(${lib} PRIVATE -Wno-error=incompatible-pointer-types) -idf_component_get_property(lib esp-libopus COMPONENT_LIB) -target_compile_options(${lib} PRIVATE -Wno-error=maybe-uninitialized) -target_compile_options(${lib} PRIVATE -Wno-error=stringop-overread) +set(TARGET_COMPONENT "") + +idf_build_get_property(build_components BUILD_COMPONENTS) +foreach(COMPONENT ${build_components}) + if(COMPONENT MATCHES "gmf_core") + set(TARGET_COMPONENT ${COMPONENT}) + break() + endif() +endforeach() + +if(TARGET_COMPONENT STREQUAL "") + target_compile_definitions(${COMPONENT_TARGET} PRIVATE "-D CONFIG_OPENAI_WITH_GMF=0") +else() + idf_component_get_property(lib ${TARGET_COMPONENT} COMPONENT_LIB) + target_compile_options(${lib} PRIVATE -Wno-error=incompatible-pointer-types) + target_compile_definitions(${COMPONENT_TARGET} PRIVATE "-D CONFIG_OPENAI_WITH_GMF=1") +endif() diff --git a/src/idf_component.yml b/src/idf_component.yml index a494de4..02e2d23 100644 --- a/src/idf_component.yml +++ b/src/idf_component.yml @@ -1,3 +1,56 @@ dependencies: idf: version: ">=4.1.0" + + sepfy/srtp: + version: "^2.3.0" + + 78/esp-opus: + version: "^1.0.4" + rules: + - if: ${BUILD_BOARD} not in ["esp32s3_korvo_2", "esp-box"] + + espressif/esp32_s3_korvo_2: + version: "^3.0.0" + rules: + - if: "${BUILD_BOARD} == esp32s3_korvo_2" + + espressif/esp-box: + version: "^3.1.0" + rules: + - if: "${BUILD_BOARD} == esp-box" + + espressif/gmf_core: + path: ./gmf_core + git: https://github.com/espressif/esp-gmf.git + version: "main" + rules: + - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"] + + espressif/gmf_io: + path: ./gmf_elements/gmf_io + git: https://github.com/espressif/esp-gmf.git + version: "main" + rules: + - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"] + + espressif/gmf_misc: + path: ./gmf_elements/gmf_misc + git: https://github.com/espressif/esp-gmf.git + version: "main" + rules: + - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"] + + espressif/gmf_audio: + path: ./gmf_elements/gmf_audio + git: https://github.com/espressif/esp-gmf.git + version: "main" + rules: + - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"] + + espressif/esp_audio_simple_player: + path: ./examples/esp_audio_simple_player + git: https://github.com/espressif/esp-gmf.git + version: "main" + rules: + - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"] diff --git a/src/main.h b/src/main.h index 198ef5c..123302c 100644 --- a/src/main.h +++ b/src/main.h @@ -3,6 +3,10 @@ #define LOG_TAG "realtimeapi-sdk" #define MAX_HTTP_OUTPUT_BUFFER 2048 +#ifdef __cplusplus +extern "C" { +#endif + void oai_wifi(void); void oai_init_audio_capture(void); void oai_init_audio_decoder(void); @@ -11,3 +15,7 @@ void oai_send_audio(PeerConnection *peer_connection); void oai_audio_decode(uint8_t *data, size_t size); void oai_webrtc(); void oai_http_request(char *offer, char *answer); + +#ifdef __cplusplus +} +#endif diff --git a/src/media.cpp b/src/media.cpp index 3ca44c0..a3b417b 100644 --- a/src/media.cpp +++ b/src/media.cpp @@ -1,3 +1,4 @@ +#if (CONFIG_OPENAI_WITH_GMF == 0) #include #include @@ -142,3 +143,4 @@ void oai_send_audio(PeerConnection *peer_connection) { peer_connection_send_audio(peer_connection, encoder_output_buffer, encoded_size); } +#endif diff --git a/src/media_gmf.cpp b/src/media_gmf.cpp new file mode 100644 index 0000000..990342e --- /dev/null +++ b/src/media_gmf.cpp @@ -0,0 +1,409 @@ +#if (CONFIG_OPENAI_WITH_GMF == 1) +#include +#include +#include + +#include "bsp/esp-bsp.h" +#include "driver/i2s_pdm.h" +#include "driver/sdmmc_host.h" +#include "esp_audio_enc_default.h" +#include "esp_audio_simple_player.h" +#include "esp_audio_simple_player_advance.h" +#include "esp_codec_dev.h" +#include "esp_err.h" +#include "esp_gmf_audio_enc.h" +#include "esp_gmf_audio_helper.h" +#include "esp_gmf_data_bus.h" +#include "esp_gmf_element.h" +#include "esp_gmf_err.h" +#include "esp_gmf_fifo.h" +#include "esp_gmf_io_codec_dev.h" +#include "esp_gmf_io_i2s_pdm.h" +#include "esp_gmf_oal_mem.h" +#include "esp_gmf_oal_thread.h" +#include "esp_gmf_pipeline.h" +#include "esp_gmf_pool.h" +#include "esp_log.h" +#include "esp_opus_enc.h" +#include "main.h" + +static const char *TAG = "gmf"; + +#define BUFFER_SAMPLES (320 * 1) +#define SAMPLE_RATE (8000 * 1) + +static esp_gmf_fifo_handle_t oai_plr_dec_fifo = NULL; +static esp_gmf_fifo_handle_t oai_rec_enc_fifo = NULL; + +static esp_codec_dev_handle_t oai_plr_handle = NULL; +static esp_codec_dev_handle_t oai_rec_handle = NULL; + +static int16_t *oai_encoder_input_buffer = NULL; + +static esp_gmf_err_t oai_record_event_callback(esp_gmf_event_pkt_t *event, + void *ctx) { + ESP_LOGI(TAG, "RECV el:%s-%p, type:%x, sub:%s, payload:%p, size:%d,%p", + "OBJ_GET_TAG(event->from)", event->from, (int)event->type, + esp_gmf_event_get_state_str((esp_gmf_event_state_t)event->sub), + event->payload, event->payload_size, ctx); + return ESP_GMF_ERR_OK; +} + +static int oai_player_event_callback(esp_asp_event_pkt_t *event, void *ctx) { + if (event->type == ESP_ASP_EVENT_TYPE_MUSIC_INFO) { + esp_asp_music_info_t info; + memcpy(&info, event->payload, event->payload_size); + ESP_LOGI(TAG, "Get info, rate:%d, channels:%d, bits:%d", info.sample_rate, + info.channels, info.bits); + } else if (event->type == ESP_ASP_EVENT_TYPE_STATE) { + esp_asp_state_t st = ESP_ASP_STATE_NONE; + memcpy(&st, event->payload, event->payload_size); + ESP_LOGI(TAG, "Get State, %d,%s", st, + esp_audio_simple_player_state_to_str(st)); + if (ctx && + ((st == ESP_ASP_STATE_STOPPED) || (st == ESP_ASP_STATE_FINISHED) || + (st == ESP_ASP_STATE_ERROR))) { + xSemaphoreGive((SemaphoreHandle_t)ctx); + } + } + return 0; +} + +static int oai_record_read_enc(uint8_t *data, int data_size) { + esp_gmf_data_bus_block_t blk; + + int ret = esp_gmf_fifo_acquire_read(oai_rec_enc_fifo, &blk, data_size, + portMAX_DELAY); + memcpy(data, blk.buf, data_size); + esp_gmf_fifo_release_read(oai_rec_enc_fifo, &blk, 0); + + return ret; +} + +static void oai_record_write_enc(uint8_t *data, int len) { + esp_gmf_data_bus_block_t blk; + + esp_gmf_fifo_acquire_write(oai_rec_enc_fifo, &blk, len, portMAX_DELAY); + memcpy((void *)blk.buf, (void *)data, len); + blk.valid_size = len; + if (len == 0) { + blk.is_last = true; + } + esp_gmf_fifo_release_write(oai_rec_enc_fifo, &blk, portMAX_DELAY); +} + +static int oai_encoder_acquire_write(void *handle, + esp_gmf_data_bus_block_t *blk, + uint32_t wanted_size, int block_ticks) { + if (blk->buf) { + return wanted_size; + } + return wanted_size; +} + +static int oai_encoder_release_write(void *handle, + esp_gmf_data_bus_block_t *blk, + int block_ticks) { + int ret = 0; + if (blk->valid_size) { + oai_record_write_enc(blk->buf, blk->valid_size); + ret = blk->valid_size; + } + return ret; +} + +static int oai_player_read_dec(uint8_t *data, int data_size, void *ctx) { + esp_gmf_data_bus_block_t blk; + + int ret = esp_gmf_fifo_acquire_read(oai_plr_dec_fifo, &blk, data_size, + portMAX_DELAY); + memcpy(data, blk.buf, data_size); + esp_gmf_fifo_release_read(oai_plr_dec_fifo, &blk, 0); + + return ret; +} + +static void oai_player_write_dec(uint8_t *data, int len) { + esp_gmf_data_bus_block_t blk; + + esp_gmf_fifo_acquire_write(oai_plr_dec_fifo, &blk, len, portMAX_DELAY); + memcpy((void *)blk.buf, (void *)data, len); + blk.valid_size = len; + if (len == 0) { + blk.is_last = true; + } + esp_gmf_fifo_release_write(oai_plr_dec_fifo, &blk, portMAX_DELAY); +} + +static int oai_player_write_out(uint8_t *data, int data_size, void *ctx) { + esp_codec_dev_handle_t dev = (esp_codec_dev_handle_t)ctx; + esp_codec_dev_write(dev, data, data_size); + return data_size; +} + +static esp_gmf_err_t oai_record_pipeline_create( + esp_gmf_pipeline_handle_t *pipe_rec, esp_gmf_pool_handle_t pool) { + esp_gmf_err_t ret = ESP_GMF_ERR_OK; + esp_gmf_pipeline_handle_t pipe = NULL; + esp_gmf_task_handle_t work_rec_task = NULL; + + do { + ret = esp_gmf_pool_new_pipeline( + pool, "codec_dev_rx", (const char *[]){"encoder"}, 1, NULL, &pipe); + if (ret != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_pool_new_pipeline failed(0x%x)", ret); + break; + } + + esp_gmf_port_handle_t out_port; + out_port = (esp_gmf_port_handle_t)NEW_ESP_GMF_PORT_OUT_BYTE( + (void *)oai_encoder_acquire_write, (void *)oai_encoder_release_write, + NULL, &out_port, 0, ESP_GMF_MAX_DELAY); + ret = esp_gmf_pipeline_reg_el_port(pipe, "encoder", ESP_GMF_IO_DIR_WRITER, + out_port); + if (ret != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_pipeline_reg_el_port failed(0x%x)", ret); + break; + } + + esp_gmf_task_cfg_t cfg_rec; + cfg_rec.name = "rec_task"; + cfg_rec.thread.prio = DEFAULT_ESP_GMF_TASK_PRIO; + cfg_rec.thread.stack_in_ext = false; + cfg_rec.thread.stack = 30 * 1024; + cfg_rec.thread.core = 1; + + ret = esp_gmf_task_init(&cfg_rec, &work_rec_task); + if (ret != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_task_init failed(0x%x)", ret); + break; + } + + esp_gmf_pipeline_bind_task(pipe, work_rec_task); + esp_gmf_pipeline_loading_jobs(pipe); + esp_gmf_pipeline_set_event(pipe, oai_record_event_callback, NULL); + + esp_gmf_element_handle_t enc_handle = NULL; + esp_gmf_pipeline_get_el_by_name(pipe, "encoder", &enc_handle); + + esp_gmf_info_sound_t info; + info.sample_rates = SAMPLE_RATE; + info.channels = 1; + info.bits = 16; + esp_gmf_audio_helper_reconfig_enc_by_type( + ESP_AUDIO_TYPE_OPUS, &info, + (esp_audio_enc_config_t *)OBJ_GET_CFG(enc_handle)); + + esp_audio_enc_config_t *cfg = + (esp_audio_enc_config_t *)OBJ_GET_CFG(enc_handle); + esp_opus_enc_config_t *opus_enc_cfg = (esp_opus_enc_config_t *)cfg->cfg; + opus_enc_cfg->bitrate = 10000 * 3; + opus_enc_cfg->enable_vbr = true; + + ret = esp_gmf_pipeline_report_info(pipe, ESP_GMF_INFO_SOUND, &info, + sizeof(info)); + if (ret != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_pipeline_report_info failed(0x%x)", ret); + break; + } + + *pipe_rec = pipe; + return ret; + } while (0); + + if (pipe) { + esp_gmf_pipeline_stop(pipe); + esp_gmf_pipeline_destroy(pipe); + } + if (work_rec_task) { + esp_gmf_task_deinit(work_rec_task); + } + return ret; +} + +void pool_register_codec_dev_io(esp_gmf_pool_handle_t pool, void *play_dev, + void *record_dev) { + if (play_dev != NULL) { + codec_dev_io_cfg_t tx_codec_dev_cfg = ESP_GMF_IO_CODEC_DEV_CFG_DEFAULT(); + tx_codec_dev_cfg.dir = ESP_GMF_IO_DIR_WRITER; + tx_codec_dev_cfg.dev = play_dev; + tx_codec_dev_cfg.name = "codec_dev_tx"; + esp_gmf_io_handle_t tx_dev = NULL; + esp_gmf_io_codec_dev_init(&tx_codec_dev_cfg, &tx_dev); + esp_gmf_pool_register_io(pool, tx_dev, NULL); + } + if (record_dev != NULL) { + codec_dev_io_cfg_t rx_codec_dev_cfg = ESP_GMF_IO_CODEC_DEV_CFG_DEFAULT(); + rx_codec_dev_cfg.dir = ESP_GMF_IO_DIR_READER; + rx_codec_dev_cfg.dev = record_dev; + rx_codec_dev_cfg.name = "codec_dev_rx"; + esp_gmf_io_handle_t rx_dev = NULL; + esp_gmf_io_codec_dev_init(&rx_codec_dev_cfg, &rx_dev); + esp_gmf_pool_register_io(pool, rx_dev, NULL); + } +} + +void pool_register_audio_codecs(esp_gmf_pool_handle_t pool) { + esp_audio_enc_register_default(); + esp_audio_enc_config_t es_enc_cfg = DEFAULT_ESP_GMF_AUDIO_ENC_CONFIG(); + esp_gmf_element_handle_t enc_handle = NULL; + esp_gmf_audio_enc_init(&es_enc_cfg, &enc_handle); + esp_gmf_pool_register_element(pool, enc_handle, NULL); +} + +void oai_init_audio_capture(void) { + /* Initialize speaker */ + oai_plr_handle = bsp_audio_codec_speaker_init(); + assert(oai_plr_handle); + /* Speaker output volume */ + esp_codec_dev_set_out_vol(oai_plr_handle, 60); + ESP_LOGI(TAG, "Initialize speaker:%p", oai_plr_handle); + + /* Initialize microphone */ + oai_rec_handle = bsp_audio_codec_microphone_init(); + assert(oai_rec_handle); + /* Microphone input gain */ + esp_codec_dev_set_in_gain(oai_rec_handle, 50.0); + ESP_LOGI(TAG, "Initialize microphone:%p", oai_rec_handle); + + esp_codec_dev_sample_info_t fs_record = { + .bits_per_sample = 16, + .channel = 1, + .channel_mask = 0, + .sample_rate = SAMPLE_RATE, + .mclk_multiple = I2S_MCLK_MULTIPLE_384, + }; + esp_codec_dev_open(oai_rec_handle, &fs_record); + + esp_codec_dev_sample_info_t fs_play = { + .bits_per_sample = 16, + .channel = 2, + .channel_mask = 0, + .sample_rate = SAMPLE_RATE, + .mclk_multiple = I2S_MCLK_MULTIPLE_384, + }; + esp_codec_dev_open(oai_plr_handle, &fs_play); +} + +void oai_init_audio_decoder(void) { + esp_err_t err = ESP_GMF_ERR_OK; + esp_asp_handle_t player_handle = NULL; + + do { + err = esp_gmf_fifo_create(10, 1, &oai_plr_dec_fifo); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "oai_plr_dec_fifo init failed (0x%x)", err); + break; + } + + esp_asp_cfg_t player_cfg; + player_cfg.in.cb = oai_player_read_dec; + player_cfg.in.user_ctx = oai_plr_dec_fifo; + player_cfg.out.cb = oai_player_write_out; + player_cfg.out.user_ctx = oai_plr_handle; + player_cfg.task_prio = 5; + player_cfg.task_stack = 30 * 1024; + player_cfg.task_core = 1; + + err = esp_audio_simple_player_new(&player_cfg, &player_handle); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "simple_player init failed (0x%x)", err); + break; + } + + err = esp_audio_simple_player_set_event(player_handle, + oai_player_event_callback, NULL); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "set_event failed (0x%x)", err); + break; + } + + esp_asp_music_info_t music_info; + music_info.sample_rate = SAMPLE_RATE; + music_info.channels = 2; + music_info.bits = 16; + music_info.bitrate = 0; + + err = esp_audio_simple_player_run(player_handle, "raw://sdcard/openAI.opus", + &music_info); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "run failed (0x%x)", err); + break; + } + + return; + } while (0); + + if (player_handle) { + esp_audio_simple_player_destroy(player_handle); + } + if (oai_plr_dec_fifo) { + esp_gmf_fifo_destroy(oai_plr_dec_fifo); + } +} + +void oai_audio_decode(uint8_t *data, size_t size) { + oai_player_write_dec(data, size); +} + +void oai_init_audio_encoder(void) { + esp_err_t err = ESP_GMF_ERR_OK; + esp_gmf_pool_handle_t pool_handle = NULL; + + do { + err = esp_gmf_fifo_create(10, 1, &oai_rec_enc_fifo); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "oai_rec_enc_fifo init failed (0x%x)", err); + break; + } + + esp_gmf_pipeline_handle_t record_pipeline = NULL; + err = esp_gmf_pool_init(&pool_handle); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_pool_init failed (0x%x)", err); + break; + } + + pool_register_audio_codecs(pool_handle); + pool_register_codec_dev_io(pool_handle, oai_plr_handle, oai_rec_handle); + ESP_GMF_POOL_SHOW_ITEMS(pool_handle); + + err = oai_record_pipeline_create(&record_pipeline, pool_handle); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "oai_record_pipeline_create failed (0x%x)", err); + break; + } + + err = esp_gmf_pipeline_run(record_pipeline); + if (err != ESP_GMF_ERR_OK) { + ESP_LOGE(TAG, "esp_gmf_pipeline_run failed (0x%x)", err); + break; + } + + oai_encoder_input_buffer = (int16_t *)malloc(BUFFER_SAMPLES); + if (!oai_encoder_input_buffer) { + ESP_LOGE(TAG, "Failed to allocate memory for encoder input buffer"); + break; + } + + return; + } while (0); + + if (oai_rec_enc_fifo) { + esp_gmf_fifo_destroy(oai_rec_enc_fifo); + } + if (pool_handle) { + esp_gmf_pool_deinit(pool_handle); + } + if (oai_encoder_input_buffer) { + free(oai_encoder_input_buffer); + } +} + +void oai_send_audio(PeerConnection *peer_connection) { + int encoded_size = + oai_record_read_enc((uint8_t *)oai_encoder_input_buffer, BUFFER_SAMPLES); + peer_connection_send_audio( + peer_connection, (const uint8_t *)oai_encoder_input_buffer, encoded_size); +} +#endif diff --git a/src/webrtc.cpp b/src/webrtc.cpp index bc1782d..e5859e2 100644 --- a/src/webrtc.cpp +++ b/src/webrtc.cpp @@ -1,4 +1,4 @@ -#ifndef LINUX_BUILD +#if (CONFIG_OPENAI_WITH_GMF == 0) && !defined(LINUX_BUILD) #include #include #endif @@ -73,6 +73,13 @@ static void oai_on_icecandidate_task(char *description, void *user_data) { peer_connection_set_remote_description(peer_connection, local_buffer); } +static void oai_peer_loop_task(void *arg) { + while (1) { + peer_connection_loop(peer_connection); + vTaskDelay(pdMS_TO_TICKS(TICK_INTERVAL)); + } +} + void oai_webrtc() { PeerConfiguration peer_connection_config = { .ice_servers = {}, @@ -106,8 +113,6 @@ void oai_webrtc() { peer_connection_create_offer(peer_connection); - while (1) { - peer_connection_loop(peer_connection); - vTaskDelay(pdMS_TO_TICKS(TICK_INTERVAL)); - } + xTaskCreatePinnedToCore(&oai_peer_loop_task, "peer connect task", 1024 * 16, + NULL, 4, NULL, 0); }