Merge branch 'master' into BLEAdvertising-return-codes

Author: FidelSch

.github/ISSUE_TEMPLATE/Issue-report.yml

@@ -41,6 +41,8 @@ body:
       options:
         - latest master (checkout manually)
         - latest development Release Candidate (RC-X)
+        - v3.1.2
+        - v3.1.1
         - v3.1.0
         - v3.0.7
         - v3.0.6

.github/scripts/set_push_chunks.sh

@@ -2,6 +2,93 @@
 
 build_all=false
 chunks_count=0
+last_check_files=""
+last_check_result=""
+gh_output=""
+
+# Define the file patterns
+core_files=(
+    '\.github/.*'
+    'cores/.*'
+    'package/.*'
+    'tools/.*'
+    'platform\.txt'
+    'programmers\.txt'
+    'variants/esp32/.*'
+    'variants/esp32c3/.*'
+    'variants/esp32c6/.*'
+    'variants/esp32h2/.*'
+    'variants/esp32p4/.*'
+    'variants/esp32s2/.*'
+    'variants/esp32s3/.*'
+)
+library_files=(
+    'libraries/.*/examples/.*'
+    'libraries/.*/src/.*'
+)
+networking_files=(
+    'libraries/Network/src/.*'
+)
+fs_files=(
+    'libraries/FS/src/.*'
+)
+static_sketches_files=(
+    'libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure\.ino'
+    'libraries/BLE/examples/Server/Server\.ino'
+    'libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer\.ino'
+    'libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics\.ino'
+    'libraries/NetworkClientSecure/src/.*'
+    'libraries/BLE/src/.*'
+    'libraries/Insights/src/.*'
+)
+idf_files=(
+    'idf_component\.yml'
+    'Kconfig\.projbuild'
+    'CMakeLists\.txt'
+    'variants/esp32c2/.*'
+)
+
+# Function to check if any files match the patterns
+check_files() {
+    local patterns=("$@")
+    local files_found=""
+    for pattern in "${patterns[@]}"; do
+        echo "Checking pattern: $pattern"
+        matched_files=$(echo "$gh_output" | grep -E "$pattern")
+        echo "matched_files: $matched_files"
+        files_found+="$matched_files "
+    done
+
+    last_check_files=$(echo "$files_found" | xargs)
+    if [[ -n $last_check_files ]]; then
+        last_check_result="true"
+    else
+        last_check_result="false"
+    fi
+    echo "last_check_result: $last_check_result"
+}
+
+if [[ $IS_PR != 'true' ]]; then
+    gh_output=$(gh api repos/espressif/arduino-esp32/commits/"$GITHUB_SHA" --jq '.files[].filename')
+else
+    gh_output=$(gh pr diff "$PR_NUM" --name-only)
+fi
+echo "gh_output: $gh_output"
+
+# Output the results
+check_files "${core_files[@]}"
+CORE_CHANGED=$last_check_result
+check_files "${library_files[@]}"
+LIB_CHANGED=$last_check_result
+LIB_FILES=$last_check_files
+check_files "${networking_files[@]}"
+NETWORKING_CHANGED=$last_check_result
+check_files "${fs_files[@]}"
+FS_CHANGED=$last_check_result
+check_files "${static_sketches_files[@]}"
+STATIC_SKETCHES_CHANGED=$last_check_result
+check_files "${idf_files[@]}"
+IDF_CHANGED=$last_check_result
 
 if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then
     echo "Core files changed or not a PR. Building all."
@@ -76,9 +163,9 @@ chunks+="]"
 
 {
     echo "build_all=$build_all"
-    echo "build_libraries=$BUILD_LIBRARIES"
-    echo "build_static_sketches=$BUILD_STATIC_SKETCHES"
-    echo "build_idf=$BUILD_IDF"
+    echo "build_libraries=$LIB_CHANGED"
+    echo "build_static_sketches=$STATIC_SKETCHES_CHANGED"
+    echo "build_idf=$IDF_CHANGED"
     echo "chunk_count=$chunks_count"
     echo "chunks=$chunks"
 } >> "$GITHUB_OUTPUT"

.github/scripts/sketch_utils.sh

@@ -244,7 +244,7 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
         build_dir="$ARDUINO_BUILD_DIR"
     elif [ "$len" -eq 1 ]; then
         # build_dir="$sketchdir/build"
-        build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
+        build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp"
     fi
 
     output_file="$HOME/.arduino/cli_compile_output.txt"
@@ -254,7 +254,7 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
     for i in $(seq 0 $((len - 1))); do
         if [ "$len" -ne 1 ]; then
             # build_dir="$sketchdir/build$i"
-            build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
+            build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp"
         fi
         rm -rf "$build_dir"
         mkdir -p "$build_dir"

.github/scripts/tests_run.sh

@@ -11,9 +11,11 @@ function run_test {
     local error=0
     local sdkconfig_path
     local extra_args
+    local test_type
 
     sketchdir=$(dirname "$sketch")
     sketchname=$(basename "$sketchdir")
+    test_type=$(basename "$(dirname "$sketchdir")")
 
     if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then
         len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json)
@@ -25,9 +27,9 @@ function run_test {
     fi
 
     if [ "$len" -eq 1 ]; then
-        sdkconfig_path="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
+        sdkconfig_path="$HOME/.arduino/tests/$target/$sketchname/build.tmp/sdkconfig"
     else
-        sdkconfig_path="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
+        sdkconfig_path="$HOME/.arduino/tests/$target/$sketchname/build0.tmp/sdkconfig"
     fi
 
     if [ -f "$sketchdir"/ci.json ]; then
@@ -43,22 +45,22 @@ function run_test {
     fi
 
     if [ ! -f "$sdkconfig_path" ]; then
-        printf "\033[93mSketch %s not built\nMight be due to missing target requirements or build failure\033[0m\n" "$sketchname"
+        printf "\033[93mSketch %s build not found in %s\nMight be due to missing target requirements or build failure\033[0m\n" "$(dirname "$sdkconfig_path")" "$sketchname"
         printf "\n\n\n"
         return 0
     fi
 
-    local right_target
-    right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path")
-    if [ -z "$right_target" ]; then
-        printf "\033[91mError: Sketch %s compiled for different target\n\033[0m\n" "$sketchname"
+    local compiled_target
+    compiled_target=$(grep -E "CONFIG_IDF_TARGET=" "$sdkconfig_path" | cut -d'"' -f2)
+    if [ "$compiled_target" != "$target" ]; then
+        printf "\033[91mError: Sketch %s compiled for %s, expected %s\033[0m\n" "$sketchname" "$compiled_target" "$target"
         printf "\n\n\n"
         return 1
     fi
 
     if [ "$len" -eq 1 ]; then
         # build_dir="$sketchdir/build"
-        build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
+        build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp"
         report_file="$sketchdir/$target/$sketchname.xml"
     fi
 
@@ -81,7 +83,7 @@ function run_test {
 
         if [ "$len" -ne 1 ]; then
             # build_dir="$sketchdir/build$i"
-            build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
+            build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp"
             report_file="$sketchdir/$target/$sketchname$i.xml"
         fi
 
@@ -113,14 +115,14 @@ function run_test {
         rm "$sketchdir"/diagram.json 2>/dev/null || true
 
         result=0
-        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
-        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
         printf "\n"
         if [ $result -ne 0 ]; then
             result=0
             printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
-            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
-            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
             printf "\n"
             if [ $result -ne 0 ]; then
                 printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"

.github/workflows/build_py_tools.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   find-changed-tools:
     name: Check if tools have been changed
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       any_changed: ${{ steps.verify-changed-files.outputs.any_changed }}
       all_changed_files: ${{ steps.verify-changed-files.outputs.all_changed_files }}
@@ -30,16 +30,16 @@ jobs:
           echo "Make sure you are using a branch inside the repository and not a fork."
 
       - name: Verify Python Tools Changed
-        uses: tj-actions/changed-files@v41
         id: verify-changed-files
-        with:
-          fetch_depth: "2"
-          since_last_remote_commit: "true"
-          files: |
-            tools/get.py
-            tools/espota.py
-            tools/gen_esp32part.py
-            tools/gen_insights_package.py
+        run: |
+          CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r ^HEAD -- tools/get.py tools/espota.py tools/gen_esp32part.py tools/gen_insights_package.py | xargs)
+          echo "all_changed_files=$CHANGED_FILES" >> $GITHUB_OUTPUT
+          if [ -n "$CHANGED_FILES" ]; then
+            echo "any_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "any_changed=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: List all changed files
         shell: bash
         run: |
@@ -55,7 +55,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, macos-latest, ubuntu-20.04, ARM]
+        os: [windows-latest, macos-latest, ubuntu-latest, ubuntu-24.04-arm]
         include:
           - os: windows-latest
             TARGET: win64
@@ -64,14 +64,12 @@ jobs:
           - os: macos-latest
             TARGET: macos
             SEPARATOR: ":"
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             TARGET: linux-amd64
             SEPARATOR: ":"
-          - os: ARM
-            CONTAINER: python:3.8-bullseye
+          - os: ubuntu-24.04-arm
             TARGET: arm
             SEPARATOR: ":"
-    container: ${{ matrix.CONTAINER }} # use python container on ARM
     env:
       DISTPATH: pytools-${{ matrix.TARGET }}
       PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi"
@@ -96,8 +94,6 @@ jobs:
           token: ${{ secrets.TOOLS_UPLOAD_PAT }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Set up Python 3.8
-        # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
-        if: matrix.os != 'ARM'
         uses: actions/setup-python@master
         with:
           python-version: 3.8

.github/workflows/pre-commit.yml

@@ -58,7 +58,22 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/[email protected]
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUM: ${{ github.event.pull_request.number }}
+          IS_PR: ${{ github.event_name == 'pull_request' }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          if [[ $IS_PR != 'true' ]]; then
+              files_changed=$(gh api repos/espressif/arduino-esp32/commits/"$GITHUB_SHA" --jq '.files[].filename' | xargs)
+          else
+              files_changed=$(gh pr diff "$PR_NUM" --name-only | xargs)
+          fi
+          echo "all_changed_files=$files_changed" >> $GITHUB_OUTPUT
+          echo "Changed files:"
+          for file in $files_changed; do
+              echo "  $file"
+          done
 
       - name: Run pre-commit hooks in changed files
         run: pre-commit run --color=always --show-diff-on-failure --files ${{ steps.changed-files.outputs.all_changed_files }}