fix(ci): Fix tests and workflows (#12024)

lucasssvaz · pre-commit-ci-lite[bot] · web-flow · commit 5ab1db62ef4c · 2025-11-20T12:10:21.000+02:00
* fix(ci): Fix tests and workflows

* feat(tests): Add job ID fixture

* fix(tests): Fix performance test result generation

* ci(pre-commit): Apply automatic fixes

---------

Co-authored-by: pre-commit-ci-lite[bot] &lt;117423508+pre-commit-ci-lite[bot]@users.noreply.github.com&gt;
diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh
@@ -111,15 +111,23 @@ function run_test {
 
         rm "$sketchdir"/diagram.json 2>/dev/null || true
 
+        local wifi_args=""
+        if [ -n "$wifi_ssid" ]; then
+            wifi_args="--wifi-ssid \"$wifi_ssid\""
+        fi
+        if [ -n "$wifi_password" ]; then
+            wifi_args="$wifi_args --wifi-password \"$wifi_password\""
+        fi
+
         result=0
-        printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-        bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
+        printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args"
+        bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$?
         printf "\n"
         if [ $result -ne 0 ]; then
             result=0
             printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
-            printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-            bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
+            printf "\033[95mpytest -s \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" "$wifi_args"
+            bash -c "set +e; pytest -s \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q} $wifi_args; exit \$?" || result=$?
             printf "\n"
             if [ $result -ne 0 ]; then
                 printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"
@@ -137,6 +145,8 @@ platform="hardware"
 chunk_run=0
 options=0
 erase=0
+wifi_ssid=""
+wifi_password=""
 
 while [ -n "$1" ]; do
     case $1 in
@@ -188,6 +198,14 @@ while [ -n "$1" ]; do
         shift
         test_type=$1
         ;;
+    -wifi-ssid )
+        shift
+        wifi_ssid=$1
+        ;;
+    -wifi-password )
+        shift
+        wifi_password=$1
+        ;;
     * )
         break
         ;;
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -68,7 +68,7 @@ jobs:
           mkdir -p ${{ github.workspace }}/hosted
 
           # Copy hosted binaries to proper directory without overwriting existing files
-          cp -n ${{ github.workspace }}/hosted-latest/*.bin ${{ github.workspace }}/hosted/
+          cp --update=none ${{ github.workspace }}/hosted-latest/*.bin ${{ github.workspace }}/hosted/
 
           # Commit the changes
           git config user.name "github-actions[bot]"
diff --git a/.github/workflows/tests_hw_wokwi.yml b/.github/workflows/tests_hw_wokwi.yml
@@ -361,7 +361,14 @@ jobs:
           download_artifacts: 'true'
           download_artifacts_on_failure: 'true'
           download_path: './gitlab-artifacts'
-          variables: '{"TEST_TYPES":"${{ steps.prepare-variables.outputs.test_types }}","TEST_CHIPS":"${{ steps.prepare-variables.outputs.test_chips }}","PIPELINE_ID":"${{ env.id }}","BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}","GITHUB_REPOSITORY":"${{ github.repository }}"}'
+          variables: >-
+            {
+              "TEST_TYPES":"${{ steps.prepare-variables.outputs.test_types }}",
+              "TEST_CHIPS":"${{ steps.prepare-variables.outputs.test_chips }}",
+              "PIPELINE_ID":"${{ env.id }}",
+              "BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}",
+              "GITHUB_REPOSITORY":"${{ github.repository }}"
+            }
 
       - name: Process Downloaded Artifacts
         if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
@@ -536,8 +543,11 @@ jobs:
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         env:
           WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
+          WOKWI_WIFI_SSID: "Wokwi-GUEST"
+          # The Wokwi Wi-Fi does not have a password, so we use an empty string
+          WOKWI_WIFI_PASSWORD: ""
         run: |
-          bash .github/scripts/tests_run.sh -c -type ${{ matrix.type }} -t ${{ matrix.chip }} -i 0 -m 1 -W
+          bash .github/scripts/tests_run.sh -c -type "${{ matrix.type }}" -t "${{ matrix.chip }}" -i 0 -m 1 -W -wifi-ssid "${{ env.WOKWI_WIFI_SSID }}" -wifi-password "${{ env.WOKWI_WIFI_PASSWORD }}"
 
       - name: Upload ${{ matrix.chip }} ${{ matrix.type }} Wokwi results as cache
         uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
diff --git a/.gitlab/workflows/hw_test_template.yml b/.gitlab/workflows/hw_test_template.yml
@@ -50,7 +50,7 @@ hw-test-template:
         [ -z "$d" ] && continue;
         sketch=$(basename "$d");
         echo Running $sketch in $d;
-        bash .github/scripts/tests_run.sh -t $TEST_CHIP -s $sketch -e || rc=$?;
+        bash .github/scripts/tests_run.sh -t "$TEST_CHIP" -s "$sketch" -e -wifi-ssid "$RUNNER_WIFI_SSID" -wifi-password "$RUNNER_WIFI_PASSWORD" || rc=$?;
       done <<< "$TEST_LIST"; exit $rc
 
   artifacts:
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,22 @@
+import pytest
+import os
+
+
+def pytest_addoption(parser):
+    parser.addoption("--wifi-password", action="store", default=None, help="Wi-Fi password.")
+    parser.addoption("--wifi-ssid", action="store", default=None, help="Wi-Fi SSID.")
+
+
+@pytest.fixture(scope="session")
+def wifi_ssid(request):
+    return request.config.getoption("--wifi-ssid")
+
+
+@pytest.fixture(scope="session")
+def wifi_pass(request):
+    return request.config.getoption("--wifi-password")
+
+
+@pytest.fixture(scope="session")
+def ci_job_id(request):
+    return os.environ.get("CI_JOB_ID")
diff --git a/tests/performance/coremark/test_coremark.py b/tests/performance/coremark/test_coremark.py
@@ -45,13 +45,14 @@ def test_coremark(dut, request):
     results = {"coremark": {"runs": runs, "cores": cores, "avg_score": avg_score}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_coremark" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/fibonacci/test_fibonacci.py b/tests/performance/fibonacci/test_fibonacci.py
@@ -67,13 +67,14 @@ def test_fibonacci(dut, request):
     results = {"fibonacci": {"runs": runs, "fib_n": fib_n, "avg_time": avg_time}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_fibonacci" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/linpack_double/test_linpack_double.py b/tests/performance/linpack_double/test_linpack_double.py
@@ -48,13 +48,14 @@ def test_linpack_double(dut, request):
     results = {"linpack_double": {"runs": runs, "avg_score": avg_score, "min_score": min_score, "max_score": max_score}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_linpack_double" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/linpack_float/test_linpack_float.py b/tests/performance/linpack_float/test_linpack_float.py
@@ -48,13 +48,14 @@ def test_linpack_float(dut, request):
     results = {"linpack_float": {"runs": runs, "avg_score": avg_score, "min_score": min_score, "max_score": max_score}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_linpack_float" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/psramspeed/test_psramspeed.py b/tests/performance/psramspeed/test_psramspeed.py
@@ -92,13 +92,14 @@ def test_psramspeed(dut, request):
     results = {"psramspeed": {"runs": runs, "copies": copies, "max_test_size": max_test_size, "results": avg_results}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_psramspeed" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/ramspeed/ramspeed.ino b/tests/performance/ramspeed/ramspeed.ino
@@ -234,8 +234,8 @@ void setup() {
     delay(10);
   }
 
-  void *dest = malloc(MAX_TEST_SIZE);
-  const void *src = malloc(MAX_TEST_SIZE);
+  void *dest = heap_caps_malloc(MAX_TEST_SIZE, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL);
+  const void *src = heap_caps_malloc(MAX_TEST_SIZE, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL);
 
   if (!dest || !src) {
     Serial.println("Memory allocation failed");
diff --git a/tests/performance/ramspeed/test_ramspeed.py b/tests/performance/ramspeed/test_ramspeed.py
@@ -92,13 +92,14 @@ def test_ramspeed(dut, request):
     results = {"ramspeed": {"runs": runs, "copies": copies, "max_test_size": max_test_size, "results": avg_results}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_ramspeed" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/performance/superpi/test_superpi.py b/tests/performance/superpi/test_superpi.py
@@ -40,13 +40,14 @@ def test_superpi(dut, request):
     results = {"superpi": {"runs": runs, "digits": digits, "avg_time": avg_time}}
 
     current_folder = os.path.dirname(request.path)
+    os.makedirs(os.path.join(current_folder, dut.app.target), exist_ok=True)
     file_index = 0
     report_file = os.path.join(current_folder, dut.app.target, "result_superpi" + str(file_index) + ".json")
     while os.path.exists(report_file):
         report_file = report_file.replace(str(file_index) + ".json", str(file_index + 1) + ".json")
         file_index += 1
 
-    with open(report_file, "w") as f:
+    with open(report_file, "w+") as f:
         try:
             f.write(json.dumps(results))
         except Exception as e:
diff --git a/tests/pytest.ini b/tests/pytest.ini
@@ -1,5 +1,8 @@
 [pytest]
 addopts = --embedded-services esp,arduino,wokwi,qemu
+junit_family = xunit1
+filterwarnings =
+    ignore::pytest.PytestExperimentalApiWarning
 
 # log related
 log_cli = True
diff --git a/tests/validation/wifi/test_wifi.py b/tests/validation/wifi/test_wifi.py
@@ -1,13 +1,36 @@
 import logging
+import pytest
 
 
-def test_wifi(dut):
+def test_wifi(dut, wifi_ssid, wifi_pass):
     LOGGER = logging.getLogger(__name__)
 
+    # Fail if no WiFi SSID is provided
+    if not wifi_ssid:
+        pytest.fail("WiFi SSID is required but not provided. Use --wifi-ssid argument.")
+
+    # Wait for device to be ready and send WiFi credentials
+    LOGGER.info("Waiting for device to be ready...")
+    dut.expect_exact("Device ready for WiFi credentials")
+
+    dut.expect_exact("Send SSID:")
+    LOGGER.info(f"Sending WiFi credentials: SSID={wifi_ssid}")
+    dut.write(f"{wifi_ssid}")
+
+    dut.expect_exact("Send Password:")
+    LOGGER.info(f"Sending WiFi password: Password={wifi_pass}")
+    dut.write(f"{wifi_pass or ''}")
+
+    # Verify credentials were received
+    dut.expect_exact(f"SSID: {wifi_ssid}")
+    dut.expect_exact(f"Password: {wifi_pass or ''}")
+
     LOGGER.info("Starting WiFi Scan")
     dut.expect_exact("Scan start")
     dut.expect_exact("Scan done")
-    dut.expect_exact("Wokwi-GUEST")
+
+    LOGGER.info(f"Looking for WiFi network: {wifi_ssid}")
+    dut.expect_exact(wifi_ssid)
     LOGGER.info("WiFi Scan done")
 
     LOGGER.info("Connecting to WiFi")
diff --git a/tests/validation/wifi/wifi.ino b/tests/validation/wifi/wifi.ino
@@ -38,8 +38,8 @@
 
 #include <WiFi.h>
 
-const char *ssid = "Wokwi-GUEST";
-const char *password = "";
+String ssid = "";
+String password = "";
 
 // WARNING: This function is called from a separate FreeRTOS task (thread)!
 void WiFiEvent(WiFiEvent_t event) {
@@ -87,14 +87,56 @@ void WiFiGotIP(WiFiEvent_t event, WiFiEventInfo_t info) {
   Serial.println(IPAddress(info.got_ip.ip_info.ip.addr));
 }
 
+void readWiFiCredentials() {
+  Serial.println("Waiting for WiFi credentials...");
+  Serial.println("Send SSID:");
+
+  // Wait for SSID
+  while (ssid.length() == 0) {
+    if (Serial.available()) {
+      ssid = Serial.readStringUntil('\n');
+      ssid.trim();
+    }
+    delay(100);
+  }
+
+  Serial.println("Send Password:");
+
+  // Wait for password (allow empty password)
+  bool password_received = false;
+  while (!password_received) {
+    if (Serial.available()) {
+      password = Serial.readStringUntil('\n');
+      password.trim();
+      password_received = true;  // Accept even empty password
+    }
+    delay(100);
+  }
+
+  Serial.print("SSID: ");
+  Serial.println(ssid);
+  Serial.print("Password: ");
+  Serial.println(password);
+}
+
 void setup() {
   Serial.begin(115200);
 
+  while (!Serial) {
+    delay(100);
+  }
+
   // delete old config
   WiFi.disconnect(true);
 
   delay(1000);
 
+  // Wait for test to be ready
+  Serial.println("Device ready for WiFi credentials");
+
+  // Read WiFi credentials from serial
+  readWiFiCredentials();
+
   // Examples of different ways to register wifi events;
   // these handlers will be called from another thread.
   WiFi.onEvent(WiFiEvent);
@@ -134,7 +176,7 @@ void setup() {
   // Delete the scan result to free memory for code below.
   WiFi.scanDelete();
 
-  WiFi.begin(ssid, password);
+  WiFi.begin(ssid.c_str(), password.c_str());
 
   Serial.println();
   Serial.println();
