dynamic crossenv + python3*-wheels + python310-313 updates

.github/actions/build.sh

@@ -61,7 +61,7 @@ if [ -n "$API_KEY" ] && [ "$PUBLISH" == "true" ]; then
 fi
 
 # Build
-PACKAGES_TO_KEEP="synocli-videodriver ffmpeg5 ffmpeg7 python310 python311"
+PACKAGES_TO_KEEP="synocli-videodriver ffmpeg5 ffmpeg7 python310 python311 python312  python313"
 for package in ${build_packages}
 do
     echo "::group:: ---- build ${package}"

.github/actions/prepare.sh

@@ -5,9 +5,10 @@
 # Evaluate packages to build and referenced source files to download.
 #
 # Functions:
-# - Evaluate all packages to build depending on files defined in ${GH_FILES}.
+# - Build all packages defined by ${USER_SPK_TO_BUILD} and ${GH_SPK_PACKAGES}
+# - Evaluate additional packages to build depending on changed folders defined in ${GH_DEPENDENT_PACKAGES}.
 # - synocli-videodriver is moved to head of packages to build first if triggered by its ffmpeg5-7
-# - python310-311 and ffmpeg5-7 are moved to head of remaining packages to build when triggered by its own or a dependent.
+# - python310-313 and ffmpeg5-7 are moved to head of remaining packages to build when triggered by its own or a dependent.
 # - Referenced native and cross packages of the packages to build are added to the download list.
 
 set -o pipefail
@@ -18,12 +19,8 @@ echo "::group:: ---- find dependent packages"
 make setup-synocommunity
 DEFAULT_TC=$(grep DEFAULT_TC local.mk | cut -f2 -d= | xargs)
 
-# filter for changes made in the spk directories and take unique package name (without spk folder)
-SPK_TO_BUILD+=" "
-SPK_TO_BUILD+=$(echo "${GH_FILES}" | tr ' ' '\n' | grep -oP "^spk/\K[^\/]*" | sort -u | tr '\n' ' ')
-
-# filter for changes made in the cross and native directories and take unique package name (including cross or native folder)
-DEPENDENT_PACKAGES=$(echo "${GH_FILES}" | tr ' ' '\n' | grep -oP "(cross|native)/[^\/]*" | sort -u | tr '\n' ' ')
+# all packages to build from changes or manual definition
+SPK_TO_BUILD="${USER_SPK_TO_BUILD} ${GH_SPK_PACKAGES} "
 
 # get dependency list
 # dependencies in this list include the cross or native folder (i.e. native/python cross/glib)
@@ -37,20 +34,20 @@ do
 done
 
 # search for dependent spk packages
-for package in ${DEPENDENT_PACKAGES}
+for package in ${GH_DEPENDENCY_FOLDERS}
 do
     echo "===> Searching for dependent package: ${package}"
     packages=$(echo "${DEPENDENCY_LIST}" | grep " ${package} " | grep -o ".*:" | tr ':' ' ' | sort -u | tr '\n' ' ')
     echo "===> Found: ${packages}"
-    SPK_TO_BUILD+=${packages}
+    SPK_TO_BUILD+=" ${packages}"
 done
 
 # fix for packages with different names
-if [ "$(echo ${SPK_TO_BUILD} | grep -ow nzbdrone)" != "" ]; then
-    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -vw "nzbdrone" | tr '\n' ' ')" sonarr3"
+if [ "$(echo ${SPK_TO_BUILD} | grep -o ' nzbdrone ')" != "" ]; then
+    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -v "^nzbdrone$" | tr '\n' ' ')" sonarr3"
 fi
-if [ "$(echo ${SPK_TO_BUILD} | grep -ow python)" != "" ]; then
-    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -vw "python" | tr '\n' ' ')" python2"
+if [ "$(echo ${SPK_TO_BUILD} | grep -o ' python ')" != "" ]; then
+    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -v "^python$" | tr '\n' ' ')" python2"
 fi
 
 # remove duplicate packages
@@ -65,7 +62,7 @@ for i in {5..7}; do
     for package in ${packages}
     do
         if [ "$(echo ffmpeg${i} ${ffmpeg_dependent_packages} | grep -ow ${package})" != "" ]; then
-            packages_without_ffmpeg=$(echo "${packages}" | tr ' ' '\n' | grep -v "ffmpeg${i}" | tr '\n' ' ')
+            packages_without_ffmpeg=$(echo "${packages}" | tr ' ' '\n' | grep -v "^ffmpeg${i}\$" | tr '\n' ' ')
             packages="ffmpeg${i} ${packages_without_ffmpeg}"
             break
         fi
@@ -80,22 +77,22 @@ videodrv_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile"
 for package in ${packages}
 do
     if [ "$(echo synocli-videodriver ${videodrv_dependent_packages} | grep -ow ${package})" != "" ]; then
-        packages_without_videodrv=$(echo "${packages}" | tr ' ' '\n' | grep -v "synocli-videodriver" | tr '\n' ' ')
+        packages_without_videodrv=$(echo "${packages}" | tr ' ' '\n' | grep -v "^synocli-videodriver\$" | tr '\n' ' ')
         packages="synocli-videodriver ${packages_without_videodrv}"
         break
     fi
 done
 
-# for python (310, 311) find all packages that depend on them
-for py in python310 python311; do
+# for python (310, 311, 312, 313) find all packages that depend on them
+for py in python310 python311 python312 python313; do
     python_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "PYTHON_PACKAGE = ${py}" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
 
     # If packages contain a package that depends on python (or is python), then ensure
     # relevant python spk is first in list
     for package in ${packages}
     do
         if [ "$(echo ${py} ${python_dependent_packages} | grep -ow ${package})" != "" ]; then
-            packages_without_python=$(echo "${packages}" | tr ' ' '\n' | grep -v "${py}" | tr '\n' ' ')
+            packages_without_python=$(echo "${packages}" | tr ' ' '\n' | grep -v "^${py}\$" | tr '\n' ' ')
             packages="${py} ${packages_without_python}"
             break
         fi

.github/workflows/build.yml

@@ -84,26 +84,31 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Get changed files for pull request
+      - name: Get changed spk_packages and dependency_folders for pull request
         if: github.event_name == 'pull_request'
-        id: getfile_pr
+        id: getchanges_pr
         run: |
-          git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | xargs
-          echo "files=$(git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | xargs)" >> $GITHUB_OUTPUT
+          git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs
+          echo "spk_packages=$(git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
+          git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "(cross|native)/[^\/]*" | sort -u | xargs
+          echo "dependency_folders=$(git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "(cross|native)/[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
 
-      - name: Get changed files for push
+      - name: Get changed spk_packages and dependency_folders of last commit for push
         if: github.event_name == 'push'
-        id: getfile
+        id: getchanges_push
         run: |
-          git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs
-          echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> $GITHUB_OUTPUT
+          git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs
+          echo "spk_packages=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
+          git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "(cross|native)/[^\/]*" | sort -u | xargs
+          echo "dependency_folders=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "(cross|native)/[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
 
       - name: Evaluate dependencies
         id: dependencies
         run: ./.github/actions/prepare.sh
         env:
-          GH_FILES: ${{ steps.getfile.outputs.files }} ${{ steps.getfile_pr.outputs.files }}
-          SPK_TO_BUILD: ${{ github.event.inputs.package }}
+          GH_SPK_PACKAGES: ${{ steps.getchanges_push.outputs.spk_packages }} ${{ steps.getchanges_pr.outputs.spk_packages }}
+          GH_DEPENDENCY_FOLDERS: ${{ steps.getchanges_push.outputs.dependency_folders }} ${{ steps.getchanges_pr.outputs.dependency_folders }}
+          USER_SPK_TO_BUILD: ${{ github.event.inputs.package }}
 
       - name: Cache downloaded files
         uses: actions/cache@v4

cross/boost/Makefile

@@ -24,7 +24,7 @@ WITH_PYTHON_LIBRARY =
 ifneq ($(findstring python,$(BOOST_LIBRARIES)),)
 WITH_PYTHON_LIBRARY = 1
 # The packages depending on boost with python, must build python before.
-include $(WORK_DIR)/python-cc.mk
+include $(or $(wildcard $(WORK_DIR)/crossenv/build/python-cc.mk), $(wildcard $(PYTHON_PACKAGE_WORK_DIR)/crossenv/build/python-cc.mk))
 PYTHON_NAME = $(basename $(notdir $(PYTHON_INTERPRETER)))
 else ifneq ($(findstring $(BOOST_LIBRARIES), all),)
 # exclude python library if "all" selected

cross/boost_1.68/Makefile

@@ -26,7 +26,7 @@ WITH_PYTHON_LIBRARY =
 ifneq ($(findstring python,$(BOOST_LIBRARIES)),)
 WITH_PYTHON_LIBRARY = 1
 # The packages depending on boost with python, must build python before.
-include $(WORK_DIR)/python-cc.mk
+include $(or $(wildcard $(WORK_DIR)/crossenv/build/python-cc.mk), $(wildcard $(PYTHON_PACKAGE_WORK_DIR)/crossenv/build/python-cc.mk))
 PYTHON_NAME = $(basename $(notdir $(PYTHON_INTERPRETER)))
 else ifneq ($(findstring $(BOOST_LIBRARIES), all),)
 # exclude python library if "all" selected

cross/boost_1.70/Makefile

@@ -27,7 +27,7 @@ WITH_PYTHON_LIBRARY =
 ifneq ($(findstring python,$(BOOST_LIBRARIES)),)
 WITH_PYTHON_LIBRARY = 1
 # The packages depending on boost with python, must build python before.
-include $(WORK_DIR)/python-cc.mk
+include $(or $(wildcard $(WORK_DIR)/crossenv/build/python-cc.mk), $(wildcard $(PYTHON_PACKAGE_WORK_DIR)/crossenv/build/python-cc.mk))
 PYTHON_NAME = $(basename $(notdir $(PYTHON_INTERPRETER)))
 else ifneq ($(findstring $(BOOST_LIBRARIES), all),)
 # exclude python library if "all" selected

cross/boost_1.78/Makefile

@@ -24,7 +24,7 @@ WITH_PYTHON_LIBRARY =
 ifneq ($(findstring python,$(BOOST_LIBRARIES)),)
 WITH_PYTHON_LIBRARY = 1
 # The packages depending on boost with python, must build python before.
-include $(WORK_DIR)/python-cc.mk
+include $(or $(wildcard $(WORK_DIR)/crossenv/build/python-cc.mk), $(wildcard $(PYTHON_PACKAGE_WORK_DIR)/crossenv/build/python-cc.mk))
 PYTHON_NAME = $(basename $(notdir $(PYTHON_INTERPRETER)))
 else ifneq ($(findstring $(BOOST_LIBRARIES), all),)
 # exclude python library if "all" selected

cross/boost_1.82/Makefile

@@ -24,7 +24,7 @@ WITH_PYTHON_LIBRARY =
 ifneq ($(findstring python,$(BOOST_LIBRARIES)),)
 WITH_PYTHON_LIBRARY = 1
 # The packages depending on boost with python, must build python before.
-include $(WORK_DIR)/python-cc.mk
+include $(or $(wildcard $(WORK_DIR)/crossenv/build/python-cc.mk), $(wildcard $(PYTHON_PACKAGE_WORK_DIR)/crossenv/build/python-cc.mk))
 PYTHON_NAME = $(basename $(notdir $(PYTHON_INTERPRETER)))
 else ifneq ($(findstring $(BOOST_LIBRARIES), all),)
 # exclude python library if "all" selected
@@ -43,7 +43,7 @@ ADDITIONAL_CXXFLAGS = -Wno-deprecated-declarations
 
 ifneq ($(strip $(WITH_PYTHON_LIBRARY)),)
 CONFIGURE_ARGS += --with-python=$(WORK_DIR)/../../../native/$(PYTHON_NAME)/work-native/install/usr/local/bin/$(PYTHON_NAME)
-ADDITIONAL_CXXFLAGS += -I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) -fpermissive
+ADDITIONAL_CXXFLAGS += -I$(or $(PYTHON_STAGING_INSTALL_PREFIX),$(STAGING_INSTALL_PREFIX))/$(PYTHON_INC_DIR) -fpermissive
 endif
 
 include ../../mk/spksrc.common.mk
@@ -75,7 +75,7 @@ boost_pre_compile:
 	@rm -rf $(WORK_DIR)/$(PKG_DIR)/user-config.jam
 	@echo "using gcc : ${TC_GCC} : ${CXX} : <address-model>\"$(ADDRESS_MODEL)\" <cflags>\"$(CFLAGS)\" <cxxflags>\"$(CXXFLAGS) $(ADDITIONAL_CXXFLAGS)\" <linkflags>\"$(LDFLAGS)\" <link>\"shared\" ; " > $(WORK_DIR)/$(PKG_DIR)/user-config.jam
 ifneq ($(strip $(WITH_PYTHON_LIBRARY)),)
-	@echo "using python : $(PYTHON_VERSION) : : $(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) ;" >> $(WORK_DIR)/$(PKG_DIR)/user-config.jam
+	@echo "using python : $(PYTHON_VERSION) : : $(or $(PYTHON_STAGING_INSTALL_PREFIX),$(STAGING_INSTALL_PREFIX))/$(PYTHON_INC_DIR) ;" >> $(WORK_DIR)/$(PKG_DIR)/user-config.jam
 endif
 	@echo "project : requirements <cxxflags>$(OFLAGS) ;" >> $(WORK_DIR)/$(PKG_DIR)/user-config.jam
 

cross/cryptography/Makefile

@@ -2,20 +2,13 @@ PKG_NAME = cryptography
 PKG_VERS = 41.0.3
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
-PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/c/$(PKG_NAME)
+PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/c/cryptography
 PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
 
-DEPENDS = cross/openssl3 cross/cffi
+DEPENDS = cross/openssl3 cross/libffi
 
 HOMEPAGE = https://github.com/pyca/cryptography
 COMMENT  = Provide cryptographic recipes and primitives to Python developers
 LICENSE  = BSD/ASL2
 
-###
-### This cross/cryptography is needed to generate python-limited abi3 wheels
-### Ref: https://github.com/pyca/cryptography/issues/7671
-###
-
-WHEELS_BUILD_ARGS = --py-limited-api=$(PYTHON_LIMITED_API)
-
 include ../../mk/spksrc.python-wheel.mk

cross/dtlssocket/Makefile

@@ -1,18 +1,22 @@
-PKG_NAME = DTLSSocket
-PKG_VERS = 0.1.16
+PKG_NAME = dtlssocket
+PKG_VERS = 0.2.2
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
-PKG_DIST_SITE = https://files.pythonhosted.org/packages/58/42/0a0442118096eb9fbc9dc70b45aee2957f7546b80545e2a05bd839380519
+PKG_DIST_SITE = https://files.pythonhosted.org/packages/97/b6/702c4197c37c7036d0b7d67a2963c83a4b7098525959566dc6e2284c9f89
 PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
 
-BUILD_DEPENDS = cross/cython
-
 HOMEPAGE = https://git.fslab.de/jkonra2m/tinydtls-cython
 COMMENT  = DTLSSocket is a Cython wrapper for tinydtls with a Socket like interface.
 LICENSE  = Eclipse Public License 1.0
 
 PRE_CONFIGURE_TARGET = dtlssocket_pre_configure
 
+include ../../mk/spksrc.common.mk
+
+ifeq ($(call version_le, $(TC_GCC), 5),1)
+ADDITIONAL_CFLAGS = -std=c99
+endif
+
 include ../../mk/spksrc.python-wheel.mk
 
 .PHONY: dtlssocket_pre_configure

cross/dtlssocket/digests

@@ -1,3 +1,3 @@
-DTLSSocket-0.1.16.tar.gz SHA1 9b4eaa3e9a949f202844a27ba54b2f05d73aa410
-DTLSSocket-0.1.16.tar.gz SHA256 30b108ae45fce1c033e3ef6c2ddd7e7418062ae374228e3a7fa969b254366a39
-DTLSSocket-0.1.16.tar.gz MD5 15dfeca9678e381c7bb6e11e4c864fd5
+dtlssocket-0.2.2.tar.gz SHA1 0be2a954ca56fe7cca3a9fbc5a9e65194f0ab5a5
+dtlssocket-0.2.2.tar.gz SHA256 4e76d715726e0c46dc09e35da9b6717b063c2389b06c1723decc3ba38b7387f4
+dtlssocket-0.2.2.tar.gz MD5 d1f686febd12e67ec7a2efe81bf1eaee

cross/mariadb-connector-c/Makefile

@@ -1,5 +1,7 @@
 PKG_NAME = mariadb-connector-c
-PKG_VERS = 3.3.5
+# Version 3.3.11 is the last known version to work with aarch64-6.2.4
+# Assembly error: Error: CFI instruction used without previous .cfi_startproc
+PKG_VERS = 3.3.11
 PKG_EXT = tar.gz
 PKG_DIST_NAME = v$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_FILE = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
@@ -10,23 +12,47 @@ HOMEPAGE = https://mariadb.com/kb/en/mariadb-connector-c/
 COMMENT  = MariaDB C Connector
 LICENSE  = LGPL
 
-#USE_NATIVE_CMAKE_LEGACY = 1
-#CMAKE_USE_TOOLCHAIN_FILE = OFF
 CMAKE_DISABLE_EXE_LINKER_FLAGS = 1
 
 include ../../mk/spksrc.cross-cmake.mk
 
-ifeq ($(call version_lt, $(TC_GCC), 5.0),1)
+ifeq ($(call version_gt, $(TC_GCC), 4.9),1)
+ADDITIONAL_CFLAGS += -Wno-error=maybe-uninitialized
+# fails on evansport-6.2.4
+ADDITIONAL_CFLAGS += -Wno-error=strict-aliasing
+endif
+
+ifeq ($(call version_lt, $(TC_GCC), 4.9),1)
 ADDITIONAL_CFLAGS += -std=gnu99
+# fails on 88f6281-6.2.4
+ADDITIONAL_CFLAGS += -Wno-error=missing-field-initializers
+ADDITIONAL_CFLAGS += -Wno-error=uninitialized
+# fails on hi3535-6.2.4
+ADDITIONAL_CFLAGS += -Wno-error=array-bounds
 endif
 
 # MySQL specific:
 CMAKE_ARGS += -DWITH_UNIT_TESTS=OFF
 CMAKE_ARGS += -DDEFAULT_CHARSET=utf8
 CMAKE_ARGS += -DINSTALL_LIBDIR=lib
 
+# Creates the following two synlinks:
+#   - libmysqlclient.so -> libmariadb.so.3
+#   - libmysqlclient_r.so -> libmariadb.so.3
+# Which in turns conflict with mysql-connector-c
+#CMAKE_ARGS += -DWITH_MYSQLCOMPAT=ON
+
+# Does not handle pkgconfig well and fails to
+# find openssl3 include and libraries using
+# spksrc.python.mk thus using OPENSSL_STAGING_PREFIX
 DEPENDS += cross/openssl3
 CMAKE_ARGS += -DWITH_SSL=ON
+ifneq ($(wildcard $(OPENSSL_STAGING_PREFIX)),)
+CMAKE_ARGS += -DOPENSSL_CRYPTO_LIBRARY:FILEPATH=$(OPENSSL_STAGING_PREFIX)/lib/libcrypto.so
+CMAKE_ARGS += -DOPENSSL_SSL_LIBRARY:FILEPATH=$(OPENSSL_STAGING_PREFIX)/lib/libssl.so
+CMAKE_ARGS += -DOPENSSL_INCLUDE_DIR:PATH=$(OPENSSL_STAGING_PREFIX)/include
+endif
+
 DEPENDS += cross/zlib
 CMAKE_ARGS += -DWITH_EXTERNAL_ZLIB=ON
 DEPENDS += cross/libiconv

cross/mariadb-connector-c/digests

@@ -1,3 +1,3 @@
-mariadb-connector-c-3.3.5.tar.gz SHA1 29ad65e81a87e41a93e8984f2c3d1662dcec7c1c
-mariadb-connector-c-3.3.5.tar.gz SHA256 c0fda1fa6e52dc85de27156cd847088a72d40d9de6514f7efa57c8d93134a54c
-mariadb-connector-c-3.3.5.tar.gz MD5 b8967cf63d4fc5660f230762c0e39ee2
+mariadb-connector-c-3.3.11.tar.gz SHA1 2fa1fe042bcc65297d6d22196685701faf3e96d1
+mariadb-connector-c-3.3.11.tar.gz SHA256 6ecada1f3fc32ad95efcd2c982b319191f5e6479f3f521bc4b714e6158f5e415
+mariadb-connector-c-3.3.11.tar.gz MD5 700afe8ebff69bdf66b3c37747d1d527