Verify bundled sources using CI - boost.context & uriparser

.github/actions/verify-generated-files/action.yml

@@ -13,5 +13,5 @@ runs:
         ext/tokenizer/tokenizer_data_gen.php
         build/gen_stub.php -f --generate-optimizer-info --verify
         ext/phar/makestub.php
-        # Use the -a flag for a bug in git 2.46.0, which doesn't consider changed -diff files.
-        git add . -N && git diff -a --exit-code
+        .github/scripts/download-bundled/make-workflow-file.php
+        .github/scripts/test-directory-unchanged.sh .

.github/scripts/download-bundled/.gitignore

@@ -0,0 +1 @@
+!*.patch

.github/scripts/download-bundled/boost-context.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/boost-context
+rm -rf "$tmp_dir"
+
+revision=refs/tags/boost-1.86.0
+
+git clone --depth 1 --revision="$revision" https://github.com/boostorg/context.git "$tmp_dir"
+
+rm -rf Zend/asm
+cp -R "$tmp_dir"/src/asm Zend/asm
+
+cd Zend/asm
+
+# remove unneeded files
+rm jump_arm_aapcs_pe_armasm.asm
+rm jump_i386_ms_pe_clang_gas.S
+rm jump_i386_ms_pe_gas.asm
+rm jump_i386_x86_64_sysv_macho_gas.S
+rm jump_ppc32_ppc64_sysv_macho_gas.S
+rm jump_x86_64_ms_pe_clang_gas.S
+rm make_arm_aapcs_pe_armasm.asm
+rm make_i386_ms_pe_clang_gas.S
+rm make_i386_ms_pe_gas.asm
+rm make_i386_x86_64_sysv_macho_gas.S
+rm make_ppc32_ppc64_sysv_macho_gas.S
+rm make_x86_64_ms_pe_clang_gas.S
+rm ontop_*.S
+rm ontop_*.asm
+rm tail_ontop_ppc32_sysv.cpp
+
+# move renamed files
+# GH-13896 introduced these 2 files named as .S but since https://github.com/boostorg/context/pull/265 they are named as .asm
+mv jump_x86_64_ms_pe_gas.asm jump_x86_64_ms_pe_gas.S
+mv make_x86_64_ms_pe_gas.asm make_x86_64_ms_pe_gas.S
+
+# add extra files
+git restore LICENSE
+git restore save_xmm_x86_64_ms_masm.asm # added in GH-18352, not an upstream boost.context file

.github/scripts/download-bundled/make-workflow-file.php

@@ -0,0 +1,189 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Phpsrc\Ci\DownloadBundled;
+
+$bundles = [
+    new Bundle('boost.context', ['Zend/asm']),
+    new Bundle('PCRE2', ['ext/pcre/pcre2lib']),
+    new Bundle('uriparser', ['ext/uri/uriparser']),
+];
+
+class Bundle
+{
+    /**
+     * @param list<string> $directories
+     */
+    public function __construct(
+        public string $name,
+        public array $directories
+    ) {}
+
+    public function getNameForPath(): string
+    {
+        return preg_replace('~\W+~', '-', strtolower($this->name));
+    }
+}
+
+class Generator
+{
+    /**
+     * @param list<Bundle> $bundles
+     */
+    public function __construct(
+        public array $bundles
+    ) {}
+
+    protected function getRepoDirectory(): string
+    {
+        return dirname(__DIR__, 3);
+    }
+
+    protected function indentString(string $value, int $levels, bool $inclFirstLine): string
+    {
+        return preg_replace(
+            '~' . ($inclFirstLine ? '^|' : '') . '(?<=\n)~',
+            str_repeat('  ', $levels),
+            $value
+        );
+    }
+
+    /**
+     * @param mixed $data
+     */
+    protected function encodeYml($data): string
+    {
+        if (is_array($data)) {
+            $isList = array_is_list($data);
+            $resParts = [];
+            foreach ($data as $k => $v) {
+                $kEncoded = $isList
+                    ? '-'
+                    : $this->encodeYml($k) . ':';
+                $vEncoded = $this->encodeYml($v);
+
+                $resParts[] = $kEncoded
+                    . (!$isList && is_array($v) && $v !== [] ? "\n  " : ' ')
+                    . (is_array($v) ? $this->indentString($vEncoded, 1, false) : $vEncoded);
+            }
+
+            return implode("\n", $resParts);
+        }
+
+        if (preg_match('~^(\w+|\$\{\{[^\}]+\}\})$~', $data)) {
+            return $data;
+        }
+
+        return strpos($data, "\n") !== false
+            ? '|' . "\n" . $this->indentString($data, 1, true)
+            : '\'' . str_replace('\'', '\'\'', $data) . '\'';
+    }
+
+    public function makeWorkflowFile(): void
+    {
+        $content = <<<'EOD'
+            name: Verify Bundled Files
+
+            on:
+              push: ~
+              pull_request: ~
+              schedule:
+                - cron: "0 1 * * *"
+              workflow_dispatch: ~
+
+            permissions:
+              contents: read
+
+            jobs:
+              VERIFY_BUNDLED_FILES:
+                name: Verify Bundled Files
+                runs-on: ubuntu-24.04
+                steps:
+                  - name: git checkout
+                    uses: actions/checkout@v5
+
+                  - name: Detect changed files
+                    uses: dorny/paths-filter@v3
+                    id: changes
+                    with:
+                      base: master
+                      filters: %filters%
+
+                  %steps%
+
+            EOD;
+
+        $filters = [];
+        foreach ($this->bundles as $bundle) {
+            $filters[$bundle->getNameForPath()] = $this->makeDornyPathsFilterFilters($bundle);
+        }
+        $content = str_replace('%filters%', $this->indentString($this->encodeYml($this->encodeYml($filters)), 5, false), $content);
+
+        $steps = [];
+        foreach ($this->bundles as $bundle) {
+            $steps[] = [
+                'name' => $bundle->name,
+                'if' => '${{ !cancelled() && (steps.changes.outputs.pcre2 == \'true\' || github.event_name == \'schedule\' || github.event_name == \'workflow_dispatch\') }}',
+                'run' => implode("\n", [
+                    'echo "::group::Download"',
+                    '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh',
+                    'echo "::endgroup::"',
+                    'echo "::group::Verify files"',
+                    ...array_map(static fn ($v) => '.github/scripts/test-directory-unchanged.sh ' . escapeshellarg($v), $bundle->directories),
+                    'echo "::endgroup::"',
+                ]),
+            ];
+        }
+        $content = str_replace('%steps%', $this->indentString($this->encodeYml($steps), 3, false), $content);
+
+        file_put_contents($this->getRepoDirectory() . '/.github/workflows/verify-bundled-files.yml', $content);
+    }
+
+    protected function makeDornyPathsFilterFilters(Bundle $bundle): array
+    {
+        return [
+            '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.*',
+            ...array_map(static fn ($v) => $v . '/**', $bundle->directories),
+        ];
+    }
+
+    public function makeDownloadScriptHeaders(): void
+    {
+        foreach ($this->bundles as $bundle) {
+            $this->makeDownloadScriptHeader($bundle);
+        }
+    }
+
+    protected function makeDownloadScriptHeader(Bundle $bundle): void
+    {
+        $scriptPath = $this->getRepoDirectory() . '/.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh';
+
+        $content = !file_exists($scriptPath)
+            ? "# TODO\n"
+            : file_get_contents($scriptPath);
+
+        $header = <<<'EOD'
+            #!/bin/sh
+            set -ex
+            cd "$(dirname "$0")/../../.."
+
+            tmp_dir=%tmp_dir%
+            rm -rf "$tmp_dir"
+
+
+            EOD;
+
+        $header = str_replace('%tmp_dir%', '/tmp/php-src-download-bundled/' . $bundle->getNameForPath(), $header);
+
+        if (!str_starts_with($content, $header)) {
+            $content = $header . $content;
+        }
+
+        file_put_contents($scriptPath, $content);
+    }
+}
+
+$generator = new Generator($bundles);
+$generator->makeWorkflowFile();
+$generator->makeDownloadScriptHeaders();

.github/scripts/download-bundled/pcre2.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/pcre2
+rm -rf "$tmp_dir"
+
+revision=refs/tags/pcre2-10.44
+
+git clone --depth 1 --recurse-submodules --revision="$revision" https://github.com/PCRE2Project/pcre2.git "$tmp_dir"
+
+rm -rf ext/pcre/pcre2lib
+cp -R "$tmp_dir"/src ext/pcre/pcre2lib
+
+cd ext/pcre/pcre2lib
+
+# remove unneeded files
+rm config.h.generic
+rm pcre2.h.in
+rm pcre2_dftables.c
+rm pcre2_fuzzsupport.c
+rm pcre2_jit_test.c
+rm pcre2demo.c
+rm pcre2grep.c
+rm pcre2posix.c
+rm pcre2posix.h
+rm pcre2posix_test.c
+rm pcre2test.c
+
+# move renamed files
+mv pcre2.h.generic pcre2.h
+mv pcre2_chartables.c.dist pcre2_chartables.c
+
+# add extra files
+git restore config.h # based on config.h.generic but with many changes

.github/scripts/download-bundled/uriparser.config.patch

@@ -0,0 +1,14 @@
+diff --git a/ext/uri/uriparser/src/UriConfig.h b/ext/uri/uriparser/src/UriConfig.h
+index b9a85a8..ab78b96 100644
+--- a/ext/uri/uriparser/src/UriConfig.h
++++ b/ext/uri/uriparser/src/UriConfig.h
+@@ -41,7 +41,9 @@
+
+ #  define PACKAGE_VERSION "@PROJECT_VERSION@"
+
++/*
+ #cmakedefine HAVE_WPRINTF
+ #cmakedefine HAVE_REALLOCARRAY
++*/
+
+ #endif /* !defined(URI_CONFIG_H) */

.github/scripts/download-bundled/uriparser.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+set -ex
+cd "$(dirname "$0")/../../.."
+
+tmp_dir=/tmp/php-src-download-bundled/uriparser
+rm -rf "$tmp_dir"
+
+revision=c3b49569f1f25550a16d9a18207e498d77458b27 # refs/tags/uriparser-0.9.9 with https://github.com/uriparser/uriparser/pull/276
+
+git clone --depth 1 --revision="$revision" https://github.com/uriparser/uriparser.git "$tmp_dir"
+
+rm -rf ext/uri/uriparser
+mkdir ext/uri/uriparser
+cp -R "$tmp_dir"/src ext/uri/uriparser
+cp -R "$tmp_dir"/include ext/uri/uriparser
+cp "$tmp_dir"/COPYING.BSD-3-Clause ext/uri/uriparser
+
+cd ext/uri/uriparser
+
+# move renamed files
+mv src/UriConfig.h.in src/UriConfig.h
+
+# patch customized files
+git apply -v ../../../.github/scripts/download-bundled/uriparser.config.patch

.github/scripts/test-directory-unchanged.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -ex
+
+cd "$(dirname "$0")/../../$1"
+
+# notify git about untracked (except ignored) files
+git add -N .
+
+# display overview of changed files
+git status .
+
+# display diff of working directory vs HEAD commit and set exit code
+git diff -a --exit-code HEAD .

.github/workflows/verify-bundled-files.yml

@@ -0,0 +1,63 @@
+name: Verify Bundled Files
+
+on:
+  push: ~
+  pull_request: ~
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch: ~
+
+permissions:
+  contents: read
+
+jobs:
+  VERIFY_BUNDLED_FILES:
+    name: Verify Bundled Files
+    runs-on: ubuntu-24.04
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v5
+
+      - name: Detect changed files
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: master
+          filters: |
+            'boost-context':
+              - '.github/scripts/download-bundled/boost-context.*'
+              - 'Zend/asm/**'
+            pcre2:
+              - '.github/scripts/download-bundled/pcre2.*'
+              - 'ext/pcre/pcre2lib/**'
+            uriparser:
+              - '.github/scripts/download-bundled/uriparser.*'
+              - 'ext/uri/uriparser/**'
+
+      - name: 'boost.context'
+        if: ${{ !cancelled() && (steps.changes.outputs.pcre2 == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
+        run: |
+          echo "::group::Download"
+          .github/scripts/download-bundled/boost-context.sh
+          echo "::endgroup::"
+          echo "::group::Verify files"
+          .github/scripts/test-directory-unchanged.sh "Zend/asm"
+          echo "::endgroup::"
+      - name: PCRE2
+        if: ${{ !cancelled() && (steps.changes.outputs.pcre2 == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
+        run: |
+          echo "::group::Download"
+          .github/scripts/download-bundled/pcre2.sh
+          echo "::endgroup::"
+          echo "::group::Verify files"
+          .github/scripts/test-directory-unchanged.sh "ext/pcre/pcre2lib"
+          echo "::endgroup::"
+      - name: uriparser
+        if: ${{ !cancelled() && (steps.changes.outputs.pcre2 == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
+        run: |
+          echo "::group::Download"
+          .github/scripts/download-bundled/uriparser.sh
+          echo "::endgroup::"
+          echo "::group::Verify files"
+          .github/scripts/test-directory-unchanged.sh "ext/uri/uriparser"
+          echo "::endgroup::"