Merge branch 'main' of github.com:pytorch/torchcodec into new-theme

Author: NicolasHug

.github/workflows/build_ffmpeg.yaml

@@ -27,15 +27,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1"]
+        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1", "8.0"]
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:
       id-token: write
       contents: read
     with:
       job-name: Build
       upload-artifact: ffmpeg-lgpl-linux_x86_64-${{ matrix.ffmpeg-version }}
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       script: |
         export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}"
         export FFMPEG_ROOT="${PWD}/ffmpeg"
@@ -52,12 +52,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1"]
+        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1", "8.0"]
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     with:
       job-name: Build
       upload-artifact: ffmpeg-lgpl-macos-${{ matrix.ffmpeg-version }}
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       runner: macos-14-xlarge
       script: |
         export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}"
@@ -75,12 +75,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1"]
+        ffmpeg-version: ["4.4.4", "5.1.4", "6.1.1", "7.0.1", "8.0"]
     uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
     with:
       job-name: Build
       upload-artifact: ffmpeg-lgpl-windows_x86_64-${{ matrix.ffmpeg-version }}
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       script: |
         export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}"
         export FFMPEG_ROOT="${PWD}/ffmpeg"

.github/workflows/docs.yaml

@@ -33,7 +33,7 @@ jobs:
     name: Build and Upload wheel
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       ref: ""
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
@@ -51,10 +51,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-          # 3.9 corresponds to the minimum python version for which we build
+          # 3.10 corresponds to the minimum python version for which we build
           # the wheel unless the label cliflow/binaries/all is present in the
           # PR.
-        python-version: ['3.9']
+        python-version: ['3.10']
         cuda-version: ['12.6']
         ffmpeg-version-for-tests: ['7']
     container:
@@ -70,7 +70,7 @@ jobs:
           echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
       - uses: actions/download-artifact@v4
         with:
-          name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
           path: pytorch/torchcodec/dist/
       - name: Setup miniconda using test-infra
         uses: pytorch/test-infra/.github/actions/setup-miniconda@main

.github/workflows/linux_cuda_wheel.yaml

@@ -43,7 +43,7 @@ jobs:
     name: Build and Upload wheel
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       ref: ""
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
@@ -61,15 +61,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-          # 3.9 corresponds to the minimum python version for which we build
+          # 3.10 corresponds to the minimum python version for which we build
           # the wheel unless the label cliflow/binaries/all is present in the
           # PR.
           # For the actual release we should add that label and change this to
           # include more python versions.
-        python-version: ['3.9']
-        cuda-version: ['12.6', '12.8']
+        python-version: ['3.10']
+        # We test against 12.6 and 13.0 to avoid having too big of a CI matrix,
+        # but for releases we should add 12.8.
+        cuda-version: ['12.6', '13.0']
         # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
-        ffmpeg-version-for-tests: ['4.4.2', '6', '7']
+        ffmpeg-version-for-tests: ['4.4.2', '6', '7', '8.0']
 
     container:
       image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
@@ -84,14 +86,15 @@ jobs:
           echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
       - uses: actions/download-artifact@v4
         with:
-          name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
           path: pytorch/torchcodec/dist/
       - name: Setup miniconda using test-infra
         uses: pytorch/test-infra/.github/actions/setup-miniconda@main
         with:
           python-version: ${{ matrix.python-version }}
           # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
-          default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}"
+          # Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it.
+          default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau"
       - name: Check env
         run: |
           ${CONDA_RUN} env
@@ -137,7 +140,7 @@ jobs:
           ls
       - name: Run Python tests
         run: |
-          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short
+          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
       - name: Run Python benchmark
         run: |
           ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none

.github/workflows/linux_wheel.yaml

@@ -44,7 +44,7 @@ jobs:
     name: Build and Upload Linux wheel
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       ref: ""
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
@@ -62,26 +62,30 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
-        ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
+        python-version: ['3.10']
+        ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0']
     needs: build
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
           path: pytorch/torchcodec/dist/
       - name: Setup conda env
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
-          miniconda-version: "latest"
+          # Using miniforge instead of miniconda ensures that the default
+          # conda channel is conda-forge instead of main/default. This ensures
+          # ABI consistency between dependencies:
+          # https://conda-forge.org/docs/user/transitioning_from_defaults/
+          miniforge-version: latest
           activate-environment: test
           python-version: ${{ matrix.python-version }}
       - name: Update pip
         run: python -m pip install --upgrade pip
       - name: Install PyTorch
         run: |
-          python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+          python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
       - name: Install torchcodec from the wheel
         run: |
           wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
@@ -123,4 +127,4 @@ jobs:
           ls
       - name: Run Python tests
         run: |
-          pytest test -vvv
+          pytest --override-ini="addopts=-v" test

.github/workflows/macos_wheel.yaml

@@ -44,15 +44,16 @@ jobs:
     name: Build and Upload Mac wheel
     uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
     with:
-      repository: pytorch/torchcodec
+      repository: meta-pytorch/torchcodec
       ref: ""
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
       pre-script: packaging/pre_build_script.sh
       post-script: packaging/post_build_script.sh
       smoke-test-script: packaging/fake_smoke_test.py
-      runner-type: macos-m1-stable
+      runner-type: macos-14
+      setup-miniconda: true
       package-name: torchcodec
       trigger-event: ${{ github.event_name }}
       build-platform: "python-build-package"
@@ -63,14 +64,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
-        ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
+        python-version: ['3.10']
+        ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1', '8.0']
     needs: build
     steps:
       - name: Download wheel
         uses: actions/download-artifact@v4
         with:
-          name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_
           path: pytorch/torchcodec/dist/
 
       - name: Setup conda env
@@ -85,7 +86,7 @@ jobs:
 
       - name: Install PyTorch
         run: |
-          python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+          python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
 
       - name: Install torchcodec from the wheel
         run: |
@@ -122,4 +123,4 @@ jobs:
 
       - name: Run Python tests
         run: |
-          pytest test -vvv
+          pytest --override-ini="addopts=-v" test

.github/workflows/reference_resources.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - test/generate_reference_resources.sh
+      - test/generate_reference_resources.py
       - .github/workflows/reference_resources.yaml # self reference
   schedule:
     - cron: '0 0 * * 0'  # on sunday
@@ -14,14 +14,51 @@ defaults:
     shell: bash -l -eo pipefail {0}
 
 jobs:
+  generate-matrix:
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: linux
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-xpu: disable
+      with-rocm: disable
+      with-cuda: disable
+      build-python-only: "disable"
+
+  build:
+    needs: generate-matrix
+    strategy:
+      fail-fast: false
+    name: Build and Upload Linux wheel
+    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
+    with:
+      repository: meta-pytorch/torchcodec
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      pre-script: packaging/pre_build_script.sh
+      post-script: packaging/post_build_script.sh
+      smoke-test-script: packaging/fake_smoke_test.py
+      package-name: torchcodec
+      trigger-event: ${{ github.event_name }}
+      build-platform: "python-build-package"
+      build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation"
+
   test-reference-resource-generation:
+    needs: build
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
+        python-version: ['3.10']
         ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
     steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64
+          path: pytorch/torchcodec/dist/
       - name: Setup conda env
         uses: conda-incubator/setup-miniconda@v2
         with:
@@ -38,16 +75,21 @@ jobs:
       - name: Update pip
         run: python -m pip install --upgrade pip
 
-      - name: Instal generation dependencies
+      - name: Install generation dependencies
         run: |
           # Note that we're installing stable - this is for running a script where we're a normal PyTorch
           # user, not for building TorhCodec.
           python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
-          python -m pip install numpy pillow
+          python -m pip install numpy pillow pytest
 
+      - name: Install torchcodec from the wheel
+        run: |
+          wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
+          echo Installing $wheel_path
+          python -m pip install $wheel_path -vvv
       - name: Check out repo
         uses: actions/checkout@v3
 
       - name: Run generation reference resources
         run: |
-          test/generate_reference_resources.sh
+          python -m test.generate_reference_resources