Fix hanging Hindcast tests (#311)

ssolson · web-flow · commit 184a5ab46941 · 2024-04-26T20:38:02.000-06:00
This PR addresses 2 issues with the hindcast tests:

* On PRs tests were only being run on changes to hindcast test files.
  - Fixed to run on changes to hindcast test and module files.
* The prepare cache job was failing due to 6 hour job limit.
  - Fixed by breaking out each hindcast cache test into its own job.

* Additionally this PR updates action packages to the latest version where possible.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -30,14 +30,15 @@ jobs:
       should-run-hindcast: ${{ steps.hindcast-logic.outputs.should-run-hindcast }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check for changes in wave/io/hindcast
         id: changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         with:
           filters: |
             wave_io_hindcast:
+              - 'mhkit/wave/io/hindcast/**'
               - 'mhkit/tests/wave/io/hindcast/**'
 
       - id: hindcast-logic
@@ -48,14 +49,13 @@ jobs:
             echo "should-run-hindcast=false" >> "$GITHUB_OUTPUT"
           fi
 
-  prepare-cache:
-    needs: [check-changes]
+  prepare-nonhindcast-cache:
     runs-on: ubuntu-latest
     env:
       PYTHON_VER: 3.9
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
@@ -68,7 +68,7 @@ jobs:
         run: |
           conda create --name TEST python=${{ env.PYTHON_VER }} numpy cython pip hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
           source activate TEST
-          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
+          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH"
           pip install -e . --no-deps --force-reinstall
 
       - name: Install dependencies
@@ -78,30 +78,106 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install coverage pytest coveralls .
 
-      - name: Prepare data
+      - name: Prepare non-hindcast API data
         run: |
           source activate TEST
           pytest mhkit/tests/river/test_io_usgs.py
           pytest mhkit/tests/tidal/test_io.py
           pytest mhkit/tests/wave/io/test_cdip.py
 
-      - name: Prepare Hindcast data
-        if: (needs.check-changes.outputs.should-run-hindcast == 'true')
+      - name: Upload data as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: data
+          path: ~/.cache/mhkit
+
+  prepare-wave-hindcast-cache:
+    needs: [check-changes]
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VER: 3.9
+    if: (needs.check-changes.outputs.should-run-hindcast == 'true')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Conda
+        uses: s-weigand/setup-conda@v1
+        with:
+          activate-conda: false
+          conda-channels: conda-forge
+
+      - name: Python setup
+        shell: bash -l {0}
+        run: |
+          conda create --name TEST python=${{ env.PYTHON_VER }} numpy cython pip hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
+          source activate TEST
+          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH"
+          pip install -e . --no-deps --force-reinstall
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          python -m pip install --upgrade pip wheel
+          pip install coverage pytest coveralls .
+
+      - name: Prepare Wave Hindcast data
         run: |
-          # pytest tests/test_specific_file.py::TestClass::test_function
           source activate TEST
           pytest mhkit/tests/wave/io/hindcast/test_hindcast.py
+
+      - name: Upload Wave Hindcast data as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wave-hindcast-data
+          path: ~/.cache/mhkit
+
+  prepare-wind-hindcast-cache:
+    needs: [check-changes]
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VER: 3.9
+    if: (needs.check-changes.outputs.should-run-hindcast == 'true')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Conda
+        uses: s-weigand/setup-conda@v1
+        with:
+          activate-conda: false
+          conda-channels: conda-forge
+
+      - name: Python setup
+        shell: bash -l {0}
+        run: |
+          conda create --name TEST python=${{ env.PYTHON_VER }} numpy cython pip hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
+          source activate TEST
+          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH"
+          pip install -e . --no-deps --force-reinstall
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          python -m pip install --upgrade pip wheel
+          pip install coverage pytest coveralls .
+
+      - name: Prepare Wind Hindcast data
+        run: |
+          source activate TEST
           pytest mhkit/tests/wave/io/hindcast/test_wind_toolkit.py
 
-      - name: Upload data as artifact
-        uses: actions/upload-artifact@v3
+      - name: Upload Wind Hindcast data as artifact
+        uses: actions/upload-artifact@v4
         with:
-          name: data
+          name: wind-hindcast-data
           path: ~/.cache/mhkit
 
   conda-build:
     name: conda-${{ matrix.os }}/${{ matrix.python-version }}
-    needs: [set-os, prepare-cache]
+    needs: [set-os, prepare-nonhindcast-cache]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -112,7 +188,7 @@ jobs:
       PYTHON_VER: ${{ matrix.python-version }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
@@ -136,7 +212,7 @@ jobs:
           pip install coverage pytest coveralls .
 
       - name: Download data from artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: data
           path: ~/.cache/mhkit
@@ -159,7 +235,7 @@ jobs:
 
   pip-build:
     name: pip-${{ matrix.os }}/${{ matrix.python-version }}
-    needs: [set-os, prepare-cache]
+    needs: [set-os, prepare-nonhindcast-cache]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -168,18 +244,18 @@ jobs:
         python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Set up Git repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download data from artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: data
           path: ~/.cache/mhkit
@@ -202,7 +278,13 @@ jobs:
 
   hindcast-calls:
     name: hindcast-${{ matrix.os }}/${{ matrix.python-version }}
-    needs: [check-changes, prepare-cache, set-os]
+    needs:
+      [
+        check-changes,
+        prepare-wave-hindcast-cache,
+        prepare-wind-hindcast-cache,
+        set-os,
+      ]
     if: (needs.check-changes.outputs.should-run-hindcast == 'true')
 
     runs-on: ${{ matrix.os }}
@@ -214,7 +296,7 @@ jobs:
         python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
@@ -230,11 +312,24 @@ jobs:
           export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
           pip install -e . --no-deps --force-reinstall
 
-      - name: Download data from artifact
-        uses: actions/download-artifact@v3
+      - name: Download Wave Hindcast data from artifact
+        uses: actions/download-artifact@v4
         with:
-          name: data
-          path: ~/.cache/mhkit
+          name: wave-hindcast-data
+          path: ~/.cache/mhkit/wave-hindcast
+
+      - name: Download Wind Hindcast data from artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wind-hindcast-data
+          path: ~/.cache/mhkit/wind-hindcast
+
+      - name: Consolidate hindcast data
+        run: |
+          mkdir -p ~/.cache/mhkit/hindcast
+          mv ~/.cache/mhkit/wave-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
+          mv ~/.cache/mhkit/wind-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
+        shell: bash
 
       - name: Install MHKiT and run pytest
         shell: bash -l {0}
diff --git a/mhkit/wave/io/hindcast/hindcast.py b/mhkit/wave/io/hindcast/hindcast.py
@@ -105,7 +105,7 @@ def request_wpto_point_data(
 
     Note: To access the WPTO hindcast data, you will need to configure
     h5pyd for data access on HSDS. Please see the WPTO_hindcast_example
-    notebook for more information.
+    notebook for setup instructions.
 
     Parameters
     ----------
