From ac78f1bd04c14993d688e5657f12e58231c4cddd Mon Sep 17 00:00:00 2001 From: jmcvey3 <53623232+jmcvey3@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:30:32 -0800 Subject: [PATCH 01/18] Remove netCDF4 constraint --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b4f8bbe98..e3dae1831 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ NREL-rex>=0.2.63 six>=1.13.0 h5py>=3.6.0 h5pyd>=0.7.0 -netCDF4>=1.5.8 +netCDF4 xarray statsmodels bottleneck From d2ad9978c6d5c413087763ef5c55dfb1f61ca87e Mon Sep 17 00:00:00 2001 From: jmcvey3 <53623232+jmcvey3@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:45:46 -0700 Subject: [PATCH 02/18] Small fixes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3dae1831..b4f8bbe98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ NREL-rex>=0.2.63 six>=1.13.0 h5py>=3.6.0 h5pyd>=0.7.0 -netCDF4 +netCDF4>=1.5.8 xarray statsmodels bottleneck From 2023f12bfe877f7acd9cb6f49b045d35f1919dee Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 10:02:51 -0600 Subject: [PATCH 03/18] Actions: Install h5 and netCDF system libraries --- .github/workflows/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e2e1d18f..717fd520b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -286,12 +286,20 @@ jobs: name: data path: ~/.cache/mhkit + - name: Install system dependencies + if: runner.os == 'Linux' + run: sudo apt-get install -y libhdf5-dev libnetcdf-dev + - name: Update and install packages shell: bash -l {0} run: | python -m pip install --upgrade pip wheel pip install -e ".[all,dev]" + - name: Reinstall h5py and netCDF4 with system libraries + if: runner.os == 'Linux' + run: "pip install --force-reinstall --no-binary=:all: h5py netCDF4" + - name: Install setuptools for Python 3.12 if: matrix.python-version == '3.12' run: pip install setuptools @@ -417,12 +425,18 @@ jobs: name: data path: ~/.cache/mhkit + - name: Install system dependencies + run: sudo apt-get install -y libhdf5-dev libnetcdf-dev + - name: Install MHKiT with optional dependency run: | python -m pip install --upgrade pip pip install "mhkit[${{ matrix.module }}]" pip install pytest + - name: Reinstall h5py and netCDF4 with system libraries + run: "pip install --force-reinstall --no-binary=:all: h5py netCDF4" + - name: Run tests for ${{ matrix.module }} env: MPLBACKEND: Agg From db2573f2dd4933a6d821d8cf1cf16b28778f64dc Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 10:41:04 -0600 Subject: [PATCH 04/18] Modules: Lazy load wave.io This removes automatic module loading of the heavy wave.io module and is meant to fix cross module loading of unnecessary submodules. One example: `import loads` when installed via pip install "mhkit[loads]" imports `wave.resource.frequency_moment`, which also imports the unnecessary wave.io module which has a dependency on requests. This fails if the user doesn't have `requests` installed, but that module is not necessary to run `frequency_moment`. --- mhkit/wave/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mhkit/wave/__init__.py b/mhkit/wave/__init__.py index f84c667cd..9a4744de6 100644 --- a/mhkit/wave/__init__.py +++ b/mhkit/wave/__init__.py @@ -1,5 +1,12 @@ from mhkit.wave import resource -from mhkit.wave import io from mhkit.wave import graphics from mhkit.wave import performance from mhkit.wave import contours + + +def __getattr__(name): + """Lazy import for wave.io to avoid loading heavy dependencies unless needed.""" + if name == "io": + from mhkit.wave import io + return io + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") From a68f0e3d76c1544fb68b26e7de6bb334561c95b4 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 10:49:41 -0600 Subject: [PATCH 05/18] Dev: Black fmt --- mhkit/wave/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mhkit/wave/__init__.py b/mhkit/wave/__init__.py index 9a4744de6..be2679211 100644 --- a/mhkit/wave/__init__.py +++ b/mhkit/wave/__init__.py @@ -8,5 +8,6 @@ def __getattr__(name): """Lazy import for wave.io to avoid loading heavy dependencies unless needed.""" if name == "io": from mhkit.wave import io + return io raise AttributeError(f"module '{__name__}' has no attribute '{name}'") From 4f70b6312e30cfbe63d5d9278ca72b9e729c250d Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 11:04:45 -0600 Subject: [PATCH 06/18] Modules: Fix recursive wave.io import --- mhkit/wave/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mhkit/wave/__init__.py b/mhkit/wave/__init__.py index be2679211..0c3cf69ad 100644 --- a/mhkit/wave/__init__.py +++ b/mhkit/wave/__init__.py @@ -7,7 +7,9 @@ def __getattr__(name): """Lazy import for wave.io to avoid loading heavy dependencies unless needed.""" if name == "io": - from mhkit.wave import io + from mhkit.wave import io as _io - return io + # Cache it in the module namespace to avoid repeated imports + globals()[name] = _io + return _io raise AttributeError(f"module '{__name__}' has no attribute '{name}'") From 5ab7ce2047e3b1221fc29a2e8fbcac8b5932bfe7 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 13:03:42 -0600 Subject: [PATCH 07/18] Modules: Fix wave.io by using non recursive import strategy --- mhkit/wave/__init__.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/mhkit/wave/__init__.py b/mhkit/wave/__init__.py index 0c3cf69ad..7fbaf30bc 100644 --- a/mhkit/wave/__init__.py +++ b/mhkit/wave/__init__.py @@ -1,3 +1,5 @@ +import importlib + from mhkit.wave import resource from mhkit.wave import graphics from mhkit.wave import performance @@ -5,11 +7,31 @@ def __getattr__(name): - """Lazy import for wave.io to avoid loading heavy dependencies unless needed.""" + """ + Lazy load the wave.io submodule using PEP 562 module-level __getattr__. + + This defers importing heavy wave.io dependencies (rex, netCDF4, etc,) until + they are actually accessed, improving import time for users who don't need + all wave submodules, and avoiding import errors for users who have specified + module level installs that need wave module functions, but not wave.io functions. + """ if name == "io": - from mhkit.wave import io as _io + # This uses importlib.import_module() here, not "from mhkit.wave import io" + # because when Python executes getattr(), it looks for 'io' as an attribute of + # mhkit.wave. At this point in the module loading code 'io' doesn't exist yet and + # Python calls __getattr__('io') again. This triggers the same "from" statement, + # which calls __getattr__('io') again yielding a RecursionError. + # + # To fix this uses importlib.import_module("mhkit.wave.io") which loads the module directly + # using the absolute path without doing attribute lookup on the parent. + # + # The statement "from mhkit.wave import io" is equivalent to: + # io = getattr(mhkit.wave, 'io') + # + io = importlib.import_module("mhkit.wave.io") + + # Cache the module so subsequent accesses bypass __getattr__ entirely + globals()[name] = io + return io - # Cache it in the module namespace to avoid repeated imports - globals()[name] = _io - return _io raise AttributeError(f"module '{__name__}' has no attribute '{name}'") From c8db4b915898c95fe33c4827e2acbdc01aaf22e0 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 13:10:08 -0600 Subject: [PATCH 08/18] Actions: Use exclusion strategy system binary installs --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 717fd520b..efdfd3e0f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -426,6 +426,7 @@ jobs: path: ~/.cache/mhkit - name: Install system dependencies + if: matrix.module != 'river' || matrix.module != 'power' || matrix.module != 'utils' || matrix.module != 'loads' run: sudo apt-get install -y libhdf5-dev libnetcdf-dev - name: Install MHKiT with optional dependency @@ -435,6 +436,7 @@ jobs: pip install pytest - name: Reinstall h5py and netCDF4 with system libraries + if: matrix.module != 'river' || matrix.module != 'power' || matrix.module != 'utils' || matrix.module != 'loads' run: "pip install --force-reinstall --no-binary=:all: h5py netCDF4" - name: Run tests for ${{ matrix.module }} From 55cde623d57f3e6bb1c3ba07d0fe707d8218f2b9 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 13:32:42 -0600 Subject: [PATCH 09/18] Actions: Temporarily enable concurrency in main.yml --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index efdfd3e0f..1a877dc03 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,9 +10,9 @@ on: - main - develop -concurrency: - group: ${{ github.event.pull_request.number }} # Use the pull request number as the concurrency group - cancel-in-progress: true # Cancel any in-progress jobs for the same group. Prevents duplicate workflows on PRs from develop into main or from main into develop. +# concurrency: +# group: ${{ github.event.pull_request.number }} # Use the pull request number as the concurrency group +# cancel-in-progress: true # Cancel any in-progress jobs for the same group. Prevents duplicate workflows on PRs from develop into main or from main into develop. jobs: set-os: From 9270a5ec020311cdf704a91a40851d83222784c3 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 14:30:31 -0600 Subject: [PATCH 10/18] Deps: Add `statsmodels` to loads Loads module tests call wave/contours.py which requires `statsmodels` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 09fc8d1e0..688247fbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ power = [ loads = [ "fatpack" + "statsmodels" ] mooring = [ From c3d4ccd6e5801976b8731bbece9de9ef8f57c4ea Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 14:33:25 -0600 Subject: [PATCH 11/18] Dev: Fix syntax error in pyproject.toml loads module deps --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 688247fbf..380c7fc07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,8 @@ power = [ ] loads = [ - "fatpack" - "statsmodels" + "fatpack", + "statsmodels", ] mooring = [ From 4badbcf93239e5a90b7c94a2aa1fe6a9243a37a5 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 14:40:41 -0600 Subject: [PATCH 12/18] Actions: always run all matrix iterations in test-optional-pip-dependencies --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a877dc03..2013522de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -406,6 +406,7 @@ jobs: needs: [set-os, prepare-nonhindcast-cache] runs-on: ubuntu-latest strategy: + fail-fast: false matrix: module: [wave, tidal, river, dolfyn, power, loads, mooring, acoustics, utils] From 3ea483f755856d621484fc35c321e1ed51f601f0 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 16:20:52 -0600 Subject: [PATCH 13/18] Deps: Use same version string for statsmodels --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 380c7fc07..e441e54ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,8 @@ power = [ ] loads = [ - "fatpack", - "statsmodels", + "fatpack", + "statsmodels>=0.14.2" ] mooring = [ From 210f7492a039d4f2433e0f79cf1e5ec93faf84b4 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 16:50:50 -0600 Subject: [PATCH 14/18] Deps: Require `requests` for all modules `requests` is small, common python library. There are some modules that pull from multiple modules (loads calls wave) which require `requests`. This always includes requests for all modules. --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e441e54ed..28fbb9e65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "xarray>=2024.6.0", "matplotlib>=3.9.1", "pecos>=0.3.0", + "requests", ] [project.optional-dependencies] @@ -38,20 +39,17 @@ wave = [ "pytz", "NREL-rex>=0.2.63", "beautifulsoup4", - "requests", "bottleneck", "lxml" ] tidal = [ "netCDF4>=1.7.1.post1", - "requests", "bottleneck" ] river = [ "netCDF4>=1.7.1.post1", - "requests", "bottleneck", ] From a8c242bd3918bb3dd16b947244f5894bbb3fa19f Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 16:53:28 -0600 Subject: [PATCH 15/18] Deps: Add scikit-learn to loads deps Loads tests call wave.contours, which need scikit-learn --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 28fbb9e65..7e6c211da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,8 @@ power = [ loads = [ "fatpack", - "statsmodels>=0.14.2" + "statsmodels>=0.14.2", + "scikit-learn>=1.5.1", ] mooring = [ From f5135f0a94f6137083e740db281c3d56bf59e3af Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Wed, 15 Oct 2025 17:25:27 -0600 Subject: [PATCH 16/18] Actions: Use pip install -e .[module] for module tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2013522de..2e6dc4dbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -433,7 +433,7 @@ jobs: - name: Install MHKiT with optional dependency run: | python -m pip install --upgrade pip - pip install "mhkit[${{ matrix.module }}]" + pip install -e ".[${{ matrix.module }}]" pip install pytest - name: Reinstall h5py and netCDF4 with system libraries From 789e7cada44da13e016b4504975fee624346224c Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Thu, 16 Oct 2025 09:09:36 -0600 Subject: [PATCH 17/18] Actions: remove out of scope pip install fixes This fix is implemented in PR 421 and is related, but not in scope of this change --- .github/workflows/main.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e6dc4dbf..094f93ef8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -286,20 +286,12 @@ jobs: name: data path: ~/.cache/mhkit - - name: Install system dependencies - if: runner.os == 'Linux' - run: sudo apt-get install -y libhdf5-dev libnetcdf-dev - - name: Update and install packages shell: bash -l {0} run: | python -m pip install --upgrade pip wheel pip install -e ".[all,dev]" - - name: Reinstall h5py and netCDF4 with system libraries - if: runner.os == 'Linux' - run: "pip install --force-reinstall --no-binary=:all: h5py netCDF4" - - name: Install setuptools for Python 3.12 if: matrix.python-version == '3.12' run: pip install setuptools From b0b9bc64cbb13505c2a64b0c78ed71522b499a20 Mon Sep 17 00:00:00 2001 From: Andrew Simms Date: Thu, 16 Oct 2025 09:57:07 -0600 Subject: [PATCH 18/18] Actions: Reenable concurrency in main.yml --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 094f93ef8..21ac57bc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,9 +10,9 @@ on: - main - develop -# concurrency: -# group: ${{ github.event.pull_request.number }} # Use the pull request number as the concurrency group -# cancel-in-progress: true # Cancel any in-progress jobs for the same group. Prevents duplicate workflows on PRs from develop into main or from main into develop. +concurrency: + group: ${{ github.event.pull_request.number }} # Use the pull request number as the concurrency group + cancel-in-progress: true # Cancel any in-progress jobs for the same group. Prevents duplicate workflows on PRs from develop into main or from main into develop. jobs: set-os: