HSDS

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: Py 3.7 3.8, 3.9 | Windows Mac Linux
 
 on:
   push:
-    branches: 
+    branches:
       - master
       - Develop
   pull_request:
@@ -102,9 +102,60 @@ jobs:
           COVERALLS_FLAG_NAME: ${{ matrix.python-version }}
           COVERALLS_PARALLEL: true
 
+  hindcast-calls:
+    name: hindcast-${{ matrix.os }}/${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        os: ["windows-latest", "macos-latest"]
+        python-version: [3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Conda
+        uses: s-weigand/setup-conda@v1
+        with:
+          activate-conda: false
+          conda-channels: conda-forge
+
+      - name: Python ${{ matrix.python-version }}
+        shell: bash -l {0}
+        run: |
+          conda create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime netcdf4 coverage --strict-channel-priority
+          source activate TEST
+          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
+          pip install -e . --no-deps --force-reinstall
+
+      - name: Install MHKiT
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          python -m pip install --upgrade pip wheel
+          pip install coveralls
+          pip install .
+
+      - name: Run pytest
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          coverage run --rcfile=.github/workflows/.coveragehindcastrc -m  pytest -c  .github/workflows/pytest-hindcast.ini
+
+      - name: Upload coverage data to coveralls.io
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          coveralls --service=github
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_FLAG_NAME: ${{ matrix.python-version }}
+          COVERALLS_PARALLEL: true
+
   coveralls:
     name: Indicate completion to coveralls.io
-    needs: [conda-build, pip-build]
+    needs: [conda-build, pip-build, hindcast-calls]
     runs-on: ubuntu-latest
     container: python:3-slim
     steps:

.hscfg

@@ -1,4 +1,4 @@
-hs_endpoint = https://developer.nrel.gov/api/hsds   
-hs_username =    
-hs_password =    
+hs_endpoint = https://developer.nrel.gov/api/hsds
+hs_username =
+hs_password =
 hs_api_key = 3K3JQbjZmWctY0xmIfSYvYgtIcM3CN0cb1Y2w9bf 

mhkit/tests/wave/io/test_hindcast.py

@@ -90,18 +90,27 @@ def test_multi_year(self):
         assert_frame_equal(self.my_swh,wave_multiyear)
         assert_frame_equal(self.my_meta,meta)
 
+
     def test_multi_loc(self):
         data_type = '3-hour'
         years = [1995]
         lat_lon = ((44.624076,-124.280097),(43.489171,-125.152137))
         parameters = 'mean_absolute_period'
-        wave_multiloc, meta=(wave.io.hindcast
-                            .request_wpto_point_data(data_type,
-                                            parameters,lat_lon,years))
-        (dir_multiyear,
-        meta_dir)=(wave.io.hindcast
-                    .request_wpto_directional_spectrum(lat_lon,year='1995'))
-        dir_multiyear = dir_multiyear.sel(time_index=slice(dir_multiyear.time_index[0],dir_multiyear.time_index[99]))
+        wave_multiloc, meta=wave.io.hindcast.request_wpto_point_data(
+            data_type,
+            parameters,
+            lat_lon,
+            years
+        )
+        dir_multiyear, meta_dir = (wave.io.hindcast
+            .request_wpto_directional_spectrum(lat_lon,year='1995')
+        )
+        dir_multiyear = dir_multiyear.sel(
+            time_index=slice(
+                dir_multiyear.time_index[0],
+                dir_multiyear.time_index[99]
+            )
+        )
         dir_multiyear = dir_multiyear.rename_vars({87:'87',58:'58'})
 
         assert_frame_equal(self.ml,wave_multiloc)

mhkit/wave/io/hindcast.py

@@ -2,6 +2,8 @@
 import numpy as np
 from rex import MultiYearWaveX, WaveX
 import sys
+from time import sleep
+
 
 
 def region_selection(lat_lon):
@@ -216,13 +218,69 @@ def request_wpto_directional_spectrum(lat_lon, year, tree=None,
     waveKwargs = {'tree':tree,'unscale':unscale,'str_decode':str_decode, 'hsds':hsds}
 
     with WaveX(wave_path, **waveKwargs) as rex_waves:
-        # get data
-        data_raw = rex_waves.get_lat_lon_df(parameter,lat_lon)
-        # get metadata
-        col = data_raw.columns[:]
-        meta = rex_waves.meta.loc[col,:]
-        meta = meta.reset_index(drop=True) 
+        # Get graphical identifier
+        gid = rex_waves.lat_lon_gid(lat_lon)  
+
+        # Setup index and columns
+        if isinstance(gid, (int, np.integer)):
+            columns = [gid]
+        else:
+            columns = gid   
+        time_index = rex_waves.time_index
+        frequency = rex_waves['frequency']
+        direction = rex_waves['direction']
+        index = pd.MultiIndex.from_product(
+            [time_index, frequency, direction],
+            names=['time_index', 'frequency', 'direction']
+        )                    
+
+        # Create bins for multiple smaller API dataset requests
+        N=6
+        length = len(rex_waves)        
+        quotient=length//N
+        remainder=length%N    
+        bins = [i*quotient for i in range(N+1) ]
+        bins[-1]+= remainder        
+        index_bins = (np.array(bins)*len(frequency)*len(direction)).tolist()
+
+        # Request multiple datasets and add to dictionary
+        datas={}
+        for i in range(len(bins)-1):
+            idx=index[index_bins[i]:index_bins[i+1]] 
+
+            # Request with exponential back off wait time
+            sleep_time = 2
+            num_retries = 4
+            for x in range(0, num_retries):  
+                try:
+                    data_array = rex_waves[parameter, bins[i]:bins[i+1], :, :, gid]
+                    str_error = None
+
+                except Exception as e:
+                    str_error = str(e)
+
+                if str_error:
+                    sleep(sleep_time)
+                    sleep_time *= 2  
+                else:
+                    break
+
+            ax1 = np.product(data_array.shape[:3])
+            ax2 = data_array.shape[-1] if len(data_array.shape) == 4 else 1
+            data_array = data_array.reshape(ax1, ax2)        
+
+            df = pd.DataFrame(data_array, columns=columns, index=idx)
+            df.name = parameter
+            datas[i]=df
+
+        # Append each request into an xarray
+        data_raw=datas[0]
+        for i in list(datas.keys())[1:]:
+            data_raw =  pd.concat([data_raw,datas[i]])        
+        data = data_raw.to_xarray()
 
-    data = data_raw.to_xarray()
+        # Get metadata
+        meta = rex_waves.meta.loc[columns,:]
+        meta = meta.reset_index(drop=True)  
 
-    return data, meta    
+    return data, meta