diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3751cf39d..be16399de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,7 @@ name: Py 3.7 3.8, 3.9 | Windows Mac Linux on: push: - branches: + branches: - master - Develop pull_request: @@ -102,9 +102,60 @@ jobs: COVERALLS_FLAG_NAME: ${{ matrix.python-version }} COVERALLS_PARALLEL: true + hindcast-calls: + name: hindcast-${{ matrix.os }}/${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: ["windows-latest", "macos-latest"] + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Setup Conda + uses: s-weigand/setup-conda@v1 + with: + activate-conda: false + conda-channels: conda-forge + + - name: Python ${{ matrix.python-version }} + shell: bash -l {0} + run: | + conda create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime netcdf4 coverage --strict-channel-priority + source activate TEST + export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config + pip install -e . --no-deps --force-reinstall + + - name: Install MHKiT + shell: bash -l {0} + run: | + source activate TEST + python -m pip install --upgrade pip wheel + pip install coveralls + pip install . + + - name: Run pytest + shell: bash -l {0} + run: | + source activate TEST + coverage run --rcfile=.github/workflows/.coveragehindcastrc -m pytest -c .github/workflows/pytest-hindcast.ini + + - name: Upload coverage data to coveralls.io + shell: bash -l {0} + run: | + source activate TEST + coveralls --service=github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: ${{ matrix.python-version }} + COVERALLS_PARALLEL: true + coveralls: name: Indicate completion to coveralls.io - needs: [conda-build, pip-build] + needs: [conda-build, pip-build, hindcast-calls] runs-on: ubuntu-latest container: python:3-slim steps: diff --git a/.hscfg b/.hscfg index 33d8d9c33..f6f00424b 100644 --- a/.hscfg +++ b/.hscfg @@ -1,4 +1,4 @@ -hs_endpoint = https://developer.nrel.gov/api/hsds -hs_username = -hs_password = +hs_endpoint = https://developer.nrel.gov/api/hsds +hs_username = +hs_password = hs_api_key = 3K3JQbjZmWctY0xmIfSYvYgtIcM3CN0cb1Y2w9bf diff --git a/examples/WPTO_hindcast_example.ipynb b/examples/WPTO_hindcast_example.ipynb index 99d6fa33e..1881dd6e4 100644 --- a/examples/WPTO_hindcast_example.ipynb +++ b/examples/WPTO_hindcast_example.ipynb @@ -912,7 +912,9 @@ " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", - " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", + " fig.send_message('set_device_pixel_ratio', {\n", + " device_pixel_ratio: fig.ratio,\n", + " });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", @@ -1291,22 +1293,7 @@ "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", - " var cursor = msg['cursor'];\n", - " switch (cursor) {\n", - " case 0:\n", - " cursor = 'pointer';\n", - " break;\n", - " case 1:\n", - " cursor = 'default';\n", - " break;\n", - " case 2:\n", - " cursor = 'crosshair';\n", - " break;\n", - " case 3:\n", - " cursor = 'move';\n", - " break;\n", - " }\n", - " fig.rubberband_canvas.style.cursor = cursor;\n", + " fig.rubberband_canvas.style.cursor = msg['cursor'];\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", @@ -1418,7 +1405,7 @@ " };\n", "};\n", "\n", - "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", + "// from https://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", @@ -1446,7 +1433,7 @@ "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", - " * http://stackoverflow.com/a/24161582/3208463\n", + " * https://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", @@ -1535,7 +1522,7 @@ "///////////////// REMAINING CONTENT GENERATED BY embed_js.py /////////////////\n", "// prettier-ignore\n", "var _JSXTOOLS_RESIZE_OBSERVER=function(A){var t,i=new WeakMap,n=new WeakMap,a=new WeakMap,r=new WeakMap,o=new Set;function s(e){if(!(this instanceof s))throw new TypeError(\"Constructor requires 'new' operator\");i.set(this,e)}function h(){throw new TypeError(\"Function is not a constructor\")}function c(e,t,i,n){e=0 in arguments?Number(arguments[0]):0,t=1 in arguments?Number(arguments[1]):0,i=2 in arguments?Number(arguments[2]):0,n=3 in arguments?Number(arguments[3]):0,this.right=(this.x=this.left=e)+(this.width=i),this.bottom=(this.y=this.top=t)+(this.height=n),Object.freeze(this)}function d(){t=requestAnimationFrame(d);var s=new WeakMap,p=new Set;o.forEach((function(t){r.get(t).forEach((function(i){var r=t instanceof window.SVGElement,o=a.get(t),d=r?0:parseFloat(o.paddingTop),f=r?0:parseFloat(o.paddingRight),l=r?0:parseFloat(o.paddingBottom),u=r?0:parseFloat(o.paddingLeft),g=r?0:parseFloat(o.borderTopWidth),m=r?0:parseFloat(o.borderRightWidth),w=r?0:parseFloat(o.borderBottomWidth),b=u+f,F=d+l,v=(r?0:parseFloat(o.borderLeftWidth))+m,W=g+w,y=r?0:t.offsetHeight-W-t.clientHeight,E=r?0:t.offsetWidth-v-t.clientWidth,R=b+v,z=F+W,M=r?t.width:parseFloat(o.width)-R-E,O=r?t.height:parseFloat(o.height)-z-y;if(n.has(t)){var k=n.get(t);if(k[0]===M&&k[1]===O)return}n.set(t,[M,O]);var S=Object.create(h.prototype);S.target=t,S.contentRect=new c(u,d,M,O),s.has(i)||(s.set(i,[]),p.add(i)),s.get(i).push(S)}))})),p.forEach((function(e){i.get(e).call(e,s.get(e),e)}))}return s.prototype.observe=function(i){if(i instanceof window.Element){r.has(i)||(r.set(i,new Set),o.add(i),a.set(i,window.getComputedStyle(i)));var n=r.get(i);n.has(this)||n.add(this),cancelAnimationFrame(t),t=requestAnimationFrame(d)}},s.prototype.unobserve=function(i){if(i instanceof window.Element&&r.has(i)){var n=r.get(i);n.has(this)&&(n.delete(this),n.size||(r.delete(i),o.delete(i))),n.size||r.delete(i),o.size||cancelAnimationFrame(t)}},A.DOMRectReadOnly=c,A.ResizeObserver=s,A.ResizeObserverEntry=h,A}; // eslint-disable-line\n", - "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", + "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pgf\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", @@ -1769,11 +1756,6 @@ "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager) {\n", - " manager = IPython.keyboard_manager;\n", - " }\n", - "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", @@ -1830,7 +1812,7 @@ { "data": { "text/html": [ - "" + "
" ], "text/plain": [ "" @@ -1912,36 +1894,23 @@ "output_type": "stream", "text": [ "\n", - "Dimensions: (direction: 24, frequency: 29, time_index: 8748)\n", + "Dimensions: (time_index: 8748, frequency: 29, direction: 24)\n", "Coordinates:\n", " * time_index (time_index) object 725850000000000000 ... 757378800000000000\n", " * frequency (frequency) float64 0.035 0.0385 0.0424 ... 0.4173 0.4591 0.505\n", " * direction (direction) float64 7.5 22.5 37.5 52.5 ... 322.5 337.5 352.5\n", "Data variables:\n", - " 87 (time_index, frequency, direction) float32 0.0 0.0 ... 4.535e-05\n" + " 58 (time_index, frequency, direction) float32 0.0 0.0 ... 7.968e-05\n" ] } ], "source": [ "year = '1993' # only one year can be passed at a time as a string\n", + "lat_lon=(43.489171,-125.152137)\n", "dir_spectra,meta = wave.io.hindcast.request_wpto_directional_spectrum(lat_lon,year)\n", "\n", "print(dir_spectra)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n" - ] } ], "metadata": { @@ -1949,7 +1918,7 @@ "hash": "29d4f3f9417e26053c15fead58122d92d3e4d33b81eac28eefb5cabca755f3aa" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1963,7 +1932,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5" + "version": "3.9.12" } }, "nbformat": 4, diff --git a/mhkit/tests/wave/io/test_hindcast.py b/mhkit/tests/wave/io/test_hindcast.py index d65e2c109..da55589ed 100644 --- a/mhkit/tests/wave/io/test_hindcast.py +++ b/mhkit/tests/wave/io/test_hindcast.py @@ -90,18 +90,27 @@ def test_multi_year(self): assert_frame_equal(self.my_swh,wave_multiyear) assert_frame_equal(self.my_meta,meta) + def test_multi_loc(self): data_type = '3-hour' years = [1995] lat_lon = ((44.624076,-124.280097),(43.489171,-125.152137)) parameters = 'mean_absolute_period' - wave_multiloc, meta=(wave.io.hindcast - .request_wpto_point_data(data_type, - parameters,lat_lon,years)) - (dir_multiyear, - meta_dir)=(wave.io.hindcast - .request_wpto_directional_spectrum(lat_lon,year='1995')) - dir_multiyear = dir_multiyear.sel(time_index=slice(dir_multiyear.time_index[0],dir_multiyear.time_index[99])) + wave_multiloc, meta=wave.io.hindcast.request_wpto_point_data( + data_type, + parameters, + lat_lon, + years + ) + dir_multiyear, meta_dir = (wave.io.hindcast + .request_wpto_directional_spectrum(lat_lon,year='1995') + ) + dir_multiyear = dir_multiyear.sel( + time_index=slice( + dir_multiyear.time_index[0], + dir_multiyear.time_index[99] + ) + ) dir_multiyear = dir_multiyear.rename_vars({87:'87',58:'58'}) assert_frame_equal(self.ml,wave_multiloc) diff --git a/mhkit/wave/io/hindcast.py b/mhkit/wave/io/hindcast.py index ac834def3..d716794dd 100644 --- a/mhkit/wave/io/hindcast.py +++ b/mhkit/wave/io/hindcast.py @@ -2,6 +2,8 @@ import numpy as np from rex import MultiYearWaveX, WaveX import sys +from time import sleep + def region_selection(lat_lon): @@ -216,13 +218,69 @@ def request_wpto_directional_spectrum(lat_lon, year, tree=None, waveKwargs = {'tree':tree,'unscale':unscale,'str_decode':str_decode, 'hsds':hsds} with WaveX(wave_path, **waveKwargs) as rex_waves: - # get data - data_raw = rex_waves.get_lat_lon_df(parameter,lat_lon) - # get metadata - col = data_raw.columns[:] - meta = rex_waves.meta.loc[col,:] - meta = meta.reset_index(drop=True) + # Get graphical identifier + gid = rex_waves.lat_lon_gid(lat_lon) + + # Setup index and columns + if isinstance(gid, (int, np.integer)): + columns = [gid] + else: + columns = gid + time_index = rex_waves.time_index + frequency = rex_waves['frequency'] + direction = rex_waves['direction'] + index = pd.MultiIndex.from_product( + [time_index, frequency, direction], + names=['time_index', 'frequency', 'direction'] + ) + + # Create bins for multiple smaller API dataset requests + N=6 + length = len(rex_waves) + quotient=length//N + remainder=length%N + bins = [i*quotient for i in range(N+1) ] + bins[-1]+= remainder + index_bins = (np.array(bins)*len(frequency)*len(direction)).tolist() + + # Request multiple datasets and add to dictionary + datas={} + for i in range(len(bins)-1): + idx=index[index_bins[i]:index_bins[i+1]] + + # Request with exponential back off wait time + sleep_time = 2 + num_retries = 4 + for x in range(0, num_retries): + try: + data_array = rex_waves[parameter, bins[i]:bins[i+1], :, :, gid] + str_error = None + + except Exception as e: + str_error = str(e) + + if str_error: + sleep(sleep_time) + sleep_time *= 2 + else: + break + + ax1 = np.product(data_array.shape[:3]) + ax2 = data_array.shape[-1] if len(data_array.shape) == 4 else 1 + data_array = data_array.reshape(ax1, ax2) + + df = pd.DataFrame(data_array, columns=columns, index=idx) + df.name = parameter + datas[i]=df + + # Append each request into an xarray + data_raw=datas[0] + for i in list(datas.keys())[1:]: + data_raw = pd.concat([data_raw,datas[i]]) + data = data_raw.to_xarray() - data = data_raw.to_xarray() + # Get metadata + meta = rex_waves.meta.loc[columns,:] + meta = meta.reset_index(drop=True) - return data, meta \ No newline at end of file + return data, meta \ No newline at end of file