diff --git a/examples/data/dolfyn/Sig1000_dp_echo.ad2cp b/examples/data/dolfyn/Sig1000_dp_echo.ad2cp new file mode 100644 index 000000000..37011fdbe Binary files /dev/null and b/examples/data/dolfyn/Sig1000_dp_echo.ad2cp differ diff --git a/examples/data/dolfyn/Sig1000_online.ad2cp b/examples/data/dolfyn/Sig1000_online.ad2cp new file mode 100644 index 000000000..c6bd14c35 Binary files /dev/null and b/examples/data/dolfyn/Sig1000_online.ad2cp differ diff --git a/examples/data/dolfyn/Sig100_avg.ad2cp b/examples/data/dolfyn/Sig100_avg.ad2cp new file mode 100644 index 000000000..e16b2dd98 Binary files /dev/null and b/examples/data/dolfyn/Sig100_avg.ad2cp differ diff --git a/examples/data/dolfyn/Sig100_raw_avg.ad2cp b/examples/data/dolfyn/Sig100_raw_avg.ad2cp new file mode 100644 index 000000000..3099caa53 Binary files /dev/null and b/examples/data/dolfyn/Sig100_raw_avg.ad2cp differ diff --git a/examples/data/dolfyn/dual_profile.ad2cp b/examples/data/dolfyn/Sig500_dp_ice.ad2cp similarity index 100% rename from examples/data/dolfyn/dual_profile.ad2cp rename to examples/data/dolfyn/Sig500_dp_ice.ad2cp diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc new file mode 100644 index 000000000..28cef003d Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc new file mode 100644 index 000000000..80290da36 Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_online.nc b/examples/data/dolfyn/test_data/Sig1000_online.nc new file mode 100644 index 000000000..d028a6ffe Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig1000_online.nc differ diff --git a/examples/data/dolfyn/test_data/Sig100_avg.nc b/examples/data/dolfyn/test_data/Sig100_avg.nc new file mode 100644 index 000000000..ff5981ea1 Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig100_avg.nc differ diff --git a/examples/data/dolfyn/test_data/Sig100_raw_avg.nc b/examples/data/dolfyn/test_data/Sig100_raw_avg.nc new file mode 100644 index 000000000..43c91de9f Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig100_raw_avg.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc new file mode 100644 index 000000000..2ac223c12 Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc differ diff --git a/examples/data/dolfyn/test_data/dual_profile.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc similarity index 76% rename from examples/data/dolfyn/test_data/dual_profile.nc rename to examples/data/dolfyn/test_data/Sig500_dp_ice2.nc index ab63f99da..0c9944c1f 100644 Binary files a/examples/data/dolfyn/test_data/dual_profile.nc and b/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc differ diff --git a/mhkit/dolfyn/io/base.py b/mhkit/dolfyn/io/base.py index c31f5fc04..15688d7fe 100644 --- a/mhkit/dolfyn/io/base.py +++ b/mhkit/dolfyn/io/base.py @@ -164,6 +164,14 @@ def _create_dataset(data): """ tag = ["_avg", "_b5", "_echo", "_bt", "_gps", "_altraw", "_altraw_avg", "_sl"] + # If burst velocity not measured + if "vel" not in data["data_vars"]: + # dual profile where burst velocity is not measured but echo sounder is + if "vel_avg" in data["data_vars"]: + data["coords"]["time"] = data["coords"]["time_avg"] + else: + t_vars = [t for t in data["coords"] if "time" in t] + data["coords"]["time"] = data["coords"][t_vars[0]] ds_dict = {} for key in data["coords"]: diff --git a/mhkit/dolfyn/io/nortek2.py b/mhkit/dolfyn/io/nortek2.py index ea56d40c3..ba67c5ae5 100644 --- a/mhkit/dolfyn/io/nortek2.py +++ b/mhkit/dolfyn/io/nortek2.py @@ -1,9 +1,9 @@ -import numpy as np -from struct import unpack, calcsize +import json +import logging import warnings +from struct import unpack, calcsize from pathlib import Path -import logging -import json +import numpy as np from . import nortek2_defs as defs from . import nortek2_lib as lib @@ -14,9 +14,6 @@ from ..time import epoch2dt64, _fill_time_gaps -int32_max = np.iinfo(np.int32).max - - def read_signature( filename, userdata=True, @@ -24,7 +21,7 @@ def read_signature( rebuild_index=False, debug=False, dual_profile=False, - **kwargs + **kwargs, ): """ Read a Nortek Signature (.ad2cp) datafile @@ -143,30 +140,22 @@ def read_signature( class _Ad2cpReader: - def __init__( - self, - fname, - endian=None, - bufsize=None, - rebuild_index=False, - debug=False, - dual_profile=False, - ): + def __init__(self, fname, rebuild_index, debug, dual_profile): self.fname = fname self.debug = debug - self._check_nortek(endian) - self.f.seek(0, 2) # Seek to end - self._eof = self.f.tell() - self.start_pos = self._check_header() + # Open file, check endianess, and find filelength + self._check_nortek2() + # Generate indexing file self._index, self._dp = lib.get_index( fname, - pos=self.start_pos, + pos=0, eof=self._eof, rebuild=rebuild_index, debug=debug, dp=dual_profile, ) - self._open(bufsize) + # Open file for reading + self._open() self.filehead_config = self._read_filehead_config_string() self._ens_pos = self._index["pos"][ lib._boolarray_firstensemble_ping(self._index) @@ -176,52 +165,18 @@ def __init__( self._init_burst_readers() self.unknown_ID_count = {} - def _calc_lastblock_iswhole( - self, - ): - blocksize, blocksize_count = np.unique( - np.diff(self._ens_pos), return_counts=True - ) - standard_blocksize = blocksize[blocksize_count.argmax()] - return (self._eof - self._ens_pos[-1]) == standard_blocksize - - def _check_nortek(self, endian): + def _check_nortek2(self): self._open(10) byts = self.f.read(2) - if endian is None: - if unpack("<" + "BB", byts) == (165, 10): - endian = "<" - elif unpack(">" + "BB", byts) == (165, 10): - endian = ">" - else: - raise Exception( - "I/O error: could not determine the 'endianness' " - "of the file. Are you sure this is a Nortek " - "AD2CP file?" - ) - self.endian = endian - - def _check_header(self): - def find_all(s, c): - idx = s.find(c) - while idx != -1: - yield idx - idx = s.find(c, idx + 1) - - # Open the entire file to find start header - if self._eof >= int32_max: - init_buffer = int32_max - else: - init_buffer = self._eof - self._open(init_buffer) - pk = self.f.peek(1) - # Search for multiple saved headers - found = [i for i in find_all(pk, b"GETCLOCKSTR")] - if len(found) < 2: - return 0 - else: - start_idx = found[-1] - 11 - return start_idx + if not (unpack("= int32_max: + buffer = int32_max + else: + buffer = eof + fin = open(_abspath(infile_obj.name), "rb", buffer) + fin.seek(current_pos, 1) + + # Search for multiple saved headers + pk = fin.peek(1) + found = [i for i in find_all(pk, b"GETCLOCKSTR")] + if found: + start_idx = found[0] - 11 # assuming next header is 10 bytes + else: + start_idx = 0 + + return fin, start_idx + + def _create_index(infile, outfile, init_pos, eof, debug): logging = getLogger() print("Indexing {}...".format(infile), end="") @@ -122,39 +153,77 @@ def _create_index(infile, outfile, init_pos, eof, debug): config = 0 last_ens = dict.fromkeys(ids, -1) seek_2ens = { - 21: 40, - 22: 40, - 23: 42, - 24: 40, - 26: 40, - 28: 40, # 23 starts from "42" - 27: 40, - 29: 40, - 30: 40, - 31: 40, - 35: 40, - 36: 40, + 21: 40, # 0x15 burst + 22: 40, # 0x16 average + 23: 42, # 0x17 bottom track, starts from "42" + 24: 40, # 0x18 interleaved burst (beam 5) + 26: 40, # 0x1A burst altimeter + 27: 40, # 0x1B DVL bottom track + 28: 40, # 0x1C echo sounder + 29: 40, # 0x1D DVL water track + 30: 40, # 0x1E altimeter + 31: 40, # 0x1F avg altimeter + 35: 40, # 0x23 raw echo sounder + 36: 40, # 0x24 raw tx echo sounder + 48: 40, # 0x30 processed wave + # 160: 40, # 0xA0 string (header info, GPS NMEA data) + # 192: 40, # 0xC0 Nortek Data format 8 record } pos = 0 - while pos <= eof: - pos = fin.tell() + header_check_flag = 0 + # leave room for header plus other data (12 + 76) + while pos <= (eof - 88): if init_pos and not pos: fin.seek(init_pos, 1) try: - dat = _hdr.unpack(fin.read(_hdr.size)) - except: + dat = struct.unpack(" 0: @@ -211,25 +280,29 @@ def _create_index(infile, outfile, init_pos, eof, debug): logging.info("Invalid skip byte at pos: %10d\n" % (pos)) break fin.seek(dat[4], 1) + # Update for while loop check + pos = fin.tell() + fin.close() fout.close() print(" Done.") def _check_index(idx, infile, fix_hw_ens=False, dp=False): + logging = getLogger() uid = np.unique(idx["ID"]) if fix_hw_ens: hwe = idx["hw_ens"] else: hwe = idx["hw_ens"].copy() - period = hwe.max() ens = idx["ens"] N_id = len(uid) - FLAG = False # Are there better ways to detect dual profile? - if (21 in uid) and (22 in uid): - warnings.warn("Dual Profile detected... Two datasets will be returned.") + if (22 in uid) and ((21 in uid) or (28 in uid)): + msg = "Dual Profile detected... Two datasets will be returned." + warnings.warn(msg) + logging.warning(msg) dp = True # This loop fixes 'skips' inside the file @@ -238,6 +311,7 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): inds = np.nonzero(idx["ID"] == id)[0] # These are bad steps in the indices for this ID ibad = np.nonzero(np.diff(inds) > N_id)[0] + # Check if spacing is equal for dual profiling ADCPs if dp: skip_size = np.diff(ibad) @@ -250,10 +324,9 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): mask = np.append(skip_size, 0).astype(bool) if any(skip_size) else [] ibad = ibad[mask] for ib in ibad: - FLAG = True # The ping number reported here may not be quite right if # the ensemble count is wrong. - warnings.warn( + logging.warning( "Skipped ping (ID: {}) in file {} at ensemble {}.".format( id, infile, idx["ens"][inds[ib + 1] - 1] ) @@ -270,7 +343,8 @@ def _boolarray_firstensemble_ping(index): each ensemble. """ dens = np.ones(index["ens"].shape, dtype="bool") - dens[1:] = np.diff(index["ens"]) != 0 + if any(index["ens"]): + dens[1:] = np.diff(index["ens"]) != 0 return dens diff --git a/mhkit/dolfyn/rotate/base.py b/mhkit/dolfyn/rotate/base.py index 9ffa4f282..ad4d3d946 100644 --- a/mhkit/dolfyn/rotate/base.py +++ b/mhkit/dolfyn/rotate/base.py @@ -49,7 +49,7 @@ def _set_coords(ds, ref_frame, forced=False): XYZ = ["X", "Y", "Z"] ENU = ["E", "N", "U"] - beam = ds.beam.values + beam = ds["beam"].values principal = ["streamwise", "x-stream", "vert"] # check make/model diff --git a/mhkit/dolfyn/rotate/vector.py b/mhkit/dolfyn/rotate/vector.py index e390322f8..df37b0f9c 100644 --- a/mhkit/dolfyn/rotate/vector.py +++ b/mhkit/dolfyn/rotate/vector.py @@ -345,7 +345,7 @@ def _euler2orient(time, heading, pitch, roll, units="degree"): ) return xr.DataArray( omat, - coords={"earth": earth, "inst": inst, "time": time}, - dims=["earth", "inst", "time"], + coords={"earth": earth, "inst": inst, time.name: time}, + dims=["earth", "inst", time.name], attrs={"units": "1", "long_name": "Orientation Matrix"}, ) diff --git a/mhkit/tests/dolfyn/test_clean.py b/mhkit/tests/dolfyn/test_clean.py index 663cae98b..0541435f5 100644 --- a/mhkit/tests/dolfyn/test_clean.py +++ b/mhkit/tests/dolfyn/test_clean.py @@ -62,7 +62,7 @@ def test_clean_upADCP(self): td_awac = tp.dat_awac.copy(deep=True) td_sig = tp.dat_sig_tide.copy(deep=True) td_rdi = tp.dat_rdi.copy(deep=True) - td_dual = tp.dat_sig_dp2.copy(deep=True) + td_dual = tp.dat_sig_dp1_ice.copy(deep=True) apm.clean.water_depth_from_pressure(td_awac, salinity=30) apm.clean.remove_surface_interference(td_awac, beam_angle=20, inplace=True) @@ -95,7 +95,7 @@ def test_clean_upADCP(self): def test_clean_downADCP(self): td = tp.dat_sig_ie.copy(deep=True) - td_dual = tp.dat_sig_dp2.copy(deep=True) + td_dual = tp.dat_sig_dp1_ice.copy(deep=True) # First remove bad data td["vel"] = apm.clean.val_exceeds_thresh(td.vel, thresh=3) diff --git a/mhkit/tests/dolfyn/test_read_adp.py b/mhkit/tests/dolfyn/test_read_adp.py index 3cba999b2..688b52cd3 100644 --- a/mhkit/tests/dolfyn/test_read_adp.py +++ b/mhkit/tests/dolfyn/test_read_adp.py @@ -32,10 +32,16 @@ dat_sig_ieb = load("VelEchoBT01.nc") dat_sig_ie = load("Sig500_Echo.nc") dat_sig_tide = load("Sig1000_tidal.nc") +dat_sig_raw_avg = load("Sig100_raw_avg.nc") +dat_sig_avg = load("Sig100_avg.nc") +dat_sig_rt = load("Sig1000_online.nc") dat_sig_skip = load("Sig_SkippedPings01.nc") dat_sig_badt = load("Sig1000_BadTime01.nc") dat_sig5_leiw = load("Sig500_last_ensemble_is_whole.nc") -dat_sig_dp2 = load("dual_profile.nc") +dat_sig_dp1_all = load("Sig500_dp_ice1.nc") +dat_sig_dp1_ice = load("Sig500_dp_ice2.nc") +dat_sig_dp2_echo = load("Sig1000_dp_echo1.nc") +dat_sig_dp2_avg = load("Sig1000_dp_echo2.nc") class io_adp_testcase(unittest.TestCase): @@ -104,8 +110,13 @@ def test_io_nortek2(self): td_sig_ieb = read("VelEchoBT01.ad2cp", nens=nens, rebuild_index=True) td_sig_ie = read("Sig500_Echo.ad2cp", nens=nens, rebuild_index=True) td_sig_tide = read("Sig1000_tidal.ad2cp", nens=nens, rebuild_index=True) - # Only need to test 2nd dataset - td_sig_dp1, td_sig_dp2 = read("dual_profile.ad2cp") + td_sig_raw_avg = read("Sig100_raw_avg.ad2cp", nens=nens, rebuild_index=True) + td_sig_avg = read("Sig100_avg.ad2cp", nens=nens, rebuild_index=True) + td_sig_rt = read("Sig1000_online.ad2cp", nens=nens, rebuild_index=True) + td_sig_dp1_all, td_sig_dp1_ice = read("Sig500_dp_ice.ad2cp", rebuild_index=True) + td_sig_dp2_echo, td_sig_dp2_avg = read( + "Sig1000_dp_echo.ad2cp", rebuild_index=True + ) with pytest.warns(UserWarning): # This issues a warning... @@ -123,10 +134,14 @@ def test_io_nortek2(self): os.remove(tb.exdt("VelEchoBT01.ad2cp.index")) os.remove(tb.exdt("Sig500_Echo.ad2cp.index")) os.remove(tb.exdt("Sig1000_tidal.ad2cp.index")) + os.remove(tb.exdt("Sig100_raw_avg.ad2cp.index")) + os.remove(tb.exdt("Sig100_avg.ad2cp.index")) + os.remove(tb.exdt("Sig1000_online.ad2cp.index")) os.remove(tb.exdt("Sig_SkippedPings01.ad2cp.index")) os.remove(tb.exdt("Sig500_last_ensemble_is_whole.ad2cp.index")) os.remove(tb.rfnm("Sig1000_BadTime01.ad2cp.index")) - os.remove(tb.exdt("dual_profile.ad2cp.index")) + os.remove(tb.exdt("Sig500_dp_ice.ad2cp.index")) + os.remove(tb.exdt("Sig1000_dp_echo.ad2cp.index")) if make_data: save(td_sig, "BenchFile01.nc") @@ -135,10 +150,16 @@ def test_io_nortek2(self): save(td_sig_ieb, "VelEchoBT01.nc") save(td_sig_ie, "Sig500_Echo.nc") save(td_sig_tide, "Sig1000_tidal.nc") + save(td_sig_raw_avg, "Sig100_raw_avg.nc") + save(td_sig_avg, "Sig100_avg.nc") + save(td_sig_rt, "Sig1000_online.nc") save(td_sig_skip, "Sig_SkippedPings01.nc") save(td_sig_badt, "Sig1000_BadTime01.nc") save(td_sig5_leiw, "Sig500_last_ensemble_is_whole.nc") - save(td_sig_dp2, "dual_profile.nc") + save(td_sig_dp1_all, "Sig500_dp_ice1.nc") + save(td_sig_dp1_ice, "Sig500_dp_ice2.nc") + save(td_sig_dp2_echo, "Sig1000_dp_echo1.nc") + save(td_sig_dp2_avg, "Sig1000_dp_echo2.nc") return assert_allclose(td_sig, dat_sig, atol=1e-6) @@ -147,10 +168,16 @@ def test_io_nortek2(self): assert_allclose(td_sig_ieb, dat_sig_ieb, atol=1e-6) assert_allclose(td_sig_ie, dat_sig_ie, atol=1e-6) assert_allclose(td_sig_tide, dat_sig_tide, atol=1e-6) + assert_allclose(td_sig_raw_avg, dat_sig_raw_avg, atol=1e-6) + assert_allclose(td_sig_avg, dat_sig_avg, atol=1e-6) + assert_allclose(td_sig_rt, dat_sig_rt, atol=1e-6) assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6) assert_allclose(td_sig_skip, dat_sig_skip, atol=1e-6) assert_allclose(td_sig_badt, dat_sig_badt, atol=1e-6) - assert_allclose(td_sig_dp2, dat_sig_dp2, atol=1e-6) + assert_allclose(td_sig_dp1_all, dat_sig_dp1_all, atol=1e-6) + assert_allclose(td_sig_dp1_ice, dat_sig_dp1_ice, atol=1e-6) + assert_allclose(td_sig_dp2_echo, dat_sig_dp2_echo, atol=1e-6) + assert_allclose(td_sig_dp2_avg, dat_sig_dp2_avg, atol=1e-6) def test_nortek2_crop(self): # Test file cropping function diff --git a/mhkit/tests/dolfyn/test_rotate_adp.py b/mhkit/tests/dolfyn/test_rotate_adp.py index 0e9598bfb..5f026e0f6 100644 --- a/mhkit/tests/dolfyn/test_rotate_adp.py +++ b/mhkit/tests/dolfyn/test_rotate_adp.py @@ -118,6 +118,8 @@ def test_rotate_earth2inst(self): rotate2(td_sig, "inst", inplace=True) td_sig_i = load("Sig1000_IMU_rotate_inst2earth.nc") rotate2(td_sig_i, "inst", inplace=True) + td_sig_avg = load("Sig100_avg") + rotate2(td_sig_avg, "inst", inplace=True) cd_rdi = load("RDI_test01_rotate_beam2inst.nc") cd_wr2 = tr.dat_wr2