Merge pull request #290 from jmcvey3/online

Skip junk headers in AD2CP files

Author: jmcvey3

mhkit/dolfyn/io/nortek.py

@@ -56,10 +56,8 @@ def read_nortek(
 
     userdata = _find_userdata(filename, userdata)
 
-    with _NortekReader(
-        filename, debug=debug, do_checksum=do_checksum, nens=nens
-    ) as rdr:
-        rdr.readfile()
+    rdr = _NortekReader(filename, debug=debug, do_checksum=do_checksum, nens=nens)
+    rdr.readfile()
     rdr.dat2sci()
     dat = rdr.data
 
@@ -300,14 +298,10 @@ def filesize(
         return self._filesz
 
     @property
-    def pos(
-        self,
-    ):
+    def pos(self):
         return self.f.tell()
 
-    def init_ADV(
-        self,
-    ):
+    def init_ADV(self):
         dat = self.data = {
             "data_vars": {},
             "coords": {},
@@ -335,9 +329,7 @@ def init_ADV(
         self.n_samp_guess = int(self.filesize / dlta + 1)
         self.n_samp_guess *= int(self.config["fs"])
 
-    def init_AWAC(
-        self,
-    ):
+    def init_AWAC(self):
         dat = self.data = {
             "data_vars": {},
             "coords": {},
@@ -392,9 +384,7 @@ def findnext(self, do_cs=True):
             sum += cs
             cs = val
 
-    def read_id(
-        self,
-    ):
+    def read_id(self):
         """Read the next 'ID' from the file."""
         self._thisid_bytes = bts = self.read(2)
         tmp = unpack(self.endian + "BB", bts)
@@ -414,9 +404,7 @@ def read_id(
             return val
         return tmp[1]
 
-    def readnext(
-        self,
-    ):
+    def readnext(self):
         id = "0x%02x" % self.read_id()
         if id in self.fun_map:
             func_name = self.fun_map[id]
@@ -503,9 +491,7 @@ def checksum(self, byts):
         else:
             self.f.seek(2, 1)
 
-    def read_user_cfg(
-        self,
-    ):
+    def read_user_cfg(self):
         # ID: '0x00 = 00
         if self.debug:
             logging.info(
@@ -607,9 +593,7 @@ def read_user_cfg(
             int(Mode1[2])
         ]  # noqa
 
-    def read_head_cfg(
-        self,
-    ):
+    def read_head_cfg(self):
         # ID: '0x04 = 04
         if self.debug:
             logging.info(
@@ -631,9 +615,7 @@ def read_head_cfg(
         )
         self.checksum(byts)
 
-    def read_hw_cfg(
-        self,
-    ):
+    def read_hw_cfg(self):
         # ID 0x05 = 05
         if self.debug:
             logging.info(
@@ -700,9 +682,7 @@ def _init_data(self, vardict):
                     if va.standard_name:
                         self.data["standard_name"][nm] = va.standard_name
 
-    def read_vec_data(
-        self,
-    ):
+    def read_vec_data(self):
         # ID: 0x10 = 16
         c = self.c
         dat = self.data
@@ -741,9 +721,7 @@ def read_vec_data(
         self.checksum(byts)
         self.c += 1
 
-    def read_vec_checkdata(
-        self,
-    ):
+    def read_vec_checkdata(self):
         # ID: 0x07 = 07
         if self.debug:
             logging.info(
@@ -794,9 +772,7 @@ def _sci_data(self, vardict):
             if retval is not None:
                 dat[nm] = retval
 
-    def sci_vec_data(
-        self,
-    ):
+    def sci_vec_data(self):
         self._sci_data(nortek_defs.vec_data)
         dat = self.data
 
@@ -814,9 +790,7 @@ def sci_vec_data(
         # Apply velocity scaling (1 or 0.1)
         dat["data_vars"]["vel"] *= self.config["vel_scale_mm"]
 
-    def read_vec_hdr(
-        self,
-    ):
+    def read_vec_hdr(self):
         # ID: '0x12 = 18
         if self.debug:
             logging.info(
@@ -846,9 +820,7 @@ def read_vec_hdr(
                 self.config["data_header"] = [self.config["data_header"]]
             self.config["data_header"] += [hdrnow]
 
-    def read_vec_sysdata(
-        self,
-    ):
+    def read_vec_sysdata(self):
         # ID: 0x11 = 17
         c = self.c
         if self.debug:
@@ -884,9 +856,7 @@ def read_vec_sysdata(
         ) = unpack(self.endian + "2H3hH2BH", byts[8:])
         self.checksum(byts)
 
-    def sci_vec_sysdata(
-        self,
-    ):
+    def sci_vec_sysdata(self):
         """Translate the data in the vec_sysdata structure into
         scientific units.
         """
@@ -937,9 +907,7 @@ def sci_vec_sysdata(
         tbx.interpgaps(dv["roll"], t)
         tbx.interpgaps(dv["temp"], t)
 
-    def read_microstrain(
-        self,
-    ):
+    def read_microstrain(self):
         """Read ADV microstrain sensor (IMU) data"""
 
         def update_defs(dat, mag=False, orientmat=False):
@@ -1058,9 +1026,7 @@ def update_defs(dat, mag=False, orientmat=False):
         self.checksum(byts0 + byts)
         self.c += 1  # reset the increment
 
-    def sci_microstrain(
-        self,
-    ):
+    def sci_microstrain(self):
         """Rotate orientation data into ADV coordinate system."""
         # MS = MicroStrain
         dv = self.data["data_vars"]
@@ -1085,9 +1051,7 @@ def sci_microstrain(
             dv["angrt"] *= self.config["fs"]
             dv["accel"] *= self.config["fs"]
 
-    def read_awac_profile(
-        self,
-    ):
+    def read_awac_profile(self):
         # ID: '0x20' = 32
         dat = self.data
         if self.debug:
@@ -1135,9 +1099,7 @@ def read_awac_profile(
         self.checksum(byts)
         self.c += 1
 
-    def sci_awac_profile(
-        self,
-    ):
+    def sci_awac_profile(self):
         self._sci_data(nortek_defs.awac_profile)
         # Calculate the ranges.
         cs_coefs = {2000: 0.0239, 1000: 0.0478, 600: 0.0797, 400: 0.1195}
@@ -1158,9 +1120,7 @@ def sci_awac_profile(
         self.data["attrs"]["cell_size"] = cs
         self.data["attrs"]["blank_dist"] = bd
 
-    def read_awac_waves_hdr(
-        self,
-    ):
+    def read_awac_waves_hdr(self):
         # ID: '0x31'
         c = self.c
         if self.debug:
@@ -1205,9 +1165,7 @@ def read_awac_waves_hdr(
                 self.config["data_header"] = [self.config["data_header"]]
             self.config["data_header"] += [hdrnow]
 
-    def read_awac_waves(
-        self,
-    ):
+    def read_awac_waves(self):
         """Read awac wave and suv data"""
         # IDs: 0x30 & 0x36
         c = self.c
@@ -1244,21 +1202,13 @@ def read_awac_waves(
         self.checksum(byts)
         self.c += 1
 
-    def dat2sci(
-        self,
-    ):
+    def dat2sci(self):
         for nm in self._dtypes:
             getattr(self, "sci_" + nm)()
         for nm in ["data_header", "checkdata"]:
             if nm in self.config and isinstance(self.config[nm], list):
                 self.config[nm] = _recatenate(self.config[nm])
 
-    def __exit__(self, type, value, trace):
-        self.close()
-
-    def __enter__(self):
-        return self
-
 
 def _crop_data(obj, range, n_lastdim):
     for nm, dat in obj.items():

mhkit/dolfyn/io/nortek2.py

@@ -132,7 +132,10 @@ def __init__(
         self._check_nortek(endian)
         self.f.seek(0, 2)  # Seek to end
         self._eof = self.f.tell()
-        self._index = lib.get_index(fname, reload=rebuild_index, debug=debug)
+        self.start_pos = self._check_header()
+        self._index = lib.get_index(
+            fname, pos=self.start_pos, eof=self._eof, reload=rebuild_index, debug=debug
+        )
         self._reopen(bufsize)
         self.filehead_config = self._read_filehead_config_string()
         self._ens_pos = self._index["pos"][
@@ -168,6 +171,24 @@ def _check_nortek(self, endian):
                 )
         self.endian = endian
 
+    def _check_header(self):
+        def find_all(s, c):
+            idx = s.find(c)
+            while idx != -1:
+                yield idx
+                idx = s.find(c, idx + 1)
+
+        # Open the entire file
+        self._reopen(self._eof)
+        pk = self.f.peek(1)
+        # Search for multiple saved headers
+        found = [i for i in find_all(pk, b"GETCLOCKSTR")]
+        if len(found) < 2:
+            return 0
+        else:
+            start_idx = found[-1] - 11
+            return start_idx
+
     def _reopen(self, bufsize=None):
         if bufsize is None:
             bufsize = 1000000

mhkit/dolfyn/io/nortek2_lib.py

@@ -107,7 +107,7 @@ def _calc_time(year, month, day, hour, minute, second, usec, zero_is_bad=True):
     return dt
 
 
-def _create_index(infile, outfile, N_ens, debug):
+def _create_index(infile, outfile, init_pos, eof, debug):
     logging = getLogger()
     print("Indexing {}...".format(infile), end="")
     fin = open(_abspath(infile), "rb")
@@ -135,15 +135,21 @@ def _create_index(infile, outfile, N_ens, debug):
         35: 40,
         36: 40,
     }
-    while N[21] < N_ens:  # Will fail if velocity ping isn't saved first
+    pos = 0
+    while pos <= eof:
         pos = fin.tell()
+        if init_pos and not pos:
+            fin.seek(init_pos, 1)
         try:
             dat = _hdr.unpack(fin.read(_hdr.size))
         except:
             break
         if dat[2] in ids:
             idk = dat[2]
             d_ver, d_off, config = struct.unpack("<BBH", fin.read(4))
+            if d_ver not in [1, 3]:
+                # 1 for bottom track, 3 for all others
+                continue
             fin.seek(4, 1)
             yr, mo, dy, h, m, s, u = struct.unpack("6BH", fin.read(8))
             fin.seek(14, 1)
@@ -181,7 +187,7 @@ def _create_index(infile, outfile, N_ens, debug):
             fin.seek(dat[4] - (36 + seek_2ens[idk]), 1)
             last_ens[idk] = ens[idk]
 
-            if debug and N[idk] < 5:
+            if debug:
                 # hex: [18, 15, 1C, 17] = [vel_b5, vel, echo, bt]
                 logging.info(
                     "%10d: %02X, %d, %02X, %d, %d, %d, %d\n"
@@ -197,6 +203,10 @@ def _create_index(infile, outfile, N_ens, debug):
                     )
                 )
         else:
+            if dat[4] < 0:
+                if debug:
+                    logging.info("Invalid skip byte at pos: %10d\n" % (pos))
+                break
             fin.seek(dat[4], 1)
     fin.close()
     fout.close()
@@ -254,7 +264,7 @@ def _boolarray_firstensemble_ping(index):
     return dens
 
 
-def get_index(infile, reload=False, debug=False):
+def get_index(infile, pos=0, eof=2**32, reload=False, debug=False):
     """
     This function reads ad2cp.index files
 
@@ -275,7 +285,7 @@ def get_index(infile, reload=False, debug=False):
 
     index_file = infile + ".index"
     if not path.isfile(index_file) or reload:
-        _create_index(infile, index_file, 2**32, debug)
+        _create_index(infile, index_file, pos, eof, debug)
     f = open(_abspath(index_file), "rb")
     file_head = f.read(12)
     if file_head[:10] == b"Index Ver:":