diff --git a/examples/data/dolfyn/test_data/BenchFile01.nc b/examples/data/dolfyn/test_data/BenchFile01.nc index fc3f4bcdc..57a73e44d 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01.nc and b/examples/data/dolfyn/test_data/BenchFile01.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01.repr.txt b/examples/data/dolfyn/test_data/BenchFile01.repr.txt index 557298fc2..5e5d563c7 100644 --- a/examples/data/dolfyn/test_data/BenchFile01.repr.txt +++ b/examples/data/dolfyn/test_data/BenchFile01.repr.txt @@ -4,6 +4,7 @@ . (100 pings @ 2Hz) Variables: - time ('time',) + - time_altraw ('time_altraw',) - time_b5 ('time_b5',) - vel ('dir', 'range', 'time') - vel_b5 ('range_b5', 'time_b5') @@ -14,6 +15,8 @@ - roll ('time',) - temp ('time',) - pressure ('time',) + - pressure_alt ('time',) + - pressure_altraw ('time_altraw',) - amp ('beam', 'range', 'time') - amp_b5 ('range_b5', 'time_b5') - corr ('beam', 'range', 'time') diff --git a/examples/data/dolfyn/test_data/BenchFile01_avg.nc b/examples/data/dolfyn/test_data/BenchFile01_avg.nc index ddd0e0327..0fa4e236b 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_avg.nc and b/examples/data/dolfyn/test_data/BenchFile01_avg.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_crop.nc b/examples/data/dolfyn/test_data/BenchFile01_crop.nc new file mode 100644 index 000000000..278a872bf Binary files /dev/null and b/examples/data/dolfyn/test_data/BenchFile01_crop.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc index dc10c6ef6..d36b432d1 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc index 5acd192b1..2a1ff37df 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc index 2a8e4b67c..194dee8a2 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc index 790d1e578..de929e2e4 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc and b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU.nc b/examples/data/dolfyn/test_data/Sig1000_IMU.nc index bee45bdb1..397475f51 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_bin.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_bin.nc index 317293a7f..2b0dcbbdc 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_bin.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_bin.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc index 51c45f146..b46762be3 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc index 2edd41f60..6c3337157 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc index 9944aeb7a..f7364c5f0 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc index f0f847653..05663f604 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal.nc b/examples/data/dolfyn/test_data/Sig1000_tidal.nc index f5bf6ba34..759fe4775 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc index f5148a480..8cb9e6678 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo.nc b/examples/data/dolfyn/test_data/Sig500_Echo.nc index e80a279a5..78dbdd06f 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo.nc and b/examples/data/dolfyn/test_data/Sig500_Echo.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc index bf913bfa0..099334d74 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc index ed7fbe512..80d4bca81 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc b/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc index a5e0f5d7d..b44b1d8ca 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc b/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc index 711cdeff1..8e12197df 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc index b9aa117df..f52d8df45 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc and b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc differ diff --git a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc index 2b2f0857c..90b41d3c7 100644 Binary files a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc and b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01.nc b/examples/data/dolfyn/test_data/VelEchoBT01.nc index d7ff5dbcb..f2fb6183e 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01.nc and b/examples/data/dolfyn/test_data/VelEchoBT01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc index 2462311e5..6cefd020c 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc differ diff --git a/mhkit/dolfyn/io/api.py b/mhkit/dolfyn/io/api.py index e540d53d0..23a68a308 100644 --- a/mhkit/dolfyn/io/api.py +++ b/mhkit/dolfyn/io/api.py @@ -76,7 +76,7 @@ def read(fname, userdata=True, nens=None, **kwargs): userdata : True, False, or string of userdata.json filename (default ``True``) Whether to read the '.userdata.json' file. nens : None, int or 2-element tuple (start, stop) - Number of pings or ensembles to read from the file. + Number of pings or ensembles to read from the file. Default is None, read entire file **kwargs : dict Passed to instrument-specific parser. @@ -173,8 +173,8 @@ def save(ds, filename, ds.attrs['complex_vars'] = [] for var in ds.data_vars: if np.iscomplexobj(ds[var]): - ds[var+'_real'] = ds[var].real - ds[var+'_imag'] = ds[var].imag + ds[var + '_real'] = ds[var].real + ds[var + '_imag'] = ds[var].imag ds = ds.drop_vars(var) ds.attrs['complex_vars'].append(var) @@ -183,15 +183,25 @@ def save(ds, filename, elif ds[var].dtype == np.float64: ds[var] = ds[var].astype('float32') - if compression: - enc = dict() - for ky in ds.variables: - enc[ky] = dict(zlib=True, complevel=1) - if 'encoding' in kwargs: - # Overwrite ('update') values in enc with whatever is in kwargs['encoding'] - enc.update(kwargs['encoding']) - else: - kwargs['encoding'] = enc + # Write variable encoding + enc = dict() + if 'encoding' in kwargs: + enc.update(kwargs['encoding']) + for ky in ds.variables: + # Save prior encoding + enc[ky] = ds[ky].encoding + # Remove unexpected netCDF4 encoding parameters + # https://github.com/pydata/xarray/discussions/5709 + params = ['szip', 'zstd', 'bzip2', 'blosc', 'contiguous', 'chunksizes'] + [enc[ky].pop(p) for p in params if p in enc[ky]] + + if compression: + # New netcdf4-c cannot compress variable length strings + if isinstance(ds[ky].data[0], str): + continue + enc[ky].update(dict(zlib=True, complevel=1)) + + kwargs['encoding'] = enc # Fix encoding on datetime64 variables. ds = _decode_cf(ds) @@ -220,19 +230,20 @@ def load(filename): # Convert numpy arrays and strings back to lists for nm in ds.attrs: - if type(ds.attrs[nm]) == np.ndarray and ds.attrs[nm].size > 1: + if isinstance(ds.attrs[nm], np.ndarray) and ds.attrs[nm].size > 1: ds.attrs[nm] = list(ds.attrs[nm]) - elif type(ds.attrs[nm]) == str and nm in ['rotate_vars']: + elif isinstance(ds.attrs[nm], str) and nm in ['rotate_vars']: ds.attrs[nm] = [ds.attrs[nm]] # Rejoin complex numbers - if hasattr(ds, 'complex_vars') and len(ds.complex_vars): - if len(ds.complex_vars[0]) == 1: - ds.attrs['complex_vars'] = [ds.complex_vars] - for var in ds.complex_vars: - ds[var] = ds[var+'_real'] + ds[var+'_imag'] * 1j - ds = ds.drop_vars([var+'_real', var+'_imag']) - ds.attrs.pop('complex_vars') + if hasattr(ds, 'complex_vars'): + if len(ds.complex_vars): + if len(ds.complex_vars[0]) == 1: + ds.attrs['complex_vars'] = [ds.complex_vars] + for var in ds.complex_vars: + ds[var] = ds[var + '_real'] + ds[var + '_imag'] * 1j + ds = ds.drop_vars([var + '_real', var + '_imag']) + ds.attrs.pop('complex_vars') return ds @@ -318,7 +329,7 @@ def load_mat(filename, datenum=True): filename : str Filename and/or path with the '.mat' extension datenum : bool - If true, converts time from datenum. If false, converts time from + If true, converts time from datenum. If false, converts time from "epoch time". Returns @@ -351,12 +362,12 @@ def load_mat(filename, datenum=True): # Convert numpy arrays and strings back to lists for nm in ds.attrs: - if type(ds.attrs[nm]) == np.ndarray and ds.attrs[nm].size > 1: + if isinstance(ds.attrs[nm], np.ndarray) and ds.attrs[nm].size > 1: try: ds.attrs[nm] = [x.strip(' ') for x in list(ds.attrs[nm])] except: ds.attrs[nm] = list(ds.attrs[nm]) - elif type(ds.attrs[nm]) == str and nm in ['time_coords', 'time_data_vars', 'rotate_vars']: + elif isinstance(ds.attrs[nm], str) and nm in ['time_coords', 'time_data_vars', 'rotate_vars']: ds.attrs[nm] = [ds.attrs[nm]] if hasattr(ds, 'orientation_down'): diff --git a/mhkit/dolfyn/io/base.py b/mhkit/dolfyn/io/base.py index 8f3b4469a..897013700 100644 --- a/mhkit/dolfyn/io/base.py +++ b/mhkit/dolfyn/io/base.py @@ -118,7 +118,7 @@ def _create_dataset(data): Direction 'dir' coordinates are set in `set_coords` """ ds = xr.Dataset() - tag = ['_avg', '_b5', '_echo', '_bt', '_gps', '_ast', '_sl'] + tag = ['_avg', '_b5', '_echo', '_bt', '_gps', '_altraw', '_sl'] FoR = {} try: @@ -205,6 +205,11 @@ def _create_dataset(data): 'dim_1': 'time_echo'}) ds[key] = ds[key].assign_coords({'range_echo': data['coords']['range_echo'], 'time_echo': data['coords']['time_echo']}) + elif key == 'samp_altraw': # raw altimeter samples + ds[key] = ds[key].rename({'dim_0': 'n_altraw', + 'dim_1': 'time_altraw'}) + ds[key] = ds[key].assign_coords({'time_altraw': data['coords']['time_altraw']}) + # ADV/ADCP instrument vector data, bottom tracking elif shp[0] == n_beams and not any(val in key for val in tag[:3]): if 'bt' in key and 'time_bt' in data['coords']: diff --git a/mhkit/dolfyn/io/nortek.py b/mhkit/dolfyn/io/nortek.py index 4709df7aa..3a4c4e2f2 100644 --- a/mhkit/dolfyn/io/nortek.py +++ b/mhkit/dolfyn/io/nortek.py @@ -165,8 +165,11 @@ class _NortekReader(): '0x10': 'read_vec_data', '0x11': 'read_vec_sysdata', '0x12': 'read_vec_hdr', - '0x71': 'read_microstrain', '0x20': 'read_awac_profile', + '0x30': 'read_awac_waves', + '0x31': 'read_awac_waves_hdr', + '0x36': 'read_awac_waves', # "SUV" + '0x71': 'read_microstrain', } def __init__(self, fname, endian=None, debug=False, @@ -1026,6 +1029,77 @@ def sci_awac_profile(self,): self.data['attrs']['cell_size'] = cs self.data['attrs']['blank_dist'] = bd + def read_awac_waves_hdr(self,): + # ID: '0x31' + c = self.c + if self.debug: + print('Reading vector header data (0x31) ping #{} @ {}...' + .format(self.c, self.pos)) + hdrnow = {} + dat = self.data + ds = dat['sys'] + dv = dat['data_vars'] + if 'time' not in dat['coords']: + self._init_data(nortek_defs.waves_hdrdata) + byts = self.read(56) + # The first two are size, the next 6 are time. + tmp = unpack(self.endian + '8x4H3h2HhH4B6H5h', byts) + dat['coords']['time'][c] = self.rd_time(byts[2:8]) + hdrnow['n_records_alt'] = tmp[0] + hdrnow['blank_dist_alt'] = tmp[1] # counts + ds['batt_alt'][c] = tmp[2] # voltage (0.1 V) + dv['c_sound_alt'][c] = tmp[3] # c (0.1 m/s) + dv['heading_alt'][c] = tmp[4] # (0.1 deg) + dv['pitch_alt'][c] = tmp[5] # (0.1 deg) + dv['roll_alt'][c] = tmp[6] # (0.1 deg) + dv['pressure1_alt'][c] = tmp[7] # min pressure previous profile (0.001 dbar) + dv['pressure2_alt'][c] = tmp[8] # max pressure previous profile (0.001 dbar) + dv['temp_alt'][c] = tmp[9] # (0.01 deg C) + hdrnow['cell_size_alt'][c] = tmp[10] # (counts of T3) + hdrnow['noise_alt'][c] = tmp[11:15] # noise amplitude beam 1-4 (counts) + hdrnow['proc_magn_alt'][c] = tmp[15:19] # processing magnitude beam 1-4 + hdrnow['n_past_window_alt'] = tmp[19] # number of samples of AST window past boundary + hdrnow['n_window_alt'] = tmp[20] # AST window size (# samples) + hdrnow['Spare1'] = tmp[21:] + self.checksum(byts) + if 'data_header' not in self.config: + self.config['data_header'] = hdrnow + else: + if not isinstance(self.config['data_header'], list): + self.config['data_header'] = [self.config['data_header']] + self.config['data_header'] += [hdrnow] + + def read_awac_waves(self,): + """Read awac wave and suv data + """ + # IDs: 0x30 & 0x36 + c = self.c + dat = self.data + if self.debug: + print('Reading awac wave data (0x30) ping #{} @ {}...' + .format(self.c, self.pos)) + if 'dist1_alt' not in dat['data_vars']: + self._init_data(nortek_defs.wave_data) + self._dtypes += ['wave_data'] + # The first two are size + byts = self.read(20) + ds = dat['sys'] + dv = dat['data_vars'] + (dv['pressure'][c], # (0.001 dbar) + dv['dist1_alt'][c], # distance 1 to surface, vertical beam (mm) + ds['AnaIn_alt'][c], # analog input 1 + dv['vel_alt'][0, c], # velocity beam 1 (mm/s) East for SUV + dv['vel_alt'][1, c], # North for SUV + dv['vel_alt'][2, c], # Up for SUV + dv['dist2_alt'][c], # distance 2 to surface, vertical beam (mm) or vel 4 for non-AST + dv['amp_alt'][0, c], # amplitude beam 1 (counts) + dv['amp_alt'][1, c], # amplitude beam 2 (counts) + dv['amp_alt'][2, c], # amplitude beam 3 (counts) + # AST quality (counts) or amplitude beam 4 for non-AST + dv['quality_alt'][c]) = unpack(self.endian + '3H4h4B', byts) + self.checksum(byts) + self.c += 1 + def dat2sci(self,): for nm in self._dtypes: getattr(self, 'sci_' + nm)() diff --git a/mhkit/dolfyn/io/nortek2.py b/mhkit/dolfyn/io/nortek2.py index fe4e3c9e7..11dd80ee8 100644 --- a/mhkit/dolfyn/io/nortek2.py +++ b/mhkit/dolfyn/io/nortek2.py @@ -26,7 +26,7 @@ def read_signature(filename, userdata=True, nens=None, rebuild_index=False, userdata : bool To search for and use a .userdata.json or not nens : None, int or 2-element tuple (start, stop) - Number of pings or ensembles to read from the file. + Number of pings or ensembles to read from the file. Default is None, read entire file rebuild_index : bool Force rebuild of dolfyn-written datafile index. Useful for code updates. @@ -218,9 +218,14 @@ def _init_burst_readers(self, ): def init_data(self, ens_start, ens_stop): outdat = {} nens = int(ens_stop - ens_start) + + # ID 26 usually only recorded in first ensemble n26 = ((self._index['ID'] == 26) & (self._index['ens'] >= ens_start) & (self._index['ens'] < ens_stop)).sum() + if not n26 and 26 in self._burst_readers: + self._burst_readers.pop(26) + for ky in self._burst_readers: if ky == 26: n = n26 @@ -234,6 +239,7 @@ def init_data(self, ens_start, ens_stop): outdat[ky]['units'] = self._burst_readers[ky].data_units() outdat[ky]['long_name'] = self._burst_readers[ky].data_longnames() outdat[ky]['standard_name'] = self._burst_readers[ky].data_stdnames() + return outdat def _read_hdr(self, do_cs=False): @@ -273,13 +279,16 @@ def readfile(self, ens_start=0, ens_stop=None): except IOError: return outdat id = hdr['id'] - if id in [21, 22, 23, 24, 28]: # vel, bt, vel_b5, echo + if id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), + # "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt), + # "interleaved burst data record" (vel_b5), "echosounder record" (echo) self._read_burst(id, outdat[id], c) - elif id in [26]: # alt_raw (altimeter burst) + elif id in [26]: + # "burst altimeter raw record" (alt_raw) - recorded on nens==0 rdr = self._burst_readers[26] if not hasattr(rdr, '_nsamp_index'): first_pass = True - tmp_idx = rdr._nsamp_index = rdr._names.index('altraw_nsamp') # noqa + tmp_idx = rdr._nsamp_index = rdr._names.index('nsamp_alt') shift = rdr._nsamp_shift = calcsize( defs._format(rdr._format[:tmp_idx], rdr._N[:tmp_idx])) @@ -301,9 +310,9 @@ def readfile(self, ens_start=0, ens_stop=None): rdr._cs_struct = defs.Struct( '<' + '{}H'.format(int(rdr.nbyte // 2))) # Initialize the array - outdat[26]['altraw_samp'] = defs._nans( + outdat[26]['samp_alt'] = defs._nans( [rdr._N[tmp_idx], - len(outdat[26]['altraw_samp'])], + len(outdat[26]['samp_alt'])], dtype=np.uint16) else: if sz != rdr._N[tmp_idx]: @@ -314,8 +323,9 @@ def readfile(self, ens_start=0, ens_stop=None): outdat[id]['ensemble'][c26] = c c26 += 1 - elif id in [27, 29, 30, 31, 35, 36]: # bt record, DVL, - # alt record, avg alt_raw record, raw echo, raw echo transmit + elif id in [27, 29, 30, 31, 35, 36]: # unknown how to handle + # "bottom track record", DVL, "altimeter record", "avg altimeter raw record", + # "raw echosounder data record", "raw echosounder transmit data record" if self.debug: logging.debug( "Skipped ID: 0x{:02X} ({:02d})\n".format(id, id)) @@ -401,8 +411,8 @@ def _reorg(dat): cfg['inst_make'] = 'Nortek' cfg['inst_type'] = 'ADCP' - for id, tag in [(21, ''), (22, '_avg'), (23, '_bt'), - (24, '_b5'), (26, '_ast'), (28, '_echo')]: + for id, tag in [(21, ''), (22, '_avg'), (23, '_bt'), + (24, '_b5'), (26, 'raw'), (28, '_echo')]: if id in [24, 26]: collapse_exclude = [0] else: @@ -454,10 +464,10 @@ def _reorg(dat): outdat['standard_name'][ky + tag] = 'number_of_observations' for ky in ['vel', 'amp', 'corr', 'prcnt_gd', 'echo', 'dist', - 'orientmat', 'angrt', 'quaternions', 'ast_pressure', - 'alt_dist', 'alt_quality', 'alt_status', - 'ast_dist', 'ast_quality', 'ast_offset_time', - 'altraw_nsamp', 'altraw_dsamp', 'altraw_samp', + 'orientmat', 'angrt', 'quaternions', 'pressure_alt', + 'le_dist_alt', 'le_quality_alt', 'status_alt', + 'ast_dist_alt', 'ast_quality_alt', 'ast_offset_time_alt', + 'nsamp_alt', 'dsamp_alt', 'samp_alt', 'status0', 'fom', 'temp_press', 'press_std', 'pitch_std', 'roll_std', 'heading_std', 'xmit_energy', ]: @@ -466,20 +476,19 @@ def _reorg(dat): # Move 'altimeter raw' data to its own down-sampled structure if 26 in dat: - ard = outdat['altraw'] for ky in list(outdat['data_vars']): - if ky.endswith('_ast'): - grp = ky.split('.')[0] - if '.' in ky and grp not in ard: - ard[grp] = {} - ard[ky.rstrip('_ast')] = outdat['data_vars'].pop(ky) + if ky.endswith('raw') and not ky.endswith('_altraw'): + outdat['data_vars'].pop(ky) + outdat['coords']['time_altraw'] = outdat['coords'].pop('timeraw') + outdat['data_vars']['samp_altraw'] = outdat['data_vars']['samp_altraw'].astype('float32') / 2**8 # convert "signed fractional" to float # Read altimeter status - alt_status = lib._alt_status2data(outdat['data_vars']['alt_status']) - for ky in alt_status: + outdat['data_vars'].pop('status_altraw') + status_alt = lib._alt_status2data(outdat['data_vars']['status_alt']) + for ky in status_alt: outdat['attrs'][ky] = lib._collapse( - alt_status[ky].astype('uint8'), name=ky) - outdat['data_vars'].pop('alt_status') + status_alt[ky].astype('uint8'), name=ky) + outdat['data_vars'].pop('status_alt') # Power level index power = {0: 'high', 1: 'med-high', 2: 'med-low', 3: 'low'} @@ -555,7 +564,7 @@ def _reduce(data): --- from different data structures within the same ensemble --- by averaging. """ - + dv = data['data_vars'] dc = data['coords'] da = data['attrs'] @@ -571,14 +580,14 @@ def _reduce(data): if 'vel' in dv: dc['range'] = ((np.arange(dv['vel'].shape[1])+1) * - da['cell_size'] + - da['blank_dist']) + da['cell_size'] + + da['blank_dist']) da['fs'] = da['filehead_config']['BURST']['SR'] tmat = da['filehead_config']['XFBURST'] if 'vel_avg' in dv: dc['range_avg'] = ((np.arange(dv['vel_avg'].shape[1])+1) * - da['cell_size_avg'] + - da['blank_dist_avg']) + da['cell_size_avg'] + + da['blank_dist_avg']) dv['orientmat'] = dv.pop('orientmat_avg') tmat = da['filehead_config']['XFAVG'] da['fs'] = da['filehead_config']['PLAN']['MIAVG'] @@ -605,7 +614,7 @@ def _reduce(data): theta = da['filehead_config']['BEAMCFGLIST'][0] if 'THETA=' in theta: da['beam_angle'] = int(theta[13:15]) - + tm = np.zeros((tmat['ROWS'], tmat['COLS']), dtype=np.float32) for irow in range(tmat['ROWS']): for icol in range(tmat['COLS']): diff --git a/mhkit/dolfyn/io/nortek2_defs.py b/mhkit/dolfyn/io/nortek2_defs.py index 6b9b1d8f2..8ec776b38 100644 --- a/mhkit/dolfyn/io/nortek2_defs.py +++ b/mhkit/dolfyn/io/nortek2_defs.py @@ -296,11 +296,11 @@ def _calc_echo_struct(config, nc): flags = lib._headconfig_int2dict(config) dd = copy(_burst_hdr) dd[19] = ('blank_dist', 'H', [], _LinFunc(0.001)) # m - if any([flags[nm] for nm in ['vel', 'amp', 'corr', 'alt', 'ast', - 'alt_raw', 'p_gd', 'std']]): + if any([flags[nm] for nm in ['vel', 'amp', 'corr', 'le', 'ast', + 'altraw', 'p_gd', 'std']]): raise Exception("Echosounder ping contains invalid data?") if flags['echo']: - dd += [('echo', 'H', [nc], _LinFunc(0.01, dtype=dt32), 'dB', + dd += [('echo', 'H', [nc], _LinFunc(0.01, dtype=dt32), 'dB', 'Echo Sounder Acoustic Signal Backscatter', 'acoustic_target_strength_in_sea_water')] if flags['ahrs']: dd += _ahrs_def @@ -320,29 +320,33 @@ def _calc_burst_struct(config, nb, nc): if flags['corr']: dd.append(('corr', 'B', [nb, nc], None, '%', 'Acoustic Signal Correlation', 'beam_consistency_indicator_from_multibeam_acoustic_doppler_velocity_profiler_in_sea_water')) - if flags['alt']: + if flags['le']: # There may be a problem here with reading 32bit floats if # nb and nc are odd - dd += [('alt_dist', 'f', [], _LinFunc(dtype=dt32), 'm', 'Altimeter Range', 'altimeter_range'), - ('alt_quality', 'H', [], _LinFunc(0.01, dtype=dt32), '1', 'Altimeter Quality Indicator'), - ('alt_status', 'H', [], None, '1', 'Altimeter Status')] + dd += [('le_dist_alt', 'f', [], _LinFunc(dtype=dt32), 'm', 'Altimeter Range Leading Edge Algorithm', + 'altimeter_range'), + ('le_quality_alt', 'H', [], _LinFunc(0.01, dtype=dt32), 'dB', + 'Altimeter Quality Indicator Leading Edge Algorithm'), + ('status_alt', 'H', [], None, '1', 'Altimeter Status')] if flags['ast']: dd += [ - ('ast_dist', 'f', [], _LinFunc(dtype=dt32), 'm', 'Acoustic Surface Tracking Range'), - ('ast_quality', 'H', [], _LinFunc(0.01, dtype=dt32), '1', - 'Acoustic Surface Tracking Quality Indicator'), - ('ast_offset_time', 'h', [], _LinFunc(0.0001, dtype=dt32), + ('ast_dist_alt', 'f', [], _LinFunc(dtype=dt32), 'm', 'Altimeter Range Acoustic Surface Tracking', + 'altimeter_range'), + ('ast_quality_alt', 'H', [], _LinFunc(0.01, dtype=dt32), 'dB', + 'Altimeter Quality Indicator Acoustic Surface Tracking'), + ('ast_offset_time_alt', 'h', [], _LinFunc(0.0001, dtype=dt32), 's', 'Acoustic Surface Tracking Time Offset to Velocity Ping'), - ('ast_pressure', 'f', [], None, 'dbar', 'Pressure measured during AST ping', + ('pressure_alt', 'f', [], None, 'dbar', 'Pressure measured during AST ping', 'sea_water_pressure'), - ('ast_spare', 'B7x', [], None), + # This use of 'x' here is a hack + ('spare', 'B7x', [], None), ] - if flags['alt_raw']: + if flags['altraw']: dd += [ - ('altraw_nsamp', 'I', [], None, '1', 'Number of Altimeter Samples'), - ('altraw_dsamp', 'H', [], _LinFunc(0.0001, dtype=dt32), 'm', + ('nsamp_alt', 'I', [], None, '1', 'Number of Altimeter Samples'), + ('dsamp_alt', 'H', [], _LinFunc(0.0001, dtype=dt32), 'm', 'Altimeter Distance between Samples'), - ('altraw_samp', 'h', [], None), + ('samp_alt', 'h', [], None, '1', 'Altimeter Samples'), ] if flags['ahrs']: dd += _ahrs_def diff --git a/mhkit/dolfyn/io/nortek2_lib.py b/mhkit/dolfyn/io/nortek2_lib.py index f3575b6e6..d5ee385f3 100644 --- a/mhkit/dolfyn/io/nortek2_lib.py +++ b/mhkit/dolfyn/io/nortek2_lib.py @@ -101,8 +101,8 @@ def _create_index(infile, outfile, N_ens, debug): fout = open(_abspath(outfile), 'wb') fout.write(b'Index Ver:') fout.write(struct.pack('