From 919acad8397a890f8c66652a255363d2cc4b1d8c Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Tue, 14 Jun 2022 22:45:16 -0700 Subject: [PATCH 01/37] Move varaibles from beam to vendor --- echopype/convert/set_groups_ad2cp.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 525251e6e..4587a5c68 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -279,17 +279,6 @@ def set_beam(self) -> xr.Dataset: "correlation_data_average": "correlation_average", "correlation_data_echosounder": "correlation_echosounder", "echosounder_data": "amplitude_echosounder", - "figure_of_merit_data": "figure_of_merit", - "altimeter_distance": "altimeter_distance", - "altimeter_quality": "altimeter_quality", - "ast_distance": "ast_distance", - "ast_quality": "ast_quality", - "ast_offset_100us": "ast_offset_100us", - "ast_pressure": "ast_pressure", - "altimeter_spare": "altimeter_spare", - "altimeter_raw_data_num_samples": "altimeter_raw_data_num_samples", - "altimeter_raw_data_sample_distance": "altimeter_raw_data_sample_distance", - "altimeter_raw_data_samples": "altimeter_raw_data_samples", } ) ds = ds.assign_attrs({"pulse_compressed": self.pulse_compressed}) @@ -335,6 +324,17 @@ def set_vendor(self) -> xr.Dataset: "echosounder_raw_transmit_samples_q": "echosounder_raw_transmit_samples_q", "echosounder_raw_beam": "echosounder_raw_beam", "echosounder_raw_echogram": "echosounder_raw_echogram", + "figure_of_merit_data": "figure_of_merit", + "altimeter_distance": "altimeter_distance", + "altimeter_quality": "altimeter_quality", + "ast_distance": "ast_distance", + "ast_quality": "ast_quality", + "ast_offset_100us": "ast_offset_100us", + "ast_pressure": "ast_pressure", + "altimeter_spare": "altimeter_spare", + "altimeter_raw_data_num_samples": "altimeter_raw_data_num_samples", + "altimeter_raw_data_sample_distance": "altimeter_raw_data_sample_distance", + "altimeter_raw_data_samples": "altimeter_raw_data_samples", } ) ds = ds.assign_attrs({"pulse_compressed": self.pulse_compressed}) From 7b0657fb522bc9e05cf6f6ef29314a8c9e570fe5 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Tue, 14 Jun 2022 23:04:56 -0700 Subject: [PATCH 02/37] Move variables from vendor to beam --- echopype/convert/set_groups_ad2cp.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 4587a5c68..4f0760f64 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -279,6 +279,12 @@ def set_beam(self) -> xr.Dataset: "correlation_data_average": "correlation_average", "correlation_data_echosounder": "correlation_echosounder", "echosounder_data": "amplitude_echosounder", + "echosounder_raw_samples_i": "backscatter_r", + "echosounder_raw_samples_q": "backscatter_i", + "echosounder_raw_transmit_samples_i": "transmit_pulse_r", + "echosounder_raw_transmit_samples_q": "transmit_pulse_i", + "echosounder_raw_beam": "echosounder_raw_beam", + "echosounder_raw_echogram": "echosounder_raw_echogram", } ) ds = ds.assign_attrs({"pulse_compressed": self.pulse_compressed}) @@ -318,12 +324,6 @@ def set_vendor(self) -> xr.Dataset: "temperature_sensor_valid": "temperature_sensor_valid", "compass_sensor_valid": "compass_sensor_valid", "tilt_sensor_valid": "tilt_sensor_valid", - "echosounder_raw_samples_i": "echosounder_raw_samples_i", - "echosounder_raw_samples_q": "echosounder_raw_samples_q", - "echosounder_raw_transmit_samples_i": "echosounder_raw_transmit_samples_i", - "echosounder_raw_transmit_samples_q": "echosounder_raw_transmit_samples_q", - "echosounder_raw_beam": "echosounder_raw_beam", - "echosounder_raw_echogram": "echosounder_raw_echogram", "figure_of_merit_data": "figure_of_merit", "altimeter_distance": "altimeter_distance", "altimeter_quality": "altimeter_quality", @@ -337,7 +337,6 @@ def set_vendor(self) -> xr.Dataset: "altimeter_raw_data_samples": "altimeter_raw_data_samples", } ) - ds = ds.assign_attrs({"pulse_compressed": self.pulse_compressed}) # FIXME: this is a hack because the current file saving # mechanism requires that the vendor group have ping_time as a dimension, From e44ede0f978c8a654d6c5c13165a258db57203ac Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Tue, 14 Jun 2022 23:05:06 -0700 Subject: [PATCH 03/37] Separate variables into multiple beam groups --- echopype/convert/set_groups_ad2cp.py | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 4f0760f64..0a715c7f7 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -258,6 +258,102 @@ def set_beam(self) -> xr.Dataset: # TODO: should we divide beam into burst/average (e.g., beam_burst, beam_average) # like was done for range_bin (we have range_bin_burst, range_bin_average, # and range_bin_echosounder)? + beam_groups = [] + + # burst + for packet in self.parser_obj.packets: + if packet.is_burst(): + beam_groups.append( + self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "velocity_data_burst": "velocity_burst", + "amplitude_data_burst": "amplitude_burst", + "correlation_data_burst": "correlation_burst", + } + ) + ) + break + # average + for packet in self.parser_obj.packets: + if packet.is_average(): + beam_groups.append( + self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "velocity_data_average": "velocity_average", + "amplitude_data_average": "amplitude_average", + "correlation_data_average": "correlation_average", + } + ) + ) + break + # echosounder + for packet in self.parser_obj.packets: + if packet.is_echosounder(): + self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "correlation_data_echosounder": "correlation_echosounder", + "echosounder_data": "amplitude_echosounder", + } + ) + # echosounder raw + for packet in self.parser_obj.packets: + if packet.is_echosounder_raw(): + self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "echosounder_raw_samples_i": "backscatter_r", + "echosounder_raw_samples_q": "backscatter_i", + "echosounder_raw_transmit_samples_i": "transmit_pulse_r", + "echosounder_raw_transmit_samples_q": "transmit_pulse_i", + "echosounder_raw_beam": "echosounder_raw_beam", + "echosounder_raw_echogram": "echosounder_raw_echogram", + } + ) + break + ds = self._make_dataset( { "num_beams": "number_of_beams", From 146bddf732b53f261df246976a18c0cb293fc9de Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Tue, 14 Jun 2022 23:18:16 -0700 Subject: [PATCH 04/37] Fix beam group separation, add pulse_compression vector --- echopype/convert/set_groups_ad2cp.py | 106 +++++++++++---------------- 1 file changed, 42 insertions(+), 64 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 0a715c7f7..06b636b22 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -254,7 +254,7 @@ def set_platform(self) -> xr.Dataset: ) return set_encodings(ds) - def set_beam(self) -> xr.Dataset: + def set_beam(self) -> List[xr.Dataset]: # TODO: should we divide beam into burst/average (e.g., beam_burst, beam_average) # like was done for range_bin (we have range_bin_burst, range_bin_average, # and range_bin_echosounder)? @@ -277,9 +277,9 @@ def set_beam(self) -> xr.Dataset: "dataset_description": "data_set_description", "transmit_energy": "transmit_energy", "velocity_scaling": "velocity_scaling", - "velocity_data_burst": "velocity_burst", - "amplitude_data_burst": "amplitude_burst", - "correlation_data_burst": "correlation_burst", + "velocity_data_burst": "velocity", + "amplitude_data_burst": "amplitude", + "correlation_data_burst": "correlation", } ) ) @@ -301,9 +301,9 @@ def set_beam(self) -> xr.Dataset: "dataset_description": "data_set_description", "transmit_energy": "transmit_energy", "velocity_scaling": "velocity_scaling", - "velocity_data_average": "velocity_average", - "amplitude_data_average": "amplitude_average", - "correlation_data_average": "correlation_average", + "velocity_data_average": "velocity", + "amplitude_data_average": "amplitude", + "correlation_data_average": "correlation", } ) ) @@ -311,7 +311,7 @@ def set_beam(self) -> xr.Dataset: # echosounder for packet in self.parser_obj.packets: if packet.is_echosounder(): - self._make_dataset( + ds = self._make_dataset( { "num_beams": "number_of_beams", "coordinate_system": "coordinate_system", @@ -324,75 +324,53 @@ def set_beam(self) -> xr.Dataset: "dataset_description": "data_set_description", "transmit_energy": "transmit_energy", "velocity_scaling": "velocity_scaling", - "correlation_data_echosounder": "correlation_echosounder", - "echosounder_data": "amplitude_echosounder", + "correlation_data_echosounder": "correlation", + "echosounder_data": "amplitude", } ) + ds = ds.assign_coords({"echogram": np.arange(3)}) + pulse_compressed = np.zeros(3) + pulse_compressed[self.pulse_compressed - 1] = 1 + ds["pulse_compressed"] = (("echogram",), pulse_compressed) + beam_groups.append(ds) + break # echosounder raw for packet in self.parser_obj.packets: if packet.is_echosounder_raw(): - self._make_dataset( - { - "num_beams": "number_of_beams", - "coordinate_system": "coordinate_system", - "num_cells": "number_of_cells", - "blanking": "blanking", - "cell_size": "cell_size", - "velocity_range": "velocity_range", - "echosounder_frequency": "echosounder_frequency", - "ambiguity_velocity": "ambiguity_velocity", - "dataset_description": "data_set_description", - "transmit_energy": "transmit_energy", - "velocity_scaling": "velocity_scaling", - "echosounder_raw_samples_i": "backscatter_r", - "echosounder_raw_samples_q": "backscatter_i", - "echosounder_raw_transmit_samples_i": "transmit_pulse_r", - "echosounder_raw_transmit_samples_q": "transmit_pulse_i", - "echosounder_raw_beam": "echosounder_raw_beam", - "echosounder_raw_echogram": "echosounder_raw_echogram", - } + beam_groups.append( + self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "echosounder_raw_samples_i": "backscatter_r", + "echosounder_raw_samples_q": "backscatter_i", + "echosounder_raw_transmit_samples_i": "transmit_pulse_r", + "echosounder_raw_transmit_samples_q": "transmit_pulse_i", + "echosounder_raw_beam": "echosounder_raw_beam", + "echosounder_raw_echogram": "echosounder_raw_echogram", + } + ) ) break - ds = self._make_dataset( - { - "num_beams": "number_of_beams", - "coordinate_system": "coordinate_system", - "num_cells": "number_of_cells", - "blanking": "blanking", - "cell_size": "cell_size", - "velocity_range": "velocity_range", - "echosounder_frequency": "echosounder_frequency", - "ambiguity_velocity": "ambiguity_velocity", - "dataset_description": "data_set_description", - "transmit_energy": "transmit_energy", - "velocity_scaling": "velocity_scaling", - "velocity_data_burst": "velocity_burst", - "velocity_data_average": "velocity_average", - "amplitude_data_burst": "amplitude_burst", - "amplitude_data_average": "amplitude_average", - "correlation_data_burst": "correlation_burst", - "correlation_data_average": "correlation_average", - "correlation_data_echosounder": "correlation_echosounder", - "echosounder_data": "amplitude_echosounder", - "echosounder_raw_samples_i": "backscatter_r", - "echosounder_raw_samples_q": "backscatter_i", - "echosounder_raw_transmit_samples_i": "transmit_pulse_r", - "echosounder_raw_transmit_samples_q": "transmit_pulse_i", - "echosounder_raw_beam": "echosounder_raw_beam", - "echosounder_raw_echogram": "echosounder_raw_echogram", - } - ) - ds = ds.assign_attrs({"pulse_compressed": self.pulse_compressed}) - # FIXME: this is a hack because the current file saving # mechanism requires that the beam group have ping_time as a dimension, # but ping_time might not be a dimension if the dataset is completely # empty - if "ping_time" not in ds.dims: - ds = ds.expand_dims(dim="ping_time") + for i, ds in enumerate(beam_groups): + if "ping_time" not in ds.dims: + beam_groups[i] = ds.expand_dims(dim="ping_time") - return set_encodings(ds) + return [set_encodings(ds) for ds in beam_groups] def set_vendor(self) -> xr.Dataset: ds = self._make_dataset( From dc3dffa01ca5ceddd5884c760b6276b2b9ef1695 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Wed, 15 Jun 2022 21:49:29 -0700 Subject: [PATCH 05/37] Rename range_sample_X to range_sample, rename sample to range_sample, rename sample_transmit to transmit_sample --- echopype/convert/parse_ad2cp.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 91a46384c..1e5f4e8fa 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -94,7 +94,6 @@ def default_dtype() -> np.dtype: } -@unique class Dimension(Enum): """ Determines the dimensions of the data in the output dataset @@ -107,12 +106,12 @@ class Dimension(Enum): PING_TIME_ECHOSOUNDER_RAW = "ping_time_echosounder_raw" PING_TIME_ECHOSOUNDER_RAW_TRANSMIT = "ping_time_echosounder_raw_transmit" BEAM = "beam" - RANGE_SAMPLE_BURST = "range_sample_burst" - RANGE_SAMPLE_AVERAGE = "range_sample_average" - RANGE_SAMPLE_ECHOSOUNDER = "range_sample_echosounder" + RANGE_SAMPLE_BURST = "range_sample" + RANGE_SAMPLE_AVERAGE = "range_sample" + RANGE_SAMPLE_ECHOSOUNDER = "range_sample" NUM_ALTIMETER_SAMPLES = "num_altimeter_samples" - SAMPLE = "sample" - SAMPLE_TRANSMIT = "sample_transmit" + SAMPLE = "range_sample" + SAMPLE_TRANSMIT = "transmit_sample" MIJ = "mij" XYZ = "xyz" WXYZ = "wxyz" From 83db5596c2a02119c1e45d14f14d4d0907c75798 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Wed, 15 Jun 2022 22:08:17 -0700 Subject: [PATCH 06/37] Rename ping_time to time1, rename ping_time_X to ping_time --- echopype/convert/parse_ad2cp.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 1e5f4e8fa..39e051298 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -99,12 +99,12 @@ class Dimension(Enum): Determines the dimensions of the data in the output dataset """ - PING_TIME = "ping_time" - PING_TIME_AVERAGE = "ping_time_average" - PING_TIME_BURST = "ping_time_burst" - PING_TIME_ECHOSOUNDER = "ping_time_echosounder" - PING_TIME_ECHOSOUNDER_RAW = "ping_time_echosounder_raw" - PING_TIME_ECHOSOUNDER_RAW_TRANSMIT = "ping_time_echosounder_raw_transmit" + PING_TIME = "time1" + PING_TIME_AVERAGE = "ping_time" + PING_TIME_BURST = "ping_time" + PING_TIME_ECHOSOUNDER = "ping_time" + PING_TIME_ECHOSOUNDER_RAW = "ping_time" + PING_TIME_ECHOSOUNDER_RAW_TRANSMIT = "ping_time" BEAM = "beam" RANGE_SAMPLE_BURST = "range_sample" RANGE_SAMPLE_AVERAGE = "range_sample" From fc95e5db38aba0ef2b68357a1ce5a4a8472f4c6a Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Wed, 15 Jun 2022 22:09:57 -0700 Subject: [PATCH 07/37] Move magnetometer_raw from platform to vendor --- echopype/convert/set_groups_ad2cp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 06b636b22..b4d21d87e 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -242,7 +242,6 @@ def set_platform(self) -> xr.Dataset: "heading": "heading", "pitch": "pitch", "roll": "roll", - "magnetometer_raw": "magnetometer_raw", } ) ds = ds.assign_attrs( @@ -409,6 +408,7 @@ def set_vendor(self) -> xr.Dataset: "altimeter_raw_data_num_samples": "altimeter_raw_data_num_samples", "altimeter_raw_data_sample_distance": "altimeter_raw_data_sample_distance", "altimeter_raw_data_samples": "altimeter_raw_data_samples", + "magnetometer_raw": "magnetometer_raw", } ) From 917168c68363727d613fb9d2678cd213db17398f Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Thu, 16 Jun 2022 09:46:01 -0700 Subject: [PATCH 08/37] Add self._beamgroups with beam group descriptions --- echopype/convert/set_groups_ad2cp.py | 52 ++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index b4d21d87e..9f1a35570 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -17,16 +17,6 @@ class SetGroupsAd2cp(SetGroupsBase): """Class for saving groups to netcdf or zarr from Ad2cp data files.""" - beamgroups_possible = [ - { - "name": "Beam_group1", - "descr": ( - "contains velocity, correlation, and backscatter power (uncalibrated)" - " data and other data derived from acoustic data." - ), - } - ] - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.pulse_compressed = self.parser_obj.get_pulse_compressed() @@ -258,6 +248,7 @@ def set_beam(self) -> List[xr.Dataset]: # like was done for range_bin (we have range_bin_burst, range_bin_average, # and range_bin_echosounder)? beam_groups = [] + self._beamgroups = [] # burst for packet in self.parser_obj.packets: @@ -282,6 +273,16 @@ def set_beam(self) -> List[xr.Dataset]: } ) ) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains echo intensity, velocity and correlation data " + "as well as other configuration parameters from the Burst mode." + ), + } + ) break # average for packet in self.parser_obj.packets: @@ -306,6 +307,16 @@ def set_beam(self) -> List[xr.Dataset]: } ) ) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains echo intensity, velocity and correlation data " + "as well as other configuration parameters from the Average mode." + ), + } + ) break # echosounder for packet in self.parser_obj.packets: @@ -332,6 +343,16 @@ def set_beam(self) -> List[xr.Dataset]: pulse_compressed[self.pulse_compressed - 1] = 1 ds["pulse_compressed"] = (("echogram",), pulse_compressed) beam_groups.append(ds) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains backscatter echo intensity and other configuration parameters from the Echosounder mode. " + "Data can be pulse compressed or raw intensity." + ), + } + ) break # echosounder raw for packet in self.parser_obj.packets: @@ -359,6 +380,16 @@ def set_beam(self) -> List[xr.Dataset]: } ) ) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains complex backscatter raw samples and other configuration parameters from the Echosounder mode, " + "including complex data from the transmit pulse." + ), + } + ) break # FIXME: this is a hack because the current file saving @@ -426,7 +457,6 @@ def set_sonar(self) -> xr.Dataset: # Add beam_group and beam_group_descr variables sharing a common dimension # (beam_group), using the information from self._beamgroups - self._beamgroups = self.beamgroups_possible beam_groups_vars, beam_groups_coord = self._beam_groups_vars() ds = xr.Dataset(beam_groups_vars, coords=beam_groups_coord) From 4c0eba877f65246e647b7daff6261e155bf3a4c7 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 16:49:32 -0700 Subject: [PATCH 09/37] Add auto-generated yaml --- echopype/convert/ad2cp_fields.yaml | 820 +++++++++++++++++++++++++++++ 1 file changed, 820 insertions(+) create mode 100644 echopype/convert/ad2cp_fields.yaml diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml new file mode 100644 index 000000000..aff9d7db9 --- /dev/null +++ b/echopype/convert/ad2cp_fields.yaml @@ -0,0 +1,820 @@ +BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: + version: + long_name: Field "Version" in the data specification. + units: '' + comment: '' + offset_of_data: + long_name: Field "Offset Of Data" in the data specification. + units: '# of bytes' + comment: ' + + Data was converted to the specified units.' + serial_number: + long_name: Field "Serial Number" in the data specification. + units: '' + comment: '' + configuration: + long_name: Field "Configuration" in the data specification. + units: '' + comment: '' + year: + long_name: Field "Year" in the data specification. + units: '' + comment: '' + month: + long_name: Field "Month" in the data specification. + units: '' + comment: '' + day: + long_name: Field "Day" in the data specification. + units: '' + comment: '' + hour: + long_name: Field "Hour" in the data specification. + units: '' + comment: '' + minute: + long_name: Field "Minute" in the data specification. + units: '' + comment: '' + seconds: + long_name: Field "Seconds" in the data specification. + units: '' + comment: '' + microsec100: + long_name: Field "Microsec100" in the data specification. + units: '' + comment: '' + speed_of_sound: + long_name: Field "Speed Of Sound" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + temperature: + long_name: Field "Temperature" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + pressure: + long_name: Field "Pressure" in the data specification. + units: dBar + comment: ' + + Data was converted to the specified units.' + heading: + long_name: Field "Heading" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + pitch: + long_name: Field "Pitch" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + roll: + long_name: Field "Roll" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + error: + long_name: Field "Error" in the data specification. + units: '' + comment: '' + status: + long_name: Field "Status" in the data specification. + units: '' + comment: '' + num_beams_and_coordinate_system_and_num_cells: + long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + units: '' + comment: '' + cell_size: + long_name: Field "Cell Size" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + blanking: + long_name: Field "Blanking" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + velocity_range: + long_name: Field "Velocity Range" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + battery_voltage: + long_name: Field "Battery Voltage" in the data specification. + units: V + comment: ' + + Data was converted to the specified units.' + magnetometer_raw: + long_name: Field "Magnetometer Raw" in the data specification. + units: '' + comment: '' + accelerometer_raw_x_axis: + long_name: Field "Accelerometer Raw X Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_y_axis: + long_name: Field "Accelerometer Raw Y Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_z_axis: + long_name: Field "Accelerometer Raw Z Axis" in the data specification. + units: '' + comment: '' + ambiguity_velocity: + long_name: Field "Ambiguity Velocity" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + dataset_description: + long_name: Field "Dataset Description" in the data specification. + units: '' + comment: '' + transmit_energy: + long_name: Field "Transmit Energy" in the data specification. + units: '' + comment: '' + velocity_scaling: + long_name: Field "Velocity Scaling" in the data specification. + units: '' + comment: '' + power_level: + long_name: Field "Power Level" in the data specification. + units: dB + comment: ' + + Data was converted to the specified units.' + velocity_data_burst: + long_name: Field "Velocity Data Burst" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + velocity_data_average: + long_name: Field "Velocity Data Average" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + velocity_data_echosounder: + long_name: Field "Velocity Data Echosounder" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + amplitude_data_burst: + long_name: Field "Amplitude Data Burst" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + amplitude_data_average: + long_name: Field "Amplitude Data Average" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + amplitude_data_echosounder: + long_name: Field "Amplitude Data Echosounder" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + correlation_data_burst: + long_name: Field "Correlation Data Burst" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' + correlation_data_average: + long_name: Field "Correlation Data Average" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' + correlation_data_echosounder: + long_name: Field "Correlation Data Echosounder" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' +BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: + version: + long_name: Field "Version" in the data specification. + units: '' + comment: '' + offset_of_data: + long_name: Field "Offset Of Data" in the data specification. + units: '# of bytes' + comment: ' + + Data was converted to the specified units.' + configuration: + long_name: Field "Configuration" in the data specification. + units: '' + comment: '' + serial_number: + long_name: Field "Serial Number" in the data specification. + units: '' + comment: '' + year: + long_name: Field "Year" in the data specification. + units: '' + comment: '' + month: + long_name: Field "Month" in the data specification. + units: '' + comment: '' + day: + long_name: Field "Day" in the data specification. + units: '' + comment: '' + hour: + long_name: Field "Hour" in the data specification. + units: '' + comment: '' + minute: + long_name: Field "Minute" in the data specification. + units: '' + comment: '' + seconds: + long_name: Field "Seconds" in the data specification. + units: '' + comment: '' + microsec100: + long_name: Field "Microsec100" in the data specification. + units: '' + comment: '' + speed_of_sound: + long_name: Field "Speed Of Sound" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + temperature: + long_name: Field "Temperature" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + pressure: + long_name: Field "Pressure" in the data specification. + units: dBar + comment: ' + + Data was converted to the specified units.' + heading: + long_name: Field "Heading" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + pitch: + long_name: Field "Pitch" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + roll: + long_name: Field "Roll" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + num_beams_and_coordinate_system_and_num_cells: + long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + units: '' + comment: '' + cell_size: + long_name: Field "Cell Size" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + blanking: + long_name: Field "Blanking" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + nominal_correlation: + long_name: Field "Nominal Correlation" in the data specification. + units: '%' + comment: ' + + Data was converted to the specified units.' + temperature_from_pressure_sensor: + long_name: Field "Temperature From Pressure Sensor" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + battery_voltage: + long_name: Field "Battery Voltage" in the data specification. + units: V + comment: ' + + Data was converted to the specified units.' + magnetometer_raw: + long_name: Field "Magnetometer Raw" in the data specification. + units: '' + comment: '' + accelerometer_raw_x_axis: + long_name: Field "Accelerometer Raw X Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_y_axis: + long_name: Field "Accelerometer Raw Y Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_z_axis: + long_name: Field "Accelerometer Raw Z Axis" in the data specification. + units: '' + comment: '' + ambiguity_velocity_or_echosounder_frequency: + long_name: Field "Ambiguity Velocity Or Echosounder Frequency" in the data specification. + units: '' + comment: '' + dataset_description: + long_name: Field "Dataset Description" in the data specification. + units: '' + comment: '' + transmit_energy: + long_name: Field "Transmit Energy" in the data specification. + units: '' + comment: '' + velocity_scaling: + long_name: Field "Velocity Scaling" in the data specification. + units: '' + comment: '' + power_level: + long_name: Field "Power Level" in the data specification. + units: dB + comment: ' + + Data was converted to the specified units.' + magnetometer_temperature: + long_name: Field "Magnetometer Temperature" in the data specification. + units: '' + comment: '' + real_time_clock_temperature: + long_name: Field "Real Time Clock Temperature" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + error: + long_name: Field "Error" in the data specification. + units: '' + comment: '' + status0: + long_name: Field "Status0" in the data specification. + units: '' + comment: '' + status: + long_name: Field "Status" in the data specification. + units: '' + comment: '' + ensemble_counter: + long_name: Field "Ensemble Counter" in the data specification. + units: '' + comment: '' + velocity_data_burst: + long_name: Field "Velocity Data Burst" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + velocity_data_average: + long_name: Field "Velocity Data Average" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + velocity_data_echosounder: + long_name: Field "Velocity Data Echosounder" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + amplitude_data_burst: + long_name: Field "Amplitude Data Burst" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + amplitude_data_average: + long_name: Field "Amplitude Data Average" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + amplitude_data_echosounder: + long_name: Field "Amplitude Data Echosounder" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + correlation_data_burst: + long_name: Field "Correlation Data Burst" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' + correlation_data_average: + long_name: Field "Correlation Data Average" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' + correlation_data_echosounder: + long_name: Field "Correlation Data Echosounder" in the data specification. + units: 0-100 + comment: ' + + Data was converted to the specified units.' + altimeter_distance: + long_name: Field "Altimeter Distance" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + altimeter_quality: + long_name: Field "Altimeter Quality" in the data specification. + units: '' + comment: '' + ast_distance: + long_name: Field "Ast Distance" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + ast_quality: + long_name: Field "Ast Quality" in the data specification. + units: '' + comment: '' + ast_offset_100us: + long_name: Field "Ast Offset 100us" in the data specification. + units: "100 \u03BCs" + comment: ' + + Data was converted to the specified units.' + ast_pressure: + long_name: Field "Ast Pressure" in the data specification. + units: dBar + comment: ' + + Data was converted to the specified units.' + altimeter_spare: + long_name: Field "Altimeter Spare" in the data specification. + units: '' + comment: '' + altimeter_raw_data_num_samples: + long_name: Field "Altimeter Raw Data Num Samples" in the data specification. + units: '' + comment: '' + altimeter_raw_data_sample_distance: + long_name: Field "Altimeter Raw Data Sample Distance" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + altimeter_raw_data_samples: + long_name: Field "Altimeter Raw Data Samples" in the data specification. + units: '' + comment: '' + echosounder_data: + long_name: Field "Echosounder Data" in the data specification. + units: dB/count + comment: ' + + Data was converted to the specified units.' + ahrs_rotation_matrix: + long_name: Field "Ahrs Rotation Matrix" in the data specification. + units: '' + comment: '' + ahrs_quaternions: + long_name: Field "Ahrs Quaternions" in the data specification. + units: '' + comment: '' + ahrs_gyro: + long_name: Field "Ahrs Gyro" in the data specification. + units: '' + comment: '' + percentage_good_data: + long_name: Field "Percentage Good Data" in the data specification. + units: '%' + comment: ' + + Data was converted to the specified units.' + std_dev_pitch: + long_name: Field "Std Dev Pitch" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + std_dev_roll: + long_name: Field "Std Dev Roll" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + std_dev_heading: + long_name: Field "Std Dev Heading" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + std_dev_pressure: + long_name: Field "Std Dev Pressure" in the data specification. + units: dBar + comment: ' + + Data was converted to the specified units.' +BOTTOM_TRACK_DATA_RECORD_FORMAT: + version: + long_name: Field "Version" in the data specification. + units: '' + comment: '' + offset_of_data: + long_name: Field "Offset Of Data" in the data specification. + units: '# of bytes' + comment: ' + + Data was converted to the specified units.' + configuration: + long_name: Field "Configuration" in the data specification. + units: '' + comment: '' + serial_number: + long_name: Field "Serial Number" in the data specification. + units: '' + comment: '' + year: + long_name: Field "Year" in the data specification. + units: '' + comment: '' + month: + long_name: Field "Month" in the data specification. + units: '' + comment: '' + day: + long_name: Field "Day" in the data specification. + units: '' + comment: '' + hour: + long_name: Field "Hour" in the data specification. + units: '' + comment: '' + minute: + long_name: Field "Minute" in the data specification. + units: '' + comment: '' + seconds: + long_name: Field "Seconds" in the data specification. + units: '' + comment: '' + microsec100: + long_name: Field "Microsec100" in the data specification. + units: '' + comment: '' + speed_of_sound: + long_name: Field "Speed Of Sound" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + temperature: + long_name: Field "Temperature" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + pressure: + long_name: Field "Pressure" in the data specification. + units: dBar + comment: ' + + Data was converted to the specified units.' + heading: + long_name: Field "Heading" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + pitch: + long_name: Field "Pitch" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + roll: + long_name: Field "Roll" in the data specification. + units: degrees + comment: ' + + Data was converted to the specified units.' + num_beams_and_coordinate_system_and_num_cells: + long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + units: '' + comment: '' + cell_size: + long_name: Field "Cell Size" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + blanking: + long_name: Field "Blanking" in the data specification. + units: m + comment: ' + + Data was converted to the specified units.' + nominal_correlation: + long_name: Field "Nominal Correlation" in the data specification. + units: '%' + comment: ' + + Data was converted to the specified units.' + battery_voltage: + long_name: Field "Battery Voltage" in the data specification. + units: V + comment: ' + + Data was converted to the specified units.' + magnetometer_raw: + long_name: Field "Magnetometer Raw" in the data specification. + units: '' + comment: '' + accelerometer_raw_x_axis: + long_name: Field "Accelerometer Raw X Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_y_axis: + long_name: Field "Accelerometer Raw Y Axis" in the data specification. + units: '' + comment: '' + accelerometer_raw_z_axis: + long_name: Field "Accelerometer Raw Z Axis" in the data specification. + units: '' + comment: '' + ambiguity_velocity: + long_name: Field "Ambiguity Velocity" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + dataset_description: + long_name: Field "Dataset Description" in the data specification. + units: '' + comment: '' + transmit_energy: + long_name: Field "Transmit Energy" in the data specification. + units: '' + comment: '' + velocity_scaling: + long_name: Field "Velocity Scaling" in the data specification. + units: '' + comment: '' + power_level: + long_name: Field "Power Level" in the data specification. + units: dB + comment: ' + + Data was converted to the specified units.' + magnetometer_temperature: + long_name: Field "Magnetometer Temperature" in the data specification. + units: '' + comment: '' + real_time_clock_temperature: + long_name: Field "Real Time Clock Temperature" in the data specification. + units: degrees Celsius + comment: ' + + Data was converted to the specified units.' + error: + long_name: Field "Error" in the data specification. + units: '' + comment: '' + status: + long_name: Field "Status" in the data specification. + units: '' + comment: '' + ensemble_counter: + long_name: Field "Ensemble Counter" in the data specification. + units: '' + comment: '' + velocity_data: + long_name: Field "Velocity Data" in the data specification. + units: m/s + comment: ' + + Data was converted to the specified units.' + distance_data: + long_name: Field "Distance Data" in the data specification. + units: '' + comment: '' + figure_of_merit_data: + long_name: Field "Figure Of Merit Data" in the data specification. + units: '' + comment: '' +ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: + version: + long_name: Field "Version" in the data specification. + units: '' + comment: '' + offset_of_data: + long_name: Field "Offset Of Data" in the data specification. + units: '# of bytes' + comment: ' + + Data was converted to the specified units.' + year: + long_name: Field "Year" in the data specification. + units: '' + comment: '' + month: + long_name: Field "Month" in the data specification. + units: '' + comment: '' + day: + long_name: Field "Day" in the data specification. + units: '' + comment: '' + hour: + long_name: Field "Hour" in the data specification. + units: '' + comment: '' + minute: + long_name: Field "Minute" in the data specification. + units: '' + comment: '' + seconds: + long_name: Field "Seconds" in the data specification. + units: '' + comment: '' + microsec100: + long_name: Field "Microsec100" in the data specification. + units: '' + comment: '' + error: + long_name: Field "Error" in the data specification. + units: '' + comment: '' + status: + long_name: Field "Status" in the data specification. + units: '' + comment: '' + serial_number: + long_name: Field "Serial Number" in the data specification. + units: '' + comment: '' + num_complex_samples: + long_name: Field "Num Complex Samples" in the data specification. + units: '' + comment: '' + ind_start_samples: + long_name: Field "Ind Start Samples" in the data specification. + units: '' + comment: '' + freq_raw_sample_data: + long_name: Field "Freq Raw Sample Data" in the data specification. + units: '' + comment: '' + echosounder_raw_samples: + long_name: Field "Echosounder Raw Samples" in the data specification. + units: '' + comment: '' + echosounder_raw_samples_i: + long_name: Field "Echosounder Raw Samples I" in the data specification. + units: '' + comment: '' + echosounder_raw_samples_q: + long_name: Field "Echosounder Raw Samples Q" in the data specification. + units: '' + comment: '' + echosounder_raw_transmit_samples: + long_name: Field "Echosounder Raw Transmit Samples" in the data specification. + units: '' + comment: '' + echosounder_raw_transmit_samples_i: + long_name: Field "Echosounder Raw Transmit Samples I" in the data specification. + units: '' + comment: '' + echosounder_raw_transmit_samples_q: + long_name: Field "Echosounder Raw Transmit Samples Q" in the data specification. + units: '' + comment: '' From 968e967419d40402cf8e38b362e0a5a2329d09ee Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:32:53 -0700 Subject: [PATCH 10/37] Add burst/average data record comments --- echopype/convert/ad2cp_fields.yaml | 366 +++++++++++++++++++---------- 1 file changed, 245 insertions(+), 121 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index aff9d7db9..29f37212f 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -2,83 +2,88 @@ BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: version: long_name: Field "Version" in the data specification. units: '' - comment: '' + comment: 'Version number of the Data Record Definition. +Should be 3' offset_of_data: long_name: Field "Offset Of Data" in the data specification. units: '# of bytes' - comment: ' + comment: 'Number of bytes from start of the record to start of +the actual data. Data was converted to the specified units.' serial_number: long_name: Field "Serial Number" in the data specification. units: '' - comment: '' + comment: 'Instrument serial number from factory.' configuration: long_name: Field "Configuration" in the data specification. units: '' - comment: '' + comment: 'Record Configuration Bit Mask' year: long_name: Field "Year" in the data specification. - units: '' - comment: '' + units: 'year' + comment: 'Is given as years from 1900.' month: long_name: Field "Month" in the data specification. - units: '' - comment: '' + units: 'month' + comment: 'January is 0.' day: long_name: Field "Day" in the data specification. - units: '' + units: 'day' comment: '' hour: long_name: Field "Hour" in the data specification. - units: '' + units: 'hour' comment: '' minute: long_name: Field "Minute" in the data specification. - units: '' + units: 'minute' comment: '' seconds: long_name: Field "Seconds" in the data specification. - units: '' + units: 's' comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: '' - comment: '' + units: 'us' + comment: 'Remaining micro seconds (Date object has +milliseconds resolution)' speed_of_sound: long_name: Field "Speed Of Sound" in the data specification. units: m/s - comment: ' + comment: 'Speed of sound used by the instrument. +Raw data given as 0.1m/s Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius - comment: ' + comment: 'Reading from the temperature sensor. +Raw data given as 0.01 °C Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: ' + comment: 'Raw data given as 0.001 dBar Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' error: @@ -90,13 +95,14 @@ BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: units: '' comment: '' num_beams_and_coordinate_system_and_num_cells: - long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' comment: '' cell_size: long_name: Field "Cell Size" in the data specification. units: m - comment: ' + comment: 'Size of each cell (resolution) on the beam. +Raw data given as mm Data was converted to the specified units.' blanking: @@ -114,194 +120,227 @@ BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: ' + comment: 'Raw value given in 0.1 Volt Data was converted to the specified units.' magnetometer_raw: long_name: Field "Magnetometer Raw" in the data specification. units: '' - comment: '' + comment: 'flux raw value in last measurement interval' accelerometer_raw_x_axis: long_name: Field "Accelerometer Raw X Axis" in the data specification. units: '' - comment: '' + comment: 'raw X axis value in last measurement interval' accelerometer_raw_y_axis: long_name: Field "Accelerometer Raw Y Axis" in the data specification. units: '' - comment: '' + comment: 'raw Y axis value in last measurement interval' accelerometer_raw_z_axis: long_name: Field "Accelerometer Raw Z Axis" in the data specification. units: '' - comment: '' + comment: 'raw Z axis value in last measurement interval' ambiguity_velocity: long_name: Field "Ambiguity Velocity" in the data specification. units: m/s - comment: ' + comment: 'Ambiguity velocity, corrected forsound velocity, +scaled according to Velocity scaling. +Data given as 10^(Velocity scaling) m/s Data was converted to the specified units.' dataset_description: long_name: Field "Dataset Description" in the data specification. units: '' - comment: '' + comment: 'Data set description.' transmit_energy: long_name: Field "Transmit Energy" in the data specification. units: '' - comment: '' + comment: 'Transmitted energy.' velocity_scaling: long_name: Field "Velocity Scaling" in the data specification. units: '' - comment: '' + comment: 'Velocity scaling used to scale velocity data.' power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: ' + comment: 'Configured power level Data was converted to the specified units.' velocity_data_burst: long_name: Field "Velocity Data Burst" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' velocity_data_average: long_name: Field "Velocity Data Average" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' velocity_data_echosounder: long_name: Field "Velocity Data Echosounder" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' amplitude_data_burst: long_name: Field "Amplitude Data Burst" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the amplitude data included bit of +the Config byte is set +Data given as 0.5 dB/count Data was converted to the specified units.' amplitude_data_average: long_name: Field "Amplitude Data Average" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the amplitude data included bit of +the Config byte is set +Data given as 0.5 dB/count Data was converted to the specified units.' amplitude_data_echosounder: long_name: Field "Amplitude Data Echosounder" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the amplitude data included bit of +the Config byte is set +Data given as 0.5 dB/count Data was converted to the specified units.' correlation_data_burst: long_name: Field "Correlation Data Burst" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +[0 – 100 %] Data was converted to the specified units.' correlation_data_average: long_name: Field "Correlation Data Average" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +[0 – 100 %] Data was converted to the specified units.' correlation_data_echosounder: long_name: Field "Correlation Data Echosounder" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +[0 – 100 %] Data was converted to the specified units.' BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: version: long_name: Field "Version" in the data specification. units: '' - comment: '' + comment: 'Version number of the Data Record Definition. +Should be 3' offset_of_data: long_name: Field "Offset Of Data" in the data specification. units: '# of bytes' - comment: ' + comment: 'Number of bytes from start of the record to start of +the actual data. Data was converted to the specified units.' configuration: long_name: Field "Configuration" in the data specification. units: '' - comment: '' + comment: 'Record Configuration Bit Mask' serial_number: long_name: Field "Serial Number" in the data specification. units: '' - comment: '' + comment: 'Instrument serial number from factory.' year: long_name: Field "Year" in the data specification. - units: '' - comment: '' + units: 'year' + comment: 'Is given as years from 1900.' month: long_name: Field "Month" in the data specification. - units: '' - comment: '' + units: 'month' + comment: 'January is 0.' day: long_name: Field "Day" in the data specification. - units: '' + units: 'day' comment: '' hour: long_name: Field "Hour" in the data specification. - units: '' + units: 'hour' comment: '' minute: long_name: Field "Minute" in the data specification. - units: '' + units: 'minute' comment: '' seconds: long_name: Field "Seconds" in the data specification. - units: '' + units: 's' comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: '' - comment: '' + units: 'us' + comment: 'Remaining micro seconds (Date object has +milliseconds resolution)' speed_of_sound: long_name: Field "Speed Of Sound" in the data specification. units: m/s - comment: ' + comment: 'Speed of sound used by the instrument. +Raw data given as 0.1m/s Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius - comment: ' + comment: 'Reading from the temperature sensor. +Raw data given as 0.01 °C Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: ' + comment: 'Raw data given as 0.001 dBar Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' num_beams_and_coordinate_system_and_num_cells: - long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' - comment: '' + comment: 'Bit 11-10 (2 bits): Coordinate system +b00:ENU, b01:XYZ, b10:BEAM + +Bit 15–12 (4 bits): Number of Beams (NB) +Active beams represented as a 4 charstring of 1s +and 0s. + +Bit 9-0 (10 bits): Number of Cells (NC).' cell_size: long_name: Field "Cell Size" in the data specification. units: m - comment: ' + comment: 'Size of each cell (resolution) on the beam. +Raw data given as mm Data was converted to the specified units.' blanking: @@ -313,7 +352,8 @@ BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: nominal_correlation: long_name: Field "Nominal Correlation" in the data specification. units: '%' - comment: ' + comment: 'The nominal correlation for the configured +combination of cell size and velocity range Data was converted to the specified units.' temperature_from_pressure_sensor: @@ -325,157 +365,232 @@ BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: ' + comment: 'Raw value given in 0.1 Volt Data was converted to the specified units.' magnetometer_raw: long_name: Field "Magnetometer Raw" in the data specification. units: '' - comment: '' + comment: 'flux raw value in last measurement interval' accelerometer_raw_x_axis: long_name: Field "Accelerometer Raw X Axis" in the data specification. units: '' - comment: '' + comment: 'raw X axis value in last measurement interval' accelerometer_raw_y_axis: long_name: Field "Accelerometer Raw Y Axis" in the data specification. units: '' - comment: '' + comment: 'raw Y axis value in last measurement interval' accelerometer_raw_z_axis: long_name: Field "Accelerometer Raw Z Axis" in the data specification. units: '' - comment: '' + comment: 'raw Z axis value in last measurement interval' ambiguity_velocity_or_echosounder_frequency: long_name: Field "Ambiguity Velocity Or Echosounder Frequency" in the data specification. units: '' - comment: '' + comment: 'Ambiguity velocity, corrected forsound velocity, +scaled according to Velocity scaling. +10^(Velocity scaling) m/s OR Echosounder frequency [Hz] + ' dataset_description: long_name: Field "Dataset Description" in the data specification. units: '' - comment: '' + comment: 'Data set description. +0-3 Physical beam used for 1st data set. +4-7 Physical beam used for 2nd data set. +8-11 Physical beam used for 3th data set. +12-16 Physical beam used for 4th data set.' transmit_energy: long_name: Field "Transmit Energy" in the data specification. units: '' - comment: '' + comment: 'Transmitted energy.' velocity_scaling: long_name: Field "Velocity Scaling" in the data specification. units: '' - comment: '' + comment: 'Velocity scaling used to scale velocity data.' power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: ' + comment: 'Configured power level Data was converted to the specified units.' magnetometer_temperature: long_name: Field "Magnetometer Temperature" in the data specification. - units: '' - comment: '' + units: degrees Celsius + comment: 'Magnetometer temperature reading. +Uncalibrated +Raw data in 1/1000 °C' real_time_clock_temperature: long_name: Field "Real Time Clock Temperature" in the data specification. units: degrees Celsius - comment: ' + comment: 'Real Time Clock temperature reading Data was converted to the specified units.' error: long_name: Field "Error" in the data specification. units: '' - comment: '' + comment: 'Error bit mask' status0: long_name: Field "Status0" in the data specification. units: '' - comment: '' + comment: 'Extended status bit mask + +0 bit: Indicates that the processor Idles less than 3 +percent +1 bit: Indicates that the processor idles less than 6 +percent +2 bit: Indicates that the processor idles less than 12 +percent +15 bit: If this bit is set the rest of the word/ extended +status should be interpreted +' # 2 bit is also 1 bit in the spec but should be 2 bit status: long_name: Field "Status" in the data specification. units: '' - comment: '' + comment: ' +Bit 31-28: Wakeup State +1111 0000 0000 0000 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm + +Bit 27-25: Orientation +0000 1110 0000 0000 0000 0000 0000 0000 +0: "XUP" Instrument x-axis defined up, heading +reference axis is Z positive +1: "XDOWN" Instrument x-axis defined down, +heading reference axis is Z positive +2: "YUP" Instrument y-axis defined up, heading +reference axis is Z positive +3: "YDOWN" Instrument y-axis defined down, +heading reference axis is Z positive +4: "ZUP" Instrument z-axis defined up, heading +reference axis is X positive +5: "ZDOWN" Instrument z-axis defined down, +heading reference axis is X positive +7: "AHRS" AHRS reports orientation any way it +points. Example: Z down -> Roll = 180 deg. + +Bit 24-22: autoOrientation +0000 0001 1100 0000 0000 0000 0000 0000 +0: "Fixed" Fixed orientation +1: "Auto" Auto Up Down +3: "AHRS3D" AHRS3D' ensemble_counter: long_name: Field "Ensemble Counter" in the data specification. units: '' - comment: '' + comment: 'Counts the number of ensembles in both averaged +and burst data' velocity_data_burst: long_name: Field "Velocity Data Burst" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' velocity_data_average: long_name: Field "Velocity Data Average" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' velocity_data_echosounder: long_name: Field "Velocity Data Echosounder" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' amplitude_data_burst: long_name: Field "Amplitude Data Burst" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data given as 10^(Velocity Scaling) Data was converted to the specified units.' amplitude_data_average: long_name: Field "Amplitude Data Average" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the amplitude data included bit of +the Config byte is set +Data given as 0.5 dB/count Data was converted to the specified units.' amplitude_data_echosounder: long_name: Field "Amplitude Data Echosounder" in the data specification. units: dB/count - comment: ' + comment: 'This field exists if the amplitude data included bit of +the Config byte is set +Data given as 0.5 dB/count Data was converted to the specified units.' correlation_data_burst: long_name: Field "Correlation Data Burst" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +Data given as [0 – 100 %] Data was converted to the specified units.' correlation_data_average: long_name: Field "Correlation Data Average" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +Data given as [0 – 100 %] Data was converted to the specified units.' correlation_data_echosounder: long_name: Field "Correlation Data Echosounder" in the data specification. - units: 0-100 - comment: ' + units: 0-100% + comment: 'This field exists if the Correlation data included bit +of the Config byte is set +Data given as [0 – 100 %] Data was converted to the specified units.' altimeter_distance: long_name: Field "Altimeter Distance" in the data specification. units: m - comment: ' + comment: 'Distance to surface from Leading Edge algorithm Data was converted to the specified units.' altimeter_quality: long_name: Field "Altimeter Quality" in the data specification. units: '' - comment: '' + comment: 'Result of LE algorithm. When quality is deemed too +low according to instrument specific limits. Distance +and Quality set to 0.' ast_distance: long_name: Field "Ast Distance" in the data specification. units: m - comment: ' + comment: 'Distance to surface from Max Peak algorithm Data was converted to the specified units.' ast_quality: long_name: Field "Ast Quality" in the data specification. - units: '' - comment: '' + units: dB + comment: 'Amplitude at which surface is detected +Raw data in steps of 0.01 dB, i.e. quality of 8000 = 80 +dB + Data was converted to the specified units.' ast_offset_100us: long_name: Field "Ast Offset 100us" in the data specification. - units: "100 \u03BCs" - comment: ' + units: "100 μs" + comment: 'Offset in step of measurement to velocity +measurement +Raw data given in 100 µs Data was converted to the specified units.' ast_pressure: long_name: Field "Ast Pressure" in the data specification. units: dBar - comment: ' + comment: 'Pressure value measured during the AST/altimeter +ping Data was converted to the specified units.' altimeter_spare: @@ -485,63 +600,72 @@ BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: altimeter_raw_data_num_samples: long_name: Field "Altimeter Raw Data Num Samples" in the data specification. units: '' - comment: '' + comment: 'Altimeter Raw Data – Number of Samples' altimeter_raw_data_sample_distance: long_name: Field "Altimeter Raw Data Sample Distance" in the data specification. units: m - comment: ' + comment: 'Distance between samples +Raw data given in 0.1mm Data was converted to the specified units.' altimeter_raw_data_samples: long_name: Field "Altimeter Raw Data Samples" in the data specification. units: '' - comment: '' + comment: 'Altimeter Raw Data – Samples +Raw data given as 16 bits Signed fract' echosounder_data: long_name: Field "Echosounder Data" in the data specification. units: dB/count - comment: ' + comment: 'Echosounder amplitude Data +Raw data given as 0.01 dB/count Data was converted to the specified units.' ahrs_rotation_matrix: - long_name: Field "Ahrs Rotation Matrix" in the data specification. + long_name: Field "AHRS Rotation Matrix" in the data specification. units: '' - comment: '' + comment: 'AHRS Rotation Matrix [3x3]' ahrs_quaternions: - long_name: Field "Ahrs Quaternions" in the data specification. + long_name: Field "AHRS Quaternions" in the data specification. units: '' - comment: '' + comment: 'quaternions' ahrs_gyro: - long_name: Field "Ahrs Gyro" in the data specification. - units: '' - comment: '' + long_name: Field "AHRS Gyro" in the data specification. + units: 'dps' + comment: 'Gyro in X direction in degrees prsecond' percentage_good_data: long_name: Field "Percentage Good Data" in the data specification. units: '%' - comment: ' + comment: 'Percent Good Estimate per cell +These fields exist if the Percentage Good data +included Data was converted to the specified units.' std_dev_pitch: long_name: Field "Std Dev Pitch" in the data specification. units: degrees - comment: ' + comment: 'Standard deviation on pitch data +Raw data in 0.01 degrees Data was converted to the specified units.' std_dev_roll: long_name: Field "Std Dev Roll" in the data specification. units: degrees - comment: ' + comment: 'Standard deviation on roll data +Raw data in 0.01 degrees Data was converted to the specified units.' std_dev_heading: long_name: Field "Std Dev Heading" in the data specification. units: degrees - comment: ' + comment: 'Standard deviation on heading data +Raw data in 0.01 degrees Data was converted to the specified units.' std_dev_pressure: long_name: Field "Std Dev Pressure" in the data specification. units: dBar - comment: ' + comment: 'Standard deviation on pressure data +Raw data in 0.001 Bar Data was converted to the specified units.' BOTTOM_TRACK_DATA_RECORD_FORMAT: From 1ec9ea183c62060f2df75edeaca2eabe30d25c5f Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:46:23 -0700 Subject: [PATCH 11/37] Add bottom track comments --- echopype/convert/ad2cp_fields.yaml | 236 ++++++++++++++++++++++------- 1 file changed, 184 insertions(+), 52 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index 29f37212f..04bdf0396 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -45,7 +45,7 @@ the actual data. comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: 'us' + units: 'μs' comment: 'Remaining micro seconds (Date object has milliseconds resolution)' speed_of_sound: @@ -89,15 +89,51 @@ Raw data given as 0.01 °C error: long_name: Field "Error" in the data specification. units: '' - comment: '' + comment: 'Error bit mask' status: long_name: Field "Status" in the data specification. units: '' - comment: '' + comment: ' +Bit 31-28: Wakeup State +1111 0000 0000 0000 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm + +Bit 27-25: Orientation +0000 1110 0000 0000 0000 0000 0000 0000 +0: "XUP" Instrument x-axis defined up, heading +reference axis is Z positive +1: "XDOWN" Instrument x-axis defined down, +heading reference axis is Z positive +2: "YUP" Instrument y-axis defined up, heading +reference axis is Z positive +3: "YDOWN" Instrument y-axis defined down, +heading reference axis is Z positive +4: "ZUP" Instrument z-axis defined up, heading +reference axis is X positive +5: "ZDOWN" Instrument z-axis defined down, +heading reference axis is X positive +7: "AHRS" AHRS reports orientation any way it +points. Example: Z down -> Roll = 180 deg. + +Bit 24-22: autoOrientation +0000 0001 1100 0000 0000 0000 0000 0000 +0: "Fixed" Fixed orientation +1: "Auto" Auto Up Down +3: "AHRS3D" AHRS3D' num_beams_and_coordinate_system_and_num_cells: long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' - comment: '' + comment: 'Bit 11-10 (2 bits): Coordinate system +b00:ENU, b01:XYZ, b10:BEAM + +Bit 15–12 (4 bits): Number of Beams (NB) +Active beams represented as a 4 charstring of 1s +and 0s. + +Bit 9-0 (10 bits): Number of Cells (NC).' cell_size: long_name: Field "Cell Size" in the data specification. units: m @@ -150,7 +186,12 @@ Data given as 10^(Velocity scaling) m/s dataset_description: long_name: Field "Dataset Description" in the data specification. units: '' - comment: 'Data set description.' + comment: 'Data set description. +0-2 Physical beam used for 1st data set. +3-5 Physical beam used for 2nd data set. +6-8 Physical beam used for 3th data set. +9-11 Physical beam used for 4th data set. +12-14 Physical beam used for 4th data set.' transmit_energy: long_name: Field "Transmit Energy" in the data specification. units: '' @@ -284,7 +325,7 @@ the actual data. comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: 'us' + units: 'μs' comment: 'Remaining micro seconds (Date object has milliseconds resolution)' speed_of_sound: @@ -672,189 +713,251 @@ BOTTOM_TRACK_DATA_RECORD_FORMAT: version: long_name: Field "Version" in the data specification. units: '' - comment: '' + comment: 'Version number of the Data Record Definition. +Should be 3' offset_of_data: long_name: Field "Offset Of Data" in the data specification. units: '# of bytes' - comment: ' + comment: 'Number of bytes from start of the record to start of +the actual data. Data was converted to the specified units.' configuration: long_name: Field "Configuration" in the data specification. units: '' - comment: '' + comment: 'Record Configuration Bit Mask' serial_number: long_name: Field "Serial Number" in the data specification. units: '' - comment: '' + comment: 'Instrument serial number from factory.' year: long_name: Field "Year" in the data specification. - units: '' - comment: '' + units: 'year' + comment: 'Is given as years from 1900.' month: long_name: Field "Month" in the data specification. - units: '' - comment: '' + units: 'month' + comment: 'January is 0.' day: long_name: Field "Day" in the data specification. - units: '' + units: 'day' comment: '' hour: long_name: Field "Hour" in the data specification. - units: '' + units: 'hour' comment: '' minute: long_name: Field "Minute" in the data specification. - units: '' + units: 'minute' comment: '' seconds: long_name: Field "Seconds" in the data specification. - units: '' + units: 's' comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: '' - comment: '' + units: 'μs' + comment: 'Remaining micro seconds (Date object has +milliseconds resolution)' speed_of_sound: long_name: Field "Speed Of Sound" in the data specification. units: m/s - comment: ' + comment: 'Speed of sound used by the instrument. +Raw data given as 0.1m/s Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius - comment: ' + comment: 'Reading from the temperature sensor. +Raw data given as 0.01 °C Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: ' + comment: 'Raw data given as 0.001 dBar Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: ' + comment: 'Raw data given as 0.01 degrees Data was converted to the specified units.' num_beams_and_coordinate_system_and_num_cells: - long_name: Field "Num Beams And Coordinate System And Num Cells" in the data specification. + long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' - comment: '' + comment: 'Bit 11-10 (2 bits) describes the coordinate system +used. +b00:ENU, b01:XYZ, b10:BEAM + +Bit 15–12 (4 bits) represent the number of beams +(NB) +Active beams represented as a 4 charstring of 1s +and 0s. + +Bit 9-0 (10 bits) represent the number of cells (NC).' cell_size: long_name: Field "Cell Size" in the data specification. units: m - comment: ' + comment: 'Size of each cell (resolution) on the beam. +Raw data given as mm Data was converted to the specified units.' blanking: long_name: Field "Blanking" in the data specification. units: m - comment: ' + comment: 'Distance from instrument to first data point on the +beam. +Raw data given as mm Data was converted to the specified units.' nominal_correlation: long_name: Field "Nominal Correlation" in the data specification. units: '%' - comment: ' + comment: 'The nominal correlation for the configured +combination of cell size and velocity range Data was converted to the specified units.' battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: ' + comment: 'Raw value given in 0.1 Volt Data was converted to the specified units.' magnetometer_raw: long_name: Field "Magnetometer Raw" in the data specification. units: '' - comment: '' + comment: 's flux raw value in last measurement interval' accelerometer_raw_x_axis: long_name: Field "Accelerometer Raw X Axis" in the data specification. units: '' - comment: '' + comment: 'raw X axis value in last measurement interval' accelerometer_raw_y_axis: long_name: Field "Accelerometer Raw Y Axis" in the data specification. units: '' - comment: '' + comment: 'raw Y axis value in last measurement interval' accelerometer_raw_z_axis: long_name: Field "Accelerometer Raw Z Axis" in the data specification. units: '' - comment: '' + comment: 'raw Z axis value in last measurement interval' ambiguity_velocity: long_name: Field "Ambiguity Velocity" in the data specification. units: m/s - comment: ' + comment: 'Ambiguity velocity, corrected forsound velocity, +scaled according to Velocity scaling. +10^(Velocity scaling) m/s Data was converted to the specified units.' dataset_description: long_name: Field "Dataset Description" in the data specification. units: '' - comment: '' + comment: 'Data set description. +0-3 Physical beam used for 1st data set. +4-7 Physical beam used for 2nd data set. +8-11 Physical beam used for 3th data set. +12-16 Physical beam used for 4th data set' transmit_energy: long_name: Field "Transmit Energy" in the data specification. units: '' - comment: '' + comment: 'Transmitted energy.' velocity_scaling: long_name: Field "Velocity Scaling" in the data specification. units: '' - comment: '' + comment: 'Velocity scaling used to scale velocity data.' power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: ' + comment: 'Configured power level Data was converted to the specified units.' magnetometer_temperature: long_name: Field "Magnetometer Temperature" in the data specification. - units: '' - comment: '' + units: degrees Celsius + comment: 'Magnetometer temperature reading. +Uncalibrated +Raw data in 1/1000 °C + + ' real_time_clock_temperature: long_name: Field "Real Time Clock Temperature" in the data specification. units: degrees Celsius - comment: ' + comment: 'Real Time Clock temperature reading Data was converted to the specified units.' error: long_name: Field "Error" in the data specification. units: '' - comment: '' + comment: 'Error bit mask' status: long_name: Field "Status" in the data specification. units: '' - comment: '' + comment: ' +Bit 31-28: Wakeup State +1111 0000 0000 0000 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm + +Bit 27-25: Orientation +0000 1110 0000 0000 0000 0000 0000 0000 +0: "XUP" Instrument x-axis defined up, heading +reference axis is Z positive +1: "XDOWN" Instrument x-axis defined down, +heading reference axis is Z positive +2: "YUP" Instrument y-axis defined up, heading +reference axis is Z positive +3: "YDOWN" Instrument y-axis defined down, +heading reference axis is Z positive +4: "ZUP" Instrument z-axis defined up, heading +reference axis is X positive +5: "ZDOWN" Instrument z-axis defined down, +heading reference axis is X positive +7: "AHRS" AHRS reports orientation any way it +points. Example: Z down -> Roll = 180 deg. + +Bit 24-22: autoOrientation +0000 0001 1100 0000 0000 0000 0000 0000 +0: "Fixed" Fixed orientation +1: "Auto" Auto Up Down +3: "AHRS3D" AHRS3D' ensemble_counter: long_name: Field "Ensemble Counter" in the data specification. units: '' - comment: '' + comment: 'Counts the number of ensembles in both averaged +and burst data' velocity_data: long_name: Field "Velocity Data" in the data specification. units: m/s - comment: ' + comment: 'This field exists if the Velocity data included bit of +the Config byte is set. +Data scaled as product of 10^(Velocity Scaling) Data was converted to the specified units.' distance_data: long_name: Field "Distance Data" in the data specification. units: '' - comment: '' + comment: 'This field exists if the Distance data included bit of +the Config byte is set.' figure_of_merit_data: long_name: Field "Figure Of Merit Data" in the data specification. units: '' - comment: '' + comment: 'This field exists if the hasFigureOfMeritData bit of +the Config byte is set.' ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: version: long_name: Field "Version" in the data specification. @@ -892,7 +995,7 @@ ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: comment: '' microsec100: long_name: Field "Microsec100" in the data specification. - units: '' + units: 'μs' comment: '' error: long_name: Field "Error" in the data specification. @@ -901,7 +1004,36 @@ ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: status: long_name: Field "Status" in the data specification. units: '' - comment: '' + comment: ' +Bit 31-28: Wakeup State +1111 0000 0000 0000 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm + +Bit 27-25: Orientation +0000 1110 0000 0000 0000 0000 0000 0000 +0: "XUP" Instrument x-axis defined up, heading +reference axis is Z positive +1: "XDOWN" Instrument x-axis defined down, +heading reference axis is Z positive +2: "YUP" Instrument y-axis defined up, heading +reference axis is Z positive +3: "YDOWN" Instrument y-axis defined down, +heading reference axis is Z positive +4: "ZUP" Instrument z-axis defined up, heading +reference axis is X positive +5: "ZDOWN" Instrument z-axis defined down, +heading reference axis is X positive +7: "AHRS" AHRS reports orientation any way it +points. Example: Z down -> Roll = 180 deg. + +Bit 24-22: autoOrientation +0000 0001 1100 0000 0000 0000 0000 0000 +0: "Fixed" Fixed orientation +1: "Auto" Auto Up Down +3: "AHRS3D" AHRS3D' serial_number: long_name: Field "Serial Number" in the data specification. units: '' From 8d781fe975cba10acc1ed484364e48017b4c9df7 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:53:30 -0700 Subject: [PATCH 12/37] Add echosounder raw comments --- echopype/convert/ad2cp_fields.yaml | 38 +++++++++++++++++------------- echopype/convert/parse_ad2cp.py | 15 ++++++++++-- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index 04bdf0396..2dd8c8610 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -962,45 +962,48 @@ ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: version: long_name: Field "Version" in the data specification. units: '' - comment: '' + comment: 'Version number of the Data Record Definition. +Should be 3' offset_of_data: long_name: Field "Offset Of Data" in the data specification. units: '# of bytes' - comment: ' + comment: 'Number of bytes from start of the record to start of +the actual data. Data was converted to the specified units.' year: long_name: Field "Year" in the data specification. - units: '' - comment: '' + units: 'uear' + comment: 'Is given as years from 1900.' month: long_name: Field "Month" in the data specification. - units: '' - comment: '' + units: 'month' + comment: 'January is 0.' day: long_name: Field "Day" in the data specification. - units: '' + units: 'day' comment: '' hour: long_name: Field "Hour" in the data specification. - units: '' + units: 'hour' comment: '' minute: long_name: Field "Minute" in the data specification. - units: '' + units: 'minute' comment: '' seconds: long_name: Field "Seconds" in the data specification. - units: '' + units: 's' comment: '' microsec100: long_name: Field "Microsec100" in the data specification. units: 'μs' - comment: '' + comment: 'Remaining micro seconds (Date object has +milliseconds resolution)' error: long_name: Field "Error" in the data specification. units: '' - comment: '' + comment: 'Error bit mask' status: long_name: Field "Status" in the data specification. units: '' @@ -1037,19 +1040,20 @@ Bit 24-22: autoOrientation serial_number: long_name: Field "Serial Number" in the data specification. units: '' - comment: '' + comment: 'Instrument serial number from factory.' num_complex_samples: long_name: Field "Num Complex Samples" in the data specification. units: '' - comment: '' + comment: 'Number of following Complex Samples' ind_start_samples: long_name: Field "Ind Start Samples" in the data specification. units: '' - comment: '' + comment: 'Sample number where the position is equal to the +configured blanking distance.' freq_raw_sample_data: long_name: Field "Freq Raw Sample Data" in the data specification. - units: '' - comment: '' + units: 'Hz' + comment: 'Sample Rate' echosounder_raw_samples: long_name: Field "Echosounder Raw Samples" in the data specification. units: '' diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 39e051298..dfd90a000 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -1279,7 +1279,12 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec F("transmit_energy", 2, UNSIGNED_INTEGER), F("velocity_scaling", 1, SIGNED_INTEGER), F("power_level", 1, SIGNED_INTEGER, field_units="dB"), - F("magnetometer_temperature", 2, SIGNED_INTEGER), + F( + "magnetometer_temperature", + 2, + SIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x * 1000, + ), F( "real_time_clock_temperature", 2, @@ -1477,6 +1482,7 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "ast_quality", 2, UNSIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["ast_data_included"], ), F( @@ -1731,7 +1737,12 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec F("transmit_energy", 2, UNSIGNED_INTEGER), F("velocity_scaling", 1, SIGNED_INTEGER), F("power_level", 1, SIGNED_INTEGER, field_units="dB"), - F("magnetometer_temperature", 2, SIGNED_INTEGER), + F( + "magnetometer_temperature", + 2, + SIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x * 1000, + ), F( "real_time_clock_temperature", 2, From d31165ec7d2584aa71b42d172c91e0d5d93328cf Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 18:46:09 -0700 Subject: [PATCH 13/37] Remove field_units from parse_ad2cp --- echopype/convert/ad2cp_fields.yaml | 8 +- echopype/convert/parse_ad2cp.py | 113 ++++++--------------------- echopype/convert/set_groups_ad2cp.py | 23 +++--- 3 files changed, 44 insertions(+), 100 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index 2dd8c8610..094f6c1da 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -144,7 +144,9 @@ Raw data given as mm blanking: long_name: Field "Blanking" in the data specification. units: m - comment: ' + comment: 'Distance from instrument to first data point on the +beam. +Raw data given as mm Data was converted to the specified units.' velocity_range: @@ -387,7 +389,9 @@ Raw data given as mm blanking: long_name: Field "Blanking" in the data specification. units: m - comment: ' + comment: 'Distance from instrument to first data point on the +beam. +Raw data given as mm Data was converted to the specified units.' nominal_correlation: diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index dfd90a000..8b6c4003e 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -134,7 +134,6 @@ def __init__( field_dimensions: Union[List[Dimension], Callable[[DataRecordType], List[Dimension]]] = [ Dimension.PING_TIME ], - field_units: Optional[str] = None, field_unit_conversion: Callable[ ["Ad2cpDataPacket", np.ndarray], np.ndarray ] = lambda _, x: x, @@ -153,7 +152,6 @@ def __init__( n number of m length arrays, etc. field_dimensions: Dimensions of the field in the output dataset - field_units: Label for the units of the field, if any field_unit_conversion: Unit conversion function on field field_exists_predicate: Tests to see whether the field should be parsed at all """ @@ -163,7 +161,6 @@ def __init__( self.field_entry_data_type = field_entry_data_type self.field_shape = field_shape self.field_dimensions = field_dimensions - self.field_units = field_units self.field_unit_conversion = field_unit_conversion self.field_exists_predicate = field_exists_predicate @@ -185,13 +182,6 @@ def default_dimensions() -> List[Dimension]: return [Dimension.PING_TIME] - def units(self): - """ - Returns the field's units - """ - - return self.field_units - F = Field # use F instead of Field to make the repeated fields easier to read @@ -813,7 +803,8 @@ class HeaderOrDataRecordFormat: A collection of fields which represents the header format or a data record format """ - def __init__(self, fields: List[Field]): + def __init__(self, name: str, fields: List[Field]): + self.name = name self.fields = OrderedDict([(f.field_name, f) for f in fields]) def get_field(self, field_name: str) -> Optional[Field]: @@ -845,6 +836,7 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec return cls.DATA_RECORD_FORMATS[data_record_type] HEADER_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "HEADER_FORMAT", [ F("sync", 1, UNSIGNED_INTEGER), F("header_size", 1, UNSIGNED_INTEGER), @@ -860,9 +852,10 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec # in (0x23, 0x24) else UNSIGNED_INTEGER), F("data_record_checksum", 2, UNSIGNED_INTEGER), F("header_checksum", 2, UNSIGNED_INTEGER), - ] + ], ) STRING_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "STRING_DATA_RECORD_FORMAT", [ F("string_data_id", 1, UNSIGNED_INTEGER), F( @@ -870,12 +863,13 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec lambda packet: packet.data["data_record_size"] - 1, STRING, ), - ] + ], ) BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT", [ F("version", 1, UNSIGNED_INTEGER), - F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), + F("offset_of_data", 1, UNSIGNED_INTEGER), F("serial_number", 4, UNSIGNED_INTEGER), F("configuration", 2, UNSIGNED_INTEGER), F("year", 1, UNSIGNED_INTEGER), @@ -889,42 +883,36 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "speed_of_sound", 2, UNSIGNED_INTEGER, - field_units="m/s", field_unit_conversion=lambda packet, x: x / 10, ), F( "temperature", 2, SIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x / 100, ), F( "pressure", 4, UNSIGNED_INTEGER, - field_units="dBar", field_unit_conversion=lambda packet, x: x / 1000, ), F( "heading", 2, UNSIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "pitch", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "roll", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F("error", 2, UNSIGNED_INTEGER), @@ -934,28 +922,24 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "cell_size", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), F( "blanking", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), F( "velocity_range", 2, UNSIGNED_INTEGER, - field_units="m/s", field_unit_conversion=lambda packet, x: x / 1000, ), F( "battery_voltage", 2, UNSIGNED_INTEGER, - field_units="V", field_unit_conversion=lambda packet, x: x / 10, ), F( @@ -987,13 +971,12 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "ambiguity_velocity", 2, UNSIGNED_INTEGER, - field_units="m/s", field_unit_conversion=lambda packet, x: x / 10000, ), F("dataset_description", 2, UNSIGNED_INTEGER), F("transmit_energy", 2, UNSIGNED_INTEGER), F("velocity_scaling", 1, SIGNED_INTEGER), - F("power_level", 1, SIGNED_INTEGER, field_units="dB"), + F("power_level", 1, SIGNED_INTEGER), F(None, 4, UNSIGNED_INTEGER), F( # used when burst "velocity_data_burst", @@ -1008,7 +991,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_burst() @@ -1027,7 +1009,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_average() @@ -1046,7 +1027,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_echosounder() @@ -1065,7 +1045,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_burst() and packet.data["amplitude_data_included"], @@ -1083,7 +1062,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_average() and packet.data["amplitude_data_included"], @@ -1101,7 +1079,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_echosounder() and packet.data["amplitude_data_included"], @@ -1119,7 +1096,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_burst() and packet.data["correlation_data_included"], ), @@ -1136,7 +1112,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_average() and packet.data["correlation_data_included"], ), @@ -1153,16 +1128,16 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_echosounder() and packet.data["correlation_data_included"], ), - ] + ], ) BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT", [ F("version", 1, UNSIGNED_INTEGER), - F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), + F("offset_of_data", 1, UNSIGNED_INTEGER), F("configuration", 2, UNSIGNED_INTEGER), F("serial_number", 4, UNSIGNED_INTEGER), F("year", 1, UNSIGNED_INTEGER), @@ -1176,42 +1151,36 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "speed_of_sound", 2, UNSIGNED_INTEGER, - field_units="m/s", field_unit_conversion=lambda packet, x: x / 10, ), F( "temperature", 2, SIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x / 100, ), F( "pressure", 4, UNSIGNED_INTEGER, - field_units="dBar", field_unit_conversion=lambda packet, x: x / 1000, ), F( "heading", 2, UNSIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "pitch", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "roll", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F("num_beams_and_coordinate_system_and_num_cells", 2, UNSIGNED_INTEGER), @@ -1219,7 +1188,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "cell_size", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), # This field is listed to be in cm, but testing has shown that it is actually in mm. @@ -1228,22 +1196,19 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "blanking", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), - F("nominal_correlation", 1, UNSIGNED_INTEGER, field_units="%"), + F("nominal_correlation", 1, UNSIGNED_INTEGER), F( "temperature_from_pressure_sensor", 1, UNSIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x * 5, ), F( "battery_voltage", 2, UNSIGNED_INTEGER, - field_units="V", field_unit_conversion=lambda packet, x: x / 10, ), F( @@ -1278,7 +1243,7 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec F("dataset_description", 2, UNSIGNED_INTEGER), F("transmit_energy", 2, UNSIGNED_INTEGER), F("velocity_scaling", 1, SIGNED_INTEGER), - F("power_level", 1, SIGNED_INTEGER, field_units="dB"), + F("power_level", 1, SIGNED_INTEGER), F( "magnetometer_temperature", 2, @@ -1289,7 +1254,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "real_time_clock_temperature", 2, SIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x / 100, ), F("error", 2, UNSIGNED_INTEGER), @@ -1309,7 +1273,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_burst() @@ -1328,7 +1291,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_average() @@ -1347,7 +1309,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.is_echosounder() @@ -1366,7 +1327,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_burst() and packet.data["amplitude_data_included"], @@ -1384,7 +1344,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_average() and packet.data["amplitude_data_included"], @@ -1402,7 +1361,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 2, field_exists_predicate=lambda packet: packet.is_echosounder() and packet.data["amplitude_data_included"], @@ -1420,7 +1378,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_BURST, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_burst() and packet.data["correlation_data_included"], ), @@ -1437,7 +1394,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_AVERAGE, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_average() and packet.data["correlation_data_included"], ), @@ -1454,7 +1410,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.BEAM, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="0-100", field_exists_predicate=lambda packet: packet.is_echosounder() and packet.data["correlation_data_included"], ), @@ -1462,7 +1417,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "altimeter_distance", 4, FLOAT, - field_units="m", field_exists_predicate=lambda packet: packet.data["altimeter_data_included"], ), F( @@ -1475,7 +1429,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "ast_distance", 4, FLOAT, - field_units="m", field_exists_predicate=lambda packet: packet.data["ast_data_included"], ), F( @@ -1489,14 +1442,12 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "ast_offset_100us", 2, SIGNED_INTEGER, - field_units="100 μs", field_exists_predicate=lambda packet: packet.data["ast_data_included"], ), F( "ast_pressure", 4, FLOAT, - field_units="dBar", field_exists_predicate=lambda packet: packet.data["ast_data_included"], ), F( @@ -1520,7 +1471,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "altimeter_raw_data_sample_distance", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 10000, field_exists_predicate=lambda packet: packet.data["altimeter_raw_data_included"], ), @@ -1543,7 +1493,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.PING_TIME_ECHOSOUNDER, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], - field_units="dB/count", field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["echosounder_data_included"], ), @@ -1580,7 +1529,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec Dimension.PING_TIME, RANGE_SAMPLES[data_record_type], ], - field_units="%", field_exists_predicate=lambda packet: packet.data["percentage_good_data_included"], ), # Only the pitch field is labeled as included when the "std dev data included" @@ -1589,7 +1537,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "std_dev_pitch", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), @@ -1597,7 +1544,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "std_dev_roll", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), @@ -1605,7 +1551,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "std_dev_heading", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), @@ -1613,7 +1558,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "std_dev_pressure", 2, SIGNED_INTEGER, - field_units="dBar", field_unit_conversion=lambda packet, x: x / 100, field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), @@ -1623,12 +1567,13 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec RAW_BYTES, field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), - ] + ], ) BOTTOM_TRACK_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "BOTTOM_TRACK_DATA_RECORD_FORMAT", [ F("version", 1, UNSIGNED_INTEGER), - F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), + F("offset_of_data", 1, UNSIGNED_INTEGER), F("configuration", 2, UNSIGNED_INTEGER), F("serial_number", 4, UNSIGNED_INTEGER), F("year", 1, UNSIGNED_INTEGER), @@ -1642,42 +1587,36 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "speed_of_sound", 2, UNSIGNED_INTEGER, - field_units="m/s", field_unit_conversion=lambda packet, x: x / 10, ), F( "temperature", 2, SIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x / 100, ), F( "pressure", 4, UNSIGNED_INTEGER, - field_units="dBar", field_unit_conversion=lambda packet, x: x / 1000, ), F( "heading", 2, UNSIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "pitch", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F( "roll", 2, SIGNED_INTEGER, - field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, ), F("num_beams_and_coordinate_system_and_num_cells", 2, UNSIGNED_INTEGER), @@ -1685,23 +1624,20 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "cell_size", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), F( "blanking", 2, UNSIGNED_INTEGER, - field_units="m", field_unit_conversion=lambda packet, x: x / 1000, ), - F("nominal_correlation", 1, UNSIGNED_INTEGER, field_units="%"), + F("nominal_correlation", 1, UNSIGNED_INTEGER), F(None, 1, RAW_BYTES), F( "battery_voltage", 2, UNSIGNED_INTEGER, - field_units="V", field_unit_conversion=lambda packet, x: x / 10, ), F( @@ -1732,11 +1668,11 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec # Unit conversions for this field are done in Ad2cpDataPacket._postprocess # because the ambiguity velocity unit conversion requires the velocity_scaling field, # which is not known when this field is parsed - F("ambiguity_velocity", 4, UNSIGNED_INTEGER, field_units="m/s"), + F("ambiguity_velocity", 4, UNSIGNED_INTEGER), F("dataset_description", 2, UNSIGNED_INTEGER), F("transmit_energy", 2, UNSIGNED_INTEGER), F("velocity_scaling", 1, SIGNED_INTEGER), - F("power_level", 1, SIGNED_INTEGER, field_units="dB"), + F("power_level", 1, SIGNED_INTEGER), F( "magnetometer_temperature", 2, @@ -1747,7 +1683,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec "real_time_clock_temperature", 2, SIGNED_INTEGER, - field_units="degrees Celsius", field_unit_conversion=lambda packet, x: x / 100, ), F("error", 4, UNSIGNED_INTEGER), @@ -1759,7 +1694,6 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec SIGNED_INTEGER, field_shape=lambda packet: [packet.data.get("num_beams", 0)], field_dimensions=[Dimension.PING_TIME, Dimension.BEAM], - field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), field_exists_predicate=lambda packet: packet.data["velocity_data_included"], @@ -1781,12 +1715,13 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec field_dimensions=[Dimension.PING_TIME, Dimension.BEAM], field_exists_predicate=lambda packet: packet.data["figure_of_merit_data_included"], ), - ] + ], ) ECHOSOUNDER_RAW_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + "ECHOSOUNDER_RAW_DATA_RECORD_FORMAT", [ F("version", 1, UNSIGNED_INTEGER), - F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), + F("offset_of_data", 1, UNSIGNED_INTEGER), F("year", 1, UNSIGNED_INTEGER), F("month", 1, UNSIGNED_INTEGER), F("day", 1, UNSIGNED_INTEGER), @@ -1875,7 +1810,7 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec ], field_exists_predicate=lambda packet: False, ), - ] + ], ) DATA_RECORD_FORMATS = { diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 9f1a35570..75bca3034 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -1,8 +1,11 @@ +from importlib import resources from typing import Dict, List, Optional, Set, Tuple, Union import numpy as np import xarray as xr +import yaml +from .. import convert from ..utils.coding import set_encodings from .parse_ad2cp import DataType, Dimension, Field, HeaderOrDataRecordFormats from .set_groups_base import SetGroupsBase @@ -21,6 +24,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.pulse_compressed = self.parser_obj.get_pulse_compressed() self._make_time_coords() + with resources.open_text(convert, "ad2cp_fields.yaml") as f: + self.field_attrs: Dict[str, Dict[str, Dict[str, str]]] = yaml.safe_load(f) # type: ignore # noqa def _make_time_coords(self): timestamps = [] @@ -69,8 +74,8 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: dims: Dict[str, List[Dimension]] = dict() # {field_name: field dtype} dtypes: Dict[str, np.dtype] = dict() - # {field_name: units} - units: Dict[str, Optional[str]] = dict() + # {field_name: attrs} + attrs: Dict[str, Dict[str, str]] = dict() # {field_name: [idx of padding]} pad_idx: Dict[str, List[int]] = {field_name: [] for field_name in var_names.keys()} # {field_name: field exists} @@ -106,8 +111,8 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: dtypes[field_name] = field.field_entry_data_type.dtype( field_entry_size_bytes ) - if field_name not in units: - units[field_name] = field.units() + if field_name not in attrs: + attrs[field_name] = self.field_attrs[data_record_format.name][field_name] if field_name in packet.data: # field is in this packet fields[field_name].append(packet.data[field_name]) @@ -185,9 +190,7 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: var_name: ( [dim.value for dim in dims[field_name]], combined_fields[field_name], - {"Units": units[field_name]} - if field_name in units and units[field_name] is not None - else {}, + attrs.get(field_name, {}), ) if field_exists[field_name] else ((), None) @@ -348,7 +351,8 @@ def set_beam(self) -> List[xr.Dataset]: { "name": f"Beam_group{len(self._beamgroups) + 1}", "descr": ( - "contains backscatter echo intensity and other configuration parameters from the Echosounder mode. " + "contains backscatter echo intensity and other configuration " + "parameters from the Echosounder mode. " "Data can be pulse compressed or raw intensity." ), } @@ -385,7 +389,8 @@ def set_beam(self) -> List[xr.Dataset]: { "name": f"Beam_group{len(self._beamgroups) + 1}", "descr": ( - "contains complex backscatter raw samples and other configuration parameters from the Echosounder mode, " + "contains complex backscatter raw samples and other configuration " + "parameters from the Echosounder mode, " "including complex data from the transmit pulse." ), } From fdcd19d1f8db4cabfb1f16a4bbaf4bb3ec0352f3 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 19:27:13 -0700 Subject: [PATCH 14/37] Update test for pulse compression --- echopype/tests/convert/test_convert_ad2cp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index b91ec59be..4eeb00715 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -170,7 +170,9 @@ def _check_raw_output( if base.attrs[f"Instrument_echo_pulseComp{i}"]: pulse_compressed = i break - assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed + pulse_compressed_vector = np.zeros(3) + pulse_compressed_vector[pulse_compressed - 1] = 1 + assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed_vector base.close() # check raw data transmit samples From 57b402477d7c4f4f563e1acad05350afcd6ed27f Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 21:00:13 -0700 Subject: [PATCH 15/37] Fix issue with non-unique dim enum values overwriting each other as dictionary keys, change transmit dim from ping_time to ping_time_transmit --- echopype/convert/parse_ad2cp.py | 56 ++++++++++++++------ echopype/convert/set_groups_ad2cp.py | 10 ++-- echopype/tests/convert/test_convert_ad2cp.py | 27 +++++----- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 8b6c4003e..3b8e856bf 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -94,27 +94,51 @@ def default_dtype() -> np.dtype: } +@unique class Dimension(Enum): """ Determines the dimensions of the data in the output dataset """ - PING_TIME = "time1" - PING_TIME_AVERAGE = "ping_time" - PING_TIME_BURST = "ping_time" - PING_TIME_ECHOSOUNDER = "ping_time" - PING_TIME_ECHOSOUNDER_RAW = "ping_time" - PING_TIME_ECHOSOUNDER_RAW_TRANSMIT = "ping_time" - BEAM = "beam" - RANGE_SAMPLE_BURST = "range_sample" - RANGE_SAMPLE_AVERAGE = "range_sample" - RANGE_SAMPLE_ECHOSOUNDER = "range_sample" - NUM_ALTIMETER_SAMPLES = "num_altimeter_samples" - SAMPLE = "range_sample" - SAMPLE_TRANSMIT = "transmit_sample" - MIJ = "mij" - XYZ = "xyz" - WXYZ = "wxyz" + PING_TIME = auto() + PING_TIME_AVERAGE = auto() + PING_TIME_BURST = auto() + PING_TIME_ECHOSOUNDER = auto() + PING_TIME_ECHOSOUNDER_RAW = auto() + PING_TIME_ECHOSOUNDER_RAW_TRANSMIT = auto() + BEAM = auto() + RANGE_SAMPLE_BURST = auto() + RANGE_SAMPLE_AVERAGE = auto() + RANGE_SAMPLE_ECHOSOUNDER = auto() + NUM_ALTIMETER_SAMPLES = auto() + SAMPLE = auto() + SAMPLE_TRANSMIT = auto() + MIJ = auto() + XYZ = auto() + WXYZ = auto() + + def dimension_name(self) -> str: + return DIMENSION_NAMES[self] + + +DIMENSION_NAMES = { + Dimension.PING_TIME: "time1", + Dimension.PING_TIME_AVERAGE: "ping_time", + Dimension.PING_TIME_BURST: "ping_time", + Dimension.PING_TIME_ECHOSOUNDER: "ping_time", + Dimension.PING_TIME_ECHOSOUNDER_RAW: "ping_time", + Dimension.PING_TIME_ECHOSOUNDER_RAW_TRANSMIT: "ping_time_transmit", + Dimension.BEAM: "beam", + Dimension.RANGE_SAMPLE_BURST: "range_sample", + Dimension.RANGE_SAMPLE_AVERAGE: "range_sample", + Dimension.RANGE_SAMPLE_ECHOSOUNDER: "range_sample", + Dimension.NUM_ALTIMETER_SAMPLES: "num_altimeter_samples", + Dimension.SAMPLE: "range_sample", + Dimension.SAMPLE_TRANSMIT: "transmit_sample", + Dimension.MIJ: "mij", + Dimension.XYZ: "xyz", + Dimension.WXYZ: "wxyz", +} class Field: diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 75bca3034..c317765e7 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -188,7 +188,7 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: Union[Tuple[List[str], np.ndarray, Dict[str, str]], Tuple[Tuple[()], None]], ] = { var_name: ( - [dim.value for dim in dims[field_name]], + [dim.dimension_name() for dim in dims[field_name]], combined_fields[field_name], attrs.get(field_name, {}), ) @@ -199,15 +199,15 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: coords: Dict[str, np.ndarray] = dict() for time_dim, time_idxs in self.times_idx.items(): if time_dim in used_dims: - coords[time_dim.value] = self.timestamps[time_idxs] + coords[time_dim.dimension_name()] = self.timestamps[time_idxs] for ahrs_dim, ahrs_coords in AHRS_COORDS.items(): if ahrs_dim in used_dims: - coords[ahrs_dim.value] = ahrs_coords + coords[ahrs_dim.dimension_name()] = ahrs_coords if Dimension.BEAM in used_dims and beam_coords is not None: - coords[Dimension.BEAM.value] = beam_coords + coords[Dimension.BEAM.dimension_name()] = beam_coords ds = xr.Dataset(data_vars=data_vars, coords=coords) # make arange coords for the remaining dims - non_coord_dims = {dim.value for dim in used_dims} - set(ds.coords.keys()) + non_coord_dims = {dim.dimension_name() for dim in used_dims} - set(ds.coords.keys()) ds = ds.assign_coords({dim: np.arange(ds.dims[dim]) for dim in non_coord_dims}) return ds diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index 4eeb00715..ed27f8e0a 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -170,6 +170,8 @@ def _check_raw_output( if base.attrs[f"Instrument_echo_pulseComp{i}"]: pulse_compressed = i break + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + if "pulse_compressed" in echodata[f"Sonar/Beam_group{i}"]: pulse_compressed_vector = np.zeros(3) pulse_compressed_vector[pulse_compressed - 1] = 1 assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed_vector @@ -190,30 +192,30 @@ def _check_raw_output( ) if "090" in filepath_raw.parts: assert np.allclose( - echodata.vendor[ - "echosounder_raw_transmit_samples_i" + echodata["Sonar/Beam_group2"][ + "transmit_pulse_r" ].data.flatten(), base["DataI"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( - echodata.vendor[ - "echosounder_raw_transmit_samples_q" + echodata["Sonar/Beam_group2"][ + "transmit_pulse_i" ].data.flatten(), base["DataQ"].data.flatten(), atol=absolute_tolerance, ) else: assert np.allclose( - echodata.vendor[ - "echosounder_raw_transmit_samples_i" + echodata["Sonar/Beam_group2"][ + "transmit_pulse_r" ].data.flatten(), base["Data_I"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( - echodata.vendor[ - "echosounder_raw_transmit_samples_q" + echodata["Sonar/Beam_group2"][ + "transmit_pulse_i" ].data.flatten(), base["Data_Q"].data.flatten(), atol=absolute_tolerance, @@ -226,25 +228,26 @@ def _check_raw_output( group="Data/RawEcho1_1000kHz", ) if "090" in filepath_raw.parts: + print(base["DataI"].data.shape) assert np.allclose( - echodata.vendor["echosounder_raw_samples_i"].data.flatten(), + echodata["Sonar/Beam_group2"]["backscatter_r"].data.flatten(), base["DataI"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( - echodata.vendor["echosounder_raw_samples_q"].data.flatten(), + echodata["Sonar/Beam_group2"]["backscatter_i"].data.flatten(), base["DataQ"].data.flatten(), atol=absolute_tolerance, ) else: # note the transpose assert np.allclose( - echodata.vendor["echosounder_raw_samples_i"].data.flatten(), + echodata["Sonar/Beam_group2"]["backscatter_r"].data.flatten(), base["Data_I"].data.T.flatten(), atol=absolute_tolerance, ) assert np.allclose( - echodata.vendor["echosounder_raw_samples_q"].data.flatten(), + echodata["Sonar/Beam_group2"]["backscatter_i"].data.flatten(), base["Data_Q"].data.T.flatten(), atol=absolute_tolerance, ) From 5f2068e90e0ffac96b529343f4f68f7ae420a5ee Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 21:17:28 -0700 Subject: [PATCH 16/37] Fix tests for 070 raw files --- echopype/tests/convert/test_convert_ad2cp.py | 114 ++++++++++--------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index ed27f8e0a..bf7656f16 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -172,9 +172,9 @@ def _check_raw_output( break for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): if "pulse_compressed" in echodata[f"Sonar/Beam_group{i}"]: - pulse_compressed_vector = np.zeros(3) - pulse_compressed_vector[pulse_compressed - 1] = 1 - assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed_vector + pulse_compressed_vector = np.zeros(3) + pulse_compressed_vector[pulse_compressed - 1] = 1 + assert (echodata[f"Sonar/Beam_group{i}"]["pulse_compressed"] == pulse_compressed_vector).all() base.close() # check raw data transmit samples @@ -191,35 +191,40 @@ def _check_raw_output( group="Data/RawEcho1_1000kHzTx", ) if "090" in filepath_raw.parts: - assert np.allclose( - echodata["Sonar/Beam_group2"][ - "transmit_pulse_r" - ].data.flatten(), - base["DataI"].data.flatten(), - atol=absolute_tolerance, - ) - assert np.allclose( - echodata["Sonar/Beam_group2"][ - "transmit_pulse_i" - ].data.flatten(), - base["DataQ"].data.flatten(), - atol=absolute_tolerance, - ) + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + if "transmit_pulse_r" in echodata[f"Sonar/Beam_group{i}"]: + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"][ + "transmit_pulse_r" + ].data.flatten(), + base["DataI"].data.flatten(), + atol=absolute_tolerance, + ) + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"][ + "transmit_pulse_i" + ].data.flatten(), + base["DataQ"].data.flatten(), + atol=absolute_tolerance, + ) else: - assert np.allclose( - echodata["Sonar/Beam_group2"][ - "transmit_pulse_r" - ].data.flatten(), - base["Data_I"].data.flatten(), - atol=absolute_tolerance, - ) - assert np.allclose( - echodata["Sonar/Beam_group2"][ - "transmit_pulse_i" - ].data.flatten(), - base["Data_Q"].data.flatten(), - atol=absolute_tolerance, - ) + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + if "transmit_pulse_r" in echodata[f"Sonar/Beam_group{i}"]: + # note the underscore + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"][ + "transmit_pulse_r" + ].data.flatten(), + base["Data_I"].data.flatten(), + atol=absolute_tolerance, + ) + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"][ + "transmit_pulse_i" + ].data.flatten(), + base["Data_Q"].data.flatten(), + atol=absolute_tolerance, + ) base.close() # check raw data samples @@ -228,27 +233,30 @@ def _check_raw_output( group="Data/RawEcho1_1000kHz", ) if "090" in filepath_raw.parts: - print(base["DataI"].data.shape) - assert np.allclose( - echodata["Sonar/Beam_group2"]["backscatter_r"].data.flatten(), - base["DataI"].data.flatten(), - atol=absolute_tolerance, - ) - assert np.allclose( - echodata["Sonar/Beam_group2"]["backscatter_i"].data.flatten(), - base["DataQ"].data.flatten(), - atol=absolute_tolerance, - ) + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + if "backscatter_r" in echodata[f"Sonar/Beam_group{i}"]: + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"]["backscatter_r"].data.flatten(), + base["DataI"].data.flatten(), + atol=absolute_tolerance, + ) + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"]["backscatter_i"].data.flatten(), + base["DataQ"].data.flatten(), + atol=absolute_tolerance, + ) else: - # note the transpose - assert np.allclose( - echodata["Sonar/Beam_group2"]["backscatter_r"].data.flatten(), - base["Data_I"].data.T.flatten(), - atol=absolute_tolerance, - ) - assert np.allclose( - echodata["Sonar/Beam_group2"]["backscatter_i"].data.flatten(), - base["Data_Q"].data.T.flatten(), - atol=absolute_tolerance, - ) + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + if "transmit_pulse_r" in echodata[f"Sonar/Beam_group{i}"]: + # note the transpose + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"]["backscatter_r"].data.flatten(), + base["Data_I"].data.T.flatten(), + atol=absolute_tolerance, + ) + assert np.allclose( + echodata[f"Sonar/Beam_group{i}"]["backscatter_i"].data.flatten(), + base["Data_Q"].data.T.flatten(), + atol=absolute_tolerance, + ) base.close() From d020ecdab443e9e55b881cf872c8f032f2d30d88 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Fri, 17 Jun 2022 22:05:12 -0700 Subject: [PATCH 17/37] Add beam groups 3 and 4 to 1.0.yml --- echopype/echodata/convention/1.0.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/echopype/echodata/convention/1.0.yml b/echopype/echodata/convention/1.0.yml index 51b6ef229..7b8110b27 100644 --- a/echopype/echodata/convention/1.0.yml +++ b/echopype/echodata/convention/1.0.yml @@ -45,6 +45,14 @@ groups: including split-beam angle data when they exist. Only exists if complex backscatter data are already in Sonar/Beam_group1 ep_group: Sonar/Beam_group2 + beam_group3: + name: Beam_group3 + description: '' + ep_group: Sonar/Beam_group3 + beam_group4: + name: Beam_group4 + description: '' + ep_group: Sonar/Beam_group4 vendor: name: Vendor_specific description: contains vendor-specific information about the sonar and the data. From c38ea813ff2a5163162c3ebaa04e0260007cd259 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sat, 18 Jun 2022 02:52:41 -0700 Subject: [PATCH 18/37] Save all AD2CP beam groups --- echopype/convert/api.py | 53 +++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index f3530cbe9..46b2273c9 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -142,18 +142,19 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # /Sonar/Beam_groupX group if echodata.sonar_model == "AD2CP": - io.save_file( - echodata.beam.chunk( - { - "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], - } - ), - path=output_path, - mode="a", - engine=engine, - group=f"Sonar/{BEAM_SUBGROUP_DEFAULT}", - compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, - ) + for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): + io.save_file( + echodata.beam.chunk( + { + "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], + } + ), + path=output_path, + mode="a", + engine=engine, + group=f"Sonar/Beam_group{i}", + compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, + ) else: io.save_file( echodata.beam.chunk( @@ -168,20 +169,20 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): group=f"Sonar/{BEAM_SUBGROUP_DEFAULT}", compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, ) - if echodata.beam_power is not None: - io.save_file( - echodata.beam_power.chunk( - { - "range_sample": DEFAULT_CHUNK_SIZE["range_sample"], - "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], - } - ), - path=output_path, - mode="a", - engine=engine, - group="Sonar/Beam_group2", - compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, - ) + if echodata.beam_power is not None: + io.save_file( + echodata.beam_power.chunk( + { + "range_sample": DEFAULT_CHUNK_SIZE["range_sample"], + "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], + } + ), + path=output_path, + mode="a", + engine=engine, + group="Sonar/Beam_group2", + compression_settings=COMPRESSION_SETTINGS[engine] if compress else None, + ) # Platform group io.save_file( From b2fae3451fcdf4b5627248bb8eb6d26f78cb424c Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sat, 18 Jun 2022 13:44:10 -0700 Subject: [PATCH 19/37] Set time encodings for Sonar and ping_time_transmit --- echopype/convert/set_groups_ad2cp.py | 2 +- echopype/utils/coding.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index c317765e7..155751309 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -487,4 +487,4 @@ def set_sonar(self) -> xr.Dataset: ds = ds.assign_attrs(sonar_attr_dict) - return ds + return set_encodings(ds) diff --git a/echopype/utils/coding.py b/echopype/utils/coding.py index f8583e964..ca1339ed5 100644 --- a/echopype/utils/coding.py +++ b/echopype/utils/coding.py @@ -20,11 +20,7 @@ DEFAULT_ENCODINGS = { "ping_time": DEFAULT_TIME_ENCODING, - "ping_time_burst": DEFAULT_TIME_ENCODING, - "ping_time_average": DEFAULT_TIME_ENCODING, - "ping_time_echosounder": DEFAULT_TIME_ENCODING, - "ping_time_echosounder_raw": DEFAULT_TIME_ENCODING, - "ping_time_echosounder_raw_transmit": DEFAULT_TIME_ENCODING, + "ping_time_transmit": DEFAULT_TIME_ENCODING, "time1": DEFAULT_TIME_ENCODING, "time2": DEFAULT_TIME_ENCODING, "time3": DEFAULT_TIME_ENCODING, From 5956377d7e66324a80e1f3828cc98c68a70a97e2 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sat, 18 Jun 2022 14:01:20 -0700 Subject: [PATCH 20/37] Fix beam group not saving to file --- echopype/convert/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 46b2273c9..dc1c2f340 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -144,7 +144,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): if echodata.sonar_model == "AD2CP": for i in range(1, len(echodata["Sonar"]["beam_group"]) + 1): io.save_file( - echodata.beam.chunk( + echodata[f"Sonar/Beam_group{i}"].chunk( { "ping_time": DEFAULT_CHUNK_SIZE["ping_time"], } From 8e0737bd27b7f607855647aaf17928af39ae41f2 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 14:23:05 -0700 Subject: [PATCH 21/37] Simplify beam group creation logic --- echopype/convert/set_groups_ad2cp.py | 248 ++++++++++++++------------- 1 file changed, 130 insertions(+), 118 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 155751309..530fdf689 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -1,3 +1,4 @@ +from enum import Enum, unique, auto from importlib import resources from typing import Dict, List, Optional, Set, Tuple, Union @@ -17,6 +18,14 @@ } +@unique +class BeamGroup(Enum): + AVERAGE = auto() + BURST = auto() + ECHOSOUNDER = auto() + ECHOSOUNDER_RAW = auto() + + class SetGroupsAd2cp(SetGroupsBase): """Class for saving groups to netcdf or zarr from Ad2cp data files.""" @@ -252,79 +261,22 @@ def set_beam(self) -> List[xr.Dataset]: # and range_bin_echosounder)? beam_groups = [] self._beamgroups = [] + beam_groups_exist = set() - # burst - for packet in self.parser_obj.packets: - if packet.is_burst(): - beam_groups.append( - self._make_dataset( - { - "num_beams": "number_of_beams", - "coordinate_system": "coordinate_system", - "num_cells": "number_of_cells", - "blanking": "blanking", - "cell_size": "cell_size", - "velocity_range": "velocity_range", - "echosounder_frequency": "echosounder_frequency", - "ambiguity_velocity": "ambiguity_velocity", - "dataset_description": "data_set_description", - "transmit_energy": "transmit_energy", - "velocity_scaling": "velocity_scaling", - "velocity_data_burst": "velocity", - "amplitude_data_burst": "amplitude", - "correlation_data_burst": "correlation", - } - ) - ) - - self._beamgroups.append( - { - "name": f"Beam_group{len(self._beamgroups) + 1}", - "descr": ( - "contains echo intensity, velocity and correlation data " - "as well as other configuration parameters from the Burst mode." - ), - } - ) - break - # average for packet in self.parser_obj.packets: if packet.is_average(): - beam_groups.append( - self._make_dataset( - { - "num_beams": "number_of_beams", - "coordinate_system": "coordinate_system", - "num_cells": "number_of_cells", - "blanking": "blanking", - "cell_size": "cell_size", - "velocity_range": "velocity_range", - "echosounder_frequency": "echosounder_frequency", - "ambiguity_velocity": "ambiguity_velocity", - "dataset_description": "data_set_description", - "transmit_energy": "transmit_energy", - "velocity_scaling": "velocity_scaling", - "velocity_data_average": "velocity", - "amplitude_data_average": "amplitude", - "correlation_data_average": "correlation", - } - ) - ) + beam_groups_exist.add(BeamGroup.AVERAGE) + elif packet.is_burst(): + beam_groups_exist.add(BeamGroup.BURST) + elif packet.is_echosounder(): + beam_groups_exist.add(BeamGroup.ECHOSOUNDER) + elif packet.is_echosounder_raw(): + beam_groups_exist.add(BeamGroup.ECHOSOUNDER_RAW) - self._beamgroups.append( - { - "name": f"Beam_group{len(self._beamgroups) + 1}", - "descr": ( - "contains echo intensity, velocity and correlation data " - "as well as other configuration parameters from the Average mode." - ), - } - ) - break - # echosounder - for packet in self.parser_obj.packets: - if packet.is_echosounder(): - ds = self._make_dataset( + # average + if BeamGroup.AVERAGE in beam_groups_exist: + beam_groups.append( + self._make_dataset( { "num_beams": "number_of_beams", "coordinate_system": "coordinate_system", @@ -337,65 +289,125 @@ def set_beam(self) -> List[xr.Dataset]: "dataset_description": "data_set_description", "transmit_energy": "transmit_energy", "velocity_scaling": "velocity_scaling", - "correlation_data_echosounder": "correlation", - "echosounder_data": "amplitude", + "velocity_data_average": "velocity", + "amplitude_data_average": "amplitude", + "correlation_data_average": "correlation", } ) - ds = ds.assign_coords({"echogram": np.arange(3)}) - pulse_compressed = np.zeros(3) - pulse_compressed[self.pulse_compressed - 1] = 1 - ds["pulse_compressed"] = (("echogram",), pulse_compressed) - beam_groups.append(ds) + ) - self._beamgroups.append( + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains echo intensity, velocity and correlation data " + "as well as other configuration parameters from the Average mode." + ), + } + ) + # burst + if BeamGroup.BURST in beam_groups_exist: + beam_groups.append( + self._make_dataset( { - "name": f"Beam_group{len(self._beamgroups) + 1}", - "descr": ( - "contains backscatter echo intensity and other configuration " - "parameters from the Echosounder mode. " - "Data can be pulse compressed or raw intensity." - ), + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "velocity_data_burst": "velocity", + "amplitude_data_burst": "amplitude", + "correlation_data_burst": "correlation", } ) - break - # echosounder raw - for packet in self.parser_obj.packets: - if packet.is_echosounder_raw(): - beam_groups.append( - self._make_dataset( - { - "num_beams": "number_of_beams", - "coordinate_system": "coordinate_system", - "num_cells": "number_of_cells", - "blanking": "blanking", - "cell_size": "cell_size", - "velocity_range": "velocity_range", - "echosounder_frequency": "echosounder_frequency", - "ambiguity_velocity": "ambiguity_velocity", - "dataset_description": "data_set_description", - "transmit_energy": "transmit_energy", - "velocity_scaling": "velocity_scaling", - "echosounder_raw_samples_i": "backscatter_r", - "echosounder_raw_samples_q": "backscatter_i", - "echosounder_raw_transmit_samples_i": "transmit_pulse_r", - "echosounder_raw_transmit_samples_q": "transmit_pulse_i", - "echosounder_raw_beam": "echosounder_raw_beam", - "echosounder_raw_echogram": "echosounder_raw_echogram", - } - ) - ) + ) - self._beamgroups.append( + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains echo intensity, velocity and correlation data " + "as well as other configuration parameters from the Burst mode." + ), + } + ) + # echosounder + if BeamGroup.ECHOSOUNDER in beam_groups_exist: + ds = self._make_dataset( + { + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "correlation_data_echosounder": "correlation", + "echosounder_data": "amplitude", + } + ) + ds = ds.assign_coords({"echogram": np.arange(3)}) + pulse_compressed = np.zeros(3) + pulse_compressed[self.pulse_compressed - 1] = 1 + ds["pulse_compressed"] = (("echogram",), pulse_compressed) + beam_groups.append(ds) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains backscatter echo intensity and other configuration " + "parameters from the Echosounder mode. " + "Data can be pulse compressed or raw intensity." + ), + } + ) + # echosounder raw + if BeamGroup.ECHOSOUNDER_RAW in beam_groups_exist: + beam_groups.append( + self._make_dataset( { - "name": f"Beam_group{len(self._beamgroups) + 1}", - "descr": ( - "contains complex backscatter raw samples and other configuration " - "parameters from the Echosounder mode, " - "including complex data from the transmit pulse." - ), + "num_beams": "number_of_beams", + "coordinate_system": "coordinate_system", + "num_cells": "number_of_cells", + "blanking": "blanking", + "cell_size": "cell_size", + "velocity_range": "velocity_range", + "echosounder_frequency": "echosounder_frequency", + "ambiguity_velocity": "ambiguity_velocity", + "dataset_description": "data_set_description", + "transmit_energy": "transmit_energy", + "velocity_scaling": "velocity_scaling", + "echosounder_raw_samples_i": "backscatter_r", + "echosounder_raw_samples_q": "backscatter_i", + "echosounder_raw_transmit_samples_i": "transmit_pulse_r", + "echosounder_raw_transmit_samples_q": "transmit_pulse_i", + "echosounder_raw_beam": "echosounder_raw_beam", + "echosounder_raw_echogram": "echosounder_raw_echogram", } ) - break + ) + + self._beamgroups.append( + { + "name": f"Beam_group{len(self._beamgroups) + 1}", + "descr": ( + "contains complex backscatter raw samples and other configuration " + "parameters from the Echosounder mode, " + "including complex data from the transmit pulse." + ), + } + ) # FIXME: this is a hack because the current file saving # mechanism requires that the beam group have ping_time as a dimension, From 3c18636fc4b9f426d14299fb751575a05211e905 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 19 Jun 2022 21:23:39 +0000 Subject: [PATCH 22/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- echopype/convert/set_groups_ad2cp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 530fdf689..2d1882996 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -1,4 +1,4 @@ -from enum import Enum, unique, auto +from enum import Enum, auto, unique from importlib import resources from typing import Dict, List, Optional, Set, Tuple, Union From feaf06bc1a9bdf7fb55f6a964c46453edf38cd3f Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 14:28:30 -0700 Subject: [PATCH 23/37] Add missing periods to comments --- echopype/convert/ad2cp_fields.yaml | 151 +++++++++++++++-------------- 1 file changed, 76 insertions(+), 75 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index 094f6c1da..ae9e77b18 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -52,38 +52,38 @@ milliseconds resolution)' long_name: Field "Speed Of Sound" in the data specification. units: m/s comment: 'Speed of sound used by the instrument. -Raw data given as 0.1m/s +Raw data given as 0.1m/s. Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius comment: 'Reading from the temperature sensor. -Raw data given as 0.01 °C +Raw data given as 0.01 °C. Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: 'Raw data given as 0.001 dBar + comment: 'Raw data given as 0.001 dBar. Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' error: @@ -138,7 +138,7 @@ Bit 9-0 (10 bits): Number of Cells (NC).' long_name: Field "Cell Size" in the data specification. units: m comment: 'Size of each cell (resolution) on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' blanking: @@ -146,7 +146,7 @@ Raw data given as mm units: m comment: 'Distance from instrument to first data point on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' velocity_range: @@ -158,7 +158,7 @@ Raw data given as mm battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: 'Raw value given in 0.1 Volt + comment: 'Raw value given in 0.1 Volt. Data was converted to the specified units.' magnetometer_raw: @@ -182,7 +182,7 @@ Raw data given as mm units: m/s comment: 'Ambiguity velocity, corrected forsound velocity, scaled according to Velocity scaling. -Data given as 10^(Velocity scaling) m/s +Data given as 10^(Velocity scaling) m/s. Data was converted to the specified units.' dataset_description: @@ -205,7 +205,7 @@ Data given as 10^(Velocity scaling) m/s power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: 'Configured power level + comment: 'Configured power level. Data was converted to the specified units.' velocity_data_burst: @@ -213,7 +213,7 @@ Data given as 10^(Velocity scaling) m/s units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' velocity_data_average: @@ -221,7 +221,7 @@ Data given as 10^(Velocity Scaling) units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' velocity_data_echosounder: @@ -229,7 +229,7 @@ Data given as 10^(Velocity Scaling) units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' amplitude_data_burst: @@ -237,7 +237,7 @@ Data given as 10^(Velocity Scaling) units: dB/count comment: 'This field exists if the amplitude data included bit of the Config byte is set -Data given as 0.5 dB/count +Data given as 0.5 dB/count. Data was converted to the specified units.' amplitude_data_average: @@ -245,7 +245,7 @@ Data given as 0.5 dB/count units: dB/count comment: 'This field exists if the amplitude data included bit of the Config byte is set -Data given as 0.5 dB/count +Data given as 0.5 dB/count. Data was converted to the specified units.' amplitude_data_echosounder: @@ -253,14 +253,14 @@ Data given as 0.5 dB/count units: dB/count comment: 'This field exists if the amplitude data included bit of the Config byte is set -Data given as 0.5 dB/count +Data given as 0.5 dB/count. Data was converted to the specified units.' correlation_data_burst: long_name: Field "Correlation Data Burst" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. [0 – 100 %] Data was converted to the specified units.' @@ -268,7 +268,7 @@ of the Config byte is set long_name: Field "Correlation Data Average" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. [0 – 100 %] Data was converted to the specified units.' @@ -276,7 +276,7 @@ of the Config byte is set long_name: Field "Correlation Data Echosounder" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. [0 – 100 %] Data was converted to the specified units.' @@ -334,38 +334,38 @@ milliseconds resolution)' long_name: Field "Speed Of Sound" in the data specification. units: m/s comment: 'Speed of sound used by the instrument. -Raw data given as 0.1m/s +Raw data given as 0.1m/s. Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius comment: 'Reading from the temperature sensor. -Raw data given as 0.01 °C +Raw data given as 0.01 °C. Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: 'Raw data given as 0.001 dBar + comment: 'Raw data given as 0.001 dBar. Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' num_beams_and_coordinate_system_and_num_cells: @@ -383,7 +383,7 @@ Bit 9-0 (10 bits): Number of Cells (NC).' long_name: Field "Cell Size" in the data specification. units: m comment: 'Size of each cell (resolution) on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' blanking: @@ -391,14 +391,14 @@ Raw data given as mm units: m comment: 'Distance from instrument to first data point on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' nominal_correlation: long_name: Field "Nominal Correlation" in the data specification. units: '%' comment: 'The nominal correlation for the configured -combination of cell size and velocity range +combination of cell size and velocity range. Data was converted to the specified units.' temperature_from_pressure_sensor: @@ -410,7 +410,7 @@ combination of cell size and velocity range battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: 'Raw value given in 0.1 Volt + comment: 'Raw value given in 0.1 Volt. Data was converted to the specified units.' magnetometer_raw: @@ -455,7 +455,7 @@ scaled according to Velocity scaling. power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: 'Configured power level + comment: 'Configured power level. Data was converted to the specified units.' magnetometer_temperature: @@ -467,7 +467,7 @@ Raw data in 1/1000 °C' real_time_clock_temperature: long_name: Field "Real Time Clock Temperature" in the data specification. units: degrees Celsius - comment: 'Real Time Clock temperature reading + comment: 'Real Time Clock temperature reading. Data was converted to the specified units.' error: @@ -531,7 +531,7 @@ and burst data' units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' velocity_data_average: @@ -539,7 +539,7 @@ Data given as 10^(Velocity Scaling) units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' velocity_data_echosounder: @@ -547,7 +547,7 @@ Data given as 10^(Velocity Scaling) units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' amplitude_data_burst: @@ -555,7 +555,7 @@ Data given as 10^(Velocity Scaling) units: dB/count comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data given as 10^(Velocity Scaling) +Data given as 10^(Velocity Scaling). Data was converted to the specified units.' amplitude_data_average: @@ -563,7 +563,7 @@ Data given as 10^(Velocity Scaling) units: dB/count comment: 'This field exists if the amplitude data included bit of the Config byte is set -Data given as 0.5 dB/count +Data given as 0.5 dB/count. Data was converted to the specified units.' amplitude_data_echosounder: @@ -571,14 +571,14 @@ Data given as 0.5 dB/count units: dB/count comment: 'This field exists if the amplitude data included bit of the Config byte is set -Data given as 0.5 dB/count +Data given as 0.5 dB/count. Data was converted to the specified units.' correlation_data_burst: long_name: Field "Correlation Data Burst" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. Data given as [0 – 100 %] Data was converted to the specified units.' @@ -586,7 +586,7 @@ Data given as [0 – 100 %] long_name: Field "Correlation Data Average" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. Data given as [0 – 100 %] Data was converted to the specified units.' @@ -594,14 +594,14 @@ Data given as [0 – 100 %] long_name: Field "Correlation Data Echosounder" in the data specification. units: 0-100% comment: 'This field exists if the Correlation data included bit -of the Config byte is set +of the Config byte is set. Data given as [0 – 100 %] Data was converted to the specified units.' altimeter_distance: long_name: Field "Altimeter Distance" in the data specification. units: m - comment: 'Distance to surface from Leading Edge algorithm + comment: 'Distance to surface from Leading Edge algorithm. Data was converted to the specified units.' altimeter_quality: @@ -613,7 +613,7 @@ and Quality set to 0.' ast_distance: long_name: Field "Ast Distance" in the data specification. units: m - comment: 'Distance to surface from Max Peak algorithm + comment: 'Distance to surface from Max Peak algorithm. Data was converted to the specified units.' ast_quality: @@ -621,21 +621,22 @@ and Quality set to 0.' units: dB comment: 'Amplitude at which surface is detected Raw data in steps of 0.01 dB, i.e. quality of 8000 = 80 -dB +dB. + Data was converted to the specified units.' ast_offset_100us: long_name: Field "Ast Offset 100us" in the data specification. units: "100 μs" comment: 'Offset in step of measurement to velocity -measurement -Raw data given in 100 µs +measurement. +Raw data given in 100 µs. Data was converted to the specified units.' ast_pressure: long_name: Field "Ast Pressure" in the data specification. units: dBar comment: 'Pressure value measured during the AST/altimeter -ping +ping. Data was converted to the specified units.' altimeter_spare: @@ -649,8 +650,8 @@ ping altimeter_raw_data_sample_distance: long_name: Field "Altimeter Raw Data Sample Distance" in the data specification. units: m - comment: 'Distance between samples -Raw data given in 0.1mm + comment: 'Distance between samples. +Raw data given in 0.1mm. Data was converted to the specified units.' altimeter_raw_data_samples: @@ -661,8 +662,8 @@ Raw data given as 16 bits Signed fract' echosounder_data: long_name: Field "Echosounder Data" in the data specification. units: dB/count - comment: 'Echosounder amplitude Data -Raw data given as 0.01 dB/count + comment: 'Echosounder amplitude Data. +Raw data given as 0.01 dB/count. Data was converted to the specified units.' ahrs_rotation_matrix: @@ -680,37 +681,37 @@ Raw data given as 0.01 dB/count percentage_good_data: long_name: Field "Percentage Good Data" in the data specification. units: '%' - comment: 'Percent Good Estimate per cell + comment: 'Percent Good Estimate per cell. These fields exist if the Percentage Good data -included +included. Data was converted to the specified units.' std_dev_pitch: long_name: Field "Std Dev Pitch" in the data specification. units: degrees - comment: 'Standard deviation on pitch data -Raw data in 0.01 degrees + comment: 'Standard deviation on pitch data. +Raw data in 0.01 degrees. Data was converted to the specified units.' std_dev_roll: long_name: Field "Std Dev Roll" in the data specification. units: degrees - comment: 'Standard deviation on roll data -Raw data in 0.01 degrees + comment: 'Standard deviation on roll data. +Raw data in 0.01 degrees. Data was converted to the specified units.' std_dev_heading: long_name: Field "Std Dev Heading" in the data specification. units: degrees - comment: 'Standard deviation on heading data -Raw data in 0.01 degrees + comment: 'Standard deviation on heading data. +Raw data in 0.01 degrees. Data was converted to the specified units.' std_dev_pressure: long_name: Field "Std Dev Pressure" in the data specification. units: dBar - comment: 'Standard deviation on pressure data -Raw data in 0.001 Bar + comment: 'Standard deviation on pressure data. +Raw data in 0.001 Bar. Data was converted to the specified units.' BOTTOM_TRACK_DATA_RECORD_FORMAT: @@ -767,38 +768,38 @@ milliseconds resolution)' long_name: Field "Speed Of Sound" in the data specification. units: m/s comment: 'Speed of sound used by the instrument. -Raw data given as 0.1m/s +Raw data given as 0.1m/s. Data was converted to the specified units.' temperature: long_name: Field "Temperature" in the data specification. units: degrees Celsius comment: 'Reading from the temperature sensor. -Raw data given as 0.01 °C +Raw data given as 0.01 °C. Data was converted to the specified units.' pressure: long_name: Field "Pressure" in the data specification. units: dBar - comment: 'Raw data given as 0.001 dBar + comment: 'Raw data given as 0.001 dBar. Data was converted to the specified units.' heading: long_name: Field "Heading" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' pitch: long_name: Field "Pitch" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' roll: long_name: Field "Roll" in the data specification. units: degrees - comment: 'Raw data given as 0.01 degrees + comment: 'Raw data given as 0.01 degrees. Data was converted to the specified units.' num_beams_and_coordinate_system_and_num_cells: @@ -818,7 +819,7 @@ Bit 9-0 (10 bits) represent the number of cells (NC).' long_name: Field "Cell Size" in the data specification. units: m comment: 'Size of each cell (resolution) on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' blanking: @@ -826,20 +827,20 @@ Raw data given as mm units: m comment: 'Distance from instrument to first data point on the beam. -Raw data given as mm +Raw data given as mm. Data was converted to the specified units.' nominal_correlation: long_name: Field "Nominal Correlation" in the data specification. units: '%' comment: 'The nominal correlation for the configured -combination of cell size and velocity range +combination of cell size and velocity range. Data was converted to the specified units.' battery_voltage: long_name: Field "Battery Voltage" in the data specification. units: V - comment: 'Raw value given in 0.1 Volt + comment: 'Raw value given in 0.1 Volt. Data was converted to the specified units.' magnetometer_raw: @@ -863,7 +864,7 @@ combination of cell size and velocity range units: m/s comment: 'Ambiguity velocity, corrected forsound velocity, scaled according to Velocity scaling. -10^(Velocity scaling) m/s +Data given as 10^(Velocity scaling) m/s. Data was converted to the specified units.' dataset_description: @@ -885,7 +886,7 @@ scaled according to Velocity scaling. power_level: long_name: Field "Power Level" in the data specification. units: dB - comment: 'Configured power level + comment: 'Configured power level. Data was converted to the specified units.' magnetometer_temperature: @@ -899,7 +900,7 @@ Raw data in 1/1000 °C real_time_clock_temperature: long_name: Field "Real Time Clock Temperature" in the data specification. units: degrees Celsius - comment: 'Real Time Clock temperature reading + comment: 'Real Time Clock temperature reading. Data was converted to the specified units.' error: @@ -949,7 +950,7 @@ and burst data' units: m/s comment: 'This field exists if the Velocity data included bit of the Config byte is set. -Data scaled as product of 10^(Velocity Scaling) +Data scaled as product of 10^(Velocity Scaling). Data was converted to the specified units.' distance_data: From cde47ceb1a6d40293374b038331ac8045ecb7de8 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 14:33:29 -0700 Subject: [PATCH 24/37] Remove time1 from beam groups --- echopype/convert/set_groups_ad2cp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 2d1882996..8be341ceb 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -417,6 +417,10 @@ def set_beam(self) -> List[xr.Dataset]: if "ping_time" not in ds.dims: beam_groups[i] = ds.expand_dims(dim="ping_time") + # remove time1 from beam groups + for i, ds in enumerate(beam_groups): + beam_groups[i] = ds.sel(time1=ds["ping_time"]) + return [set_encodings(ds) for ds in beam_groups] def set_vendor(self) -> xr.Dataset: From 3d28666e29d0ce7bbc26a808d56497ca38b8b99f Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 14:39:11 -0700 Subject: [PATCH 25/37] Remove unnecessary ping_time dim expansion and change chunking dimension from ping_time to time1 --- echopype/convert/api.py | 4 ---- echopype/convert/set_groups_ad2cp.py | 14 -------------- 2 files changed, 18 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index dc1c2f340..7e7c86ade 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -119,11 +119,7 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): # Environment group io.save_file( echodata.environment.chunk( - # Making chunking w.r.t. ping_time for AD2CP - # and w.r.t. time1 for the rest of the sensors {"time1": DEFAULT_CHUNK_SIZE["ping_time"]} - if echodata.top.attrs["keywords"] != "AD2CP" - else {"ping_time": DEFAULT_CHUNK_SIZE["ping_time"]} ), # TODO: chunking necessary? path=output_path, mode="a", diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 8be341ceb..0255cabdd 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -229,13 +229,6 @@ def set_env(self) -> xr.Dataset: } ) - # FIXME: this is a hack because the current file saving - # mechanism requires that the env group have ping_time as a dimension, - # but ping_time might not be a dimension if the dataset is completely - # empty - if "ping_time" not in ds.dims: - ds = ds.expand_dims(dim="ping_time") - return set_encodings(ds) def set_platform(self) -> xr.Dataset: @@ -464,13 +457,6 @@ def set_vendor(self) -> xr.Dataset: } ) - # FIXME: this is a hack because the current file saving - # mechanism requires that the vendor group have ping_time as a dimension, - # but ping_time might not be a dimension if the dataset is completely - # empty - if "ping_time" not in ds.dims: - ds = ds.expand_dims(dim="ping_time") - return set_encodings(ds) def set_sonar(self) -> xr.Dataset: From 2165d8dbcd390caf59c92d220a7c3986360ca735 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:53:15 -0700 Subject: [PATCH 26/37] Add attrs for postprocessed fields --- echopype/convert/ad2cp_fields.yaml | 232 ++++++++++++++++++++++++++- echopype/convert/set_groups_ad2cp.py | 36 ++++- 2 files changed, 264 insertions(+), 4 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index ae9e77b18..5af0beb6d 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -180,7 +180,7 @@ Raw data given as mm. ambiguity_velocity: long_name: Field "Ambiguity Velocity" in the data specification. units: m/s - comment: 'Ambiguity velocity, corrected forsound velocity, + comment: 'Ambiguity velocity, corrected for sound velocity, scaled according to Velocity scaling. Data given as 10^(Velocity scaling) m/s. @@ -432,7 +432,7 @@ combination of cell size and velocity range. ambiguity_velocity_or_echosounder_frequency: long_name: Field "Ambiguity Velocity Or Echosounder Frequency" in the data specification. units: '' - comment: 'Ambiguity velocity, corrected forsound velocity, + comment: 'Ambiguity velocity, corrected for sound velocity, scaled according to Velocity scaling. 10^(Velocity scaling) m/s OR Echosounder frequency [Hz] ' @@ -862,7 +862,7 @@ combination of cell size and velocity range. ambiguity_velocity: long_name: Field "Ambiguity Velocity" in the data specification. units: m/s - comment: 'Ambiguity velocity, corrected forsound velocity, + comment: 'Ambiguity velocity, corrected for sound velocity, scaled according to Velocity scaling. Data given as 10^(Velocity scaling) m/s. @@ -1083,3 +1083,229 @@ configured blanking distance.' long_name: Field "Echosounder Raw Transmit Samples Q" in the data specification. units: '' comment: '' +POSTPROCESSED: + pressure_sensor_valid: + long_name: Field "Pressure Sensor Valid" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + temperature_sensor_valid: + long_name: Field "Temperature Sensor Valid" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + compass_sensor_valid: + long_name: Field "Compass Sensor Valid" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + tilt_sensor_valid: + long_name: Field "Tilt Sensor Valid" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + velocity_data_included: + long_name: Field "Velocity Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + amplitude_data_included: + long_name: Field "Amplitude Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + correlation_data_included: + long_name: Field "Correlation Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + altimeter_data_included: + long_name: Field "Altimeter Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + altimeter_raw_data_included: + long_name: Field "Altimeter Raw Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + ast_data_included: + long_name: Field "Ast Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + echosounder_data_included: + long_name: Field "Echosounder Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + ahrs_data_included: + long_name: Field "Ahrs Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + percentage_good_data_included: + long_name: Field "Percentage Good Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + std_dev_data_included: + long_name: Field "Std Dev Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + distance_data_included: + long_name: Field "Distance Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + figure_of_merit_data_included: + long_name: Field "Figure Of Merit Data Included" parsed from field "Configuration" + units: '' + comment: Derived from field "Configuration" + num_cells: + long_name: Field "Number Of Cells" parsed from field "Beams, Coordinates And Cells" + units: '' + comment: Derived from field "Beams, Coordinates And Cells" + num_echosounder_cells: + long_name: Field "Number Of Echosounder Cells" parsed from field "Beams, Coordinates And Cells" + units: '' + comment: Derived from field "Beams, Coordinates And Cells" + coordinate_system: + long_name: Field "Coordinate System" parsed from field "Beams, Coordinates And Cells" + units: '' + comment: Derived from field "Beams, Coordinates And Cells" + num_beams: + long_name: Field "Number Of Beams" parsed from field "Beams, Coordinates And Cells" + units: '' + comment: Derived from field "Beams, Coordinates And Cells" + echosounder_frequency: + long_name: Field "Echosounder Frequency" parsed from field "Ambiguity Velocity Or Echosounder Frequency" + units: Hz + comment: Derived from field "Ambiguity Velocity Or Echosounder Frequency" + ambiguity_velocity: + long_name: Field "Ambiguity Velocity" parsed from field "Ambiguity Velocity Or Echosounder Frequency" + units: m/s + comment: 'Derived from field "Ambiguity Velocity Or Echosounder Frequency" + + Ambiguity velocity, corrected for sound velocity, +scaled according to Velocity scaling. +Data given as 10^(Velocity scaling) m/s. + + Data was converted to the specified units.' + procidle3: + long_name: Field "Processor Idles <3%" parsed from field "Status0" + units: '' + comment: 'Derived from field "Status0" + +Indicates that the processor Idles less than 3 +percent' + procidle6: + long_name: Field "Processor Idles <6%" parsed from field "Status0" + units: '' + comment: 'Derived from field "Status0" + +Indicates that the processor Idles less than 6 +percent' + procidle12: + long_name: Field "Processor Idles <12%" parsed from field "Status0" + units: '' + comment: 'Derived from field "Status0" + +Indicates that the processor Idles less than 12 +percent' + wakeup_state: + long_name: Field "Wakeup State" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 31-28: Wakeup State +1111 0000 0000 0000 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm' + orientation: + long_name: Field "Orientation" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 27-25: Orientation +0000 1110 0000 0000 0000 0000 0000 0000 +0: "XUP" Instrument x-axis defined up, heading +reference axis is Z positive +1: "XDOWN" Instrument x-axis defined down, +heading reference axis is Z positive +2: "YUP" Instrument y-axis defined up, heading +reference axis is Z positive +3: "YDOWN" Instrument y-axis defined down, +heading reference axis is Z positive +4: "ZUP" Instrument z-axis defined up, heading +reference axis is X positive +5: "ZDOWN" Instrument z-axis defined down, +heading reference axis is X positive +7: "AHRS" AHRS reports orientation any way it +points. Example: Z down -> Roll = 180 deg.' + autoorientation: + long_name: Field "Auto Orientation" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 24-22: autoOrientation +0000 0001 1100 0000 0000 0000 0000 0000 +0: "Fixed" Fixed orientation +1: "Auto" Auto Up Down +3: "AHRS3D" AHRS3D' + previous_wakeup_state: + long_name: Field "Previous Wakeup State" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 21-18: Previous wakeup state +0000 0000 0011 1100 0000 0000 0000 0000 +00 = bad power +01 = power applied +10 = break +11 = RTC alarm' + last_measurement_low_voltage_skip: + long_name: Field "Previous Measurement Skipped Due To Low Voltage" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 17: Last measurement low voltage skip +0: normal operation +1: last measurement skipped due to low input +voltage' + active_configuration: + long_name: Field "Active Configuration" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 16: Active configuration +0: Settings for PLAN,BURST,AVG +1: Settings for PLAN1,BURST1,AVG1' + echosounder_index: + long_name: Field "Echosounder Index" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 15-12: Echosounder frequency index. Valid +numbers are 0, 1 and 2 (or 0000, 0001 and 0010) +referring to frequencies 1, 2 or 3 as used in +SET-/GETECHO. +0000 0000 0000 0000 1111 0000 0000 0000' + telemetry_data: + long_name: Field "Telemetry Data" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Telemetry data' + boost_running: + long_name: Field "Boost Running" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Boost running' + echosounder_frequency_bin: + long_name: Field "Echosounder Frequency Bin" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 9-5: Echosounder frequency bin. +Used only on the Signature100 which supports up to +5 packages pr frequency index. Processing is then +done on the different frequency bands. +0000 0000 0000 0000 0000 0011 1110 0000' + bd_scaling: + long_name: Field "Blanking Distance Scaling In Cm" parsed from field "Status" + units: '' + comment: 'Derived from field "Status" + +Bit 1: Scaling of blanking distance +0: mm scaling +1: given in cm' diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 0255cabdd..8fa6351e8 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -108,6 +108,10 @@ def _make_dataset(self, var_names: Dict[str, str]) -> xr.Dataset: field_dimensions = Field.default_dimensions() # can't store in dims yet because there might be another data record format # which does have this field + + if field_name not in attrs: + if field_name in self.field_attrs["POSTPROCESSED"]: + attrs[field_name] = self.field_attrs["POSTPROCESSED"][field_name] else: field_dimensions = field.dimensions(packet.data_record_type) @@ -420,15 +424,45 @@ def set_vendor(self) -> xr.Dataset: ds = self._make_dataset( { "version": "data_record_version", + "pressure_sensor_valid": "pressure_sensor_valid", + "temperature_sensor_valid": "temperature_sensor_valid", + "compass_sensor_valid": "compass_sensor_valid", + "tilt_sensor_valid": "tilt_sensor_valid", + "velocity_data_included": "velocity_data_included", + "amplitude_data_included": "amplitude_data_included", + "correlation_data_included": "correlation_data_included", + "altimeter_data_included": "altimeter_data_included", + "altimeter_raw_data_included": "altimeter_raw_data_included", + "ast_data_included": "ast_data_included", + "echosounder_data_included": "echosounder_data_included", + "ahrs_data_included": "ahrs_data_included", + "percentage_good_data_included": "percentage_good_data_included", + "std_dev_data_included": "std_dev_data_included", + "distance_data_included": "distance_data_included", + "figure_of_merit_data_included": "figure_of_merit_data_included", "error": "error", "status": "status", + "procidle3": "procidle3", + "procidle6": "procidle6", + "procidle12": "procidle12", "status0": "status0", + "wakeup_state": "wakeup_state", + "orientation": "orientation", + "autoorientation": "autoorientation", + "previous_wakeup_state": "previous_wakeup_state", + "last_measurement_low_voltage_skip": "last_measurement_low_voltage_skip", + "active_configuration": "active_configuration", + "echosounder_index": "echosounder_index", + "telemetry_data": "telemetry_data", + "boost_running": "boost_running", + "echosounder_frequency_bin": "echosounder_frequency_bin", + "bd_scaling": "bd_scaling", "battery_voltage": "battery_voltage", "power_level": "power_level", "temperature_from_pressure_sensor": "temperature_of_pressure_sensor", "nominal_correlation": "nominal_correlation", "magnetometer_temperature": "magnetometer_temperature", - "real_ping_time_clock_temperature": "real_ping_time_clock_temperature", + "real_time_clock_temperature": "real_time_clock_temperature", "ensemble_counter": "ensemble_counter", "ahrs_rotation_matrix": "ahrs_rotation_matrix_mij", "ahrs_quaternions": "ahrs_quaternions_wxyz", From d5f8260684d528a406a3fc63ad4518b8ec78ebe9 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:57:22 -0700 Subject: [PATCH 27/37] Only chunk env by time1 if time1 exists --- echopype/convert/api.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 7e7c86ade..c54ed7790 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -117,15 +117,24 @@ def _save_groups_to_file(echodata, output_path, engine, compress=True): ) # Environment group - io.save_file( - echodata.environment.chunk( - {"time1": DEFAULT_CHUNK_SIZE["ping_time"]} - ), # TODO: chunking necessary? - path=output_path, - mode="a", - engine=engine, - group="Environment", - ) + if "time1" in echodata.environment: + io.save_file( + echodata.environment.chunk( + {"time1": DEFAULT_CHUNK_SIZE["ping_time"]} + ), # TODO: chunking necessary? + path=output_path, + mode="a", + engine=engine, + group="Environment", + ) + else: + io.save_file( + echodata.environment, + path=output_path, + mode="a", + engine=engine, + group="Environment", + ) # Sonar group io.save_file( From 535e747d5646f284d99df866ba4b2f966377d62e Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:05:11 -0700 Subject: [PATCH 28/37] Re-order status and status0 fields --- echopype/convert/set_groups_ad2cp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 8fa6351e8..89d900620 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -441,11 +441,11 @@ def set_vendor(self) -> xr.Dataset: "distance_data_included": "distance_data_included", "figure_of_merit_data_included": "figure_of_merit_data_included", "error": "error", - "status": "status", + "status0": "status0", "procidle3": "procidle3", "procidle6": "procidle6", "procidle12": "procidle12", - "status0": "status0", + "status": "status", "wakeup_state": "wakeup_state", "orientation": "orientation", "autoorientation": "autoorientation", From 185ee5786782b0897b451b9130ab2c54c4d0b6d4 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:26:08 -0700 Subject: [PATCH 29/37] Stop searching for beam groups if they are all present --- echopype/convert/set_groups_ad2cp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 89d900620..5bb7965f4 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -270,6 +270,9 @@ def set_beam(self) -> List[xr.Dataset]: elif packet.is_echosounder_raw(): beam_groups_exist.add(BeamGroup.ECHOSOUNDER_RAW) + if len(beam_groups_exist) == len(BeamGroup): + break + # average if BeamGroup.AVERAGE in beam_groups_exist: beam_groups.append( From efbf9834383441b78892b96beba76077226e04ca Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 20:08:33 -0700 Subject: [PATCH 30/37] Drop time1 from beam groups --- echopype/convert/set_groups_ad2cp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 5bb7965f4..d33d3eb8a 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -419,7 +419,7 @@ def set_beam(self) -> List[xr.Dataset]: # remove time1 from beam groups for i, ds in enumerate(beam_groups): - beam_groups[i] = ds.sel(time1=ds["ping_time"]) + beam_groups[i] = ds.sel(time1=ds["ping_time"]).drop_vars("time1", errors="ignore") return [set_encodings(ds) for ds in beam_groups] From 854607f93a0f52c73c5cdc7cd69f0f93871e6ad4 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:03:15 -0700 Subject: [PATCH 31/37] Parse echosounder raw status, remove echosounder_raw_echogram --- echopype/convert/parse_ad2cp.py | 24 +++++++++++++++++------- echopype/convert/set_groups_ad2cp.py | 1 - 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 3b8e856bf..647136774 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -739,13 +739,6 @@ def _postprocess(self, field_name): ("bd_scaling", 1, 1), ], ) - if ( - self.parser.packets[-1].is_echosounder_raw() - or self.parser.packets[-1].is_echosounder_raw_transmit() - ): - self.parser.packets[-1].data["echosounder_raw_echogram"] = self.data[ - "echosounder_index" - ] elif self.data_record_format == HeaderOrDataRecordFormats.BOTTOM_TRACK_DATA_RECORD_FORMAT: if field_name == "configuration": self._postprocess_bitfield( @@ -796,6 +789,23 @@ def _postprocess(self, field_name): self.data["echosounder_raw_transmit_samples_q"] = self.data[ "echosounder_raw_transmit_samples" ][:, 1] + elif field_name == "status": + self._postprocess_bitfield( + self.data["status"], + [ + ("wakeup_state", 31, 28), + ("orientation", 27, 25), + ("autoorientation", 24, 22), + ("previous_wakeup_state", 21, 18), + ("last_measurement_low_voltage_skip", 17, 17), + ("active_configuration", 16, 16), + ("echosounder_index", 15, 12), + ("telemetry_data", 11, 11), + ("boost_running", 10, 10), + ("echosounder_frequency_bin", 9, 5), + ("bd_scaling", 1, 1), + ], + ) @staticmethod def checksum(data: bytes) -> int: diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index d33d3eb8a..7bd2ee069 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -393,7 +393,6 @@ def set_beam(self) -> List[xr.Dataset]: "echosounder_raw_transmit_samples_i": "transmit_pulse_r", "echosounder_raw_transmit_samples_q": "transmit_pulse_i", "echosounder_raw_beam": "echosounder_raw_beam", - "echosounder_raw_echogram": "echosounder_raw_echogram", } ) ) From f04c9153893ed98c1feb3793a23e9a840e429a72 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:29:19 -0700 Subject: [PATCH 32/37] Add echosounder_raw variables, fix serial number saving --- echopype/convert/set_groups_ad2cp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 7bd2ee069..46512e5b3 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -388,6 +388,9 @@ def set_beam(self) -> List[xr.Dataset]: "dataset_description": "data_set_description", "transmit_energy": "transmit_energy", "velocity_scaling": "velocity_scaling", + "num_complex_samples": "num_complex_samples", + "ind_start_samples": "ind_start_samples", + "freq_raw_sample_data": "freq_raw_sample_data", "echosounder_raw_samples_i": "backscatter_r", "echosounder_raw_samples_q": "backscatter_i", "echosounder_raw_transmit_samples_i": "transmit_pulse_r", @@ -515,7 +518,7 @@ def set_sonar(self) -> xr.Dataset: } for packet in self.parser_obj.packets: if "serial_number" in packet.data: - ds.attrs["sonar_serial_number"] = packet.data["serial_number"] + sonar_attr_dict["sonar_serial_number"] = packet.data["serial_number"] break firmware_version = self.parser_obj.get_firmware_version() if firmware_version is not None: From 9896e2f2a288db2e7ef63a3dba10e90234dff004 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 22:32:28 -0700 Subject: [PATCH 33/37] Remove echosounder_raw_beam --- echopype/convert/parse_ad2cp.py | 5 ----- echopype/convert/set_groups_ad2cp.py | 1 - 2 files changed, 6 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 647136774..5c3e77626 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -631,11 +631,6 @@ def _postprocess(self, field_name): self.data["dataset_description"], [(2, 0), (5, 3), (8, 6), (11, 9), (14, 12)], ) - if ( - self.parser.packets[-1].is_echosounder_raw() - or self.parser.packets[-1].is_echosounder_raw_transmit() - ): - self.parser.packets[-1].data["echosounder_raw_beam"] = self.data["beams"][0] elif ( self.data_record_format == HeaderOrDataRecordFormats.BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 46512e5b3..49b75f964 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -395,7 +395,6 @@ def set_beam(self) -> List[xr.Dataset]: "echosounder_raw_samples_q": "backscatter_i", "echosounder_raw_transmit_samples_i": "transmit_pulse_r", "echosounder_raw_transmit_samples_q": "transmit_pulse_i", - "echosounder_raw_beam": "echosounder_raw_beam", } ) ) From de669cb9228dc65ce874ff7b82bde11b0abfa9c4 Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Sun, 19 Jun 2022 23:02:21 -0700 Subject: [PATCH 34/37] Include all unique serial numbers --- echopype/convert/set_groups_ad2cp.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 49b75f964..43afe2cc1 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -509,16 +509,20 @@ def set_sonar(self) -> xr.Dataset: sonar_attr_dict = { "sonar_manufacturer": "Nortek", "sonar_model": "AD2CP", - "sonar_serial_number": "", + "sonar_serial_number": ", ".join( + np.unique( + [ + str(packet.data["serial_number"]) + for packet in self.parser_obj.packets + if "serial_number" in packet.data + ] + ) + ), "sonar_software_name": "", "sonar_software_version": "", "sonar_firmware_version": "", "sonar_type": "acoustic Doppler current profiler (ADCP)", } - for packet in self.parser_obj.packets: - if "serial_number" in packet.data: - sonar_attr_dict["sonar_serial_number"] = packet.data["serial_number"] - break firmware_version = self.parser_obj.get_firmware_version() if firmware_version is not None: sonar_attr_dict["sonar_firmware_version"] = ", ".join( From 230c2d8a433dc531c45d7d049e30bbc644af434c Mon Sep 17 00:00:00 2001 From: imranmaj <49664304+imranmaj@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:05:17 -0700 Subject: [PATCH 35/37] Return NaT on invalid timestamp --- echopype/convert/parse_ad2cp.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 5c3e77626..deffd1f87 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -325,9 +325,12 @@ def timestamp(self) -> np.datetime64: minute = self.data["minute"] seconds = self.data["seconds"] microsec100 = self.data["microsec100"] - return np.datetime64( - f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{seconds:02}.{microsec100:04}" - ) + try: + return np.datetime64( + f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{seconds:02}.{microsec100:04}" + ) # type: ignore + except ValueError: + return np.datetime64("NaT") # type: ignore def is_burst(self) -> bool: """ From b81ffef78c72edb6ef990ab7b166eca1be0beff5 Mon Sep 17 00:00:00 2001 From: leewujung Date: Mon, 20 Jun 2022 19:01:54 -0700 Subject: [PATCH 36/37] add some more missing . --- echopype/convert/ad2cp_fields.yaml | 138 ++++++++++++++--------------- 1 file changed, 67 insertions(+), 71 deletions(-) diff --git a/echopype/convert/ad2cp_fields.yaml b/echopype/convert/ad2cp_fields.yaml index 5af0beb6d..9fa426c30 100644 --- a/echopype/convert/ad2cp_fields.yaml +++ b/echopype/convert/ad2cp_fields.yaml @@ -94,15 +94,15 @@ Raw data given as 0.01 °C. long_name: Field "Status" in the data specification. units: '' comment: ' -Bit 31-28: Wakeup State -1111 0000 0000 0000 0000 0000 0000 0000 +Bit 31-28: Wakeup State. +1111 0000 0000 0000 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break 11 = RTC alarm -Bit 27-25: Orientation -0000 1110 0000 0000 0000 0000 0000 0000 +Bit 27-25: Orientation. +0000 1110 0000 0000 0000 0000 0000 0000. 0: "XUP" Instrument x-axis defined up, heading reference axis is Z positive 1: "XDOWN" Instrument x-axis defined down, @@ -118,18 +118,18 @@ heading reference axis is X positive 7: "AHRS" AHRS reports orientation any way it points. Example: Z down -> Roll = 180 deg. -Bit 24-22: autoOrientation -0000 0001 1100 0000 0000 0000 0000 0000 +Bit 24-22: autoOrientation. +0000 0001 1100 0000 0000 0000 0000 0000. 0: "Fixed" Fixed orientation 1: "Auto" Auto Up Down 3: "AHRS3D" AHRS3D' num_beams_and_coordinate_system_and_num_cells: long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' - comment: 'Bit 11-10 (2 bits): Coordinate system + comment: 'Bit 11-10 (2 bits): Coordinate system. b00:ENU, b01:XYZ, b10:BEAM -Bit 15–12 (4 bits): Number of Beams (NB) +Bit 15–12 (4 bits): Number of Beams (NB). Active beams represented as a 4 charstring of 1s and 0s. @@ -236,7 +236,7 @@ Data given as 10^(Velocity Scaling). long_name: Field "Amplitude Data Burst" in the data specification. units: dB/count comment: 'This field exists if the amplitude data included bit of -the Config byte is set +the Config byte is set. Data given as 0.5 dB/count. Data was converted to the specified units.' @@ -244,7 +244,7 @@ Data given as 0.5 dB/count. long_name: Field "Amplitude Data Average" in the data specification. units: dB/count comment: 'This field exists if the amplitude data included bit of -the Config byte is set +the Config byte is set. Data given as 0.5 dB/count. Data was converted to the specified units.' @@ -252,7 +252,7 @@ Data given as 0.5 dB/count. long_name: Field "Amplitude Data Echosounder" in the data specification. units: dB/count comment: 'This field exists if the amplitude data included bit of -the Config byte is set +the Config byte is set. Data given as 0.5 dB/count. Data was converted to the specified units.' @@ -371,10 +371,10 @@ Raw data given as 0.01 °C. num_beams_and_coordinate_system_and_num_cells: long_name: Field "Beams, Coordinates And Cells" in the data specification. units: '' - comment: 'Bit 11-10 (2 bits): Coordinate system + comment: 'Bit 11-10 (2 bits): Coordinate system. b00:ENU, b01:XYZ, b10:BEAM -Bit 15–12 (4 bits): Number of Beams (NB) +Bit 15–12 (4 bits): Number of Beams (NB). Active beams represented as a 4 charstring of 1s and 0s. @@ -443,7 +443,8 @@ scaled according to Velocity scaling. 0-3 Physical beam used for 1st data set. 4-7 Physical beam used for 2nd data set. 8-11 Physical beam used for 3th data set. -12-16 Physical beam used for 4th data set.' +12-16 Physical beam used for 4th data set. +This was parsed and used to set beam coordinate' transmit_energy: long_name: Field "Transmit Energy" in the data specification. units: '' @@ -477,14 +478,11 @@ Raw data in 1/1000 °C' status0: long_name: Field "Status0" in the data specification. units: '' - comment: 'Extended status bit mask + comment: 'Extended status bit mask. -0 bit: Indicates that the processor Idles less than 3 -percent -1 bit: Indicates that the processor idles less than 6 -percent -2 bit: Indicates that the processor idles less than 12 -percent +0 bit: Indicates that the processor Idles less than 3 percent +1 bit: Indicates that the processor idles less than 6 percent +2 bit: Indicates that the processor idles less than 12 percent 15 bit: If this bit is set the rest of the word/ extended status should be interpreted ' # 2 bit is also 1 bit in the spec but should be 2 bit @@ -492,15 +490,15 @@ status should be interpreted long_name: Field "Status" in the data specification. units: '' comment: ' -Bit 31-28: Wakeup State -1111 0000 0000 0000 0000 0000 0000 0000 +Bit 31-28: Wakeup State. +1111 0000 0000 0000 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break 11 = RTC alarm -Bit 27-25: Orientation -0000 1110 0000 0000 0000 0000 0000 0000 +Bit 27-25: Orientation. +0000 1110 0000 0000 0000 0000 0000 0000. 0: "XUP" Instrument x-axis defined up, heading reference axis is Z positive 1: "XDOWN" Instrument x-axis defined down, @@ -516,8 +514,8 @@ heading reference axis is X positive 7: "AHRS" AHRS reports orientation any way it points. Example: Z down -> Roll = 180 deg. -Bit 24-22: autoOrientation -0000 0001 1100 0000 0000 0000 0000 0000 +Bit 24-22: autoOrientation. +0000 0001 1100 0000 0000 0000 0000 0000. 0: "Fixed" Fixed orientation 1: "Auto" Auto Up Down 3: "AHRS3D" AHRS3D' @@ -562,7 +560,7 @@ Data given as 10^(Velocity Scaling). long_name: Field "Amplitude Data Average" in the data specification. units: dB/count comment: 'This field exists if the amplitude data included bit of -the Config byte is set +the Config byte is set. Data given as 0.5 dB/count. Data was converted to the specified units.' @@ -570,7 +568,7 @@ Data given as 0.5 dB/count. long_name: Field "Amplitude Data Echosounder" in the data specification. units: dB/count comment: 'This field exists if the amplitude data included bit of -the Config byte is set +the Config byte is set. Data given as 0.5 dB/count. Data was converted to the specified units.' @@ -579,7 +577,7 @@ Data given as 0.5 dB/count. units: 0-100% comment: 'This field exists if the Correlation data included bit of the Config byte is set. -Data given as [0 – 100 %] +Data given as [0 – 100 %]. Data was converted to the specified units.' correlation_data_average: @@ -587,7 +585,7 @@ Data given as [0 – 100 %] units: 0-100% comment: 'This field exists if the Correlation data included bit of the Config byte is set. -Data given as [0 – 100 %] +Data given as [0 – 100 %]. Data was converted to the specified units.' correlation_data_echosounder: @@ -595,7 +593,7 @@ Data given as [0 – 100 %] units: 0-100% comment: 'This field exists if the Correlation data included bit of the Config byte is set. -Data given as [0 – 100 %] +Data given as [0 – 100 %]. Data was converted to the specified units.' altimeter_distance: @@ -809,10 +807,8 @@ Raw data given as 0.01 °C. used. b00:ENU, b01:XYZ, b10:BEAM -Bit 15–12 (4 bits) represent the number of beams -(NB) -Active beams represented as a 4 charstring of 1s -and 0s. +Bit 15–12 (4 bits) represent the number of beams (NB). +Active beams represented as a 4 charstring of 1s and 0s. Bit 9-0 (10 bits) represent the number of cells (NC).' cell_size: @@ -911,15 +907,15 @@ Raw data in 1/1000 °C long_name: Field "Status" in the data specification. units: '' comment: ' -Bit 31-28: Wakeup State -1111 0000 0000 0000 0000 0000 0000 0000 +Bit 31-28: Wakeup State. +1111 0000 0000 0000 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break 11 = RTC alarm -Bit 27-25: Orientation -0000 1110 0000 0000 0000 0000 0000 0000 +Bit 27-25: Orientation. +0000 1110 0000 0000 0000 0000 0000 0000. 0: "XUP" Instrument x-axis defined up, heading reference axis is Z positive 1: "XDOWN" Instrument x-axis defined down, @@ -935,8 +931,8 @@ heading reference axis is X positive 7: "AHRS" AHRS reports orientation any way it points. Example: Z down -> Roll = 180 deg. -Bit 24-22: autoOrientation -0000 0001 1100 0000 0000 0000 0000 0000 +Bit 24-22: autoOrientation. +0000 0001 1100 0000 0000 0000 0000 0000. 0: "Fixed" Fixed orientation 1: "Auto" Auto Up Down 3: "AHRS3D" AHRS3D' @@ -1013,15 +1009,15 @@ milliseconds resolution)' long_name: Field "Status" in the data specification. units: '' comment: ' -Bit 31-28: Wakeup State -1111 0000 0000 0000 0000 0000 0000 0000 +Bit 31-28: Wakeup State. +1111 0000 0000 0000 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break 11 = RTC alarm -Bit 27-25: Orientation -0000 1110 0000 0000 0000 0000 0000 0000 +Bit 27-25: Orientation. +0000 1110 0000 0000 0000 0000 0000 0000. 0: "XUP" Instrument x-axis defined up, heading reference axis is Z positive 1: "XDOWN" Instrument x-axis defined down, @@ -1037,8 +1033,8 @@ heading reference axis is X positive 7: "AHRS" AHRS reports orientation any way it points. Example: Z down -> Roll = 180 deg. -Bit 24-22: autoOrientation -0000 0001 1100 0000 0000 0000 0000 0000 +Bit 24-22: autoOrientation. +0000 0001 1100 0000 0000 0000 0000 0000. 0: "Fixed" Fixed orientation 1: "Auto" Auto Up Down 3: "AHRS3D" AHRS3D' @@ -1171,7 +1167,7 @@ POSTPROCESSED: ambiguity_velocity: long_name: Field "Ambiguity Velocity" parsed from field "Ambiguity Velocity Or Echosounder Frequency" units: m/s - comment: 'Derived from field "Ambiguity Velocity Or Echosounder Frequency" + comment: 'Derived from field "Ambiguity Velocity Or Echosounder Frequency." Ambiguity velocity, corrected for sound velocity, scaled according to Velocity scaling. @@ -1181,31 +1177,31 @@ Data given as 10^(Velocity scaling) m/s. procidle3: long_name: Field "Processor Idles <3%" parsed from field "Status0" units: '' - comment: 'Derived from field "Status0" + comment: 'Derived from field "Status0". Indicates that the processor Idles less than 3 percent' procidle6: long_name: Field "Processor Idles <6%" parsed from field "Status0" units: '' - comment: 'Derived from field "Status0" + comment: 'Derived from field "Status0". Indicates that the processor Idles less than 6 percent' procidle12: long_name: Field "Processor Idles <12%" parsed from field "Status0" units: '' - comment: 'Derived from field "Status0" + comment: 'Derived from field "Status0". Indicates that the processor Idles less than 12 percent' wakeup_state: long_name: Field "Wakeup State" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 31-28: Wakeup State -1111 0000 0000 0000 0000 0000 0000 0000 +Bit 31-28: Wakeup State. +1111 0000 0000 0000 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break @@ -1215,8 +1211,8 @@ Bit 31-28: Wakeup State units: '' comment: 'Derived from field "Status" -Bit 27-25: Orientation -0000 1110 0000 0000 0000 0000 0000 0000 +Bit 27-25: Orientation. +0000 1110 0000 0000 0000 0000 0000 0000. 0: "XUP" Instrument x-axis defined up, heading reference axis is Z positive 1: "XDOWN" Instrument x-axis defined down, @@ -1234,20 +1230,20 @@ points. Example: Z down -> Roll = 180 deg.' autoorientation: long_name: Field "Auto Orientation" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 24-22: autoOrientation -0000 0001 1100 0000 0000 0000 0000 0000 +Bit 24-22: autoOrientation. +0000 0001 1100 0000 0000 0000 0000 0000. 0: "Fixed" Fixed orientation 1: "Auto" Auto Up Down 3: "AHRS3D" AHRS3D' previous_wakeup_state: long_name: Field "Previous Wakeup State" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 21-18: Previous wakeup state -0000 0000 0011 1100 0000 0000 0000 0000 +Bit 21-18: Previous wakeup state. +0000 0000 0011 1100 0000 0000 0000 0000. 00 = bad power 01 = power applied 10 = break @@ -1255,24 +1251,24 @@ Bit 21-18: Previous wakeup state last_measurement_low_voltage_skip: long_name: Field "Previous Measurement Skipped Due To Low Voltage" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 17: Last measurement low voltage skip +Bit 17: Last measurement low voltage skip. 0: normal operation 1: last measurement skipped due to low input voltage' active_configuration: long_name: Field "Active Configuration" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 16: Active configuration +Bit 16: Active configuration. 0: Settings for PLAN,BURST,AVG 1: Settings for PLAN1,BURST1,AVG1' echosounder_index: long_name: Field "Echosounder Index" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". Bit 15-12: Echosounder frequency index. Valid numbers are 0, 1 and 2 (or 0000, 0001 and 0010) @@ -1294,7 +1290,7 @@ Boost running' echosounder_frequency_bin: long_name: Field "Echosounder Frequency Bin" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". Bit 9-5: Echosounder frequency bin. Used only on the Signature100 which supports up to @@ -1304,8 +1300,8 @@ done on the different frequency bands. bd_scaling: long_name: Field "Blanking Distance Scaling In Cm" parsed from field "Status" units: '' - comment: 'Derived from field "Status" + comment: 'Derived from field "Status". -Bit 1: Scaling of blanking distance +Bit 1: Scaling of blanking distance. 0: mm scaling 1: given in cm' From 026246271221e92b47c25b714eec1ccd741aa6a4 Mon Sep 17 00:00:00 2001 From: leewujung Date: Mon, 20 Jun 2022 19:26:12 -0700 Subject: [PATCH 37/37] add TODO for sections to fix #734 --- echopype/convert/set_groups_ad2cp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index 43afe2cc1..4fc692e46 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -31,6 +31,8 @@ class SetGroupsAd2cp(SetGroupsBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # TODO: bug: 0 if not exist in first string packet + # resulting in index error in setting ds["pulse_compressed"] self.pulse_compressed = self.parser_obj.get_pulse_compressed() self._make_time_coords() with resources.open_text(convert, "ad2cp_fields.yaml") as f: @@ -358,6 +360,7 @@ def set_beam(self) -> List[xr.Dataset]: ) ds = ds.assign_coords({"echogram": np.arange(3)}) pulse_compressed = np.zeros(3) + # TODO: bug: if self.pulse_compress=0 this will set the last index to 1 pulse_compressed[self.pulse_compressed - 1] = 1 ds["pulse_compressed"] = (("echogram",), pulse_compressed) beam_groups.append(ds)