Added sonar model checkers for EK60, EK80, ER60, AZFP6, AZFP, AD2CP and tests

docs/source/contributing.md

@@ -81,7 +81,23 @@ One can replace `conda` with `mamba` in the above commands when creating the env
 
 Currently, test data are stored in a private Google Drive folder and
 made available via the [`cormorack/http`](https://hub.docker.com/r/cormorack/http)
-Docker image on Docker hub.
+Docker image on Docker hub. There’s no need to pull directly from Docker Hub, as the scripts below will handle that for you. For Linux Users: If you run into an issue where accessing the Docker daemon requires sudo privileges, which may not suit your environment, you can adjust your settings to allow non-root access to Docker:
+
+```shell
+sudo groupadd docker
+```
+
+Add the current user to the Docker group
+```shell
+sudo gpasswd -a $USER docker
+```
+
+Restart the docker daemon
+```shell
+sudo service docker restart
+```
+
+You might also need to restart your computer if the steps above don't resolve the issue.
 The image is rebuilt daily when new test data are added.
 If your tests require adding new test data, ping the maintainers (@leewujung, @ctuguinay)
 to get them added to the the Google Drive.

echopype/convert/parse_azfp6.py

@@ -695,4 +695,4 @@ def _calc_Sv_offset(freq, pulse_len):
                 "and raise an issue in the echopype repository."
             )
 
-        return SV_OFFSET[freq][pulse_len]
+        return SV_OFFSET[freq][pulse_len]

echopype/convert/sonar_model_checker.py

@@ -0,0 +1,125 @@
+import os
+import xml.etree.ElementTree as ET
+
+import numpy as np
+
+from .utils.ek_raw_io import RawSimradFile
+
+
+def is_EK80(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK80 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+
+        # Return True if "configuration" exists in config_datagram
+        return "configuration" in config_datagram
+
+
+def is_EK60(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK60 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+
+        try:
+            # Return True if the sounder name matches "EK60"
+            return config_datagram["sounder_name"] in {"ER60", "EK60"}
+        except KeyError:
+            return False
+
+
+def is_AZFP6(raw_file):
+    """
+    Check if the provided file has a .azfp extension.
+
+    Parameters:
+    raw_file (str): The name of the file to check.
+
+    Returns:
+    bool: True if the file has a .azfp extension, False otherwise.
+    """
+
+    # Check if the input is a string
+    if not isinstance(raw_file, str):
+        return False  # Return False if the input is not a string
+
+    # Use the str.lower() method to check for the .azfp extension
+    has_azfp_extension = raw_file.lower().endswith(".azfp")
+
+    # Return the result of the check
+    return has_azfp_extension
+
+
+def is_AZFP(raw_file):
+    """
+    Check if the specified XML file contains an <InstrumentType> with string="AZFP".
+
+    Parameters:
+    raw_file (str): The base name of the XML file (with or without extension).
+
+    Returns:
+    bool: True if <InstrumentType> with string="AZFP" is found, False otherwise.
+    """
+
+    # Check if the filename ends with .xml or .XML, and strip the extension if it does
+    base_filename = raw_file.rstrip(".xml").rstrip(".XML")
+
+    # Create a list of possible filenames with both extensions
+    possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"]
+
+    for full_filename in possible_files:
+        if os.path.isfile(full_filename):
+            try:
+                # Parse the XML file
+                tree = ET.parse(full_filename)
+                root = tree.getroot()
+
+                # Check for <InstrumentType> elements
+                for instrument in root.findall(".//InstrumentType"):
+                    if instrument.get("string") == "AZFP":
+                        return True
+            except ET.ParseError:
+                print(f"Error parsing the XML file: {full_filename}.")
+
+    return False
+
+
+def is_AD2CP(raw_file):
+    """
+    Check if the provided file has a .ad2cp extension.
+
+    Parameters:
+    raw_file (str): The name of the file to check.
+
+    Returns:
+    bool: True if the file has a .ad2cp extension, False otherwise.
+    """
+
+    # Check if the input is a string
+    if not isinstance(raw_file, str):
+        return False  # Return False if the input is not a string
+
+    # Use the str.lower() method to check for the .ad2cp extension
+    has_ad2cp_extension = raw_file.lower().endswith(".ad2cp")
+
+    # Return the result of the check
+    return has_ad2cp_extension
+
+
+def is_ER60(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK60 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+        # Return True if the sounder name matches "ER60"
+        try:
+            return config_datagram["sounder_name"] in {"ER60", "EK60"}
+        except KeyError:
+            return False

echopype/test_data/README.md

@@ -11,8 +11,6 @@ Most of these files are stored on Git LFS but the ones that aren't (due to file
 - 2019118 group2survey-D20191214-T081342.raw: Contains 6 channels but only 2 of those channels collect ping data
 - D20200528-T125932.raw: Data collected from WBT mini (instead of WBT), from @emlynjdavies
 - Green2.Survey2.FM.short.slow.-D20191004-T211557.raw: Contains 2-in-1 transducer, from @FletcherFT (reduced from 104.9 MB to 765 KB in test data updates)
-- raw4-D20220514-T172704.raw: Contains RAW4 datagram, 1 channel only, from @cornejotux
-- D20210330-T123857.raw: do not contain filter coefficients
 
 
 ### EA640
@@ -24,7 +22,6 @@ Most of these files are stored on Git LFS but the ones that aren't (due to file
 - Winter2017-D20170115-T150122.raw: Contains a change of recording length in the middle of the file
 - 2015843-D20151023-T190636.raw: Not used in tests but contains ranges are not constant across ping times
 - SH1701_consecutive_files_w_range_change: Not used in tests. [Folder](https://drive.google.com/drive/u/1/folders/1PaDtL-xnG5EK3N3P1kGlXa5ub16Yic0f) on shared drive that contains sequential files with ranges that are not constant across ping times.
-- NBP_B050N-D20180118-T090228.raw: split-beam setup without angle data
 
 
 ### AZFP

echopype/tests/convert/test_convert_ad2cp.py

@@ -5,7 +5,6 @@
 whereas files under "raw" additionally contain the IQ samples.
 """
 
-
 import xarray as xr
 import numpy as np
 import netCDF4
@@ -16,7 +15,6 @@
 from echopype import open_raw, open_converted
 from echopype.testing import TEST_DATA_FOLDER
 
-
 @pytest.fixture
 def ocean_contour_export_dir(test_path):
     return test_path["AD2CP"] / "ocean-contour"
@@ -261,4 +259,4 @@ def _check_raw_output(
                         base["Data_Q"].data.T.flatten(),
                         atol=absolute_tolerance,
                     )
-        base.close()
+        base.close()

echopype/tests/convert/test_convert_azfp.py

@@ -275,4 +275,4 @@ def test_load_parse_azfp_xml(azfp_path):
     assert parseAZFP.parameters['pulse_len_phase1'] == [1000, 1000, 1000, 1000]
     assert parseAZFP.parameters['pulse_len_phase2'] == [0, 0, 0, 0]
     assert parseAZFP.parameters['range_samples_phase1'] == [8273, 8273, 8273, 8273]
-    assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750]
+    assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750]

echopype/tests/convert/test_convert_azfp6.py

@@ -4,14 +4,12 @@
 - verify echopype converted files against those from AZFP Matlab scripts and EchoView
 - convert AZFP file with different range settings across frequency
 """
-
 import numpy as np
-import pandas as pd
 from datetime import datetime, timedelta
 from scipy.io import loadmat
 from echopype import open_raw
 import pytest
-from echopype.convert.parse_azfp6 import ParseAZFP6
+
 
 
 @pytest.fixture
@@ -165,4 +163,4 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path):
         np.stack([echodata["Platform"].latitude, echodata["Platform"].longitude])
     )
 
-    check_platform_required_scalar_vars(echodata)
+    check_platform_required_scalar_vars(echodata)

echopype/tests/convert/test_convert_ek60.py

@@ -1,5 +1,5 @@
 import warnings
-
+import glob
 import numpy as np
 import pandas as pd
 from scipy.io import loadmat
@@ -269,4 +269,4 @@ def test_converting_ek60_raw_with_missing_channel_power(ek60_missing_channel_pow
 
     # Check that all empty power channels do not exist in the EchoData Beam group
     for _, empty_power_channel_name in empty_power_chs.items():
-        assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"]
+        assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"]

echopype/tests/convert/test_convert_ek80.py

@@ -1,5 +1,4 @@
 import shutil
-
 import pytest
 import numpy as np
 import pandas as pd

echopype/tests/convert/test_sonar_model_checker.py

@@ -0,0 +1,121 @@
+import glob
+from echopype.convert.sonar_model_checker import is_AD2CP, is_AZFP, is_AZFP6, is_EK60, is_EK80, is_ER60
+import pytest
+
+@pytest.fixture(scope="session") 
+def test_is_EK80_ek80_files():
+    """Test that EK80 files are identified as EK80."""
+    # Collect all .raw files in the ek80 directory
+    ek80_files = glob.glob("echopype/test_data/ek80/*.raw")
+
+    # Check that each file in ek80 is identified as EK80
+    for test_file_path in ek80_files:
+        assert is_EK80(test_file_path, storage_options={}) == True
+
+@pytest.fixture(scope="session") 
+def test_is_EK80_non_ek80_files():
+    """Test that non-EK80 files are not identified as EK80."""
+    # Collect all .raw files in the ek60 directory (non-EK80 files)
+    ek60_files = glob.glob("echopype/test_data/ek60/*.raw")
+
+    # Check that each file in ek60 is not identified as EK80
+    for test_file_path in ek60_files:
+        assert is_EK80(test_file_path, storage_options={}) == False
+
+@pytest.fixture(scope="session")         
+def test_is_EK60_ek60_files():
+    """Check that EK60 files are identified as EK60"""
+    # Collect all .raw files in the ek60 directory
+    ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw")
+
+    # Check that each file in ek60 is identified as EK60
+    for test_file_path in ek60_files:
+        assert is_EK60(test_file_path, storage_options={}) == True
+
+@pytest.fixture(scope="session") 
+def test_is_EK60_non_ek60_files():
+    """Check that non-EK60 files are not identified as EK60"""
+    # Collect all .raw files in the ek80 directory (non-EK60 files)
+    ek80_files = glob.glob("echopype/test_data/ek80/*.raw")
+
+    # Check that each file in ek80 is not identified as EK60
+    for test_file_path in ek80_files:
+        assert is_EK60(test_file_path, storage_options={}) == False
+
+@pytest.fixture(scope="session")         
+def test_is_ER60_er60_files():
+    """Check that EK60 files are identified as EK60"""
+    # Collect all .raw files in the ek60 directory
+    ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw")
+
+    # Check that each file in ek60 is identified as EK60
+    for test_file_path in ek60_files:
+        assert is_ER60(test_file_path, storage_options={}) == True
+
+@pytest.fixture(scope="session") 
+def test_is_ER60_non_er60_files():
+    """Check that non-EK60 files are not identified as EK60"""
+    # Collect all .raw files in the ek80 directory (non-EK60 files)
+    ek80_files = glob.glob("echopype/test_data/ek80/*.raw")
+
+    # Check that each file in ek80 is not identified as EK60
+    for test_file_path in ek80_files:
+        assert is_ER60(test_file_path, storage_options={}) == False
+
+@pytest.fixture(scope="session")         
+def test_is_AZFP6_valid_files():
+    """Test that .azfp files are identified as valid AZFP files."""
+    # Collect all .azfp files in the test directory
+    azfp_files = glob.glob("echopype/test_data/azfp6/*.azfp")
+
+    # Check that each file in azfp is identified as valid AZFP
+    for test_file_path in azfp_files:
+        assert is_AZFP6(test_file_path) == True
+
+@pytest.fixture(scope="session") 
+def test_is_AZFP6_invalid_files():
+    """Test that non-.azfp files are not identified as valid AZFP files."""
+    # Collect all non-.azfp files in the test directory
+    non_azfp_files = glob.glob("echopype/test_data/azfp/*")
+
+    # Check that each file in non_azfp is not identified as valid AZFP
+    for test_file_path in non_azfp_files:
+        assert is_AZFP6(test_file_path) == False
+
+@pytest.fixture(scope="session")         
+def test_is_AZFP_valid_files():
+    """Test that XML files with <InstrumentType string="AZFP"> are identified as valid AZFP files."""
+    # Collect all valid XML files in the test directory
+    valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("echopype/test_data/azfp/*.XML")
+
+    for test_file_path in valid_files:
+        assert is_AZFP(test_file_path) == True
+
+@pytest.fixture(scope="session") 
+def test_is_AZFP_invalid_files():
+    """Test that XML files without <InstrumentType string="AZFP"> are not identified as valid AZFP files."""
+    # Collect all invalid XML files in the test directory
+    invalid_files = glob.glob("echopype/test_data/azfp6/*")
+
+    for test_file_path in invalid_files:
+        assert is_AZFP(test_file_path) == False
+
+@pytest.fixture(scope="session") 
+def test_is_AD2CP_valid_files():
+    """Test that .ad2cp files are identified as valid AD2CP files."""
+    # Collect all .ad2cp files in the test directory
+    ad2cp_files = glob.glob("echopype/test_data/ad2cp/normal/*.ad2cp")
+
+    # Check that each file in ad2cp is identified as valid AD2CP
+    for test_file_path in ad2cp_files:
+        assert is_AD2CP(test_file_path) == True
+
+@pytest.fixture(scope="session") 
+def test_is_AD2CP_invalid_files():
+    """Test that non-.ad2cp files are not identified as valid AD2CP files."""
+    # Collect all non-.ad2cp files in the test directory
+    non_ad2cp_files = glob.glob("echopype/test_data/azfp6/*")
+
+    # Check that each file in non_ad2cp is not identified as valid AD2CP
+    for test_file_path in non_ad2cp_files:
+        assert is_AD2CP(test_file_path) == False