diff --git a/docs/source/contributing.md b/docs/source/contributing.md index 446ab1d0a..5c52f48c0 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -81,7 +81,23 @@ One can replace `conda` with `mamba` in the above commands when creating the env Currently, test data are stored in a private Google Drive folder and made available via the [`cormorack/http`](https://hub.docker.com/r/cormorack/http) -Docker image on Docker hub. +Docker image on Docker hub. There’s no need to pull directly from Docker Hub, as the scripts below will handle that for you. For Linux Users: If you run into an issue where accessing the Docker daemon requires sudo privileges, which may not suit your environment, you can adjust your settings to allow non-root access to Docker: + +```shell +sudo groupadd docker +``` + +Add the current user to the Docker group +```shell +sudo gpasswd -a $USER docker +``` + +Restart the docker daemon +```shell +sudo service docker restart +``` + +You might also need to restart your computer if the steps above don't resolve the issue. The image is rebuilt daily when new test data are added. If your tests require adding new test data, ping the maintainers (@leewujung, @ctuguinay) to get them added to the the Google Drive. diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py index 37919eb83..b3e835293 100644 --- a/echopype/convert/parse_azfp6.py +++ b/echopype/convert/parse_azfp6.py @@ -695,4 +695,4 @@ def _calc_Sv_offset(freq, pulse_len): "and raise an issue in the echopype repository." ) - return SV_OFFSET[freq][pulse_len] + return SV_OFFSET[freq][pulse_len] diff --git a/echopype/convert/sonar_model_checker.py b/echopype/convert/sonar_model_checker.py new file mode 100644 index 000000000..2ebe7b9aa --- /dev/null +++ b/echopype/convert/sonar_model_checker.py @@ -0,0 +1,125 @@ +import os +import xml.etree.ElementTree as ET + +import numpy as np + +from .utils.ek_raw_io import RawSimradFile + + +def is_EK80(raw_file, storage_options): + """Check if a raw data file is from Simrad EK80 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + # Return True if "configuration" exists in config_datagram + return "configuration" in config_datagram + + +def is_EK60(raw_file, storage_options): + """Check if a raw data file is from Simrad EK60 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + try: + # Return True if the sounder name matches "EK60" + return config_datagram["sounder_name"] in {"ER60", "EK60"} + except KeyError: + return False + + +def is_AZFP6(raw_file): + """ + Check if the provided file has a .azfp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .azfp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .azfp extension + has_azfp_extension = raw_file.lower().endswith(".azfp") + + # Return the result of the check + return has_azfp_extension + + +def is_AZFP(raw_file): + """ + Check if the specified XML file contains an with string="AZFP". + + Parameters: + raw_file (str): The base name of the XML file (with or without extension). + + Returns: + bool: True if with string="AZFP" is found, False otherwise. + """ + + # Check if the filename ends with .xml or .XML, and strip the extension if it does + base_filename = raw_file.rstrip(".xml").rstrip(".XML") + + # Create a list of possible filenames with both extensions + possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"] + + for full_filename in possible_files: + if os.path.isfile(full_filename): + try: + # Parse the XML file + tree = ET.parse(full_filename) + root = tree.getroot() + + # Check for elements + for instrument in root.findall(".//InstrumentType"): + if instrument.get("string") == "AZFP": + return True + except ET.ParseError: + print(f"Error parsing the XML file: {full_filename}.") + + return False + + +def is_AD2CP(raw_file): + """ + Check if the provided file has a .ad2cp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .ad2cp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .ad2cp extension + has_ad2cp_extension = raw_file.lower().endswith(".ad2cp") + + # Return the result of the check + return has_ad2cp_extension + + +def is_ER60(raw_file, storage_options): + """Check if a raw data file is from Simrad EK60 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + # Return True if the sounder name matches "ER60" + try: + return config_datagram["sounder_name"] in {"ER60", "EK60"} + except KeyError: + return False diff --git a/echopype/test_data/README.md b/echopype/test_data/README.md index d3295604e..c79ad71f3 100644 --- a/echopype/test_data/README.md +++ b/echopype/test_data/README.md @@ -11,8 +11,6 @@ Most of these files are stored on Git LFS but the ones that aren't (due to file - 2019118 group2survey-D20191214-T081342.raw: Contains 6 channels but only 2 of those channels collect ping data - D20200528-T125932.raw: Data collected from WBT mini (instead of WBT), from @emlynjdavies - Green2.Survey2.FM.short.slow.-D20191004-T211557.raw: Contains 2-in-1 transducer, from @FletcherFT (reduced from 104.9 MB to 765 KB in test data updates) -- raw4-D20220514-T172704.raw: Contains RAW4 datagram, 1 channel only, from @cornejotux -- D20210330-T123857.raw: do not contain filter coefficients ### EA640 @@ -24,7 +22,6 @@ Most of these files are stored on Git LFS but the ones that aren't (due to file - Winter2017-D20170115-T150122.raw: Contains a change of recording length in the middle of the file - 2015843-D20151023-T190636.raw: Not used in tests but contains ranges are not constant across ping times - SH1701_consecutive_files_w_range_change: Not used in tests. [Folder](https://drive.google.com/drive/u/1/folders/1PaDtL-xnG5EK3N3P1kGlXa5ub16Yic0f) on shared drive that contains sequential files with ranges that are not constant across ping times. -- NBP_B050N-D20180118-T090228.raw: split-beam setup without angle data ### AZFP diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index 07386bdad..40ebd1b31 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -5,7 +5,6 @@ whereas files under "raw" additionally contain the IQ samples. """ - import xarray as xr import numpy as np import netCDF4 @@ -16,7 +15,6 @@ from echopype import open_raw, open_converted from echopype.testing import TEST_DATA_FOLDER - @pytest.fixture def ocean_contour_export_dir(test_path): return test_path["AD2CP"] / "ocean-contour" @@ -261,4 +259,4 @@ def _check_raw_output( base["Data_Q"].data.T.flatten(), atol=absolute_tolerance, ) - base.close() + base.close() \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 4ff718797..1860a0c4b 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -275,4 +275,4 @@ def test_load_parse_azfp_xml(azfp_path): assert parseAZFP.parameters['pulse_len_phase1'] == [1000, 1000, 1000, 1000] assert parseAZFP.parameters['pulse_len_phase2'] == [0, 0, 0, 0] assert parseAZFP.parameters['range_samples_phase1'] == [8273, 8273, 8273, 8273] - assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750] + assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750] \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_azfp6.py b/echopype/tests/convert/test_convert_azfp6.py index 29122a0bb..0f0e9a522 100644 --- a/echopype/tests/convert/test_convert_azfp6.py +++ b/echopype/tests/convert/test_convert_azfp6.py @@ -4,14 +4,12 @@ - verify echopype converted files against those from AZFP Matlab scripts and EchoView - convert AZFP file with different range settings across frequency """ - import numpy as np -import pandas as pd from datetime import datetime, timedelta from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp6 import ParseAZFP6 + @pytest.fixture @@ -165,4 +163,4 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path): np.stack([echodata["Platform"].latitude, echodata["Platform"].longitude]) ) - check_platform_required_scalar_vars(echodata) + check_platform_required_scalar_vars(echodata) \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 054e7fef6..553fc5c70 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -1,5 +1,5 @@ import warnings - +import glob import numpy as np import pandas as pd from scipy.io import loadmat @@ -269,4 +269,4 @@ def test_converting_ek60_raw_with_missing_channel_power(ek60_missing_channel_pow # Check that all empty power channels do not exist in the EchoData Beam group for _, empty_power_channel_name in empty_power_chs.items(): - assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"] + assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"] \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index ae7d5434d..19e58f886 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -1,5 +1,4 @@ import shutil - import pytest import numpy as np import pandas as pd diff --git a/echopype/tests/convert/test_sonar_model_checker.py b/echopype/tests/convert/test_sonar_model_checker.py new file mode 100644 index 000000000..64822156f --- /dev/null +++ b/echopype/tests/convert/test_sonar_model_checker.py @@ -0,0 +1,121 @@ +import glob +from echopype.convert.sonar_model_checker import is_AD2CP, is_AZFP, is_AZFP6, is_EK60, is_EK80, is_ER60 +import pytest + +@pytest.fixture(scope="session") +def test_is_EK80_ek80_files(): + """Test that EK80 files are identified as EK80.""" + # Collect all .raw files in the ek80 directory + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is identified as EK80 + for test_file_path in ek80_files: + assert is_EK80(test_file_path, storage_options={}) == True + +@pytest.fixture(scope="session") +def test_is_EK80_non_ek80_files(): + """Test that non-EK80 files are not identified as EK80.""" + # Collect all .raw files in the ek60 directory (non-EK80 files) + ek60_files = glob.glob("echopype/test_data/ek60/*.raw") + + # Check that each file in ek60 is not identified as EK80 + for test_file_path in ek60_files: + assert is_EK80(test_file_path, storage_options={}) == False + +@pytest.fixture(scope="session") +def test_is_EK60_ek60_files(): + """Check that EK60 files are identified as EK60""" + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in ek60_files: + assert is_EK60(test_file_path, storage_options={}) == True + +@pytest.fixture(scope="session") +def test_is_EK60_non_ek60_files(): + """Check that non-EK60 files are not identified as EK60""" + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_EK60(test_file_path, storage_options={}) == False + +@pytest.fixture(scope="session") +def test_is_ER60_er60_files(): + """Check that EK60 files are identified as EK60""" + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in ek60_files: + assert is_ER60(test_file_path, storage_options={}) == True + +@pytest.fixture(scope="session") +def test_is_ER60_non_er60_files(): + """Check that non-EK60 files are not identified as EK60""" + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_ER60(test_file_path, storage_options={}) == False + +@pytest.fixture(scope="session") +def test_is_AZFP6_valid_files(): + """Test that .azfp files are identified as valid AZFP files.""" + # Collect all .azfp files in the test directory + azfp_files = glob.glob("echopype/test_data/azfp6/*.azfp") + + # Check that each file in azfp is identified as valid AZFP + for test_file_path in azfp_files: + assert is_AZFP6(test_file_path) == True + +@pytest.fixture(scope="session") +def test_is_AZFP6_invalid_files(): + """Test that non-.azfp files are not identified as valid AZFP files.""" + # Collect all non-.azfp files in the test directory + non_azfp_files = glob.glob("echopype/test_data/azfp/*") + + # Check that each file in non_azfp is not identified as valid AZFP + for test_file_path in non_azfp_files: + assert is_AZFP6(test_file_path) == False + +@pytest.fixture(scope="session") +def test_is_AZFP_valid_files(): + """Test that XML files with are identified as valid AZFP files.""" + # Collect all valid XML files in the test directory + valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("echopype/test_data/azfp/*.XML") + + for test_file_path in valid_files: + assert is_AZFP(test_file_path) == True + +@pytest.fixture(scope="session") +def test_is_AZFP_invalid_files(): + """Test that XML files without are not identified as valid AZFP files.""" + # Collect all invalid XML files in the test directory + invalid_files = glob.glob("echopype/test_data/azfp6/*") + + for test_file_path in invalid_files: + assert is_AZFP(test_file_path) == False + +@pytest.fixture(scope="session") +def test_is_AD2CP_valid_files(): + """Test that .ad2cp files are identified as valid AD2CP files.""" + # Collect all .ad2cp files in the test directory + ad2cp_files = glob.glob("echopype/test_data/ad2cp/normal/*.ad2cp") + + # Check that each file in ad2cp is identified as valid AD2CP + for test_file_path in ad2cp_files: + assert is_AD2CP(test_file_path) == True + +@pytest.fixture(scope="session") +def test_is_AD2CP_invalid_files(): + """Test that non-.ad2cp files are not identified as valid AD2CP files.""" + # Collect all non-.ad2cp files in the test directory + non_ad2cp_files = glob.glob("echopype/test_data/azfp6/*") + + # Check that each file in non_ad2cp is not identified as valid AD2CP + for test_file_path in non_ad2cp_files: + assert is_AD2CP(test_file_path) == False \ No newline at end of file