From 1f9b67a3357c8b5a9681b69f8de96245e4ff31ae Mon Sep 17 00:00:00 2001 From: Lloyd Izard <76954858+LOCEANlloydizard@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:36:36 -0700 Subject: [PATCH] first test for data inventory --- .ci_helpers/check_test_data_inventory.py | 131 +++++++ docs/source/_config.yml | 3 +- docs/source/_toc.yml | 1 + docs/source/contrib_test_data.md | 44 +++ docs/source/test_data_inventory.yml | 412 +++++++++++++++++++++++ docs/source/test_data_inventory_ext.py | 75 +++++ 6 files changed, 665 insertions(+), 1 deletion(-) create mode 100644 .ci_helpers/check_test_data_inventory.py create mode 100644 docs/source/contrib_test_data.md create mode 100644 docs/source/test_data_inventory.yml create mode 100644 docs/source/test_data_inventory_ext.py diff --git a/.ci_helpers/check_test_data_inventory.py b/.ci_helpers/check_test_data_inventory.py new file mode 100644 index 000000000..45964bdd8 --- /dev/null +++ b/.ci_helpers/check_test_data_inventory.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Validate the test data inventory against the Pooch test-data bundles.""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + +CONFTST = ROOT / "echopype" / "tests" / "conftest.py" +INVENTORY = ROOT / "docs" / "source" / "test_data_inventory.yml" + + +REQUIRED_BUNDLE_KEYS = { + "documented_checksum", + "instrument", + "description", + "source", + "contributor", + "references", + "notes", + "files", +} + +REQUIRED_FILE_KEYS = { + "instrument", + "description", + "source", + "contributor", + "references", + "notes", +} + + +def load_conftest_bundles_and_registry(): + source = CONFTST.read_text(encoding="utf-8") + tree = ast.parse(source) + + bundles = None + registry = None + + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "bundles": + bundles = ast.literal_eval(node.value) + elif isinstance(target, ast.Name) and target.id == "registry": + registry = ast.literal_eval(node.value) + + if bundles is None: + raise RuntimeError(f"Could not find 'bundles' in {CONFTST}") + + if registry is None: + raise RuntimeError(f"Could not find 'registry' in {CONFTST}") + + return bundles, registry + + +def load_inventory(): + with INVENTORY.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def main(): + bundles, registry = load_conftest_bundles_and_registry() + inventory = load_inventory() + + errors = [] + + conftest_bundles = set(bundles) + inventory_bundles = set(inventory) + + missing = sorted(conftest_bundles - inventory_bundles) + if missing: + errors.append( + "Bundles listed in conftest.py but missing from inventory:\n - " + + "\n - ".join(missing) + ) + + extra = sorted(inventory_bundles - conftest_bundles) + if extra: + errors.append( + "Bundles listed in inventory but not in conftest.py:\n - " + "\n - ".join(extra) + ) + + for bundle in sorted(conftest_bundles & inventory_bundles): + metadata = inventory[bundle] + + missing_keys = REQUIRED_BUNDLE_KEYS - set(metadata) + if missing_keys: + errors.append( + f"{bundle}: missing required bundle keys: " + ", ".join(sorted(missing_keys)) + ) + + checksum = metadata.get("documented_checksum") + if checksum is not None and checksum != registry[bundle]: + errors.append( + f"{bundle}: documented_checksum does not match conftest.py registry\n" + f" inventory: {checksum}\n" + f" registry: {registry[bundle]}" + ) + + files = metadata.get("files", {}) + if files is None: + errors.append(f"{bundle}: files must be a mapping, not null") + continue + + for filename, file_metadata in files.items(): + missing_file_keys = REQUIRED_FILE_KEYS - set(file_metadata) + if missing_file_keys: + errors.append( + f"{bundle} / {filename}: missing required file keys: " + + ", ".join(sorted(missing_file_keys)) + ) + + if errors: + print("\nTEST DATA INVENTORY CHECK FAILED\n") + print("\n\n".join(errors)) + sys.exit(1) + + print("Test data inventory is valid.") + + +if __name__ == "__main__": + main() diff --git a/docs/source/_config.yml b/docs/source/_config.yml index 4bbd88988..520c086f4 100644 --- a/docs/source/_config.yml +++ b/docs/source/_config.yml @@ -45,7 +45,8 @@ sphinx: 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.githubpages', - 'sphinxcontrib.mermaid' + 'sphinxcontrib.mermaid', + 'test_data_inventory_ext', ] config: bibtex_reference_style: label diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml index b63358f37..760cb8ee8 100644 --- a/docs/source/_toc.yml +++ b/docs/source/_toc.yml @@ -30,6 +30,7 @@ parts: - file: contrib_roadmap - file: contrib_howto - file: contrib_setup + - file: test_data_inventory - caption: Help & references chapters: - file: whats-new diff --git a/docs/source/contrib_test_data.md b/docs/source/contrib_test_data.md new file mode 100644 index 000000000..a73bdd1d9 --- /dev/null +++ b/docs/source/contrib_test_data.md @@ -0,0 +1,44 @@ +(contrib:test-data)= + +# Test data inventory + +echopype uses a collection of test data bundles to support unit and integration tests across different sonar models and file formats. These bundles are distributed through GitHub release assets and downloaded automatically using Pooch during testing. + +The test data inventory is maintained in: + +```text +docs/source/test_data_inventory.yml +``` + +This inventory serves as the central metadata registry for the test data bundles currently used by the test suite. It includes, when available: + +* instrument type, +* file descriptions, +* data source, +* contributor, +* references, +* additional notes. + +The inventory is validated in CI against the list of test data bundles declared in the test suite (`echopype/tests/conftest.py`). When a new test data bundle is added, the inventory should be updated accordingly. + +## Inventory fields + +| Field | Description | +| :-------------------- | :------------------------------------------------------------------------------ | +| `documented_checksum` | SHA256 checksum of the bundle version for which the metadata has been reviewed. | +| `instrument` | Sonar or instrument type, when known. | +| `description` | Short description of the bundle or file. | +| `source` | Origin of the dataset, when known. | +| `contributor` | Contributor or data provider, when known. | +| `references` | Related publications, documentation, or external resources. | +| `notes` | Additional information or context. | +| `files` | Metadata for individual files contained in the bundle. | + +## Adding a new test data bundle + +When adding a new test data bundle: + +1. Add the bundle to the Pooch registry in `echopype/tests/conftest.py`. +2. Add a corresponding entry to `docs/source/test_data_inventory.yml`. +3. Populate the available metadata fields as completely as possible. +4. If the bundle metadata has been reviewed, update `documented_checksum` to match the bundle checksum in the Pooch registry. diff --git a/docs/source/test_data_inventory.yml b/docs/source/test_data_inventory.yml new file mode 100644 index 000000000..b945a90da --- /dev/null +++ b/docs/source/test_data_inventory.yml @@ -0,0 +1,412 @@ +########## Template +# bundle_name.zip: +# documented_checksum: null +# instrument: null +# description: null +# source: null +# contributor: null +# references: [] +# notes: null +# files: {} + +ad2cp.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +azfp.zip: + documented_checksum: null + instrument: AZFP + description: null + source: null + contributor: null + references: [] + notes: null + files: + 17082117.01A: + instrument: AZFP + description: Standard test. Used with 17041823.XML. + source: null + contributor: null + references: [] + notes: null + +azfp6.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ea640.zip: + documented_checksum: null + instrument: EA640 + description: null + source: null + contributor: null + references: [] + notes: null + files: + 0001a-D20200321-T032026.raw: + instrument: EA640 + description: Data of identical format to standard EK80 files, but with a different NME1 datagram (instead of NME0). + source: null + contributor: null + references: [] + notes: null + +ecs.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek60.zip: + documented_checksum: null + instrument: EK60 + description: null + source: null + contributor: null + references: [] + notes: null + files: + DY1801_EK60-D20180211-T164025.raw: + instrument: EK60 + description: Standard test with constant ranges across ping times + source: null + contributor: null + references: [] + notes: null + + Winter2017-D20170115-T150122.raw: + instrument: EK60 + description: Contains a change of recording length in the middle of the file + source: null + contributor: null + references: [] + notes: null + + 2015843-D20151023-T190636.raw: + instrument: EK60 + description: Not used in tests but contains ranges are not constant across ping times + source: null + contributor: null + references: [] + notes: null + + SH1701_consecutive_files_w_range_change: + instrument: EK60 + description: Not used in tests. + source: null + contributor: null + references: [] + notes: Folder on shared drive that contains sequential files with ranges that are not constant across ping times. + + NBP_B050N-D20180118-T090228.raw: + instrument: EK60 + description: split-beam setup without angle data + source: null + contributor: null + references: [] + notes: null + +ek60_calibrate_chunks.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek60_missing_channel_power.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80.zip: + documented_checksum: null + instrument: EK80 + description: null + source: null + contributor: null + references: [] + notes: null + files: + D20190822-T161221.raw: + instrument: EK80 + description: Contains channels that only record real power data + source: null + contributor: null + references: [] + notes: null + + D20170912-T234910.raw: + instrument: EK80 + description: Contains channels that only record complex power data + source: null + contributor: null + references: [] + notes: null + + Summer2018--D20180905-T033113.raw: + instrument: EK80 + description: Contains BB channels encoded in complex and CW channels encoded in power samples (reduced from 300 MB to 3.8 MB in test data updates). + source: null + contributor: null + references: [] + notes: null + + Summer2018: + instrument: EK80 + description: Contains channels with complex power data as well as channels with real power data. They can be combined. + source: null + contributor: null + references: [] + notes: null + + 2019118 group2survey-D20191214-T081342.raw: + instrument: EK80 + description: Contains 6 channels but only 2 of those channels collect ping data + source: null + contributor: null + references: [] + notes: null + + D20200528-T125932.raw: + instrument: EK80 + description: Data collected from WBT mini (instead of WBT), from @emlynjdavies + source: null + contributor: null + references: [] + notes: null + + Green2.Survey2.FM.short.slow.-D20191004-T211557.raw: + instrument: EK80 + description: Contains 2-in-1 transducer, from @FletcherFT (reduced from 104.9 MB to 765 KB in test data updates) + source: null + contributor: null + references: [] + notes: null + + raw4-D20220514-T172704.raw: + instrument: EK80 + description: Contains RAW4 datagram, 1 channel only, from @cornejotux + source: null + contributor: null + references: [] + notes: null + + D20210330-T123857.raw: + instrument: EK80 + description: do not contain filter coefficients + source: null + contributor: null + references: [] + notes: null + +ek80_bb_complex_multiplex.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_bb_with_calibration.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_duplicate_ping_times.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_ext.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_invalid_env_datagrams.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_missing_sound_velocity_profile.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_new.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ek80_sequence.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +es60.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +es70.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +es80.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +legacy_datatree.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + files: {} + +ts_spectrum_example_data.zip: + documented_checksum: null + instrument: null + description: null + source: null + contributor: null + references: + - Andersen, L. N., et al. (2024). Quantitative processing of broadband data as implemented in a scientific split-beam echosounder. Methods in Ecology and Evolution, 15(2), 317–328. https://doi.org/10.1111/2041-210X.14261 + notes: This pre-release is intended for staging new files in the release assets for an upcoming release. + files: + IMR-D20211215-T143432-TSf.raw: + instrument: null + description: Broadband EK80 FM dataset from the CRIMAC broadband processing example described in Andersen et al. (2024). + source: null + contributor: null + references: [] + notes: null + + crimac_tsf_reference_outputs.npz: + instrument: null + description: Reference outputs generated from the same dataset using the CRIMAC broadband target strength processing workflow. These outputs are provided for convenience and can be used to validate the echopype implementation against the CRIMAC reference results. + source: null + contributor: null + references: [] + notes: null + +resample_to_geometry_example_data.zip: + documented_checksum: null + instrument: null + description: Reference outputs generated using Echoview's Match Geometry algorithm after resampling all channels onto the 200 kHz channel geometry. These files are provided for validation purposes and can be used to compare the `echopype.commongrid.resample_to_geometry()` implementation against Echoview reference results. + source: null + contributor: null + references: + - Echoview Software Pty Ltd. Match Geometry Algorithm. https://support.echoview.com/WebHelp/Reference/Algorithms/Operators/Match_geometry_algorithm.htm + notes: null + files: + 18kHz_regridded.sv.csv: + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + + 120kHz_regridded.sv.csv: + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + + 200kHz.sv.csv: + instrument: null + description: null + source: null + contributor: null + references: [] + notes: null + + Hake-D20230811-T165727.raw: + instrument: null + description: EK80 CW dataset from the NOAA NCEI/WCSD public fisheries acoustics archive and also used in the echopype Getting Started notebook. + source: null + contributor: null + references: [] + notes: null diff --git a/docs/source/test_data_inventory_ext.py b/docs/source/test_data_inventory_ext.py new file mode 100644 index 000000000..febde9d4c --- /dev/null +++ b/docs/source/test_data_inventory_ext.py @@ -0,0 +1,75 @@ +"""Render the test data inventory page from YAML metadata.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +def _value(value): + if value in (None, "", []): + return "NA" + if isinstance(value, list): + return "
".join(str(v) for v in value) + return str(value).replace("\n", " ") + + +def _render_inventory(app): + inventory_path = Path(app.srcdir) / "test_data_inventory.yml" + inventory = yaml.safe_load(inventory_path.read_text(encoding="utf-8")) + + lines = [ + "(test-data-inventory)=\n", + "# Test data inventory\n", + "This page is generated automatically from `docs/source/test_data_inventory.yml`.\n", + ] + + for bundle, metadata in sorted(inventory.items()): + lines.append(f"## `{bundle}`\n") + lines.append(f"**Instrument:** {_value(metadata.get('instrument'))} \n") + lines.append(f"**Description:** {_value(metadata.get('description'))} \n") + lines.append(f"**Source:** {_value(metadata.get('source'))} \n") + lines.append(f"**Contributor:** {_value(metadata.get('contributor'))} \n") + lines.append(f"**Notes:** {_value(metadata.get('notes'))} \n") + + references = metadata.get("references", []) + if references: + lines.append("\n**References:**\n") + for ref in references: + lines.append(f"- {ref}\n") + else: + lines.append("\n**References:** NA\n") + + files = metadata.get("files", {}) + lines.append("\n### Files\n") + + if files: + lines.append("| File | Instrument | Description | Source | Contributor | Notes |\n") + lines.append("| :--- | :--------- | :---------- | :----- | :---------- | :---- |\n") + + for filename, file_metadata in sorted(files.items()): + lines.append( + f"| `{filename}` " + f"| {_value(file_metadata.get('instrument'))} " + f"| {_value(file_metadata.get('description'))} " + f"| {_value(file_metadata.get('source'))} " + f"| {_value(file_metadata.get('contributor'))} " + f"| {_value(file_metadata.get('notes'))} |\n" + ) + else: + lines.append("NA\n") + + lines.append("\n") + + output_path = Path(app.srcdir) / "test_data_inventory.md" + output_path.write_text("".join(lines), encoding="utf-8") + + +def setup(app): + app.connect("builder-inited", _render_inventory) + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + }