Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
22d2240
Switch from old stable version to pending PR for mainline adoption
BrianMichell Dec 5, 2025
eb31d96
Apply minimal updates to get working
BrianMichell Dec 5, 2025
e500bfd
Updates to support both drivers
BrianMichell Dec 8, 2025
473e475
Fix reading stats from mdio-python files
BrianMichell Dec 12, 2025
204421a
Fix fill value metadata issue for boolean types
BrianMichell Dec 12, 2025
a5e5a24
Fix chunk shape in test
BrianMichell Dec 12, 2025
f056887
Add zarr3 driver to internal dependencies
BrianMichell Dec 12, 2025
be55802
Unity v2/3 acceptance test, add support for zarr version specificatio…
BrianMichell Dec 18, 2025
b567425
Ensure all appropriate tests run for v2 and v3 drivers
BrianMichell Dec 18, 2025
db8ffb9
Suppress warning output spam
BrianMichell Dec 19, 2025
faa8393
Fix driver requirements
BrianMichell Dec 19, 2025
b259f60
Update inert bucket example to be more appropriate
BrianMichell Dec 19, 2025
2a09c99
Propogate context through internal objects to allow for credentials t…
BrianMichell Dec 22, 2025
fbf46af
Add optional code coverage and begin expanding coverage
BrianMichell Dec 30, 2025
bd0906f
Expand meaningful test coverage
BrianMichell Jan 2, 2026
649610e
Merge pull request #1 from BrianMichell/expand_coverage
BrianMichell Jan 2, 2026
528cab7
Formatting and linting
BrianMichell Jan 2, 2026
9d1f717
Begin re-unifying the zarr drivers
BrianMichell Jan 2, 2026
d5cdcf8
Fix copyright date
BrianMichell Jan 2, 2026
ff8fa55
Resolve logic error for serializing structarray with v3 driver
BrianMichell Jan 2, 2026
74b2cab
Add support for field selection of v3 datasets
BrianMichell Jan 2, 2026
1609ff8
Fix logic for fill values on v3 driver
BrianMichell Jan 2, 2026
1b7f8f5
Reduce boilerplate
BrianMichell Jan 2, 2026
b78d7bc
Begin fixing metadata serialization errors
BrianMichell Jan 2, 2026
19b2855
Resolve zarr3 outputting improper metadata, fix filesystem access to …
BrianMichell Jan 5, 2026
b1c673f
Refactor metadata handling and path utilities for Zarr drivers
BrianMichell Jan 5, 2026
dc65f2a
Linting
BrianMichell Jan 5, 2026
eba8c41
Merge pull request #2 from BrianMichell/fix_regressions
BrianMichell Jan 5, 2026
2bdeb94
Update tensorstore latest (#3)
BrianMichell Jun 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ include(CMakeDependentOption)
# provides a method to download dependencies:
include(FetchContent)
include(CMakeHelpers/MdioHelpers)
# optional code coverage support:
include(CodeCoverage)

list(APPEND mdio_DEFAULT_COPTS
"-Wno-deprecated-declarations"
Expand Down Expand Up @@ -108,6 +110,7 @@ if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(mdio_INTERNAL_DEPS
tensorstore::driver_array
tensorstore::driver_zarr
tensorstore::driver_zarr3
tensorstore::driver_json
tensorstore::kvstore_file
tensorstore::stack
Expand Down
6 changes: 6 additions & 0 deletions cmake/CMakeHelpers/MdioHelpers.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,12 @@ function(mdio_cc_test)
PRIVATE ${mdio_CC_TEST_COPTS}
)

# Add coverage flags only to mdio test targets (not dependencies)
if(MDIO_ENABLE_COVERAGE)
target_compile_options(${_NAME} PRIVATE ${MDIO_COVERAGE_COMPILE_FLAGS})
target_link_options(${_NAME} PRIVATE ${MDIO_COVERAGE_LINK_FLAGS})
endif()

target_link_libraries(${_NAME}
PUBLIC ${mdio_CC_TEST_DEPS}
PRIVATE ${mdio_CC_TEST_LINKOPTS}
Expand Down
166 changes: 166 additions & 0 deletions cmake/CodeCoverage.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# Copyright 2024 TGS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ==============================================================================
# Code Coverage Support (gcov + lcov)
# ==============================================================================
#
# Coverage instrumentation is applied ONLY to mdio test targets, not to
# dependencies like Tensorstore. This keeps build times and test execution fast.
#
# REQUIRED CMAKE FLAGS:
# -DMDIO_ENABLE_COVERAGE=ON Enable coverage instrumentation
# -DCMAKE_BUILD_TYPE=Debug Recommended for meaningful line coverage
#
# REQUIRED SYSTEM TOOLS:
# - gcov (usually bundled with GCC)
# - lcov (install via: apt install lcov / brew install lcov)
# - genhtml (included with lcov)
#
# USAGE:
# 1. Configure and build with coverage enabled:
# cd build
# cmake .. -DMDIO_ENABLE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug
# make
#
# 2. Run tests to generate coverage data (coverage accumulates across runs):
# ./mdio/mdio_variable_test # single test
# ./mdio/mdio_variable_test && ./mdio/mdio_dataset_test # multiple tests
# ctest # all registered tests
#
# 3. Generate HTML coverage report:
# make coverage-capture
#
# 4. View the report:
# Open build/coverage_report/index.html in a browser
#
# AVAILABLE TARGETS:
# make coverage - Reset counters, capture data, generate report
# make coverage-capture - Capture current data and generate report (no reset)
# make coverage-reset - Zero out all coverage counters
#
# ==============================================================================

option(MDIO_ENABLE_COVERAGE "Enable code coverage instrumentation (requires GCC or Clang)" OFF)

if(MDIO_ENABLE_COVERAGE)
message(STATUS "Code coverage enabled")

# Check for supported compiler
if(NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
message(FATAL_ERROR "Code coverage requires GCC or Clang compiler")
endif()

# Coverage compiler/linker flags - exported for use in MdioHelpers.cmake
# These are applied only to mdio targets, not to dependencies like Tensorstore
set(MDIO_COVERAGE_COMPILE_FLAGS "-fprofile-arcs;-ftest-coverage" CACHE INTERNAL "Coverage compile flags")
set(MDIO_COVERAGE_LINK_FLAGS "--coverage" CACHE INTERNAL "Coverage link flags")

# Find required tools
find_program(LCOV_PATH lcov)
find_program(GENHTML_PATH genhtml)

if(NOT LCOV_PATH)
message(WARNING "lcov not found - coverage report generation will not be available")
endif()

if(NOT GENHTML_PATH)
message(WARNING "genhtml not found - coverage report generation will not be available")
endif()

# Create coverage report target if tools are available
if(LCOV_PATH AND GENHTML_PATH)
# Custom target to generate coverage report
add_custom_target(coverage
COMMENT "Generating code coverage report..."

# Clear previous coverage data
COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters

# Run all tests (ctest must be run separately or you can uncomment below)
# COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure

# Capture coverage data
COMMAND ${LCOV_PATH}
--directory ${CMAKE_BINARY_DIR}
--capture
--output-file ${CMAKE_BINARY_DIR}/coverage.info
--ignore-errors mismatch,negative

# Remove coverage data for external dependencies
COMMAND ${LCOV_PATH}
--remove ${CMAKE_BINARY_DIR}/coverage.info
'/usr/*'
'${CMAKE_BINARY_DIR}/_deps/*'
'*/googletest/*'
'*/gtest/*'
'*/gmock/*'
'*_test.cc'
--output-file ${CMAKE_BINARY_DIR}/coverage.info
--ignore-errors unused,negative

# Generate HTML report
COMMAND ${GENHTML_PATH}
${CMAKE_BINARY_DIR}/coverage.info
--output-directory ${CMAKE_BINARY_DIR}/coverage_report
--title "MDIO Code Coverage"
--legend
--show-details

WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)

# Target to just capture coverage (without zeroing first)
add_custom_target(coverage-capture
COMMENT "Capturing coverage data..."

COMMAND ${LCOV_PATH}
--directory ${CMAKE_BINARY_DIR}
--capture
--output-file ${CMAKE_BINARY_DIR}/coverage.info
--ignore-errors mismatch,negative

COMMAND ${LCOV_PATH}
--remove ${CMAKE_BINARY_DIR}/coverage.info
'/usr/*'
'${CMAKE_BINARY_DIR}/_deps/*'
'*/googletest/*'
'*/gtest/*'
'*/gmock/*'
'*_test.cc'
--output-file ${CMAKE_BINARY_DIR}/coverage.info
--ignore-errors unused,negative

COMMAND ${GENHTML_PATH}
${CMAKE_BINARY_DIR}/coverage.info
--output-directory ${CMAKE_BINARY_DIR}/coverage_report
--title "MDIO Code Coverage"
--legend
--show-details

WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)

# Target to reset coverage counters
add_custom_target(coverage-reset
COMMENT "Resetting coverage counters..."
COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)

message(STATUS "Coverage targets available: 'coverage', 'coverage-capture', 'coverage-reset'")
message(STATUS "Coverage report will be generated at: ${CMAKE_BINARY_DIR}/coverage_report/index.html")
endif()
endif()

10 changes: 5 additions & 5 deletions cmake/FindEXT_TENSORSTORE.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ IF ( NOT TARGET tensorstore )

include(FetchContent)

FetchContent_Declare(
tensorstore
GIT_REPOSITORY
https://github.com/brian-michell/tensorstore.git
GIT_TAG v0.1.63_latest
FetchContent_Declare(
tensorstore
GIT_REPOSITORY
https://github.com/google/tensorstore.git
GIT_TAG 917edaf341217f750b7bd3b8db6e75e6db64eab8
)

FetchContent_MakeAvailable(tensorstore)
Expand Down
2 changes: 1 addition & 1 deletion cmake/Findnlohmann_json_schema_validator.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ if (NOT TARGET nlohmann_json_schema_validator)
FetchContent_Declare(
nlohmann_json_schema_validator
GIT_REPOSITORY https://github.com/pboettch/json-schema-validator.git
GIT_TAG 2.2.0
GIT_TAG 2.4.0
)

if(NOT BUILD_VALIDATOR)
Expand Down
22 changes: 11 additions & 11 deletions examples/dataset_example/src/dataset_example.cc
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ absl::Status Run() {
/// handled by returning a "Result" object; the result can be tested for
/// "ok()", there is also a convenient Tensorstore macro for doing assigment
/// while handling errors.
MDIO_ASSIGN_OR_RETURN(auto dataset, dataset_result)
MDIO_ASSIGN_OR_RETURN(auto dataset, dataset_result);

/// The dataset represent data on disk, the object holds only minimal state.
/// Its memory footprint is small.
Expand Down Expand Up @@ -113,7 +113,7 @@ absl::Status Run() {
mdio::SliceDescriptor desc1 = {"inline", 20, 120, 1};
mdio::SliceDescriptor desc2 = {"crossline", 100, 200, 1};

MDIO_ASSIGN_OR_RETURN(auto slice, dataset.isel(desc1, desc2))
MDIO_ASSIGN_OR_RETURN(auto slice, dataset.isel(desc1, desc2));
/// The slice represents a subset of the MDIO data on disk, I/O operations can
/// be made using the variables contained in this slice.
std::cout << slice << "\n\n" << std::endl;
Expand All @@ -123,7 +123,7 @@ absl::Status Run() {
/// describing data, this means that the datatype is discovered at runtime.
/// Here we request a Variable given we know its dtype of unit32:
MDIO_ASSIGN_OR_RETURN(auto variableObject,
slice.variables.get<uint32_t>("inline"))
slice.variables.get<uint32_t>("inline"));
/// The Variable object contains a collection of metadata associated with the
/// seismic, including names, and units.
std::cout << variableObject << std::endl;
Expand All @@ -143,7 +143,7 @@ absl::Status Run() {
// MDIO_ASSIGN_OR_RETURN(
// auto data, tensorstore::Read(variableObject.get_store()).result()
// )
MDIO_ASSIGN_OR_RETURN(auto data, variableObject.Read().result())
MDIO_ASSIGN_OR_RETURN(auto data, variableObject.Read().result());
auto d1 = data.get_data_accessor();
/// The read returns a SharedArray, which we can addrss like and array, in 3-d
/// we might do something like this data({0, 0, 0}), here the inline label is
Expand All @@ -163,7 +163,7 @@ absl::Status Run() {
/// Optionally, the variable can handle the I/O. In this scenario data can be
/// read into a VariableData object that retains it's dimension names and
/// other metadata.
MDIO_ASSIGN_OR_RETURN(auto variableData, variableObject.Read().result())
MDIO_ASSIGN_OR_RETURN(auto variableData, variableObject.Read().result());

/// The accessor method provides access to an underlying tensorstore
/// SharedArray.
Expand All @@ -185,7 +185,7 @@ absl::Status Run() {
auto existing_dataset =
mdio::Dataset::Open(dataset_path, mdio::constants::kOpen).result();

MDIO_ASSIGN_OR_RETURN(dataset, existing_dataset)
MDIO_ASSIGN_OR_RETURN(dataset, existing_dataset);

/// In the previous scenario used a slice to operate over the range of data
/// [20, 120), this dataset is defined over the entire range, so we should see
Expand All @@ -196,14 +196,14 @@ absl::Status Run() {
/// to uint32_t (in this case), then the get method will return a result that
/// is not OK.
MDIO_ASSIGN_OR_RETURN(variableObject,
dataset.variables.get<uint32_t>("inline"))
dataset.variables.get<uint32_t>("inline"));
inclusive_min =
variableObject.get_store().domain()[0].interval().inclusive_min();
exclusive_max =
variableObject.get_store().domain()[0].interval().exclusive_max();

/// Here we read all of the variable from disk into memory
MDIO_ASSIGN_OR_RETURN(variableData, variableObject.Read().result())
MDIO_ASSIGN_OR_RETURN(variableData, variableObject.Read().result());
tick_labels = variableData.get_data_accessor();
for (Index i = inclusive_min; i < exclusive_max; i += 20) {
std::cout << "dataset domain, " << variableData.variableName
Expand All @@ -219,10 +219,10 @@ absl::Status Run() {
}
MDIO_ASSIGN_OR_RETURN(
/// in this example, all the dims labels are unint32
variableObject, dataset.variables.get<uint32_t>(label))
variableObject, dataset.variables.get<uint32_t>(label));

/// Suppose we want to read existing values ...
MDIO_ASSIGN_OR_RETURN(variableData, variableObject.Read().result())
MDIO_ASSIGN_OR_RETURN(variableData, variableObject.Read().result());

inclusive_min =
variableObject.get_store().domain()[0].interval().inclusive_min();
Expand All @@ -242,7 +242,7 @@ absl::Status Run() {
/// For the purposes of information hiding, you can also extract a single
/// variable and its coordinates, with the other metadata. This acts like a
/// regular Dataset, but has only a single variable.
MDIO_ASSIGN_OR_RETURN(auto inline_labels, dataset["cdp-x"])
MDIO_ASSIGN_OR_RETURN(auto inline_labels, dataset["cdp-x"]);

return absl::OkStatus();
}
Expand Down
4 changes: 2 additions & 2 deletions examples/real_data_example/src/real_data_example.cc
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ absl::Status Run(const Descriptors... descriptors) {
auto dataset,
mdio::Dataset::Open(std::string(absl::GetFlag(FLAGS_dataset_path)),
mdio::constants::kOpen)
.result())
.result());

if (absl::GetFlag(FLAGS_print_dataset)) {
std::cout << dataset << std::endl;
Expand All @@ -70,7 +70,7 @@ absl::Status Run(const Descriptors... descriptors) {
return absl::InvalidArgumentError("Seismic data must be 3D");
}

MDIO_ASSIGN_OR_RETURN(auto seismic_data, ReadWithProgress(variable).result())
MDIO_ASSIGN_OR_RETURN(auto seismic_data, ReadWithProgress(variable).result());

auto seismic_accessor = seismic_data.get_data_accessor();

Expand Down
6 changes: 3 additions & 3 deletions examples/xarray_integration/src/mdio_from_xarray.cc
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@
absl::Status Run(std::string dataset_path) {
MDIO_ASSIGN_OR_RETURN(
auto dataset,
mdio::Dataset::Open(dataset_path, mdio::constants::kOpen).result())
mdio::Dataset::Open(dataset_path, mdio::constants::kOpen).result());

MDIO_ASSIGN_OR_RETURN(auto variable,
dataset.variables.get<float32_t>("image"))
dataset.variables.get<float32_t>("image"));

MDIO_ASSIGN_OR_RETURN(auto variable_data, variable.Read().result())
MDIO_ASSIGN_OR_RETURN(auto variable_data, variable.Read().result());

auto image = variable_data.get_data_accessor();

Expand Down
2 changes: 1 addition & 1 deletion examples/xarray_integration/src/xarray_integration.cc
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ absl::Status Run(std::string dataset_path) {
MDIO_ASSIGN_OR_RETURN(auto dataset,
mdio::Dataset::from_json(json_spec, dataset_path,
mdio::constants::kCreateClean)
.result())
.result());

auto populate_inline = [](SharedArray<uint32_t>& data) {
for (auto i = data.domain()[0].inclusive_min();
Expand Down
2 changes: 1 addition & 1 deletion examples/xarray_integration/src/xarray_integration.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ using SharedArray = mdio::SharedArray<T, R, OriginKind>;
template <typename T = void>
mdio::Result<mdio::VariableData<T>> from_dataset(
const mdio::Dataset& dataset, const std::string& variable_name) {
MDIO_ASSIGN_OR_RETURN(auto variable, dataset.variables.get<T>(variable_name))
MDIO_ASSIGN_OR_RETURN(auto variable, dataset.variables.get<T>(variable_name));

return mdio::from_variable<T>(variable);
}
Expand Down
Loading
Loading