Skip to content

Commit 85c022f

Browse files
committed
Make vendored ExternalData upstreamable
1 parent 40ffdac commit 85c022f

4 files changed

Lines changed: 111 additions & 80 deletions

File tree

cmake/NablaAssetManifests.cmake

Lines changed: 23 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ function(_nam_summary MESSAGE_TEXT)
2424
message(STATUS "NablaAssetManifests: ${MESSAGE_TEXT}")
2525
endfunction()
2626

27-
function(_nam_include_externaldata OUT_VAR)
27+
macro(_nam_include_externaldata OUT_VAR)
2828
if (NAM_USE_VENDORED_EXTERNALDATA)
2929
include("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/vendor/ExternalData-NAM.cmake")
3030
set(_provider "vendored")
3131
else()
3232
include(ExternalData)
3333
set(_provider "stock")
3434
endif()
35-
set(${OUT_VAR} "${_provider}" PARENT_SCOPE)
36-
endfunction()
35+
set(${OUT_VAR} "${_provider}")
36+
endmacro()
3737

3838
function(_nam_validate_file_link_mode MODE_VALUE OUT_VAR)
3939
string(TOLOWER "${MODE_VALUE}" _mode)
@@ -491,9 +491,7 @@ function(nam_add_channel_target)
491491
_nam_summary("materialization mode for file assets: `${_file_link_mode}`")
492492

493493
set(_asset_refs)
494-
set(_asset_data_refs)
495494
set(_asset_relpaths)
496-
set(_asset_target_paths)
497495
foreach(_asset IN LISTS _items)
498496
_nam_find_channel_asset(
499497
CHANNEL "${NAM_CHANNEL}"
@@ -513,82 +511,44 @@ function(nam_add_channel_target)
513511

514512
set(_data_name "${NAM_CHANNEL}/${_relative_path}")
515513
set(_data_ref "${_refs_root}/${_data_name}")
516-
set(_target_path "${NAM_DESTINATION_ROOT}/${_data_name}")
517514
get_filename_component(_link_dir "${_data_ref}" DIRECTORY)
518515
file(MAKE_DIRECTORY "${_link_dir}")
519516
file(WRITE "${_data_ref}.sha256" "${_sha256}\n")
520517
list(APPEND _asset_refs "DATA{${_data_ref}}")
521-
list(APPEND _asset_data_refs "${_data_ref}")
522518
list(APPEND _asset_relpaths "${_relative_path}")
523-
list(APPEND _asset_target_paths "${_target_path}")
524519
endforeach()
525520

526521
if (_asset_refs)
522+
set(_asset_target "${NAM_TARGET}__externaldata")
523+
set(ExternalData_SOURCE_ROOT "${_refs_root}")
527524
if (_externaldata_provider STREQUAL "vendored")
525+
set(ExternalData_BINARY_ROOT "${NAM_DESTINATION_ROOT}")
526+
set(ExternalData_STATE_ROOT "${_build_root}/state")
528527
set(ExternalData_LINK_MODE "${_file_link_mode}")
529-
set(ExternalData_TIMEOUT_INACTIVITY "")
530-
set(ExternalData_TIMEOUT_ABSOLUTE "")
531-
set(ExternalData_NO_SYMLINKS "")
532-
string(CONCAT _ExternalData_CONFIG_CODE
533-
"set(ExternalData_CUSTOM_SCRIPT_NAM [=[${_fetch_script}]=])")
534-
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${NAM_TARGET}__externaldata_config.cmake")
535-
configure_file(
536-
"${CMAKE_CURRENT_FUNCTION_LIST_DIR}/vendor/ExternalData_config.cmake.in"
537-
"${_externaldata_config}"
538-
@ONLY
539-
)
528+
else()
529+
set(ExternalData_BINARY_ROOT "${_externaldata_binary_root}")
530+
unset(ExternalData_STATE_ROOT)
531+
unset(ExternalData_LINK_MODE)
532+
endif()
533+
unset(ExternalData_NO_SYMLINKS)
534+
set(_old_suppress_dev "${CMAKE_SUPPRESS_DEVELOPER_WARNINGS}")
535+
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1)
536+
ExternalData_Expand_Arguments("${_asset_target}" _asset_expanded ${_asset_refs})
537+
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS "${_old_suppress_dev}")
538+
ExternalData_Add_Target("${_asset_target}" SHOW_PROGRESS "${NAM_SHOW_PROGRESS}")
539+
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${_asset_target}_config.cmake")
540+
if (EXISTS "${_externaldata_config}")
540541
if (NAM_VERBOSE)
541542
set(_externaldata_log_level "STATUS")
542543
else()
543544
set(_externaldata_log_level "NOTICE")
544545
endif()
545546
file(READ "${_externaldata_config}" _externaldata_config_contents)
546547
file(WRITE "${_externaldata_config}" "set(CMAKE_MESSAGE_LOG_LEVEL ${_externaldata_log_level})\n${_externaldata_config_contents}")
548+
endif()
549+
add_dependencies("${NAM_TARGET}" "${_asset_target}")
547550

548-
list(LENGTH _asset_data_refs _asset_count)
549-
math(EXPR _asset_last "${_asset_count} - 1")
550-
foreach(_index RANGE ${_asset_last})
551-
list(GET _asset_data_refs ${_index} _data_ref)
552-
list(GET _asset_target_paths ${_index} _target_path)
553-
set(_stamp "${_build_root}/file_stamps/${_index}.stamp")
554-
set(_hash_record "${_build_root}/hash_records/${_index}.txt")
555-
get_filename_component(_stamp_dir "${_stamp}" DIRECTORY)
556-
get_filename_component(_hash_record_dir "${_hash_record}" DIRECTORY)
557-
file(MAKE_DIRECTORY "${_stamp_dir}")
558-
file(MAKE_DIRECTORY "${_hash_record_dir}")
559-
add_custom_command(
560-
OUTPUT "${_stamp}" "${_target_path}" "${_hash_record}"
561-
COMMAND "${CMAKE_COMMAND}" -Drelative_top=${CMAKE_BINARY_DIR} -Dfile=${_target_path} -Dname=${_data_ref} -Dexts=.sha256 -DExternalData_STAMP_FILE=${_hash_record} -DExternalData_ACTION=fetch -DExternalData_SHOW_PROGRESS=${NAM_SHOW_PROGRESS} -DExternalData_CONFIG=${_externaldata_config} -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/vendor/ExternalData-NAM.cmake"
562-
COMMAND "${CMAKE_COMMAND}" -E touch "${_stamp}"
563-
MAIN_DEPENDENCY "${_data_ref}.sha256"
564-
DEPENDS "${_fetch_script}" "${_externaldata_config}"
565-
VERBATIM
566-
)
567-
list(APPEND _materialize_stamps "${_stamp}")
568-
endforeach()
569-
else()
570-
set(_asset_target "${NAM_TARGET}__externaldata")
571-
set(ExternalData_SOURCE_ROOT "${_refs_root}")
572-
set(ExternalData_BINARY_ROOT "${_externaldata_binary_root}")
573-
unset(ExternalData_LINK_MODE)
574-
unset(ExternalData_NO_SYMLINKS)
575-
set(_old_suppress_dev "${CMAKE_SUPPRESS_DEVELOPER_WARNINGS}")
576-
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1)
577-
ExternalData_Expand_Arguments("${_asset_target}" _asset_expanded ${_asset_refs})
578-
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS "${_old_suppress_dev}")
579-
ExternalData_Add_Target("${_asset_target}" SHOW_PROGRESS "${NAM_SHOW_PROGRESS}")
580-
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${_asset_target}_config.cmake")
581-
if (EXISTS "${_externaldata_config}")
582-
if (NAM_VERBOSE)
583-
set(_externaldata_log_level "STATUS")
584-
else()
585-
set(_externaldata_log_level "NOTICE")
586-
endif()
587-
file(READ "${_externaldata_config}" _externaldata_config_contents)
588-
file(WRITE "${_externaldata_config}" "set(CMAKE_MESSAGE_LOG_LEVEL ${_externaldata_log_level})\n${_externaldata_config_contents}")
589-
endif()
590-
add_dependencies("${NAM_TARGET}" "${_asset_target}")
591-
551+
if (NOT _externaldata_provider STREQUAL "vendored")
592552
list(LENGTH _asset_expanded _asset_expanded_count)
593553
math(EXPR _asset_last "${_asset_expanded_count} - 1")
594554
foreach(_index RANGE ${_asset_last})

cmake/README.md

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,13 @@ consumer destination tree.
119119
The NAM patch is intentionally small:
120120

121121
- it adds `ExternalData_LINK_MODE = auto|symlink|hardlink|copy`
122-
- it lets NAM call the vendored build-time script directly per asset
122+
- it adds `ExternalData_STATE_ROOT` so metadata can live outside the final
123+
destination tree
124+
- it keeps the stock `ExternalData_Expand_Arguments` plus
125+
`ExternalData_Add_Target` consumer flow
123126
- on the vendored path those modes are applied directly from the shared object
124-
store into the final destination tree
127+
store into the final destination tree while the module keeps only metadata
128+
under the separate state root
125129

126130
This keeps the shared object store model while avoiding an unnecessary physical
127131
copy when the host supports lightweight links.
@@ -142,12 +146,13 @@ The resulting model is:
142146
- one shared local object store per user
143147
- content-addressed objects under `.../objects/SHA256/<hash>`
144148
- generated `.sha256` references under `${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/refs/<channel>/...`
145-
- vendored-path metadata under `${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/file_stamps` and
146-
`${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/hash_records`
149+
- vendored-path metadata under `${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/state`
147150
- direct final outputs under `${DESTINATION_ROOT}/${CHANNEL}/...` when the
148151
vendored module is enabled
149152
- a stock-module fallback path under `${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/assets`
150153
only when `NAM_USE_VENDORED_EXTERNALDATA=OFF`
154+
- stock-path materialization stamps under
155+
`${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/file_stamps`
151156
- normal build targets for consumers
152157

153158
During configure the module probes the current host once and selects the
@@ -161,9 +166,12 @@ Current detection order is:
161166
At build time:
162167

163168
- the vendored path fetches missing objects into the shared object store and
164-
materializes final files directly from that store
169+
materializes final files directly from that store through the normal
170+
`ExternalData_Add_Target` build graph
165171
- on the default vendored path every release asset is exposed from the object
166172
store directly into the final destination root using the configured mode
173+
- on the vendored path `ExternalData` keeps only hash records and build driver
174+
stamps under `${CMAKE_CURRENT_BINARY_DIR}/.nam/<target>/state`
167175
- on the stock fallback path NAM keeps the older `.nam/<target>/assets` staging
168176
step and then materializes into the destination root
169177

cmake/vendor/ExternalData-NAM.cmake

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,23 @@ calling any of the functions provided by this module.
174174
some platforms. The ``ExternalData_NO_SYMLINKS`` variable may be set
175175
to disable use of symbolic links and enable use of copies instead.
176176
177+
.. variable:: ExternalData_LINK_MODE
178+
179+
The ``ExternalData_LINK_MODE`` variable selects how real data files are
180+
exposed under ``ExternalData_BINARY_ROOT``. Supported values are
181+
``auto``, ``hardlink``, ``symlink``, and ``copy``.
182+
183+
When set to ``auto``, the module chooses the lightest supported mode for the
184+
host. The default order is ``hardlink -> symlink -> copy`` on Windows and
185+
``symlink -> hardlink -> copy`` elsewhere.
186+
187+
.. variable:: ExternalData_STATE_ROOT
188+
189+
The ``ExternalData_STATE_ROOT`` variable may be set to place module-managed
190+
metadata outside ``ExternalData_BINARY_ROOT``. When set, ``ExternalData``
191+
stores its hash records and build driver stamps under this directory while
192+
still materializing real data files under ``ExternalData_BINARY_ROOT``.
193+
177194
.. variable:: ExternalData_OBJECT_STORES
178195
179196
The ``ExternalData_OBJECT_STORES`` variable may be set to a list of local
@@ -516,24 +533,26 @@ function(ExternalData_add_target target)
516533
string(REPLACE "|" ";" tuple "${entry}")
517534
list(GET tuple 0 file)
518535
list(GET tuple 1 name)
536+
_ExternalData_compute_build_stamp_file("${file}" build_stamp_file)
519537
if(NOT DEFINED "_ExternalData_FILE_${file}")
520538
set("_ExternalData_FILE_${file}" 1)
521539
get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
522540
if(NOT added)
523541
set_property(DIRECTORY PROPERTY "_ExternalData_FILE_${file}" 1)
524542
add_custom_command(
525543
COMMENT "Generating ${file}"
526-
OUTPUT "${file}"
544+
OUTPUT "${build_stamp_file}" "${file}"
527545
COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
528546
-Dfile=${file} -Dname=${name}
529547
-DExternalData_ACTION=local
530548
-DExternalData_SHOW_PROGRESS=${_ExternalData_add_target_SHOW_PROGRESS}
531549
-DExternalData_CONFIG=${config}
532550
-P ${CMAKE_CURRENT_FUNCTION_LIST_FILE}
551+
COMMAND ${CMAKE_COMMAND} -E touch "${build_stamp_file}"
533552
MAIN_DEPENDENCY "${name}"
534553
)
535554
endif()
536-
list(APPEND files "${file}")
555+
list(APPEND files "${build_stamp_file}")
537556
endif()
538557
endforeach()
539558

@@ -546,7 +565,8 @@ function(ExternalData_add_target target)
546565
list(GET tuple 2 exts)
547566
string(REPLACE "+" ";" exts_list "${exts}")
548567
list(GET exts_list 0 first_ext)
549-
set(stamp "-hash-stamp")
568+
_ExternalData_compute_stamp_file("${file}" stamp_file)
569+
_ExternalData_compute_build_stamp_file("${file}" build_stamp_file)
550570
if(NOT DEFINED "_ExternalData_FILE_${file}")
551571
set("_ExternalData_FILE_${file}" 1)
552572
get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
@@ -555,23 +575,25 @@ function(ExternalData_add_target target)
555575
add_custom_command(
556576
# Users care about the data file, so hide the hash/timestamp file.
557577
COMMENT "Generating ${file}"
558-
# The hash/timestamp file is the output from the build perspective.
559-
# List the real file as a second output in case it is a broken link.
578+
# Use a dedicated build stamp as the primary output so IDE
579+
# generators keep an explicit build step for each materialized file.
580+
# List the hash record and real file as secondary outputs.
560581
# The files must be listed in this order so CMake can hide from the
561582
# make tool that a symlink target may not be newer than the input.
562-
OUTPUT "${file}${stamp}" "${file}"
583+
OUTPUT "${build_stamp_file}" "${stamp_file}" "${file}"
563584
# Run the data fetch/update script.
564585
COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
565586
-Dfile=${file} -Dname=${name} -Dexts=${exts}
566587
-DExternalData_ACTION=fetch
567588
-DExternalData_SHOW_PROGRESS=${_ExternalData_add_target_SHOW_PROGRESS}
568589
-DExternalData_CONFIG=${config}
569590
-P ${CMAKE_CURRENT_FUNCTION_LIST_FILE}
591+
COMMAND ${CMAKE_COMMAND} -E touch "${build_stamp_file}"
570592
# Update whenever the object hash changes.
571593
MAIN_DEPENDENCY "${name}${first_ext}"
572594
)
573595
endif()
574-
list(APPEND files "${file}${stamp}")
596+
list(APPEND files "${build_stamp_file}")
575597
endif()
576598
endforeach()
577599

@@ -641,12 +663,56 @@ function(_ExternalData_exact_regex regex_var string)
641663
endfunction()
642664

643665
function(_ExternalData_atomic_write file content)
666+
get_filename_component(dir "${file}" DIRECTORY)
667+
file(MAKE_DIRECTORY "${dir}")
644668
_ExternalData_random(random)
645669
set(tmp "${file}.tmp${random}")
646670
file(WRITE "${tmp}" "${content}")
647671
file(RENAME "${tmp}" "${file}")
648672
endfunction()
649673

674+
function(_ExternalData_compute_stamp_file file var_stamp)
675+
if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "")
676+
if(NOT ExternalData_BINARY_ROOT)
677+
set(top_bin "${CMAKE_BINARY_DIR}")
678+
else()
679+
set(top_bin "${ExternalData_BINARY_ROOT}")
680+
endif()
681+
file(RELATIVE_PATH relfile "${top_bin}" "${file}")
682+
if(IS_ABSOLUTE "${relfile}" OR "${relfile}" MATCHES "^\\.\\./")
683+
message(FATAL_ERROR
684+
"File path is not under ExternalData_BINARY_ROOT:\n"
685+
" file = ${file}\n"
686+
" ExternalData_BINARY_ROOT = ${top_bin}")
687+
endif()
688+
set(stamp_file "${ExternalData_STATE_ROOT}/${relfile}-hash-stamp")
689+
else()
690+
set(stamp_file "${file}-hash-stamp")
691+
endif()
692+
set(${var_stamp} "${stamp_file}" PARENT_SCOPE)
693+
endfunction()
694+
695+
function(_ExternalData_compute_build_stamp_file file var_stamp)
696+
if(DEFINED ExternalData_STATE_ROOT AND NOT "${ExternalData_STATE_ROOT}" STREQUAL "")
697+
if(NOT ExternalData_BINARY_ROOT)
698+
set(top_bin "${CMAKE_BINARY_DIR}")
699+
else()
700+
set(top_bin "${ExternalData_BINARY_ROOT}")
701+
endif()
702+
file(RELATIVE_PATH relfile "${top_bin}" "${file}")
703+
if(IS_ABSOLUTE "${relfile}" OR "${relfile}" MATCHES "^\\.\\./")
704+
message(FATAL_ERROR
705+
"File path is not under ExternalData_BINARY_ROOT:\n"
706+
" file = ${file}\n"
707+
" ExternalData_BINARY_ROOT = ${top_bin}")
708+
endif()
709+
set(build_stamp_file "${ExternalData_STATE_ROOT}/${relfile}-build-stamp")
710+
else()
711+
set(build_stamp_file "${file}-build-stamp")
712+
endif()
713+
set(${var_stamp} "${build_stamp_file}" PARENT_SCOPE)
714+
endfunction()
715+
650716
function(_ExternalData_link_content name var_ext)
651717
if("${ExternalData_LINK_CONTENT}" MATCHES "^(${_ExternalData_REGEX_ALGO})$")
652718
set(algo "${ExternalData_LINK_CONTENT}")
@@ -1294,11 +1360,7 @@ if("${ExternalData_ACTION}" STREQUAL "fetch")
12941360
message(FATAL_ERROR "${errorMsg}")
12951361
endif()
12961362
# Check if file already corresponds to the object.
1297-
if(DEFINED ExternalData_STAMP_FILE AND NOT "${ExternalData_STAMP_FILE}" STREQUAL "")
1298-
set(stamp_file "${ExternalData_STAMP_FILE}")
1299-
else()
1300-
set(stamp_file "${file}-hash-stamp")
1301-
endif()
1363+
_ExternalData_compute_stamp_file("${file}" stamp_file)
13021364
set(file_up_to_date 0)
13031365
if(EXISTS "${file}" AND EXISTS "${stamp_file}")
13041366
file(READ "${stamp_file}" f_hash)

cmake/vendor/ExternalData_config.cmake.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
44
set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
55
set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@")
66
set(ExternalData_LINK_MODE "@ExternalData_LINK_MODE@")
7+
set(ExternalData_STATE_ROOT "@ExternalData_STATE_ROOT@")
78
@_ExternalData_CONFIG_CODE@

0 commit comments

Comments
 (0)