Skip to content

Commit 033cd73

Browse files
AnastaZIukAnastaZIuk
committed
Keep release assets as files
1 parent af9a59b commit 033cd73

4 files changed

Lines changed: 36 additions & 156 deletions

File tree

‎cmake/NablaAssetManifests.cmake‎

Lines changed: 25 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ endfunction()
118118

119119
function(_nam_parse_dvc_file)
120120
set(options)
121-
set(oneValueArgs DVC_FILE CHANNEL OUT_KIND OUT_RELATIVE_PATH OUT_RELEASE_ASSET OUT_KEY)
121+
set(oneValueArgs DVC_FILE CHANNEL OUT_RELATIVE_PATH OUT_RELEASE_ASSET OUT_KEY)
122122
cmake_parse_arguments(NAM "${options}" "${oneValueArgs}" "" ${ARGN})
123123

124124
if (NOT NAM_DVC_FILE)
@@ -148,15 +148,13 @@ function(_nam_parse_dvc_file)
148148
file(TO_CMAKE_PATH "${_relative_path}" _relative_path)
149149

150150
if (_md5 MATCHES "\\.dir$")
151-
set(_kind "archive")
152151
get_filename_component(_name "${_tracked_abs}" NAME)
153152
set(_release_asset "${_name}.zip")
153+
set(_relative_path "${_relative_path}.zip")
154154
else()
155-
set(_kind "file")
156155
get_filename_component(_release_asset "${_tracked_abs}" NAME)
157156
endif()
158157

159-
set(${NAM_OUT_KIND} "${_kind}" PARENT_SCOPE)
160158
set(${NAM_OUT_RELATIVE_PATH} "${_relative_path}" PARENT_SCOPE)
161159
set(${NAM_OUT_RELEASE_ASSET} "${_release_asset}" PARENT_SCOPE)
162160
set(${NAM_OUT_KEY} "${_relative_path}" PARENT_SCOPE)
@@ -180,7 +178,6 @@ function(nam_get_channel_asset_keys OUT_VAR)
180178
_nam_parse_dvc_file(
181179
DVC_FILE "${_dvc}"
182180
CHANNEL "${NAM_CHANNEL}"
183-
OUT_KIND _kind
184181
OUT_RELATIVE_PATH _relative_path
185182
OUT_RELEASE_ASSET _release_asset
186183
OUT_KEY _key
@@ -192,7 +189,7 @@ endfunction()
192189

193190
function(_nam_find_channel_asset)
194191
set(options)
195-
set(oneValueArgs CHANNEL ASSET OUT_KIND OUT_RELATIVE_PATH OUT_RELEASE_ASSET OUT_KEY)
192+
set(oneValueArgs CHANNEL ASSET OUT_RELATIVE_PATH OUT_RELEASE_ASSET OUT_KEY)
196193
cmake_parse_arguments(NAM "${options}" "${oneValueArgs}" "" ${ARGN})
197194

198195
if (NOT DEFINED NAM_CHANNEL OR "${NAM_CHANNEL}" STREQUAL "")
@@ -211,14 +208,12 @@ function(_nam_find_channel_asset)
211208
_nam_parse_dvc_file(
212209
DVC_FILE "${_dvc}"
213210
CHANNEL "${NAM_CHANNEL}"
214-
OUT_KIND _kind
215211
OUT_RELATIVE_PATH _relative_path
216212
OUT_RELEASE_ASSET _release_asset
217213
OUT_KEY _key
218214
)
219215
if (_key STREQUAL "${NAM_ASSET}" OR _release_asset STREQUAL "${NAM_ASSET}")
220216
math(EXPR _match_count "${_match_count}+1")
221-
set(_resolved_kind "${_kind}")
222217
set(_resolved_relative_path "${_relative_path}")
223218
set(_resolved_release_asset "${_release_asset}")
224219
set(_resolved_key "${_key}")
@@ -232,7 +227,6 @@ function(_nam_find_channel_asset)
232227
message(FATAL_ERROR "NablaAssetManifests: ambiguous asset selector `${NAM_ASSET}` in channel `${NAM_CHANNEL}`")
233228
endif()
234229

235-
set(${NAM_OUT_KIND} "${_resolved_kind}" PARENT_SCOPE)
236230
set(${NAM_OUT_RELATIVE_PATH} "${_resolved_relative_path}" PARENT_SCOPE)
237231
set(${NAM_OUT_RELEASE_ASSET} "${_resolved_release_asset}" PARENT_SCOPE)
238232
set(${NAM_OUT_KEY} "${_resolved_key}" PARENT_SCOPE)
@@ -417,16 +411,12 @@ function(nam_add_channel_target)
417411
endif()
418412
_nam_summary("materialization mode for file assets: `${_file_link_mode}`")
419413

420-
set(_file_refs)
421-
set(_file_relpaths)
422-
set(_archive_refs)
423-
set(_archive_relpaths)
424-
set(_archive_hashes)
414+
set(_asset_refs)
415+
set(_asset_relpaths)
425416
foreach(_asset IN LISTS _items)
426417
_nam_find_channel_asset(
427418
CHANNEL "${NAM_CHANNEL}"
428419
ASSET "${_asset}"
429-
OUT_KIND _kind
430420
OUT_RELATIVE_PATH _relative_path
431421
OUT_RELEASE_ASSET _release_asset
432422
OUT_KEY _key
@@ -440,37 +430,25 @@ function(nam_add_channel_target)
440430
CACHE_ROOT "${NAM_CACHE_ROOT}"
441431
)
442432

443-
if (_kind STREQUAL "file")
444-
set(_data_name "${NAM_CHANNEL}/${_relative_path}")
445-
get_filename_component(_link_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}" DIRECTORY)
446-
file(MAKE_DIRECTORY "${_link_dir}")
447-
file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}.sha256" "${_sha256}\n")
448-
list(APPEND _file_refs "DATA{${_data_name}}")
449-
list(APPEND _file_relpaths "${_relative_path}")
450-
elseif(_kind STREQUAL "archive")
451-
set(_data_name "${NAM_CHANNEL}/nam_archives/${_relative_path}.zip")
452-
get_filename_component(_link_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}" DIRECTORY)
453-
file(MAKE_DIRECTORY "${_link_dir}")
454-
file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}.sha256" "${_sha256}\n")
455-
list(APPEND _archive_refs "DATA{${_data_name}}")
456-
list(APPEND _archive_relpaths "${_relative_path}")
457-
list(APPEND _archive_hashes "${_sha256}")
458-
else()
459-
message(FATAL_ERROR "NablaAssetManifests: unsupported asset kind `${_kind}`")
460-
endif()
433+
set(_data_name "${NAM_CHANNEL}/${_relative_path}")
434+
get_filename_component(_link_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}" DIRECTORY)
435+
file(MAKE_DIRECTORY "${_link_dir}")
436+
file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/${_data_name}.sha256" "${_sha256}\n")
437+
list(APPEND _asset_refs "DATA{${_data_name}}")
438+
list(APPEND _asset_relpaths "${_relative_path}")
461439
endforeach()
462440

463-
if (_file_refs)
464-
set(_file_target "${NAM_TARGET}__externaldata_files")
441+
if (_asset_refs)
442+
set(_asset_target "${NAM_TARGET}__externaldata")
465443
set(ExternalData_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
466-
set(ExternalData_BINARY_ROOT "${_build_root}/files")
444+
set(ExternalData_BINARY_ROOT "${_build_root}/assets")
467445
unset(ExternalData_NO_SYMLINKS)
468446
set(_old_suppress_dev "${CMAKE_SUPPRESS_DEVELOPER_WARNINGS}")
469447
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1)
470-
ExternalData_Expand_Arguments("${_file_target}" _file_expanded ${_file_refs})
448+
ExternalData_Expand_Arguments("${_asset_target}" _asset_expanded ${_asset_refs})
471449
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS "${_old_suppress_dev}")
472-
ExternalData_Add_Target("${_file_target}" SHOW_PROGRESS "${NAM_SHOW_PROGRESS}")
473-
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${_file_target}_config.cmake")
450+
ExternalData_Add_Target("${_asset_target}" SHOW_PROGRESS "${NAM_SHOW_PROGRESS}")
451+
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${_asset_target}_config.cmake")
474452
if (EXISTS "${_externaldata_config}")
475453
if (NAM_VERBOSE)
476454
set(_externaldata_log_level "STATUS")
@@ -480,80 +458,32 @@ function(nam_add_channel_target)
480458
file(READ "${_externaldata_config}" _externaldata_config_contents)
481459
file(WRITE "${_externaldata_config}" "set(CMAKE_MESSAGE_LOG_LEVEL ${_externaldata_log_level})\n${_externaldata_config_contents}")
482460
endif()
483-
add_dependencies("${NAM_TARGET}" "${_file_target}")
461+
add_dependencies("${NAM_TARGET}" "${_asset_target}")
484462

485-
list(LENGTH _file_expanded _file_expanded_count)
486-
math(EXPR _file_last "${_file_expanded_count} - 1")
487-
foreach(_index RANGE ${_file_last})
488-
list(GET _file_expanded ${_index} _expanded_path)
489-
list(GET _file_relpaths ${_index} _relative_path)
463+
list(LENGTH _asset_expanded _asset_expanded_count)
464+
math(EXPR _asset_last "${_asset_expanded_count} - 1")
465+
foreach(_index RANGE ${_asset_last})
466+
list(GET _asset_expanded ${_index} _expanded_path)
467+
list(GET _asset_relpaths ${_index} _relative_path)
490468
set(_target_path "${NAM_DESTINATION_ROOT}/${NAM_CHANNEL}/${_relative_path}")
491469
set(_stamp "${_build_root}/file_stamps/${_index}.stamp")
492470
get_filename_component(_stamp_dir "${_stamp}" DIRECTORY)
493471
file(MAKE_DIRECTORY "${_stamp_dir}")
494472
add_custom_command(
495473
OUTPUT "${_stamp}"
496-
COMMAND "${CMAKE_COMMAND}" -DTYPE=file -DINPUT=${_expanded_path} -DDESTINATION=${_target_path} -DLINK_MODE=${_file_link_mode} -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/NablaAssetManifestsMaterialize.cmake"
497-
COMMAND "${CMAKE_COMMAND}" -E touch "${_stamp}"
498-
DEPENDS "${_expanded_path}"
499-
VERBATIM
500-
)
501-
list(APPEND _materialize_stamps "${_stamp}")
502-
endforeach()
503-
endif()
504-
505-
list(LENGTH _archive_refs _archive_count)
506-
if (_archive_refs)
507-
set(_archive_target "${NAM_TARGET}__externaldata_archives")
508-
set(ExternalData_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
509-
set(ExternalData_BINARY_ROOT "${_build_root}/archives")
510-
unset(ExternalData_NO_SYMLINKS)
511-
set(_old_suppress_dev "${CMAKE_SUPPRESS_DEVELOPER_WARNINGS}")
512-
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1)
513-
ExternalData_Expand_Arguments("${_archive_target}" _archive_expanded ${_archive_refs})
514-
set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS "${_old_suppress_dev}")
515-
ExternalData_Add_Target("${_archive_target}" SHOW_PROGRESS "${NAM_SHOW_PROGRESS}")
516-
set(_externaldata_config "${CMAKE_CURRENT_BINARY_DIR}/${_archive_target}_config.cmake")
517-
if (EXISTS "${_externaldata_config}")
518-
if (NAM_VERBOSE)
519-
set(_externaldata_log_level "STATUS")
520-
else()
521-
set(_externaldata_log_level "NOTICE")
522-
endif()
523-
file(READ "${_externaldata_config}" _externaldata_config_contents)
524-
file(WRITE "${_externaldata_config}" "set(CMAKE_MESSAGE_LOG_LEVEL ${_externaldata_log_level})\n${_externaldata_config_contents}")
525-
endif()
526-
527-
add_dependencies("${NAM_TARGET}" "${_archive_target}")
528-
529-
list(LENGTH _archive_expanded _archive_expanded_count)
530-
math(EXPR _archive_last "${_archive_expanded_count} - 1")
531-
foreach(_index RANGE ${_archive_last})
532-
list(GET _archive_relpaths ${_index} _relative_path)
533-
list(GET _archive_hashes ${_index} _archive_hash)
534-
list(GET _archive_expanded ${_index} _expanded_path)
535-
set(_target_path "${NAM_DESTINATION_ROOT}/${NAM_CHANNEL}/${_relative_path}")
536-
set(_stamp "${_build_root}/stamps/${_index}.stamp")
537-
get_filename_component(_stamp_dir "${_stamp}" DIRECTORY)
538-
file(MAKE_DIRECTORY "${_stamp_dir}")
539-
add_custom_command(
540-
OUTPUT "${_stamp}"
541-
COMMAND "${CMAKE_COMMAND}" -DTYPE=archive -DINPUT=${_expanded_path} -DDESTINATION=${_target_path} -DEXTRACT_ROOT=${_cache_root}/extracted/SHA256 -DHASH=${_archive_hash} -DLINK_MODE=${_file_link_mode} -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/NablaAssetManifestsMaterialize.cmake"
474+
COMMAND "${CMAKE_COMMAND}" -DINPUT=${_expanded_path} -DDESTINATION=${_target_path} -DLINK_MODE=${_file_link_mode} -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/NablaAssetManifestsMaterialize.cmake"
542475
COMMAND "${CMAKE_COMMAND}" -E touch "${_stamp}"
543476
DEPENDS "${_expanded_path}"
544477
VERBATIM
545478
)
546479
list(APPEND _materialize_stamps "${_stamp}")
547480
endforeach()
548-
549-
add_dependencies("${NAM_TARGET}" "${_archive_target}")
550481
endif()
551482

552483
if (_materialize_stamps)
553484
add_custom_target("${NAM_TARGET}__materialize" DEPENDS ${_materialize_stamps})
554485
add_dependencies("${NAM_TARGET}" "${NAM_TARGET}__materialize")
555486
endif()
556487

557-
math(EXPR _file_count "${_item_count} - ${_archive_count}")
558-
_nam_summary("channel target `${NAM_TARGET}` ready: total=${_item_count}, file=${_file_count}, archive=${_archive_count}, destination=`${NAM_DESTINATION_ROOT}`")
488+
_nam_summary("channel target `${NAM_TARGET}` ready: total=${_item_count}, destination=`${NAM_DESTINATION_ROOT}`")
559489
endfunction()

‎cmake/NablaAssetManifestsExtract.cmake‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎cmake/NablaAssetManifestsMaterialize.cmake‎

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
1-
if (NOT DEFINED TYPE OR "${TYPE}" STREQUAL "")
2-
message(FATAL_ERROR "NablaAssetManifestsMaterialize: TYPE is required")
3-
endif()
4-
51
if (NOT DEFINED INPUT OR "${INPUT}" STREQUAL "")
62
message(FATAL_ERROR "NablaAssetManifestsMaterialize: INPUT is required")
73
endif()
@@ -34,43 +30,4 @@ function(_nam_materialize_file SOURCE_PATH DESTINATION_PATH)
3430
endif()
3531
endfunction()
3632

37-
if (TYPE STREQUAL "file")
38-
_nam_materialize_file("${INPUT}" "${DESTINATION}")
39-
elseif(TYPE STREQUAL "archive")
40-
if (NOT DEFINED EXTRACT_ROOT OR "${EXTRACT_ROOT}" STREQUAL "")
41-
message(FATAL_ERROR "NablaAssetManifestsMaterialize: EXTRACT_ROOT is required for archive mode")
42-
endif()
43-
if (NOT DEFINED HASH OR "${HASH}" STREQUAL "")
44-
message(FATAL_ERROR "NablaAssetManifestsMaterialize: HASH is required for archive mode")
45-
endif()
46-
47-
set(_extract_dir "${EXTRACT_ROOT}/${HASH}")
48-
set(_extract_stamp "${_extract_dir}/.extract_complete")
49-
50-
if (NOT EXISTS "${_extract_stamp}")
51-
if (EXISTS "${_extract_dir}")
52-
file(REMOVE_RECURSE "${_extract_dir}")
53-
endif()
54-
file(MAKE_DIRECTORY "${_extract_dir}")
55-
file(ARCHIVE_EXTRACT INPUT "${INPUT}" DESTINATION "${_extract_dir}")
56-
file(WRITE "${_extract_stamp}" "ok\n")
57-
endif()
58-
59-
if (EXISTS "${DESTINATION}")
60-
file(REMOVE_RECURSE "${DESTINATION}")
61-
endif()
62-
file(MAKE_DIRECTORY "${DESTINATION}")
63-
64-
file(GLOB_RECURSE _files RELATIVE "${_extract_dir}" "${_extract_dir}/*")
65-
foreach(_relative IN LISTS _files)
66-
set(_source_path "${_extract_dir}/${_relative}")
67-
set(_destination_path "${DESTINATION}/${_relative}")
68-
if (IS_DIRECTORY "${_source_path}")
69-
file(MAKE_DIRECTORY "${_destination_path}")
70-
else()
71-
_nam_materialize_file("${_source_path}" "${_destination_path}")
72-
endif()
73-
endforeach()
74-
else()
75-
message(FATAL_ERROR "NablaAssetManifestsMaterialize: unsupported TYPE `${TYPE}`")
76-
endif()
33+
_nam_materialize_file("${INPUT}" "${DESTINATION}")

‎cmake/README.md‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ Building target `media` then:
2929

3030
- fetches all assets from the default `media` channel
3131
- reuses the shared local object store
32-
- materializes standalone files and bundle directories under
32+
- materializes release assets as files under
3333
`${CMAKE_CURRENT_BINARY_DIR}/media`
34+
- keeps `zip` payloads as `zip` files
3435

3536
### Explicit usage
3637

@@ -74,7 +75,6 @@ For input assets the source of truth is:
7475
The module reads those `.dvc` files and derives:
7576

7677
- the logical relative path inside the channel
77-
- whether a payload is a standalone file or a bundle directory
7878
- the expected release asset name
7979
- the content hash used by the shared `ExternalData` object store
8080

@@ -84,6 +84,12 @@ Current publishing convention:
8484
- bundle directories are published as `zip` archives with the directory basename
8585
- the current backend is `github_release`
8686

87+
Consumer-side rule:
88+
89+
- every published release asset is treated as a file
90+
- nothing is unpacked automatically
91+
- if a consumer wants to open a `zip`, it does so at runtime
92+
8793
## ExternalData model
8894

8995
The module uses only public `ExternalData` APIs:
@@ -101,7 +107,6 @@ The resulting model is:
101107
- content-addressed objects under `.../objects/SHA256/<hash>`
102108
- generated content links under `nam-data/<target>/` in the consumer source tree
103109
- normal build targets for consumers
104-
- archive extraction as an explicit build step for bundle payloads
105110

106111
During configure the module probes the current host once and selects the
107112
lightest supported file materialization mode. On the current Windows host this
@@ -110,11 +115,8 @@ resolves to `hardlink`.
110115
At build time:
111116

112117
- `ExternalData` populates the shared object store
113-
- standalone files are materialized to the destination root using the detected
114-
lightweight file mode when available
115-
- bundle archives are extracted once into a shared extracted cache and then
116-
materialized to the destination root using the same lightweight file mode for
117-
the extracted files
118+
- every release asset is materialized to the destination root exactly as it was
119+
published, using the detected lightweight file mode when available
118120

119121
Passing `NO_SYMLINKS` forces copy materialization even when the host supports
120122
lightweight links.

0 commit comments

Comments
 (0)