Skip to content

Commit ac1424e

Browse files
committed
Add smoke matrix CI
1 parent 033cd73 commit ac1424e

6 files changed

Lines changed: 248 additions & 5 deletions

File tree

.github/workflows/smoke.yml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Smoke
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- master
8+
9+
jobs:
10+
smoke:
11+
name: ${{ matrix.os }} / ${{ matrix.link_mode }}
12+
runs-on: ${{ matrix.os }}
13+
timeout-minutes: 20
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
os:
18+
- ubuntu-latest
19+
- windows-latest
20+
link_mode:
21+
- symlink
22+
- hardlink
23+
- copy
24+
include:
25+
- os: ubuntu-latest
26+
native_build_arg: -j1
27+
- os: windows-latest
28+
native_build_arg: /m:1
29+
30+
env:
31+
NAM_CACHE_ROOT: ${{ github.workspace }}/.nam-cache
32+
SMOKE_BUILD_DIR: ${{ github.workspace }}/smoke/build
33+
34+
steps:
35+
- uses: actions/checkout@v4
36+
37+
- name: Restore NAM cache
38+
uses: actions/cache@v4
39+
with:
40+
path: ${{ env.NAM_CACHE_ROOT }}
41+
key: nam-cache-${{ runner.os }}-${{ hashFiles('media/**/*.dvc') }}
42+
restore-keys: |
43+
nam-cache-${{ runner.os }}-
44+
45+
- name: Configure smoke
46+
shell: pwsh
47+
run: |
48+
cmake -E rm -rf $env:SMOKE_BUILD_DIR
49+
cmake -E make_directory $env:NAM_CACHE_ROOT
50+
cmake -S smoke -B $env:SMOKE_BUILD_DIR `
51+
-DNAM_SMOKE_CACHE_ROOT="$env:NAM_CACHE_ROOT" `
52+
-DNAM_SMOKE_LINK_MODE=${{ matrix.link_mode }}
53+
54+
- name: Build smoke
55+
shell: pwsh
56+
run: cmake --build $env:SMOKE_BUILD_DIR --config Debug --target media -- ${{ matrix.native_build_arg }}
57+
58+
- name: Verify materialization stats
59+
shell: pwsh
60+
run: |
61+
python smoke/ci/verify_materialization.py `
62+
--build-dir $env:SMOKE_BUILD_DIR `
63+
--cache-root $env:NAM_CACHE_ROOT `
64+
--expected-mode ${{ matrix.link_mode }}

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ This is the exact consumer model we want:
103103
- a shared local object store
104104
- reuse across many build directories and worktrees
105105
- reuse across many independent repositories too
106-
- normal local files materialized into build trees via symlinks or copies
106+
- normal local files materialized into build trees via symlinks, hardlinks, or copies
107107
- no requirement for consumers to know which remote backend served the blob
108108

109109
## Backends

cmake/NablaAssetManifests.cmake

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,18 @@ function(_nam_summary MESSAGE_TEXT)
1414
message(STATUS "NablaAssetManifests: ${MESSAGE_TEXT}")
1515
endfunction()
1616

17+
function(_nam_validate_file_link_mode MODE_VALUE OUT_VAR)
18+
string(TOLOWER "${MODE_VALUE}" _mode)
19+
if (
20+
NOT _mode STREQUAL "copy"
21+
AND NOT _mode STREQUAL "hardlink"
22+
AND NOT _mode STREQUAL "symlink"
23+
)
24+
message(FATAL_ERROR "NablaAssetManifests: unsupported file link mode `${MODE_VALUE}`")
25+
endif()
26+
set(${OUT_VAR} "${_mode}" PARENT_SCOPE)
27+
endfunction()
28+
1729
function(_nam_detect_file_link_mode OUT_VAR)
1830
set(_probe_root "${CMAKE_CURRENT_BINARY_DIR}/.nam_probe")
1931
file(MAKE_DIRECTORY "${_probe_root}")
@@ -54,6 +66,17 @@ function(_nam_detect_file_link_mode OUT_VAR)
5466
set(${OUT_VAR} "copy" PARENT_SCOPE)
5567
endfunction()
5668

69+
function(_nam_resolve_file_link_mode OUT_VAR)
70+
if (DEFINED NAM_INTERNAL_FORCE_FILE_LINK_MODE AND NOT "${NAM_INTERNAL_FORCE_FILE_LINK_MODE}" STREQUAL "")
71+
_nam_validate_file_link_mode("${NAM_INTERNAL_FORCE_FILE_LINK_MODE}" _forced_mode)
72+
set(${OUT_VAR} "${_forced_mode}" PARENT_SCOPE)
73+
return()
74+
endif()
75+
76+
_nam_detect_file_link_mode(_detected_mode)
77+
set(${OUT_VAR} "${_detected_mode}" PARENT_SCOPE)
78+
endfunction()
79+
5780
function(nam_get_repo_root OUT_VAR)
5881
if (DEFINED CMAKE_CURRENT_FUNCTION_LIST_DIR AND NOT "${CMAKE_CURRENT_FUNCTION_LIST_DIR}" STREQUAL "")
5982
get_filename_component(_root "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/.." ABSOLUTE)
@@ -405,7 +428,7 @@ function(nam_add_channel_target)
405428
set(ExternalData_URL_TEMPLATES "ExternalDataCustomScript://NAM/%(hash)")
406429
set(ExternalData_CUSTOM_SCRIPT_NAM "${_fetch_script}")
407430
add_custom_target("${NAM_TARGET}")
408-
_nam_detect_file_link_mode(_file_link_mode)
431+
_nam_resolve_file_link_mode(_file_link_mode)
409432
if (NAM_NO_SYMLINKS)
410433
set(_file_link_mode "copy")
411434
endif()

cmake/NablaAssetManifestsMaterialize.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@ function(_nam_materialize_file SOURCE_PATH DESTINATION_PATH)
1616
if (NOT DEFINED LINK_MODE OR "${LINK_MODE}" STREQUAL "" OR LINK_MODE STREQUAL "copy")
1717
file(COPY_FILE "${SOURCE_PATH}" "${DESTINATION_PATH}" ONLY_IF_DIFFERENT)
1818
elseif(LINK_MODE STREQUAL "hardlink")
19-
file(CREATE_LINK "${SOURCE_PATH}" "${DESTINATION_PATH}" RESULT _result COPY_ON_ERROR)
19+
file(CREATE_LINK "${SOURCE_PATH}" "${DESTINATION_PATH}" RESULT _result)
2020
if (_result)
2121
message(FATAL_ERROR "NablaAssetManifestsMaterialize: failed to create hardlink from `${SOURCE_PATH}` to `${DESTINATION_PATH}`: ${_result}")
2222
endif()
2323
elseif(LINK_MODE STREQUAL "symlink")
24-
file(CREATE_LINK "${SOURCE_PATH}" "${DESTINATION_PATH}" SYMBOLIC RESULT _result COPY_ON_ERROR)
24+
file(CREATE_LINK "${SOURCE_PATH}" "${DESTINATION_PATH}" SYMBOLIC RESULT _result)
2525
if (_result)
2626
message(FATAL_ERROR "NablaAssetManifestsMaterialize: failed to create symlink from `${SOURCE_PATH}` to `${DESTINATION_PATH}`: ${_result}")
2727
endif()

smoke/CMakeLists.txt

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,44 @@ project(NablaAssetManifestsSmoke NONE)
44
include("${CMAKE_CURRENT_LIST_DIR}/../nam.cmake")
55

66
option(NAM_SMOKE_NO_SYMLINKS "Force copy materialization in the smoke consumer." OFF)
7+
set(NAM_SMOKE_LINK_MODE "auto" CACHE STRING "Smoke materialization mode override.")
8+
set_property(CACHE NAM_SMOKE_LINK_MODE PROPERTY STRINGS auto symlink hardlink copy)
9+
set(NAM_SMOKE_CACHE_ROOT "" CACHE PATH "Shared cache root for smoke runs.")
10+
11+
string(TOLOWER "${NAM_SMOKE_LINK_MODE}" _nam_smoke_link_mode)
12+
if (
13+
NOT _nam_smoke_link_mode STREQUAL "auto"
14+
AND NOT _nam_smoke_link_mode STREQUAL "copy"
15+
AND NOT _nam_smoke_link_mode STREQUAL "hardlink"
16+
AND NOT _nam_smoke_link_mode STREQUAL "symlink"
17+
)
18+
message(FATAL_ERROR "NAM_SMOKE_LINK_MODE must be one of: auto, symlink, hardlink, copy")
19+
endif()
20+
21+
if (
22+
NAM_SMOKE_NO_SYMLINKS
23+
AND NOT _nam_smoke_link_mode STREQUAL "auto"
24+
AND NOT _nam_smoke_link_mode STREQUAL "copy"
25+
)
26+
message(FATAL_ERROR "NAM_SMOKE_NO_SYMLINKS conflicts with NAM_SMOKE_LINK_MODE=`${_nam_smoke_link_mode}`")
27+
endif()
728

829
set(_nam_extra_args)
9-
if (NAM_SMOKE_NO_SYMLINKS)
30+
if (NAM_SMOKE_CACHE_ROOT)
31+
list(APPEND _nam_extra_args CACHE_ROOT "${NAM_SMOKE_CACHE_ROOT}")
32+
endif()
33+
34+
unset(NAM_INTERNAL_FORCE_FILE_LINK_MODE)
35+
if (_nam_smoke_link_mode STREQUAL "copy" OR NAM_SMOKE_NO_SYMLINKS)
1036
list(APPEND _nam_extra_args NO_SYMLINKS)
37+
elseif(NOT _nam_smoke_link_mode STREQUAL "auto")
38+
set(NAM_INTERNAL_FORCE_FILE_LINK_MODE "${_nam_smoke_link_mode}")
1139
endif()
1240

1341
nam_add_channel_target(
1442
TARGET media
1543
DESTINATION_ROOT "${CMAKE_CURRENT_BINARY_DIR}"
1644
${_nam_extra_args}
1745
)
46+
47+
unset(NAM_INTERNAL_FORCE_FILE_LINK_MODE)

smoke/ci/verify_materialization.py

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
from __future__ import annotations
2+
3+
import argparse
4+
import json
5+
import os
6+
import stat
7+
import sys
8+
import zipfile
9+
from pathlib import Path
10+
11+
12+
def parse_args() -> argparse.Namespace:
13+
parser = argparse.ArgumentParser()
14+
parser.add_argument("--build-dir", required=True)
15+
parser.add_argument("--cache-root", required=True)
16+
parser.add_argument("--expected-mode", required=True, choices=("symlink", "hardlink", "copy"))
17+
return parser.parse_args()
18+
19+
20+
def iter_files(root: Path):
21+
for path in root.rglob("*"):
22+
if path.is_symlink() or path.is_file():
23+
yield path
24+
25+
26+
def classify_materialization(path: Path) -> tuple[str, int, int]:
27+
link_info = path.lstat()
28+
if stat.S_ISLNK(link_info.st_mode):
29+
target_size = path.stat().st_size
30+
return ("symlink", target_size, link_info.st_size)
31+
32+
direct_info = path.stat(follow_symlinks=False)
33+
if direct_info.st_nlink > 1:
34+
return ("hardlink", direct_info.st_size, 0)
35+
36+
return ("copy", direct_info.st_size, direct_info.st_size)
37+
38+
39+
def format_bytes(value: int) -> str:
40+
units = ("B", "KiB", "MiB", "GiB")
41+
size = float(value)
42+
unit = units[0]
43+
for unit in units:
44+
if size < 1024.0 or unit == units[-1]:
45+
break
46+
size /= 1024.0
47+
if unit == "B":
48+
return f"{int(size)} {unit}"
49+
return f"{size:.2f} {unit}"
50+
51+
52+
def main() -> int:
53+
args = parse_args()
54+
build_dir = Path(args.build_dir).resolve()
55+
cache_root = Path(args.cache_root).resolve()
56+
media_root = build_dir / "media"
57+
if not media_root.exists():
58+
raise SystemExit(f"Missing materialized tree: {media_root}")
59+
60+
bunny_path = media_root / "assets/mesh/standalone/stl/Stanford_Bunny.stl"
61+
yellowflower_path = media_root / "assets/mesh/bundles/obj/yellowflower.zip"
62+
required_paths = (bunny_path, yellowflower_path)
63+
for path in required_paths:
64+
if not path.exists():
65+
raise SystemExit(f"Missing required file: {path}")
66+
67+
files = list(iter_files(media_root))
68+
if not files:
69+
raise SystemExit(f"No files found under {media_root}")
70+
71+
counts = {"symlink": 0, "hardlink": 0, "copy": 0}
72+
logical_size = 0
73+
estimated_extra_size = 0
74+
for path in files:
75+
materialization, file_size, extra_size = classify_materialization(path)
76+
counts[materialization] += 1
77+
logical_size += file_size
78+
estimated_extra_size += extra_size
79+
80+
sample_modes: dict[str, str] = {}
81+
for path in required_paths:
82+
materialization, _, _ = classify_materialization(path)
83+
sample_modes[str(path.relative_to(build_dir))] = materialization
84+
if materialization != args.expected_mode:
85+
raise SystemExit(
86+
f"Expected `{args.expected_mode}` for {path.name} but found `{materialization}`"
87+
)
88+
89+
if not zipfile.is_zipfile(yellowflower_path):
90+
raise SystemExit(f"Expected a zip payload at {yellowflower_path}")
91+
92+
summary = {
93+
"build_dir": str(build_dir),
94+
"cache_root": str(cache_root),
95+
"expected_mode": args.expected_mode,
96+
"file_count": len(files),
97+
"counts": counts,
98+
"logical_size_bytes": logical_size,
99+
"logical_size_human": format_bytes(logical_size),
100+
"estimated_extra_size_bytes": estimated_extra_size,
101+
"estimated_extra_size_human": format_bytes(estimated_extra_size),
102+
"sample_modes": sample_modes,
103+
}
104+
105+
print(json.dumps(summary, indent=2, sort_keys=True))
106+
107+
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
108+
if step_summary:
109+
lines = [
110+
"## Smoke materialization summary",
111+
"",
112+
f"- Expected mode: `{args.expected_mode}`",
113+
f"- Files: `{len(files)}`",
114+
f"- Counts: `symlink={counts['symlink']}` `hardlink={counts['hardlink']}` `copy={counts['copy']}`",
115+
f"- Logical size: `{format_bytes(logical_size)}`",
116+
f"- Estimated extra size in build tree: `{format_bytes(estimated_extra_size)}`",
117+
f"- Stanford_Bunny.stl: `{sample_modes[str(bunny_path.relative_to(build_dir))]}`",
118+
f"- yellowflower.zip: `{sample_modes[str(yellowflower_path.relative_to(build_dir))]}`",
119+
]
120+
Path(step_summary).write_text("\n".join(lines) + "\n", encoding="utf-8")
121+
122+
return 0
123+
124+
125+
if __name__ == "__main__":
126+
sys.exit(main())

0 commit comments

Comments
 (0)