Skip to content

Commit 3db2cfa

Browse files
aiqiaoyaiqiaoy
authored
Add ability to cache only latest N versions and cache github/gh-aw-actions (#136)
* add ability to cache only latest N versions * fix test
1 parent 567140f commit 3db2cfa

8 files changed

Lines changed: 279 additions & 13 deletions

File tree

‎config/actions/github_gh-aw-actions.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"owner": "github",
3+
"repo": "gh-aw-actions",
4+
"patterns": [
5+
"+^master$",
6+
"+^v[0-9]+(\\.[0-9]+){0,2}$"
7+
],
8+
"branches": {},
9+
"defaultBranch": "master",
10+
"latestMajorVersions": 1,
11+
"latestVersionsPerMajor": 3,
12+
"tags": {
13+
"v0.67.4": {
14+
"commit": "2b3c275b3652caa01c2ebe31cbab50ec2df0f927",
15+
"tag": "9d6ae06250fc0ec536a0e5f35de313b35bad7246"
16+
},
17+
"v0.68.0": {
18+
"commit": "6715c81fe97e4bcbfa0734c3422491672ebda34f",
19+
"tag": "0acfb4a691fe207cd8bc982ea5cb9d750d57a702"
20+
},
21+
"v0.68.1": {
22+
"commit": "ea222e359276c0702a5f5203547ff9d88d0ddd76",
23+
"tag": "2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc"
24+
}
25+
}
26+
}

‎script/generated/github_gh-aw-actions.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
mkdir github_gh-aw-actions
2+
pushd github_gh-aw-actions
3+
curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927'
4+
curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927'
5+
curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/6715c81fe97e4bcbfa0734c3422491672ebda34f'
6+
curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/6715c81fe97e4bcbfa0734c3422491672ebda34f'
7+
curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/ea222e359276c0702a5f5203547ff9d88d0ddd76'
8+
curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/ea222e359276c0702a5f5203547ff9d88d0ddd76'
9+
popd

‎script/internal/action-config.js‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,18 @@ class ActionConfig {
4444
*/
4545
defaultBranch = 'master'
4646

47+
/**
48+
* Maximum number of latest major versions to include (default to unlimited)
49+
* @type {number|undefined}
50+
*/
51+
latestMajorVersions = undefined
52+
53+
/**
54+
* Maximum number of latest version tags per major version (default to unlimited)
55+
* @type {number|undefined}
56+
*/
57+
latestVersionsPerMajor = undefined
58+
4759
/**
4860
* Tag versions
4961
* @type {{[ref: string]: TagVersion}}
@@ -73,9 +85,11 @@ exports.TagVersion = TagVersion
7385
* @param {string[]} patternStrings
7486
* @param {string} defaultBranch
7587
* @param {string[]|undefined} ignoreTags
88+
* @param {number|undefined} latestMajorVersions
89+
* @param {number|undefined} latestVersionsPerMajor
7690
* @returns {Promise}
7791
*/
78-
async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
92+
async function add(owner, repo, patternStrings, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor) {
7993
assert.ok(owner, "Arg 'owner' must not be empty")
8094
assert.ok(repo, "Arg 'repo' must not be empty")
8195
assert.ok(patternStrings, "Arg 'patternStrings' must not be null")
@@ -94,6 +108,12 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
94108
if (ignoreTags && ignoreTags.length > 0) {
95109
config.ignoreTags = ignoreTags
96110
}
111+
if (latestMajorVersions && latestMajorVersions > 0) {
112+
config.latestMajorVersions = latestMajorVersions
113+
}
114+
if (latestVersionsPerMajor && latestVersionsPerMajor > 0) {
115+
config.latestVersionsPerMajor = latestVersionsPerMajor
116+
}
97117
config.defaultBranch = defaultBranch
98118

99119
const tempDir = path.join(paths.temp, `${owner}_${repo}`)
@@ -130,6 +150,9 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
130150
config.tags[tag] = tagVersion
131151
}
132152

153+
// Prune old tags based on version limits
154+
pruneOldTags(config)
155+
133156
// Write config
134157
await exec.exec('mkdir', ['-p', path.dirname(file)])
135158
await fs.promises.writeFile(file, JSON.stringify(config, null, ' '))
@@ -141,6 +164,75 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
141164
}
142165
exports.add = add
143166

167+
/**
168+
* Prunes old tags from the config based on latestMajorVersions and latestVersionsPerMajor.
169+
* Modifies config.tags in place.
170+
* @param {ActionConfig} config
171+
*/
172+
function pruneOldTags(config) {
173+
const maxMajors = config.latestMajorVersions || 0
174+
const maxPerMajor = config.latestVersionsPerMajor || 0
175+
176+
if (!maxMajors && !maxPerMajor) {
177+
return
178+
}
179+
180+
const tagNames = Object.keys(config.tags)
181+
const versionTags = []
182+
const keepTags = new Set()
183+
184+
for (const tag of tagNames) {
185+
const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/)
186+
if (!match) {
187+
// Always keep non-version tags
188+
keepTags.add(tag)
189+
continue
190+
}
191+
192+
const major = parseInt(match[1], 10)
193+
const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1
194+
const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1
195+
versionTags.push({ tag, major, minor, patch, isMajorOnly: minor === -1 })
196+
}
197+
198+
// Distinct major versions sorted descending (newest first)
199+
const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a)
200+
const allowedMajors = new Set(
201+
maxMajors > 0 ? majorVersions.slice(0, maxMajors) : majorVersions
202+
)
203+
204+
for (const major of allowedMajors) {
205+
const tagsForMajor = versionTags.filter(v => v.major === major)
206+
207+
// Always keep major-only pointers (e.g. "v4")
208+
for (const v of tagsForMajor.filter(v => v.isMajorOnly)) {
209+
keepTags.add(v.tag)
210+
}
211+
212+
// Sort non-major-only tags by version descending (latest first)
213+
const sorted = tagsForMajor
214+
.filter(v => !v.isMajorOnly)
215+
.sort((a, b) => {
216+
if (a.minor !== b.minor) return b.minor - a.minor
217+
return b.patch - a.patch
218+
})
219+
220+
const kept = maxPerMajor > 0 ? sorted.slice(0, maxPerMajor) : sorted
221+
for (const v of kept) {
222+
keepTags.add(v.tag)
223+
}
224+
}
225+
226+
// Remove pruned tags
227+
for (const tag of tagNames) {
228+
if (!keepTags.has(tag)) {
229+
console.log(`Pruning tag '${tag}' from config (version limit)`)
230+
delete config.tags[tag]
231+
}
232+
}
233+
}
234+
exports.pruneOldTags = pruneOldTags
235+
144236
/**
145237
* Returns the action config file path
146238
* @param {string} owner

‎script/internal/add-action.js‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ async function main() {
1515
const patterns = args.patterns
1616
const defaultBranch = args.defaultBranch || 'master'
1717
const ignoreTags = args.ignoreTags
18+
const latestMajorVersions = args.latestMajorVersions
19+
const latestVersionsPerMajor = args.latestVersionsPerMajor
1820

1921
// File exists?
2022
const file = actionConfig.getFilePath(owner, repo)
@@ -24,7 +26,7 @@ async function main() {
2426
await fsHelper.reinitTemp()
2527

2628
// Add the config
27-
await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags)
29+
await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor)
2830
}
2931
catch (err) {
3032
// Help
@@ -60,7 +62,7 @@ class Args {
6062
*/
6163
function getArgs() {
6264
// Parse
63-
const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags'])
65+
const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags', 'latest-major-versions', 'latest-versions-per-major'])
6466
if (parsedArgs.arguments.length < 1) {
6567
argHelper.throwError('Expected at least one arg')
6668
}
@@ -101,16 +103,29 @@ function getArgs() {
101103
repo: splitNwo[1],
102104
patterns: patterns,
103105
defaultBranch: parsedArgs.options['default-branch'],
104-
ignoreTags: ignoreTags
106+
ignoreTags: ignoreTags,
107+
latestMajorVersions: parseNonNegativeInt(parsedArgs.options['latest-major-versions'], 'latest-major-versions'),
108+
latestVersionsPerMajor: parseNonNegativeInt(parsedArgs.options['latest-versions-per-major'], 'latest-versions-per-major')
105109
}
106110
}
107111

112+
function parseNonNegativeInt(value, name) {
113+
if (!value) return 0
114+
const n = Number(value)
115+
if (!Number.isInteger(n) || n < 0) {
116+
argHelper.throwError(`--${name} must be a non-negative integer, got '${value}'`)
117+
}
118+
return n
119+
}
120+
108121
function printUsage() {
109-
console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] nwo [(+|-)regexp [...]]')
110-
console.error(` --default-branch Default branch name. For example: master`)
111-
console.error(` --ignore-tags Comma-separated version prefixes to ignore. For example: v1,v2`)
112-
console.error(` nwo Name with owner. For example: actions/checkout`)
113-
console.error(` regexp Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`)
122+
console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] [--latest-major-versions N] [--latest-versions-per-major N] nwo [(+|-)regexp [...]]')
123+
console.error(` --default-branch Default branch name. For example: master`)
124+
console.error(` --ignore-tags Comma-separated version prefixes to ignore. For example: v1,v2`)
125+
console.error(` --latest-major-versions Only cache the latest N major versions. For example: 3`)
126+
console.error(` --latest-versions-per-major Only cache the latest N version tags per major version. For example: 5`)
127+
console.error(` nwo Name with owner. For example: actions/checkout`)
128+
console.error(` regexp Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`)
114129
}
115130

116131
main()

‎script/internal/filter-tags.js‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Filters tags from an action config based on latestMajorVersions and latestVersionsPerMajor.
2+
// Reads JSON config from stdin, outputs allowed tag names (one per line).
3+
4+
async function main() {
5+
let input = ''
6+
for await (const chunk of process.stdin) {
7+
input += chunk
8+
}
9+
10+
const config = JSON.parse(input)
11+
const tags = Object.keys(config.tags || {})
12+
const latestMajorVersions = config.latestMajorVersions || 0 // 0 = unlimited
13+
const latestVersionsPerMajor = config.latestVersionsPerMajor || 0 // 0 = unlimited
14+
15+
if (!latestMajorVersions && !latestVersionsPerMajor) {
16+
// No filtering configured, output all tags
17+
for (const tag of tags) {
18+
console.log(tag)
19+
}
20+
return
21+
}
22+
23+
// Parse version info from tag names
24+
const versionTags = []
25+
const nonVersionTags = []
26+
27+
for (const tag of tags) {
28+
const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/)
29+
if (!match) {
30+
nonVersionTags.push(tag)
31+
continue
32+
}
33+
34+
const major = parseInt(match[1], 10)
35+
const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1
36+
const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1
37+
const isMajorOnly = minor === -1
38+
39+
versionTags.push({ tag, major, minor, patch, isMajorOnly })
40+
}
41+
42+
// Find distinct major versions sorted descending (newest first)
43+
const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a)
44+
45+
// Apply latestMajorVersions filter
46+
const allowedMajors = new Set(
47+
latestMajorVersions > 0 ? majorVersions.slice(0, latestMajorVersions) : majorVersions
48+
)
49+
50+
// Always include non-version tags
51+
const result = [...nonVersionTags]
52+
53+
for (const major of allowedMajors) {
54+
const tagsForMajor = versionTags.filter(v => v.major === major)
55+
56+
// Always include major-only pointers (e.g., "v4")
57+
for (const v of tagsForMajor.filter(v => v.isMajorOnly)) {
58+
result.push(v.tag)
59+
}
60+
61+
// Sort non-major-only tags by version descending (latest first)
62+
const sortedVersions = tagsForMajor
63+
.filter(v => !v.isMajorOnly)
64+
.sort((a, b) => {
65+
if (a.minor !== b.minor) return b.minor - a.minor
66+
return b.patch - a.patch
67+
})
68+
69+
// Apply latestVersionsPerMajor filter
70+
const kept = latestVersionsPerMajor > 0
71+
? sortedVersions.slice(0, latestVersionsPerMajor)
72+
: sortedVersions
73+
74+
for (const v of kept) {
75+
result.push(v.tag)
76+
}
77+
}
78+
79+
for (const tag of result) {
80+
console.log(tag)
81+
}
82+
}
83+
84+
main().catch(err => {
85+
console.error(err.message)
86+
process.exitCode = 1
87+
})

‎script/internal/generate-scripts.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@ for json_file in $script_dir/../../config/actions/*.json; do
4444
ignore_patterns=()
4545
IFS=$'\n' read -r -d '' -a ignore_patterns < <( echo "$json" | jq --raw-output '.ignoreTags // [] | .[]' && printf '\0' )
4646

47+
# Get version-filtered tags (applies latestMajorVersions and latestVersionsPerMajor)
48+
filtered_tags=()
49+
IFS=$'\n' read -r -d '' -a filtered_tags < <( echo "$json" | node "$script_dir/filter-tags.js" && printf '\0' )
50+
unset filtered_tag_set
51+
declare -A filtered_tag_set
52+
for t in "${filtered_tags[@]}"; do
53+
filtered_tag_set[$t]=1
54+
done
55+
4756
# Get an array of tag info. Each item contains "<tag> <commit_sha>"
4857
tag_info=()
4958
IFS=$'\n' read -r -d '' -a tag_info < <( echo "$json" | jq --raw-output '.tags | to_entries | .[] | .key + " " + .value.commit' && printf '\0' )
@@ -67,6 +76,12 @@ for json_file in $script_dir/../../config/actions/*.json; do
6776
continue
6877
fi
6978

79+
# Check if the tag passes version filter
80+
if [ -z "${filtered_tag_set[$tag]+x}" ]; then
81+
echo "Skipping tag '$tag' (filtered by version limits)"
82+
continue
83+
fi
84+
7085
# Append curl download command
7186
curl_download_commands+=("curl -s -S -L -o '$sha.tar.gz' 'https://api.github.com/repos/$owner/$repo/tarball/$sha'")
7287
curl_download_commands+=("curl -s -S -L -o '$sha.zip' 'https://api.github.com/repos/$owner/$repo/zipball/$sha'")

0 commit comments

Comments
 (0)