Skip to content

Commit d65bae3

Browse files
SatabdiGSatabdiG
authored
Merge pull request #205 from SAP/182_refactor_testing_code
[Feature]: Add custom upgrade testing framework
2 parents 185d22c + 7a788f2 commit d65bae3

14 files changed

Lines changed: 1163 additions & 430 deletions

‎.gitignore‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ dev.env
1717
secrets.env
1818
*~
1919
/test/e2e/logs/
20+
# Environment files with secrets
21+
.env
22+
test/**/.env
23+
test/upgrade/.env
24+
25+
# Upgrade Test artifacts
26+
test-upgrade-output.log
2027
test/upgrade/logs/
21-
test/upgrade/env.txt
22-
test/upgrade/upgrade.test
28+

‎Makefile‎

Lines changed: 102 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,17 @@ export UUT_CONTROLLER = $(BUILD_REGISTRY)/$(subst crossplane-,crossplane/,$(PROJ
6565
export UUT_IMAGES = {"crossplane/provider-cloudfoundry":"docker.io/$(UUT_CONFIG)","crossplane/provider-cloudfoundry-controller":"docker.io/$(UUT_CONTROLLER)"}
6666
export E2E_IMAGES = {"package":"$(UUT_CONFIG)","controller":"$(UUT_CONTROLLER)"}
6767

68+
# Import upgrade test environment variables from shell
69+
export UPGRADE_TEST_FROM_TAG
70+
export UPGRADE_TEST_TO_TAG
71+
export UPGRADE_TEST_VERIFY_TIMEOUT
72+
export UPGRADE_TEST_WAIT_FOR_PAUSE
73+
export CF_EMAIL
74+
export CF_USERNAME
75+
export CF_PASSWORD
76+
export CF_ENDPOINT
77+
78+
6879
# NOTE(hasheddan): we ensure up is installed prior to running platform-specific
6980
# build steps in parallel to avoid encountering an installation race condition.
7081
build.init: $(UP)
@@ -182,13 +193,18 @@ test-acceptance: $(KIND) $(HELM3) build
182193
# Upgrade test directory
183194
UPGRADE_TEST_DIR := test/upgrade
184195
UPGRADE_TEST_CRS_DIR := $(UPGRADE_TEST_DIR)/testdata/baseCrs
196+
UPGRADE_TEST_CUSTOM_CRS_DIR := $(UPGRADE_TEST_DIR)/testdata/customCRs
185197
UPGRADE_TEST_OUTPUT_LOG := test-upgrade-output.log
186198

187199
# If UPGRADE_TEST_CRS_TAG is not set, use UPGRADE_TEST_FROM_TAG as default
188200
UPGRADE_TEST_CRS_TAG ?= $(UPGRADE_TEST_FROM_TAG)
189201

202+
# Test filter for running specific tests
203+
UPGRADE_TEST_FILTER ?= .
204+
190205
.PHONY: check-upgrade-test-vars
191206
check-upgrade-test-vars: ## Verify required upgrade test environment variables
207+
@$(INFO) Checking required environment variables for upgrade tests are present
192208
@test -n "$(UPGRADE_TEST_FROM_TAG)" || { echo "❌ Set UPGRADE_TEST_FROM_TAG"; exit 1; }
193209
@test -n "$(UPGRADE_TEST_TO_TAG)" || { echo "❌ Set UPGRADE_TEST_TO_TAG"; exit 1; }
194210
@$(OK) required upgrade test environment variables are set
@@ -208,9 +224,47 @@ test-upgrade-compile: ## Verify upgrade tests compile
208224
@cd $(UPGRADE_TEST_DIR) && go test -c -tags=upgrade -o /dev/null
209225
@$(OK) upgrade tests compile successfully
210226

211-
.PHONY: test-upgrade
212-
test-upgrade: $(KIND) check-upgrade-test-vars build-upgrade-test-images ## Run upgrade tests
227+
# ====================================================================================
228+
# Base Upgrade Tests (Standard Resource Verification)
229+
# ====================================================================================
230+
231+
232+
.PHONY: test-upgrade-base
233+
test-upgrade-base: $(KIND) check-upgrade-test-vars build-upgrade-test-images ## Run upgrade tests (standard resource verification )
213234
@$(INFO) running upgrade tests from $(UPGRADE_TEST_FROM_TAG) to $(UPGRADE_TEST_TO_TAG)
235+
@cd $(UPGRADE_TEST_DIR) && go test -v -tags=upgrade -timeout=45m -run TestUpgradeProvider ./... 2>&1 | tee ../../$(UPGRADE_TEST_OUTPUT_LOG)
236+
@echo "==========Base Upgrade Test Summary =========="
237+
@grep -E "PASS|FAIL|ok " $(UPGRADE_TEST_OUTPUT_LOG) | tail -5
238+
@case `tail -n 1 $(UPGRADE_TEST_OUTPUT_LOG)` in \
239+
*FAIL*) echo "❌ Upgrade test failed"; exit 1 ;; \
240+
*ok*) echo "✅ Upgrade tests passed"; $(OK) upgrade tests passed ;; \
241+
*) echo "⚠️ Could not determine test result"; exit 1 ;; \
242+
esac
243+
244+
# ====================================================================================
245+
# Custom Upgrade Tests (External-Name Validation, etc.)
246+
# ====================================================================================
247+
248+
.PHONY: test-upgrade-custom
249+
test-upgrade-custom: $(KIND) check-upgrade-test-vars build-upgrade-test-images ## Run custom upgrade tests (external-name validation, etc.)
250+
@$(INFO) running custom upgrade tests from $(UPGRADE_TEST_FROM_TAG) to $(UPGRADE_TEST_TO_TAG)
251+
@$(INFO) test filter: $(UPGRADE_TEST_FILTER)
252+
@cd $(UPGRADE_TEST_DIR) && go test -v -tags=upgrade -timeout=45m -run '$(UPGRADE_TEST_FILTER)' ./... 2>&1 | tee ../../$(UPGRADE_TEST_OUTPUT_LOG)
253+
@echo "========== Custom Upgrade Test Summary =========="
254+
@grep -E "PASS|FAIL|ok " $(UPGRADE_TEST_OUTPUT_LOG) | tail -5
255+
@case `tail -n 1 $(UPGRADE_TEST_OUTPUT_LOG)` in \
256+
*FAIL*) echo "❌ Custom upgrade test failed"; exit 1 ;; \
257+
*ok*) echo "✅ Custom upgrade tests passed"; $(OK) custom upgrade tests passed ;; \
258+
*) echo "⚠️ Could not determine test result"; exit 1 ;; \
259+
esac
260+
261+
# ====================================================================================
262+
# Combined: Run All Upgrade Tests
263+
# ====================================================================================
264+
265+
.PHONY: test-upgrade
266+
test-upgrade: $(KIND) check-upgrade-test-vars build-upgrade-test-images ## Run ALL upgrade tests (base + custom)
267+
@$(INFO) running all upgrade tests from $(UPGRADE_TEST_FROM_TAG) to $(UPGRADE_TEST_TO_TAG)
214268
@cd $(UPGRADE_TEST_DIR) && go test -v -tags=upgrade -timeout=45m ./... 2>&1 | tee ../../$(UPGRADE_TEST_OUTPUT_LOG)
215269
@echo "========== Upgrade Test Summary =========="
216270
@grep -E "PASS|FAIL|ok " $(UPGRADE_TEST_OUTPUT_LOG) | tail -5
@@ -220,6 +274,11 @@ test-upgrade: $(KIND) check-upgrade-test-vars build-upgrade-test-images ## Run u
220274
*) echo "⚠️ Could not determine test result"; exit 1 ;; \
221275
esac
222276

277+
278+
# ====================================================================================
279+
# CR Preparation
280+
# ====================================================================================
281+
223282
.PHONY: test-upgrade-prepare-crs
224283
test-upgrade-prepare-crs: ## Prepare CRs from CRS_TAG version
225284
@$(INFO) preparing CRs from $(UPGRADE_TEST_CRS_TAG)
@@ -232,7 +291,7 @@ test-upgrade-prepare-crs: ## Prepare CRs from CRS_TAG version
232291
mkdir -p $(UPGRADE_TEST_CRS_DIR); \
233292
if git ls-tree -r $(UPGRADE_TEST_CRS_TAG) --name-only | grep -q "^$(UPGRADE_TEST_CRS_DIR)/"; then \
234293
$(INFO) "✅ Found $(UPGRADE_TEST_CRS_DIR)/ in $(UPGRADE_TEST_CRS_TAG)"; \
235-
git archive $(UPGRADE_TEST_CRS_TAG) $(UPGRADE_TEST_CRS_DIR)/ | tar -x --strip-components=2 -C $(UPGRADE_TEST_CRS_DIR)/; \
294+
git archive $(UPGRADE_TEST_CRS_TAG) $(UPGRADE_TEST_CRS_DIR)/ | tar -x --strip-components=4 -C $(UPGRADE_TEST_CRS_DIR)/; \
236295
$(OK) "Copied all CRs from $(UPGRADE_TEST_CRS_DIR)/"; \
237296
else \
238297
$(INFO) "⚠️ $(UPGRADE_TEST_CRS_DIR)/ not found, using hardcoded e2e paths"; \
@@ -244,6 +303,10 @@ test-upgrade-prepare-crs: ## Prepare CRs from CRS_TAG version
244303
fi; \
245304
fi
246305

306+
# ====================================================================================
307+
# Upgrade Tests with Version-Specific CRs
308+
# ====================================================================================
309+
247310
.PHONY: test-upgrade-with-version-crs
248311
test-upgrade-with-version-crs: $(KIND) check-upgrade-test-vars build-upgrade-test-images test-upgrade-prepare-crs ## Run upgrade tests with FROM version CRs
249312
@$(INFO) running upgrade tests from $(UPGRADE_TEST_FROM_TAG) to $(UPGRADE_TEST_TO_TAG)
@@ -256,13 +319,22 @@ test-upgrade-with-version-crs: $(KIND) check-upgrade-test-vars build-upgrade-tes
256319
*) echo "⚠️ Could not determine test result"; exit 1; ;; \
257320
esac
258321

322+
# ====================================================================================
323+
# Debugging Support
324+
# ====================================================================================
325+
326+
# TODO: Add test-upgrade-debug-base and test-upgrade-debug-custom variants
327+
259328
.PHONY: test-upgrade-debug
260329
test-upgrade-debug: $(KIND) check-upgrade-test-vars build-upgrade-test-images test-upgrade-prepare-crs ## Run upgrade tests with debugger
261330
@$(INFO) running upgrade tests with debugger
262331
@cd $(UPGRADE_TEST_DIR) && dlv test -tags=upgrade . --listen=:2345 --headless=true --api-version=2 --build-flags="-tags=upgrade" -- -test.v -test.timeout 45m 2>&1 | tee ../../$(UPGRADE_TEST_OUTPUT_LOG)
263332
@echo "========== Upgrade Test Summary =========="
264333
@grep -E "PASS|FAIL|ok " $(UPGRADE_TEST_OUTPUT_LOG) | tail -5
265334

335+
# ====================================================================================
336+
# Utility Targets
337+
# ====================================================================================
266338

267339
.PHONY: test-upgrade-restore-crs
268340
test-upgrade-restore-crs: ## Restore $(UPGRADE_TEST_CRS_DIR)/ to current version
@@ -284,49 +356,51 @@ test-upgrade-help: ## Show upgrade test usage examples
284356
@$(INFO) "Upgrade Test Examples:"
285357
@$(INFO) "======================"
286358
@$(INFO) ""
287-
@$(INFO) " 1. Test between two releases:"
359+
@$(INFO) " 1. Run ALL upgrade tests (base + custom):"
288360
@$(INFO) " export UPGRADE_TEST_FROM_TAG=v0.3.2"
289361
@$(INFO) " export UPGRADE_TEST_TO_TAG=v0.4.0"
290362
@$(INFO) " make test-upgrade"
291363
@$(INFO) ""
292-
@$(INFO) " 2. Test local changes (v0.3.2 -> your code):"
364+
@$(INFO) " 2. Run ONLY base upgrade tests:"
293365
@$(INFO) " export UPGRADE_TEST_FROM_TAG=v0.3.2"
294-
@$(INFO) " export UPGRADE_TEST_TO_TAG=local"
295-
@$(INFO) " make test-upgrade"
366+
@$(INFO) " export UPGRADE_TEST_TO_TAG=v0.4.0"
367+
@$(INFO) " make test-upgrade-base"
368+
@$(INFO) ""
369+
@$(INFO) " 3. Run ONLY custom upgrade tests:"
370+
@$(INFO) " export UPGRADE_TEST_FROM_TAG=v0.3.2"
371+
@$(INFO) " export UPGRADE_TEST_TO_TAG=v0.4.0"
372+
@$(INFO) " make test-upgrade-custom"
296373
@$(INFO) ""
297-
@$(INFO) " 3. Manual upgrade test (no CR checkout):"
374+
@$(INFO) " 4. Run specific custom test:"
298375
@$(INFO) " export UPGRADE_TEST_FROM_TAG=v0.3.2"
299376
@$(INFO) " export UPGRADE_TEST_TO_TAG=v0.4.0"
377+
@$(INFO) " export UPGRADE_TEST_FILTER='Test_Space_External_Name'"
378+
@$(INFO) " make test-upgrade-custom"
379+
@$(INFO) ""
380+
@$(INFO) " 5. Test local changes (v0.3.2 -> your code):"
381+
@$(INFO) " export UPGRADE_TEST_FROM_TAG=v0.3.2"
382+
@$(INFO) " export UPGRADE_TEST_TO_TAG=local"
300383
@$(INFO) " make test-upgrade"
301-
@$(INFO) " Note: Uses current $(UPGRADE_TEST_CRS_DIR) (may fail if incompatible)"
302384
@$(INFO) ""
303-
@$(INFO) " 4. Clean up test artifacts:"
385+
@$(INFO) " 6. Clean up test artifacts:"
304386
@$(INFO) " make test-upgrade-clean"
305387
@$(INFO) ""
306-
@$(INFO) " 5. Restore CRs after version checkout:"
388+
@$(INFO) " 7. Restore CRs after version checkout:"
307389
@$(INFO) " make test-upgrade-restore-crs"
308390
@$(INFO) ""
309391
@$(INFO) "Required Environment Variables:"
310392
@$(INFO) " CF_EMAIL, CF_USERNAME, CF_PASSWORD, CF_ENDPOINT"
311393
@$(INFO) " UPGRADE_TEST_FROM_TAG, UPGRADE_TEST_TO_TAG"
312394
@$(INFO) ""
313395
@$(INFO) "Optional Environment Variables:"
314-
@$(INFO) " UPGRADE_TEST_CRS_PATH (default: $(UPGRADE_TEST_CRS_DIR))"
315-
@$(INFO) " UPGRADE_TEST_VERIFY_TIMEOUT (default: 30 minutes)"
316-
@$(INFO) " UPGRADE_TEST_WAIT_FOR_PAUSE (default: 1 minute)"
396+
@$(INFO) " UPGRADE_TEST_FILTER (default: '.' - runs all tests)"
397+
@$(INFO) " UPGRADE_TEST_CRS_TAG (default: UPGRADE_TEST_FROM_TAG)"
317398
@$(INFO) ""
318-
@$(INFO) "How CRS Checkout Works (test-upgrade-with-version-crs):"
319-
@$(INFO) "========================================================"
320-
@$(INFO) " 1. If FROM_TAG is 'local': Uses current $(UPGRADE_TEST_CRS_DIR)/"
321-
@$(INFO) " 2. If FROM_TAG has $(UPGRADE_TEST_CRS_DIR)/: Copies entire directory"
399+
@$(INFO) "Test Types:"
400+
@$(INFO) "==========="
401+
@$(INFO) " Base Tests: Standard resource verification (TestUpgradeProvider)"
402+
@$(INFO) " Custom Tests: External-name validation (Test_Space_External_Name, etc.)"
322403
@$(INFO) ""
323-
@$(INFO) ""
324-
@$(INFO) "⚠️ IMPORTANT NOTES:"
325-
@$(INFO) " - test-upgrade-with-version-crs OVERWRITES $(UPGRADE_TEST_CRS_DIR)/"
326-
@$(INFO) " - test-upgrade-restore-crs will to restore your files"
327-
@$(INFO) " - E2E CRs (fallback) may have complex dependencies - test might fail"
328-
@$(INFO) ""
329-
330404

331405
# ====================================================================================
332406
# Special Targets
@@ -338,7 +412,9 @@ Crossplane Targets:
338412
run Run crossplane locally, out-of-cluster. Useful for development.
339413

340414
Upgrade Testing:
341-
test-upgrade Run upgrade tests (requires env vars)
415+
test-upgrade Run ALL upgrade tests (requires env vars, base + custom)
416+
test-upgrade-base Run base upgrade tests only
417+
test-upgrade-custom Run custom upgrade tests only (external-name validation, etc.)
342418
test-upgrade-with-version-crs Run upgrade tests with auto CR checkout
343419
test-upgrade-compile Verify upgrade tests compile
344420
test-upgrade-debug Run upgrade tests with debugger

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ kind delete cluster <cluster-name>
6969

7070
### Upgrade Tests
7171

72-
The provider comes with a set of upgrade tests that can be run locally. To learn more about it : [Upgrade Tests README](./test/upgrade/README.md).
72+
The provider comes with a set of upgrade tests that can be run locally. To learn more about it : [Upgrade Tests README](./docs/upgrade-testing.md).
7373

7474
#### Required Configuration
7575
In order for the tests to perform successfully some configuration need to be present as environment variables:

0 commit comments

Comments
 (0)