From ebe39e544968b623c70acac3e11611a6fda139e2 Mon Sep 17 00:00:00 2001 From: "blacksmith-sh[bot]" <157653362+blacksmith-sh[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 21:48:51 +0000 Subject: [PATCH 01/25] Migrate workflows to Blacksmith --- .github/workflows/cd-dgraph-nightly.yml | 7 ++++--- .github/workflows/cd-dgraph.yml | 4 ++-- .github/workflows/ci-dgraph-core-tests.yml | 2 +- .github/workflows/ci-dgraph-core-upgrade-tests.yml | 2 +- .github/workflows/ci-dgraph-fuzz.yml | 2 +- .github/workflows/ci-dgraph-integration2-tests.yml | 2 +- .github/workflows/ci-dgraph-jepsen-tests.yml | 2 +- .github/workflows/ci-dgraph-ldbc-tests.yml | 2 +- .github/workflows/ci-dgraph-load-tests.yml | 2 +- .github/workflows/ci-dgraph-system-upgrade-tests.yml | 2 +- .github/workflows/ci-dgraph-systest-tests.yml | 2 +- .../workflows/ci-dgraph-upgrade-fixed-versions-tests.yml | 2 +- .github/workflows/ci-dgraph-vector-tests.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/labeler.yml | 2 +- 15 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cd-dgraph-nightly.yml b/.github/workflows/cd-dgraph-nightly.yml index f5c9fb6216c..8d5319d21af 100644 --- a/.github/workflows/cd-dgraph-nightly.yml +++ b/.github/workflows/cd-dgraph-nightly.yml @@ -10,9 +10,10 @@ permissions: jobs: build: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - - uses: docker/setup-buildx-action@v3.11.1 + - name: Setup Blacksmith Builder + uses: useblacksmith/setup-docker-builder@v1 - name: Log in to GitHub Container Registry uses: docker/login-action@v3.5.0 @@ -22,7 +23,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@v6.18.0 + uses: useblacksmith/build-push-action@v2 with: push: true platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/cd-dgraph.yml b/.github/workflows/cd-dgraph.yml index 36e527efe94..9f27c90cd3a 100644 --- a/.github/workflows/cd-dgraph.yml +++ b/.github/workflows/cd-dgraph.yml @@ -21,7 +21,7 @@ permissions: jobs: dgraph-build-amd64: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 steps: - uses: actions/checkout@v5 @@ -222,7 +222,7 @@ jobs: graph-docker-image-and-manifests-push: needs: [dgraph-build-amd64, dgraph-build-arm64] - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-core-tests.yml index 7fedfc46ba5..78aa69e05e1 100644 --- a/.github/workflows/ci-dgraph-core-tests.yml +++ b/.github/workflows/ci-dgraph-core-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-core-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-core-upgrade-tests.yml b/.github/workflows/ci-dgraph-core-upgrade-tests.yml index 9446016f12d..5060818969c 100644 --- a/.github/workflows/ci-dgraph-core-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-core-upgrade-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-fuzz.yml b/.github/workflows/ci-dgraph-fuzz.yml index a1f2f0f61db..4e0f9015743 100644 --- a/.github/workflows/ci-dgraph-fuzz.yml +++ b/.github/workflows/ci-dgraph-fuzz.yml @@ -23,7 +23,7 @@ permissions: jobs: fuzz-test: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index cb94826215d..c3ba80eeec1 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-integration2-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-jepsen-tests.yml b/.github/workflows/ci-dgraph-jepsen-tests.yml index fa5466e9f07..781af1d5d23 100644 --- a/.github/workflows/ci-dgraph-jepsen-tests.yml +++ b/.github/workflows/ci-dgraph-jepsen-tests.yml @@ -12,7 +12,7 @@ permissions: jobs: dgraph-jepsen-tests: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: Checkout dgraph repo diff --git a/.github/workflows/ci-dgraph-ldbc-tests.yml b/.github/workflows/ci-dgraph-ldbc-tests.yml index 5e193f7eb95..6b56855ea79 100644 --- a/.github/workflows/ci-dgraph-ldbc-tests.yml +++ b/.github/workflows/ci-dgraph-ldbc-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-ldbc-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - name: Checkout Dgraph diff --git a/.github/workflows/ci-dgraph-load-tests.yml b/.github/workflows/ci-dgraph-load-tests.yml index 688fdb6770b..0aee7076e43 100644 --- a/.github/workflows/ci-dgraph-load-tests.yml +++ b/.github/workflows/ci-dgraph-load-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-load-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests.yml b/.github/workflows/ci-dgraph-system-upgrade-tests.yml index bd6a64a3985..2e5697e011d 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index a199880f46d..e73a4a8ca22 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-systest-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml index 15e0a447325..8732fec2254 100644 --- a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml +++ b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml @@ -9,7 +9,7 @@ permissions: jobs: dgraph-upgrade-fixed-versions-tests: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml index a4682c7007c..1d25fc5885c 100644 --- a/.github/workflows/ci-dgraph-vector-tests.yml +++ b/.github/workflows/ci-dgraph-vector-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-vector-tests: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ab7db66d9f1..3b84ba57608 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,7 +18,7 @@ permissions: jobs: analyze: name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 360 permissions: security-events: write diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e25d544aef3..7c1d9e41dcb 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -20,7 +20,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: actions/checkout@v5 - uses: actions/labeler@v6 From 78606de8cab83e753df3334a6982d02525de2f80 Mon Sep 17 00:00:00 2001 From: mattthew Date: Mon, 3 Nov 2025 17:51:40 -0500 Subject: [PATCH 02/25] Added all available Blacksmith runner labels (2vcpu through 32vcpu); upgrade some runners to prevent resource exhaustion --- .github/actionlint.yaml | 27 +++++++++++++++++++ .github/workflows/ci-dgraph-core-tests.yml | 2 +- .../ci-dgraph-core-upgrade-tests.yml | 2 +- .../ci-dgraph-integration2-tests.yml | 2 +- .github/workflows/ci-dgraph-jepsen-tests.yml | 2 +- .../ci-dgraph-system-upgrade-tests.yml | 2 +- .github/workflows/ci-dgraph-systest-tests.yml | 2 +- ...ci-dgraph-upgrade-fixed-versions-tests.yml | 2 +- .github/workflows/ci-dgraph-vector-tests.yml | 2 +- 9 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000..a47e40edbc3 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,27 @@ +self-hosted-runner: + # Define custom labels for Blacksmith runners + # Ubuntu 24.04 x64 runners + labels: + - blacksmith-2vcpu-ubuntu-2404 + - blacksmith-4vcpu-ubuntu-2404 + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-16vcpu-ubuntu-2404 + - blacksmith-32vcpu-ubuntu-2404 + # Ubuntu 24.04 ARM runners + - blacksmith-2vcpu-ubuntu-2404-arm + - blacksmith-4vcpu-ubuntu-2404-arm + - blacksmith-8vcpu-ubuntu-2404-arm + - blacksmith-16vcpu-ubuntu-2404-arm + - blacksmith-32vcpu-ubuntu-2404-arm + # Ubuntu 22.04 x64 runners + - blacksmith-2vcpu-ubuntu-2204 + - blacksmith-4vcpu-ubuntu-2204 + - blacksmith-8vcpu-ubuntu-2204 + - blacksmith-16vcpu-ubuntu-2204 + - blacksmith-32vcpu-ubuntu-2204 + # Ubuntu 22.04 ARM runners + - blacksmith-2vcpu-ubuntu-2204-arm + - blacksmith-4vcpu-ubuntu-2204-arm + - blacksmith-8vcpu-ubuntu-2204-arm + - blacksmith-16vcpu-ubuntu-2204-arm + - blacksmith-32vcpu-ubuntu-2204-arm diff --git a/.github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-core-tests.yml index 78aa69e05e1..c9c0225b868 100644 --- a/.github/workflows/ci-dgraph-core-tests.yml +++ b/.github/workflows/ci-dgraph-core-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-core-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-core-upgrade-tests.yml b/.github/workflows/ci-dgraph-core-upgrade-tests.yml index 5060818969c..f88eb9b8845 100644 --- a/.github/workflows/ci-dgraph-core-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-core-upgrade-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index c3ba80eeec1..40c3ef28c9d 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-integration2-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-jepsen-tests.yml b/.github/workflows/ci-dgraph-jepsen-tests.yml index 781af1d5d23..7709af4264d 100644 --- a/.github/workflows/ci-dgraph-jepsen-tests.yml +++ b/.github/workflows/ci-dgraph-jepsen-tests.yml @@ -12,7 +12,7 @@ permissions: jobs: dgraph-jepsen-tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: Checkout dgraph repo diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests.yml b/.github/workflows/ci-dgraph-system-upgrade-tests.yml index 2e5697e011d..f5adafdcc79 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests.yml @@ -24,7 +24,7 @@ permissions: jobs: dgraph-upgrade-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index e73a4a8ca22..8c1ec792ad9 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-systest-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml index 8732fec2254..13b5280fabe 100644 --- a/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml +++ b/.github/workflows/ci-dgraph-upgrade-fixed-versions-tests.yml @@ -9,7 +9,7 @@ permissions: jobs: dgraph-upgrade-fixed-versions-tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml index 1d25fc5885c..d0b6ca88201 100644 --- a/.github/workflows/ci-dgraph-vector-tests.yml +++ b/.github/workflows/ci-dgraph-vector-tests.yml @@ -26,7 +26,7 @@ permissions: jobs: dgraph-vector-tests: if: github.event.pull_request.draft == false - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 From 7ee51312519d54c780f4d8904f5bc0c331b40a5d Mon Sep 17 00:00:00 2001 From: mattthew Date: Mon, 3 Nov 2025 20:24:32 -0500 Subject: [PATCH 03/25] Add debugging to failure --- .github/workflows/ci-dgraph-systest-tests.yml | 1 + systest/backup/common/utils.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index 8c1ec792ad9..90b2b0bf3dc 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -63,6 +63,7 @@ jobs: #!/bin/bash # go env settings export GOPATH=~/go + export DEBUG_SHOW_ERROR=1 # move the binary cp dgraph/dgraph ~/go/bin/dgraph # run the unit and systests diff --git a/systest/backup/common/utils.go b/systest/backup/common/utils.go index adb2774821e..efafaf2a7b8 100644 --- a/systest/backup/common/utils.go +++ b/systest/backup/common/utils.go @@ -278,5 +278,8 @@ func CopyToLocalFsFromNFS(t *testing.T, backupDst string, copyBackupDirectory st // "docker cp" to create a copy that is not owned by the root user. require.NoError(t, os.RemoveAll(copyBackupDirectory)) srcPath := testutil.DockerPrefix + "_nfs_1:/data" + backupDst - require.NoError(t, testutil.DockerCp(srcPath, copyBackupDirectory)) + if err := testutil.DockerCp(srcPath, copyBackupDirectory); err != nil { + t.Logf("DockerCp failed: src=%s dst=%s error=%v", srcPath, copyBackupDirectory, err) + require.NoError(t, err) + } } From 3f3136f6aaa1a9ae9bc95aafc3c032ae19488a12 Mon Sep 17 00:00:00 2001 From: mattthew Date: Mon, 3 Nov 2025 21:33:04 -0500 Subject: [PATCH 04/25] Break-up upgrade tests --- ... => ci-dgraph-system-upgrade-tests-mt.yml} | 2 - ...em-upgrade-tests-mutations-and-queries.yml | 57 +++++++++++++++++++ .../ci-dgraph-system-upgrade-tests-plugin.yml | 57 +++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) rename .github/workflows/{ci-dgraph-system-upgrade-tests.yml => ci-dgraph-system-upgrade-tests-mt.yml} (90%) create mode 100644 .github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml create mode 100644 .github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml similarity index 90% rename from .github/workflows/ci-dgraph-system-upgrade-tests.yml rename to .github/workflows/ci-dgraph-system-upgrade-tests-mt.yml index f5adafdcc79..be686bc988f 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml @@ -50,8 +50,6 @@ jobs: cp dgraph/dgraph ~/go/bin/dgraph # run the sytem upgrade tests go test -v -timeout=120m -failfast -tags=upgrade \ - github.com/hypermodeinc/dgraph/v25/systest/mutations-and-queries \ - github.com/hypermodeinc/dgraph/v25/systest/plugin \ github.com/hypermodeinc/dgraph/v25/systest/multi-tenancy # clean up docker containers after test execution go clean -testcache diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml new file mode 100644 index 00000000000..f86201eda64 --- /dev/null +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml @@ -0,0 +1,57 @@ +name: ci-dgraph-system-upgrade-tests + +on: + pull_request: + paths: + - "!**/*.md" + - "**/*.go" + - "**/go.mod" + - "**/*.yml" + - "**/Dockerfile" + - "**/Makefile" + types: + - opened + - reopened + - synchronize + - ready_for_review + branches: + - main + - release/** + +permissions: + contents: read + +jobs: + dgraph-upgrade-tests: + if: github.event.pull_request.draft == false + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Make Linux Build and Docker Image + run: make docker-image + - name: Clean Up Environment + run: | + #!/bin/bash + # clean cache + go clean -testcache + - name: Run System Upgrade Tests + run: | + #!/bin/bash + # go env settings + export GOPATH=~/go + # move the binary + cp dgraph/dgraph ~/go/bin/dgraph + # run the sytem upgrade tests + go test -v -timeout=120m -failfast -tags=upgrade \ + github.com/hypermodeinc/dgraph/v25/systest/mutations-and-queries + # clean up docker containers after test execution + go clean -testcache + # sleep + sleep 5 diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml new file mode 100644 index 00000000000..eb5169fe724 --- /dev/null +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml @@ -0,0 +1,57 @@ +name: ci-dgraph-system-upgrade-tests + +on: + pull_request: + paths: + - "!**/*.md" + - "**/*.go" + - "**/go.mod" + - "**/*.yml" + - "**/Dockerfile" + - "**/Makefile" + types: + - opened + - reopened + - synchronize + - ready_for_review + branches: + - main + - release/** + +permissions: + contents: read + +jobs: + dgraph-upgrade-tests: + if: github.event.pull_request.draft == false + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Make Linux Build and Docker Image + run: make docker-image + - name: Clean Up Environment + run: | + #!/bin/bash + # clean cache + go clean -testcache + - name: Run System Upgrade Tests + run: | + #!/bin/bash + # go env settings + export GOPATH=~/go + # move the binary + cp dgraph/dgraph ~/go/bin/dgraph + # run the sytem upgrade tests + go test -v -timeout=120m -failfast -tags=upgrade \ + github.com/hypermodeinc/dgraph/v25/systest/plugin + # clean up docker containers after test execution + go clean -testcache + # sleep + sleep 5 From 0097dc4626b4b1f36f46b23b42c581338ed7ab45 Mon Sep 17 00:00:00 2001 From: mattthew Date: Mon, 3 Nov 2025 21:33:49 -0500 Subject: [PATCH 05/25] Guard against unneeded overwrite --- dgraphtest/image.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dgraphtest/image.go b/dgraphtest/image.go index a91eb4e1875..6a83d1c967a 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -199,6 +199,16 @@ func copy(src, dst string) error { return errors.Errorf("%s is not a regular file", src) } + // Check if destination already exists and matches source size + if destStat, err := os.Stat(dst); err == nil { + if destStat.Size() == sourceFileStat.Size() { + log.Printf("[INFO] destination file %s already exists with matching size, skipping copy", dst) + return nil + } + log.Printf("[WARNING] destination file %s exists but size mismatch (source=%d, dest=%d), will overwrite", + dst, sourceFileStat.Size(), destStat.Size()) + } + // Open source file source, err := os.Open(src) if err != nil { From 3e0ada1f6846ecb5b1f4b458dc0db65ce5ab9ada Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 10:52:12 -0500 Subject: [PATCH 06/25] Add resiliency to log verification --- testutil/testaudit/audit.go | 55 +++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/testutil/testaudit/audit.go b/testutil/testaudit/audit.go index 48d03c3d994..4e843f6d329 100644 --- a/testutil/testaudit/audit.go +++ b/testutil/testaudit/audit.go @@ -20,30 +20,55 @@ import ( ) func VerifyLogs(t *testing.T, path string, cmds []string) { - // to make sure that the audit log is flushed - time.Sleep(time.Second * 5) abs, err := filepath.Abs(path) require.NoError(t, err) - f, err := os.Open(abs) - require.NoError(t, err) type log struct { Msg string `json:"endpoint"` } - logMap := make(map[string]bool) - fileScanner := bufio.NewScanner(f) - for fileScanner.Scan() { - bytes := fileScanner.Bytes() - l := new(log) - require.NoError(t, json.Unmarshal(bytes, l)) - logMap[l.Msg] = true - } - for _, m := range cmds { - if !logMap[m] { - t.Fatalf("audit logs not present for command %s", m) + maxRetries := 10 + for attempt := 1; attempt <= maxRetries; attempt++ { + f, err := os.Open(abs) + if err != nil { + if attempt == maxRetries { + require.NoError(t, err, "failed to open audit log after %d attempts", maxRetries) + } + time.Sleep(time.Second) + continue + } + + logMap := make(map[string]bool) + fileScanner := bufio.NewScanner(f) + for fileScanner.Scan() { + bytes := fileScanner.Bytes() + l := new(log) + if err := json.Unmarshal(bytes, l); err != nil { + f.Close() + if attempt == maxRetries { + require.NoError(t, err, "failed to unmarshal audit log after %d attempts", maxRetries) + } + time.Sleep(time.Second) + continue + } + logMap[l.Msg] = true + } + f.Close() + + missingCmds := []string{} + for _, m := range cmds { + if !logMap[m] { + missingCmds = append(missingCmds, m) + } + } + if len(missingCmds) == 0 { + return + } + if attempt == maxRetries { + t.Fatalf("audit logs not present after %d attempts. Missing commands: %v", maxRetries, missingCmds) } + time.Sleep(time.Second) } } From fd3aeabed1af5b5253e9a95e0a9d42340bfa75fb Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 12:38:44 -0500 Subject: [PATCH 07/25] Update names of split tests --- .github/workflows/ci-dgraph-system-upgrade-tests-mt.yml | 2 +- .../ci-dgraph-system-upgrade-tests-mutations-and-queries.yml | 2 +- .github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml index be686bc988f..f30f6cba304 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml @@ -1,4 +1,4 @@ -name: ci-dgraph-system-upgrade-tests +name: ci-dgraph-system-upgrade-tests-multi-tenancy on: pull_request: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml index f86201eda64..6e8a043842b 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml @@ -1,4 +1,4 @@ -name: ci-dgraph-system-upgrade-tests +name: ci-dgraph-system-upgrade-tests-mutations-and-queries on: pull_request: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml index eb5169fe724..10dfe7767a3 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml @@ -1,4 +1,4 @@ -name: ci-dgraph-system-upgrade-tests +name: ci-dgraph-system-upgrade-tests-plugin on: pull_request: From e983334759b70c31d2d91845b3251d766be81ffb Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 16:24:25 -0500 Subject: [PATCH 08/25] Serialize commands to prevent race condition --- systest/backup/nfs-backup/docker-compose.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/systest/backup/nfs-backup/docker-compose.yml b/systest/backup/nfs-backup/docker-compose.yml index 4a06a691dba..19f75496e12 100644 --- a/systest/backup/nfs-backup/docker-compose.yml +++ b/systest/backup/nfs-backup/docker-compose.yml @@ -24,7 +24,8 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 + while ! mountpoint -q /mnt; do sleep 0.1; done /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha1_backup_clust_ha:7080 --zero=zero1_backup_clust_ha:5080,zero2_backup_clust_ha:5080,zero3_backup_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha2_backup_clust_ha: @@ -157,7 +158,8 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 + while ! mountpoint -q /mnt; do sleep 0.1; done /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha4_restore_clust_ha:7080 --zero=zero4_restore_clust_ha:5080,zero5_restore_clust_ha:5080,zero6_restore_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha5_restore_clust_ha: @@ -311,7 +313,8 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 + while ! mountpoint -q /mnt; do sleep 0.1; done /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha7_backup_clust_non_ha:7080 --zero=zero7_backup_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" #non HA restore cluster @@ -358,7 +361,8 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 + while ! mountpoint -q /mnt; do sleep 0.1; done /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha8_restore_clust_non_ha:7080 --zero=zero8_restore_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" nfs: From 1c1cdb3bdc2aa2f3fbc92379a25a15eae351c886 Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 16:50:03 -0500 Subject: [PATCH 09/25] Rework nfs mounting logic --- systest/backup/nfs-backup/docker-compose.yml | 52 +++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/systest/backup/nfs-backup/docker-compose.yml b/systest/backup/nfs-backup/docker-compose.yml index 19f75496e12..8046fe69262 100644 --- a/systest/backup/nfs-backup/docker-compose.yml +++ b/systest/backup/nfs-backup/docker-compose.yml @@ -24,8 +24,17 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 - while ! mountpoint -q /mnt; do sleep 0.1; done + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha1_backup_clust_ha:7080 --zero=zero1_backup_clust_ha:5080,zero2_backup_clust_ha:5080,zero3_backup_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha2_backup_clust_ha: @@ -158,8 +167,17 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 - while ! mountpoint -q /mnt; do sleep 0.1; done + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha4_restore_clust_ha:7080 --zero=zero4_restore_clust_ha:5080,zero5_restore_clust_ha:5080,zero6_restore_clust_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" alpha5_restore_clust_ha: @@ -313,8 +331,17 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 - while ! mountpoint -q /mnt; do sleep 0.1; done + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha7_backup_clust_non_ha:7080 --zero=zero7_backup_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" #non HA restore cluster @@ -361,8 +388,17 @@ services: - /bin/sh - -c - | - mount -v -o vers=4,loud nfs:/ /mnt 2>&1 - while ! mountpoint -q /mnt; do sleep 0.1; done + mount -v -o vers=4,loud nfs:/ /mnt 2>&1 & + for i in $(seq 1 30); do + if mountpoint -q /mnt 2>/dev/null; then + break + fi + sleep 1 + done + if ! mountpoint -q /mnt 2>/dev/null; then + echo "ERROR: Failed to mount NFS after 30 seconds" + exit 1 + fi /gobin/dgraph ${COVERAGE_OUTPUT} alpha --my=alpha8_restore_clust_non_ha:7080 --zero=zero8_restore_clust_non_ha:5080 --logtostderr -v=2 --security "whitelist=0.0.0.0/0;" nfs: From d50165861da135cdc12f6ce2857c9320b52ccfb8 Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 16:55:16 -0500 Subject: [PATCH 10/25] Disable (temporarily) all actions except systest --- .github/workflows/ci-dgraph-core-tests.yml | 3 ++- .github/workflows/ci-dgraph-core-upgrade-tests.yml | 3 ++- .github/workflows/ci-dgraph-fuzz.yml | 3 ++- .github/workflows/ci-dgraph-integration2-tests.yml | 3 ++- .github/workflows/ci-dgraph-ldbc-tests.yml | 3 ++- .github/workflows/ci-dgraph-load-tests.yml | 3 ++- .github/workflows/ci-dgraph-system-upgrade-tests-mt.yml | 3 ++- .../ci-dgraph-system-upgrade-tests-mutations-and-queries.yml | 3 ++- .github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml | 3 ++- .github/workflows/ci-dgraph-vector-tests.yml | 3 ++- 10 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-core-tests.yml index c9c0225b868..722d31c4d01 100644 --- a/.github/workflows/ci-dgraph-core-tests.yml +++ b/.github/workflows/ci-dgraph-core-tests.yml @@ -25,7 +25,8 @@ permissions: jobs: dgraph-core-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: diff --git a/.github/workflows/ci-dgraph-core-upgrade-tests.yml b/.github/workflows/ci-dgraph-core-upgrade-tests.yml index f88eb9b8845..ed2231dec47 100644 --- a/.github/workflows/ci-dgraph-core-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-core-upgrade-tests.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-upgrade-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: diff --git a/.github/workflows/ci-dgraph-fuzz.yml b/.github/workflows/ci-dgraph-fuzz.yml index 4e0f9015743..5e9d232cab9 100644 --- a/.github/workflows/ci-dgraph-fuzz.yml +++ b/.github/workflows/ci-dgraph-fuzz.yml @@ -22,7 +22,8 @@ permissions: jobs: fuzz-test: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index 40c3ef28c9d..064fdb39fdb 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-integration2-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-ldbc-tests.yml b/.github/workflows/ci-dgraph-ldbc-tests.yml index 6b56855ea79..f4c3eb12370 100644 --- a/.github/workflows/ci-dgraph-ldbc-tests.yml +++ b/.github/workflows/ci-dgraph-ldbc-tests.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-ldbc-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/.github/workflows/ci-dgraph-load-tests.yml b/.github/workflows/ci-dgraph-load-tests.yml index 0aee7076e43..be5551f64d1 100644 --- a/.github/workflows/ci-dgraph-load-tests.yml +++ b/.github/workflows/ci-dgraph-load-tests.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-load-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml index f30f6cba304..fc08d4bef7b 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-upgrade-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml index 6e8a043842b..1555604d6ba 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-upgrade-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml index 10dfe7767a3..42af5f715a5 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml @@ -23,7 +23,8 @@ permissions: jobs: dgraph-upgrade-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml index d0b6ca88201..ab5cf897dc7 100644 --- a/.github/workflows/ci-dgraph-vector-tests.yml +++ b/.github/workflows/ci-dgraph-vector-tests.yml @@ -25,7 +25,8 @@ permissions: jobs: dgraph-vector-tests: - if: github.event.pull_request.draft == false + if: false # TEMPORARILY DISABLED + # if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: From 0d2601c42508d4dce1efa889a02aaa6f668cc9cf Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 17:22:57 -0500 Subject: [PATCH 11/25] Wait for clusters to be healthy before starting test --- systest/backup/nfs-backup/backup_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 1c18257302c..95b525595be 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -40,19 +40,31 @@ var ( func TestBackupHAClust(t *testing.T) { - backupRestoreTest(t, testutil.GetSockAddr(), testutil.GetSockAddrAlpha4Http(), + backupRestoreTest(t, "alpha1_backup_clust_ha", "zero1_backup_clust_ha", + testutil.GetSockAddr(), testutil.GetSockAddrAlpha4Http(), testutil.GetSockAddrZeroHttp(), backupDstHA, testutil.GetSockAddrHttp()) } func TestBackupNonHAClust(t *testing.T) { - backupRestoreTest(t, testutil.GetSockAddrAlpha7(), testutil.GetSockAddrAlpha8Http(), + backupRestoreTest(t, "alpha7_backup_clust_non_ha", "zero7_backup_clust_non_ha", + testutil.GetSockAddrAlpha7(), testutil.GetSockAddrAlpha8Http(), testutil.GetSockAddrZero7Http(), backupDstNonHA, testutil.GetSockAddrAlpha7Http()) } -func backupRestoreTest(t *testing.T, backupAlphaSocketAddr string, restoreAlphaAddr string, +func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName string, + backupAlphaSocketAddr string, restoreAlphaAddr string, backupZeroAddr string, backupDst string, backupAlphaSocketAddrHttp string) { + // Wait for containers to be healthy before proceeding + t.Logf("Waiting for %s to be healthy...", backupAlphaName) + backupAlpha := testutil.ContainerInstance{Name: backupAlphaName, Prefix: testutil.DockerPrefix} + require.NoError(t, backupAlpha.BestEffortWaitForHealthy(8080)) + + t.Logf("Waiting for %s to be healthy...", backupZeroName) + backupZero := testutil.ContainerInstance{Name: backupZeroName, Prefix: testutil.DockerPrefix} + require.NoError(t, backupZero.BestEffortWaitForHealthy(6080)) + conn, err := grpc.NewClient(backupAlphaSocketAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) require.NoError(t, err) dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) From 11301f2f7e2c9e5da688c83f6ea9918d35a83d65 Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 17:43:26 -0500 Subject: [PATCH 12/25] Add wait for gRPC connection --- systest/backup/nfs-backup/backup_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 95b525595be..ab226f00630 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -69,6 +69,11 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri require.NoError(t, err) dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) ctx := context.Background() + + // Wait for gRPC connection to be ready with retries + t.Log("Waiting for gRPC connection to be ready...") + testutil.RetryQuery(t, dg, `{ health { status } }`) + require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) // Add schema and types. require.NoError(t, dg.Alter(ctx, &api.Operation{Schema: `movie: string . From 42f1a99eac04aa02cf5fb2505909d57c1d956eea Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 18:05:22 -0500 Subject: [PATCH 13/25] Fix call to retry function --- systest/backup/nfs-backup/backup_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index ab226f00630..7881ec54a45 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -72,7 +72,8 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri // Wait for gRPC connection to be ready with retries t.Log("Waiting for gRPC connection to be ready...") - testutil.RetryQuery(t, dg, `{ health { status } }`) + _, err = testutil.RetryQuery(dg, `{ health { status } }`) + require.NoError(t, err) require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) // Add schema and types. From 5d8d664b3865b74ae7bd78539ee6908ee76a531e Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 18:27:10 -0500 Subject: [PATCH 14/25] Add more health checks --- systest/backup/nfs-backup/backup_test.go | 38 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 7881ec54a45..326f82c05cd 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" @@ -41,15 +42,21 @@ var ( func TestBackupHAClust(t *testing.T) { backupRestoreTest(t, "alpha1_backup_clust_ha", "zero1_backup_clust_ha", - testutil.GetSockAddr(), testutil.GetSockAddrAlpha4Http(), - testutil.GetSockAddrZeroHttp(), backupDstHA, testutil.GetSockAddrHttp()) + testutil.ContainerAddr("alpha1_backup_clust_ha", 9080), + testutil.ContainerAddr("alpha4_restore_clust_ha", 8080), + testutil.ContainerAddr("zero1_backup_clust_ha", 6080), + backupDstHA, + testutil.ContainerAddr("alpha1_backup_clust_ha", 8080)) } func TestBackupNonHAClust(t *testing.T) { backupRestoreTest(t, "alpha7_backup_clust_non_ha", "zero7_backup_clust_non_ha", - testutil.GetSockAddrAlpha7(), testutil.GetSockAddrAlpha8Http(), - testutil.GetSockAddrZero7Http(), backupDstNonHA, testutil.GetSockAddrAlpha7Http()) + testutil.ContainerAddr("alpha7_backup_clust_non_ha", 9080), + testutil.ContainerAddr("alpha8_restore_clust_non_ha", 8080), + testutil.ContainerAddr("zero7_backup_clust_non_ha", 6080), + backupDstNonHA, + testutil.ContainerAddr("alpha7_backup_clust_non_ha", 8080)) } func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName string, @@ -65,15 +72,28 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri backupZero := testutil.ContainerInstance{Name: backupZeroName, Prefix: testutil.DockerPrefix} require.NoError(t, backupZero.BestEffortWaitForHealthy(6080)) - conn, err := grpc.NewClient(backupAlphaSocketAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) + var dg *dgo.Dgraph + var err error ctx := context.Background() // Wait for gRPC connection to be ready with retries t.Log("Waiting for gRPC connection to be ready...") - _, err = testutil.RetryQuery(dg, `{ health { status } }`) - require.NoError(t, err) + for i := 0; i < 30; i++ { + conn, connErr := grpc.NewClient(backupAlphaSocketAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if connErr != nil { + t.Logf("Failed to create gRPC client (attempt %d/30): %v", i+1, connErr) + time.Sleep(time.Second) + continue + } + dg = dgo.NewDgraphClient(api.NewDgraphClient(conn)) + _, err = testutil.RetryQuery(dg, `{ health { status } }`) + if err == nil { + break + } + t.Logf("Health query failed (attempt %d/30): %v", i+1, err) + time.Sleep(time.Second) + } + require.NoError(t, err, "Failed to connect to gRPC after 30 attempts") require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) // Add schema and types. From a4b4ba194272a7cc58f5e3149f077192d408dbdb Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 19:30:49 -0500 Subject: [PATCH 15/25] Update health query --- systest/backup/nfs-backup/backup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 326f82c05cd..7313e34e7b6 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -86,7 +86,7 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri continue } dg = dgo.NewDgraphClient(api.NewDgraphClient(conn)) - _, err = testutil.RetryQuery(dg, `{ health { status } }`) + _, err = testutil.RetryQuery(dg, `schema {}`) if err == nil { break } From 198c2b8640ca48bc24e2d85208a0c28339616597 Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 19:49:41 -0500 Subject: [PATCH 16/25] Guarantee containers are fully registered with Docker before trying to get their port mappings --- systest/backup/nfs-backup/backup_test.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 7313e34e7b6..40aed66becf 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -42,26 +42,17 @@ var ( func TestBackupHAClust(t *testing.T) { backupRestoreTest(t, "alpha1_backup_clust_ha", "zero1_backup_clust_ha", - testutil.ContainerAddr("alpha1_backup_clust_ha", 9080), - testutil.ContainerAddr("alpha4_restore_clust_ha", 8080), - testutil.ContainerAddr("zero1_backup_clust_ha", 6080), - backupDstHA, - testutil.ContainerAddr("alpha1_backup_clust_ha", 8080)) + "alpha4_restore_clust_ha", backupDstHA) } func TestBackupNonHAClust(t *testing.T) { backupRestoreTest(t, "alpha7_backup_clust_non_ha", "zero7_backup_clust_non_ha", - testutil.ContainerAddr("alpha7_backup_clust_non_ha", 9080), - testutil.ContainerAddr("alpha8_restore_clust_non_ha", 8080), - testutil.ContainerAddr("zero7_backup_clust_non_ha", 6080), - backupDstNonHA, - testutil.ContainerAddr("alpha7_backup_clust_non_ha", 8080)) + "alpha8_restore_clust_non_ha", backupDstNonHA) } func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName string, - backupAlphaSocketAddr string, restoreAlphaAddr string, - backupZeroAddr string, backupDst string, backupAlphaSocketAddrHttp string) { + restoreAlphaName string, backupDst string) { // Wait for containers to be healthy before proceeding t.Logf("Waiting for %s to be healthy...", backupAlphaName) @@ -72,6 +63,12 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri backupZero := testutil.ContainerInstance{Name: backupZeroName, Prefix: testutil.DockerPrefix} require.NoError(t, backupZero.BestEffortWaitForHealthy(6080)) + // Resolve addresses after containers are healthy + backupAlphaSocketAddr := testutil.ContainerAddr(backupAlphaName, 9080) + backupAlphaSocketAddrHttp := testutil.ContainerAddr(backupAlphaName, 8080) + restoreAlphaAddr := testutil.ContainerAddr(restoreAlphaName, 8080) + backupZeroAddr := testutil.ContainerAddr(backupZeroName, 6080) + var dg *dgo.Dgraph var err error ctx := context.Background() From af85cb59f4a4560055b53488deaf6a95b17df572 Mon Sep 17 00:00:00 2001 From: mattthew Date: Tue, 4 Nov 2025 20:33:35 -0500 Subject: [PATCH 17/25] Limit test for debugging --- .github/workflows/ci-dgraph-systest-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index 90b2b0bf3dc..8eb6e239cfe 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -67,7 +67,8 @@ jobs: # move the binary cp dgraph/dgraph ~/go/bin/dgraph # run the unit and systests - cd t; ./t --suite=systest + # cd t; ./t --suite=systest + cd t; ./t --test=TestBackupHAClust # clean up docker containers after test execution ./t -r # sleep From 0e54111ac275cda8146d877833e569a4b2febd30 Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 16:16:59 -0500 Subject: [PATCH 18/25] Upgrade client creation (linter blocking) --- systest/backup/nfs-backup/backup_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 40aed66becf..dad72832514 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -76,13 +76,14 @@ func backupRestoreTest(t *testing.T, backupAlphaName string, backupZeroName stri // Wait for gRPC connection to be ready with retries t.Log("Waiting for gRPC connection to be ready...") for i := 0; i < 30; i++ { - conn, connErr := grpc.NewClient(backupAlphaSocketAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + var connErr error + dg, connErr = dgo.Open(fmt.Sprintf("dgraph://%s?sslmode=disable", backupAlphaSocketAddr)) if connErr != nil { - t.Logf("Failed to create gRPC client (attempt %d/30): %v", i+1, connErr) + err = connErr + t.Logf("Failed to create Dgraph client (attempt %d/30): %v", i+1, connErr) time.Sleep(time.Second) continue } - dg = dgo.NewDgraphClient(api.NewDgraphClient(conn)) _, err = testutil.RetryQuery(dg, `schema {}`) if err == nil { break From f2f16e7ca03a1d760f53d39b6d6467be58e2ee49 Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 16:24:59 -0500 Subject: [PATCH 19/25] Update security contact --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 90c2a08dd32..e27e3140bb2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,6 +5,6 @@ vulnerability in Dgraph, we encourage you to let us know right away. We will investigate all legitimate reports and do our best to quickly fix the problem. Please report any issues or vulnerabilities via GitHub Security Advisories instead of posting a public issue in -GitHub. You can also send security communications to security@hypermode.com. +GitHub. You can also send security communications to dgraph-admin@istaridigital.com. Please include the version identifier and details on how the vulnerability can be exploited. From 2d9f07ed2fe7aeb4c224dec46c2fcfcd1ffaf034 Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 17:44:06 -0500 Subject: [PATCH 20/25] Revert single test invocation --- .github/workflows/ci-dgraph-systest-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-dgraph-systest-tests.yml b/.github/workflows/ci-dgraph-systest-tests.yml index 8eb6e239cfe..90b2b0bf3dc 100644 --- a/.github/workflows/ci-dgraph-systest-tests.yml +++ b/.github/workflows/ci-dgraph-systest-tests.yml @@ -67,8 +67,7 @@ jobs: # move the binary cp dgraph/dgraph ~/go/bin/dgraph # run the unit and systests - # cd t; ./t --suite=systest - cd t; ./t --test=TestBackupHAClust + cd t; ./t --suite=systest # clean up docker containers after test execution ./t -r # sleep From 3313ed7de97382435816dc6bb20db01771467846 Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 17:45:16 -0500 Subject: [PATCH 21/25] Revert skipped tests --- .github/workflows/ci-dgraph-core-tests.yml | 3 +-- .github/workflows/ci-dgraph-core-upgrade-tests.yml | 3 +-- .github/workflows/ci-dgraph-fuzz.yml | 3 +-- .github/workflows/ci-dgraph-integration2-tests.yml | 3 +-- .github/workflows/ci-dgraph-ldbc-tests.yml | 3 +-- .github/workflows/ci-dgraph-load-tests.yml | 3 +-- .github/workflows/ci-dgraph-system-upgrade-tests-mt.yml | 3 +-- .../ci-dgraph-system-upgrade-tests-mutations-and-queries.yml | 3 +-- .github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml | 3 +-- .github/workflows/ci-dgraph-vector-tests.yml | 3 +-- 10 files changed, 10 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci-dgraph-core-tests.yml b/.github/workflows/ci-dgraph-core-tests.yml index 722d31c4d01..c9c0225b868 100644 --- a/.github/workflows/ci-dgraph-core-tests.yml +++ b/.github/workflows/ci-dgraph-core-tests.yml @@ -25,8 +25,7 @@ permissions: jobs: dgraph-core-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: diff --git a/.github/workflows/ci-dgraph-core-upgrade-tests.yml b/.github/workflows/ci-dgraph-core-upgrade-tests.yml index ed2231dec47..f88eb9b8845 100644 --- a/.github/workflows/ci-dgraph-core-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-core-upgrade-tests.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-upgrade-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: diff --git a/.github/workflows/ci-dgraph-fuzz.yml b/.github/workflows/ci-dgraph-fuzz.yml index 5e9d232cab9..4e0f9015743 100644 --- a/.github/workflows/ci-dgraph-fuzz.yml +++ b/.github/workflows/ci-dgraph-fuzz.yml @@ -22,8 +22,7 @@ permissions: jobs: fuzz-test: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index 064fdb39fdb..40c3ef28c9d 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-integration2-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-ldbc-tests.yml b/.github/workflows/ci-dgraph-ldbc-tests.yml index f4c3eb12370..6b56855ea79 100644 --- a/.github/workflows/ci-dgraph-ldbc-tests.yml +++ b/.github/workflows/ci-dgraph-ldbc-tests.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-ldbc-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: diff --git a/.github/workflows/ci-dgraph-load-tests.yml b/.github/workflows/ci-dgraph-load-tests.yml index be5551f64d1..0aee7076e43 100644 --- a/.github/workflows/ci-dgraph-load-tests.yml +++ b/.github/workflows/ci-dgraph-load-tests.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-load-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml index fc08d4bef7b..f30f6cba304 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mt.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-upgrade-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml index 1555604d6ba..6e8a043842b 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-mutations-and-queries.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-upgrade-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml index 42af5f715a5..10dfe7767a3 100644 --- a/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml +++ b/.github/workflows/ci-dgraph-system-upgrade-tests-plugin.yml @@ -23,8 +23,7 @@ permissions: jobs: dgraph-upgrade-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 90 steps: diff --git a/.github/workflows/ci-dgraph-vector-tests.yml b/.github/workflows/ci-dgraph-vector-tests.yml index ab5cf897dc7..d0b6ca88201 100644 --- a/.github/workflows/ci-dgraph-vector-tests.yml +++ b/.github/workflows/ci-dgraph-vector-tests.yml @@ -25,8 +25,7 @@ permissions: jobs: dgraph-vector-tests: - if: false # TEMPORARILY DISABLED - # if: github.event.pull_request.draft == false + if: github.event.pull_request.draft == false runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 steps: From ad251ea44f2b2d65aff435f872b794d91a51765b Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 19:04:06 -0500 Subject: [PATCH 22/25] Remove unused imports --- systest/backup/nfs-backup/backup_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index dad72832514..2c6da0852c0 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -21,8 +21,6 @@ import ( "time" "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "github.com/dgraph-io/dgo/v250" "github.com/dgraph-io/dgo/v250/protos/api" From 54583bc52884c790dc47acc0616726fc1a42c998 Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 19:25:21 -0500 Subject: [PATCH 23/25] Skipping flaky test --- systest/backup/nfs-backup/backup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 2c6da0852c0..1c9911460e9 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -38,7 +38,7 @@ var ( ) func TestBackupHAClust(t *testing.T) { - + t.Skip("Skipping HA backup test via NFS") backupRestoreTest(t, "alpha1_backup_clust_ha", "zero1_backup_clust_ha", "alpha4_restore_clust_ha", backupDstHA) } From 4012aba052586b44c23bb3fd85fd93428f9dde4b Mon Sep 17 00:00:00 2001 From: mattthew Date: Wed, 19 Nov 2025 19:52:41 -0500 Subject: [PATCH 24/25] Skipping flaky test --- systest/backup/nfs-backup/backup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systest/backup/nfs-backup/backup_test.go b/systest/backup/nfs-backup/backup_test.go index 1c9911460e9..aa9f609428f 100644 --- a/systest/backup/nfs-backup/backup_test.go +++ b/systest/backup/nfs-backup/backup_test.go @@ -44,7 +44,7 @@ func TestBackupHAClust(t *testing.T) { } func TestBackupNonHAClust(t *testing.T) { - + t.Skip("Skipping Non-HA backup test via NFS") backupRestoreTest(t, "alpha7_backup_clust_non_ha", "zero7_backup_clust_non_ha", "alpha8_restore_clust_non_ha", backupDstNonHA) } From 6de39e235c5f5cdd25e959c22a77f553e5977f83 Mon Sep 17 00:00:00 2001 From: mattthew Date: Thu, 20 Nov 2025 13:09:58 -0500 Subject: [PATCH 25/25] Add protection for git operations --- dgraphtest/image.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dgraphtest/image.go b/dgraphtest/image.go index 6a83d1c967a..a472486e551 100644 --- a/dgraphtest/image.go +++ b/dgraphtest/image.go @@ -21,6 +21,7 @@ import ( var ( cloneOnce sync.Once + gitMutex sync.Mutex // Protects git operations on shared repoDir ) func (c *LocalCluster) dgraphImage() string { @@ -49,7 +50,23 @@ func (c *LocalCluster) setupBinary() error { return copyBinary(fromDir, c.tempBinDir, c.conf.version) } - isFileThere, err := fileExists(filepath.Join(binariesPath, fmt.Sprintf(binaryNameFmt, c.conf.version))) + binaryPath := filepath.Join(binariesPath, fmt.Sprintf(binaryNameFmt, c.conf.version)) + + // First check without lock (fast path) + isFileThere, err := fileExists(binaryPath) + if err != nil { + return err + } + if isFileThere { + return copyBinary(binariesPath, c.tempBinDir, c.conf.version) + } + + // Lock git operations to prevent parallel tests from conflicting + gitMutex.Lock() + defer gitMutex.Unlock() + + // Double-check after acquiring lock - another parallel test may have built it + isFileThere, err = fileExists(binaryPath) if err != nil { return err }