Skip to content

Commit f6e5e74

Browse files
committed
Merge branch 'main' into fix/shiva-test-mac-os
Resolve conflicts in Makefile and CONTRIBUTING.md, keeping the PR's dual-command default (unit,systest,core + integration2) and the SUITE=all example line.
2 parents 4453939 + 1a7fff6 commit f6e5e74

29 files changed

Lines changed: 2726 additions & 90 deletions

compose/cluster-viz.sh

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
#!/bin/bash
2+
# Visualize Dgraph cluster topology from docker-compose.yml
3+
4+
FILE="${1:-docker-compose.yml}"
5+
6+
if [[ ! -f ${FILE} ]]; then
7+
echo "Usage: $0 [docker-compose.yml]"
8+
exit 1
9+
fi
10+
11+
# Flatten the file (join continuation lines) for easier parsing
12+
FLAT=$(tr '\n' ' ' <"${FILE}" | sed 's/ */ /g' || true)
13+
14+
# Extract zero nodes (find --my=zeroN:port patterns near "zero")
15+
zeros=$(echo "${FLAT}" | grep -oE '\-\-my=zero[a-zA-Z0-9_-]*:[0-9]+' | sed -E 's/--my=([^:]+):.*/\1/' | sort -u || true)
16+
17+
# Extract alpha nodes with their group assignments
18+
TMPFILE=$(mktemp)
19+
20+
# Extract all alphas by their --my flag. Group assignment is optional in flags.
21+
# If no group= is found near the alpha's command, put it under UNASSIGNED.
22+
alphas=$(echo "${FLAT}" | grep -oE '\-\-my=alpha[a-zA-Z0-9_-]*:[0-9]+' | sed -E 's/--my=([^:]+):.*/\1/' | sort -u || true)
23+
for alpha in ${alphas}; do
24+
# Look for group= near this alpha occurrence in the flattened compose.
25+
# Many compose files won't specify it; in that case group is assigned dynamically by Zero.
26+
# Use awk (portable on macOS) instead of BSD sed regex intervals.
27+
group=$(awk -v s="${FLAT}" -v a="--my=${alpha}:" '
28+
BEGIN { pos = index(s, a); if (pos == 0) exit }
29+
END {
30+
win = substr(s, pos, 600)
31+
if (match(win, /group=[0-9]+/)) {
32+
g = substr(win, RSTART, RLENGTH)
33+
split(g, parts, "=")
34+
print parts[2]
35+
}
36+
}
37+
' </dev/null || true)
38+
if [[ -z ${group} ]]; then
39+
group="UNASSIGNED"
40+
fi
41+
echo "${group} ${alpha}"
42+
done >"${TMPFILE}"
43+
44+
# Get unique groups (numeric groups first, UNASSIGNED last, never duplicated)
45+
numeric_groups=$(grep -E '^[0-9]+ ' "${TMPFILE}" | cut -d' ' -f1 | sort -n | uniq || true)
46+
unique_groups="${numeric_groups}"
47+
if grep -q '^UNASSIGNED ' "${TMPFILE}"; then
48+
unique_groups="${unique_groups} UNASSIGNED"
49+
fi
50+
51+
# Extract ratel if present
52+
has_ratel=$(grep -q 'ratel' "${FILE}" && echo "yes" || echo "no")
53+
54+
# Get replica count
55+
replicas=$(grep -oE '\-\-replicas=[0-9]+' "${FILE}" | head -1 | cut -d= -f2 || true)
56+
replicas=${replicas:-1}
57+
58+
# Calculate max alphas per group for width calculation
59+
max_per_group=0
60+
for group in ${unique_groups}; do
61+
cnt=$(grep -c "^${group} " "${TMPFILE}" 2>/dev/null || echo 0)
62+
if [[ ${cnt} -gt ${max_per_group} ]]; then
63+
max_per_group=${cnt}
64+
fi
65+
done
66+
67+
# Count zeros
68+
zero_count=$(echo "${zeros}" | wc -w | tr -d ' ' || true)
69+
70+
# Box width: 14 chars per node (┌──────────┐ + 2 spaces) + margins
71+
# Use the larger of zeros or max alphas per group
72+
max_nodes=${zero_count}
73+
if [[ ${max_per_group} -gt ${max_nodes} ]]; then
74+
max_nodes=${max_per_group}
75+
fi
76+
[[ ${max_nodes} -lt 3 ]] && max_nodes=3
77+
78+
inner_width=$((max_nodes * 14 + 4))
79+
[[ ${inner_width} -lt 60 ]] && inner_width=60
80+
outer_width=$((inner_width + 4))
81+
82+
# Total line width (display columns between outer left and right border)
83+
W=$((outer_width))
84+
85+
# Print N space characters
86+
sp() {
87+
local i
88+
for ((i = 0; i < $1; i++)); do printf " "; done
89+
}
90+
91+
# Print N copies of a character
92+
rp() {
93+
local i
94+
for ((i = 0; i < $1; i++)); do printf "%s" "$2"; done
95+
}
96+
97+
# Finish outer row: pad from current column to W, then print |
98+
fin() {
99+
sp "$((W - $1))"
100+
printf "|\n"
101+
}
102+
103+
# Finish inner row (inside group box): pad then print | |
104+
# Total should be W+1 cols: content($1) + spaces + 3(| |) = W+1
105+
fini() {
106+
sp "$((W - $1 - 2))"
107+
printf "| |\n"
108+
}
109+
110+
# Separator using full width
111+
sep() {
112+
printf "%s" "$1"
113+
rp "${W}" "$2"
114+
printf "%s\n" "$3"
115+
}
116+
117+
# Print a row that starts with '|' and does NOT yet include the final right border.
118+
# Pads with spaces so the final '|' is always aligned.
119+
outln() {
120+
local s="$1"
121+
local l=${#s}
122+
# Total line length must be W + 2 (left border + interior + right border).
123+
# If s already includes the left border '|', it must be padded to length W+1,
124+
# then we print the final right border.
125+
printf "%s" "${s}"
126+
local pad=$((W + 1 - l))
127+
if [[ ${pad} -lt 0 ]]; then
128+
pad=0
129+
fi
130+
sp "${pad}"
131+
printf "|\n"
132+
}
133+
134+
# Render a row inside the group box.
135+
# The group box top uses: "| +" + GW*"-" + "+".
136+
# For interior rows we use: "| |" + <payload> + <pad-to-GW> + "|".
137+
out_group() {
138+
local payload="$1"
139+
local plen=${#payload}
140+
local pad=$((GW - plen))
141+
if [[ ${pad} -lt 0 ]]; then
142+
pad=0
143+
fi
144+
local s="| |${payload}"
145+
# pad inside the group box up to width GW
146+
for ((i = 0; i < pad; i++)); do
147+
s+=" "
148+
done
149+
s+="|"
150+
outln "${s}"
151+
}
152+
153+
echo ""
154+
155+
# === HEADER ===
156+
sep "+" "=" "+"
157+
outln "| DGRAPH CLUSTER TOPOLOGY"
158+
sep "+" "=" "+"
159+
echo ""
160+
161+
# === ZERO LAYER ===
162+
sep "+" "-" "+"
163+
outln "| ZERO LAYER (Cluster Coordination)"
164+
sep "+" "-" "+"
165+
166+
# Zero boxes: each is 14 cols (box 12 + 2 spaces)
167+
line="| "
168+
for z in ${zeros}; do line+="+----------+ "; done
169+
outln "${line}"
170+
line="| "
171+
for z in ${zeros}; do line+=$(printf "| %-8s | " "${z}"); done
172+
outln "${line}"
173+
line="| "
174+
for z in ${zeros}; do line+="| (zero) | "; done
175+
outln "${line}"
176+
line="| "
177+
for z in ${zeros}; do line+="+----------+ "; done
178+
outln "${line}"
179+
180+
sep "+" "-" "+"
181+
echo ""
182+
183+
# === ALPHA LAYER ===
184+
sep "+" "-" "+"
185+
outln "| ALPHA LAYER (Data Storage) - Replicas: ${replicas}"
186+
sep "+" "-" "+"
187+
188+
# Group box inner width
189+
GW=$((inner_width - 2))
190+
191+
for group in ${unique_groups}; do
192+
alphas=$(grep "^${group} " "${TMPFILE}" | cut -d' ' -f2 || true)
193+
count=$(echo "${alphas}" | wc -l | tr -d ' ' || true)
194+
195+
# Empty row
196+
outln "|"
197+
198+
# Group box top
199+
line="| +"
200+
for ((i = 0; i < GW; i++)); do line+="-"; done
201+
line+="+"
202+
outln "${line}"
203+
204+
# Group title row
205+
if [[ ${group} == "UNASSIGNED" ]]; then
206+
gtitle=" GROUP UNASSIGNED (Raft Replicas: ${count})"
207+
else
208+
gtitle=" GROUP ${group} (Raft Replicas: ${count})"
209+
fi
210+
out_group "${gtitle}"
211+
212+
# Alpha boxes
213+
line=" "
214+
for a in ${alphas}; do line+="+----------+ "; done
215+
out_group "${line}"
216+
line=" "
217+
for a in ${alphas}; do line+=$(printf "| %-8s | " "${a}" || true); done
218+
out_group "${line}"
219+
line=" "
220+
for a in ${alphas}; do line+="| (alpha) | "; done
221+
out_group "${line}"
222+
line=" "
223+
for a in ${alphas}; do line+="+----------+ "; done
224+
out_group "${line}"
225+
226+
# Group box bottom
227+
line="| +"
228+
for ((i = 0; i < GW; i++)); do line+="-"; done
229+
line+="+"
230+
outln "${line}"
231+
done
232+
233+
outln "|"
234+
sep "+" "-" "+"
235+
236+
# === RATEL ===
237+
if [[ ${has_ratel} == "yes" ]]; then
238+
echo ""
239+
sep "+" "-" "+"
240+
outln "| UI LAYER"
241+
sep "+" "-" "+"
242+
outln "| +----------+"
243+
outln "| | ratel | (Web UI on :8000)"
244+
outln "| +----------+"
245+
sep "+" "-" "+"
246+
fi
247+
248+
rm -f "${TMPFILE}"
249+
250+
echo ""
251+
echo "Legend: Alphas in same GROUP replicate data via Raft consensus"
252+
echo ""

compose/compose.go

Lines changed: 55 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ type options struct {
7979
DataVol bool
8080
TmpFS bool
8181
UserOwnership bool
82-
Jaeger bool
82+
Jaeger int
83+
TraceRatio string
84+
TraceService bool
8385
Metrics bool
8486
PortOffset int
8587
Verbosity int
@@ -202,8 +204,15 @@ func initService(basename string, idx, grpcPort int) service {
202204
svc.Command += fmt.Sprintf(" --cwd=/data/%s", svc.name)
203205
}
204206
svc.Command += " " + basename
205-
if opts.Jaeger {
206-
svc.Command += ` --trace "jaeger=http://jaeger:14268;"`
207+
if opts.Jaeger > 0 {
208+
traceFlag := "jaeger=http://jaeger:4318;"
209+
if opts.TraceRatio != "" {
210+
traceFlag += fmt.Sprintf(" ratio=%s;", opts.TraceRatio)
211+
}
212+
if opts.TraceService {
213+
traceFlag += fmt.Sprintf(" service=%s;", svc.name)
214+
}
215+
svc.Command += fmt.Sprintf(` --trace "%s"`, traceFlag)
207216
}
208217
return svc
209218
}
@@ -375,23 +384,44 @@ func getVolume(vol string) volume {
375384

376385
}
377386

378-
func getJaeger() service {
379-
svc := service{
380-
Image: "jaegertracing/all-in-one:1.18",
387+
func getJaeger(version int) service {
388+
if version == 2 {
389+
// Jaeger v2.x uses OpenTelemetry Collector architecture with YAML config.
390+
// OTLP receivers bind to localhost by default, so we need JAEGER_LISTEN_HOST=0.0.0.0
391+
// for container environments.
392+
return service{
393+
Image: "jaegertracing/jaeger:latest",
394+
ContainerName: containerName("jaeger"),
395+
Ports: []string{
396+
toPort(4318),
397+
toPort(16686),
398+
},
399+
Environment: []string{
400+
"JAEGER_LISTEN_HOST=0.0.0.0",
401+
},
402+
}
403+
}
404+
// Jaeger v1.60 with badger storage
405+
return service{
406+
Image: "jaegertracing/all-in-one:1.60",
381407
ContainerName: containerName("jaeger"),
382-
WorkingDir: "/working/jaeger",
408+
User: "0",
383409
Ports: []string{
384-
toPort(14268),
410+
toPort(4318),
385411
toPort(16686),
386412
},
413+
Volumes: []volume{{
414+
Type: "volume",
415+
Source: "jaeger-volume",
416+
Target: "/jaeger",
417+
}},
387418
Environment: []string{
388419
"SPAN_STORAGE_TYPE=badger",
389420
},
390421
Command: "--badger.ephemeral=false" +
391-
" --badger.directory-key /working/jaeger" +
392-
" --badger.directory-value /working/jaeger",
422+
" --badger.directory-key /jaeger" +
423+
" --badger.directory-value /jaeger",
393424
}
394-
return svc
395425
}
396426

397427
func getMinio(minioDataDir string) service {
@@ -421,12 +451,12 @@ func getRatel() service {
421451
portFlag = fmt.Sprintf(" -port=%d", opts.RatelPort)
422452
}
423453
svc := service{
424-
Image: opts.Image + ":" + opts.Tag,
454+
Image: "dgraph/ratel:latest",
425455
ContainerName: containerName("ratel"),
426456
Ports: []string{
427457
toPort(opts.RatelPort),
428458
},
429-
Command: "dgraph-ratel" + portFlag,
459+
Command: portFlag,
430460
}
431461
return svc
432462
}
@@ -543,8 +573,13 @@ func main() {
543573
"run as the current user rather than root")
544574
cmd.PersistentFlags().BoolVar(&opts.TmpFS, "tmpfs", false,
545575
"store w and zw directories on a tmpfs filesystem")
546-
cmd.PersistentFlags().BoolVarP(&opts.Jaeger, "jaeger", "j", false,
547-
"include jaeger service")
576+
cmd.PersistentFlags().IntVarP(&opts.Jaeger, "jaeger", "j", 0,
577+
"include jaeger service (1 for v1.60, 2 for v2.x)")
578+
cmd.PersistentFlags().Lookup("jaeger").NoOptDefVal = "1"
579+
cmd.PersistentFlags().StringVar(&opts.TraceRatio, "trace_ratio", "",
580+
"ratio of queries to trace (e.g., 0.01 for 1%)")
581+
cmd.PersistentFlags().BoolVar(&opts.TraceService, "trace_service", false,
582+
"use compose service name as trace service name (e.g., alpha1, zero1)")
548583
cmd.PersistentFlags().BoolVarP(&opts.Metrics, "metrics", "m", false,
549584
"include metrics (prometheus, grafana) services")
550585
cmd.PersistentFlags().IntVarP(&opts.PortOffset, "port_offset", "o", 100,
@@ -689,8 +724,11 @@ func main() {
689724
cfg.Volumes["data"] = stringMap{}
690725
}
691726

692-
if opts.Jaeger {
693-
services["jaeger"] = getJaeger()
727+
if opts.Jaeger > 0 {
728+
services["jaeger"] = getJaeger(opts.Jaeger)
729+
if opts.Jaeger == 1 {
730+
cfg.Volumes["jaeger-volume"] = stringMap{}
731+
}
694732
}
695733

696734
if opts.Ratel {

contrib/jepsen/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ var (
116116
testCount = pflag.IntP("test-count", "c", 1, "Test count per Jepsen test.")
117117
jaeger = pflag.StringP("jaeger", "j", "",
118118
"Run with Jaeger collector. Set to empty string to disable collection to Jaeger."+
119-
" Otherwise set to http://jaeger:14268.")
119+
" Otherwise set to http://jaeger:4318.")
120120
jaegerSaveTraces = pflag.Bool("jaeger-save-traces", true, "Save Jaeger traces on test error.")
121121
deferDbTeardown = pflag.Bool("defer-db-teardown", false,
122122
"Wait until user input to tear down DB nodes")

0 commit comments

Comments
 (0)