Skip to content

Commit cefc428

Browse files
johnpgarrykawasaki
johnpgarry
authored and
kawasaki
committed
md/rc: add _md_atomics_test
The stacked device atomic writes testing is currently limited. md/002 currently only tests scsi_debug. SCSI does not support atomic boundaries, so it would be nice to test NVMe (which does support them). Furthermore, the testing in md/002 for chunk boundaries is very limited, in that we test once one boundary value. Indeed, for RAID0 and RAID10, a boundary should always be set for testing. Finally, md/002 only tests md RAID0/1/10. In future we will also want to test the following stacked device personalities which support atomic writes: - md-linear (being upstreamed) - dm-linear - dm-stripe - dm-mirror To solve all those problems, add a generic test handler, _md_atomics_test(). This can be extended for more extensive testing. This test handler will accept a group of devices and test as follows: a. calculate expected atomic write limits based on device limits b. Take results from a., and refine expected limits based on any chunk size c. loop through creating a stacked device for different chunk size. We loop once for any personality which does not have a chunk size, e.g. RAID1 d. test sysfs and statx limits vs what is calculated in a. and b. e. test RWF_ATOMIC is accepted or rejected as expected Steps c, d, and e are really same as md/002. Signed-off-by: John Garry <[email protected]> [Shin'ichiro: fixed a shellcheck warning] Signed-off-by: Shin'ichiro Kawasaki <[email protected]>
1 parent 9104d53 commit cefc428

1 file changed

Lines changed: 377 additions & 0 deletions

File tree

  • tests/md

‎tests/md/rc‎

Lines changed: 377 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,386 @@
55
# Tests for md raid
66

77
. common/rc
8+
. common/xfs
89

910
group_requires() {
1011
_have_root
1112
_have_program mdadm
1213
_have_driver md-mod
1314
}
15+
16+
_stacked_atomic_test_requires() {
17+
_have_kver 6 14 0
18+
_have_xfs_io_atomic_write
19+
_have_driver raid0
20+
_have_driver raid1
21+
_have_driver raid10
22+
}
23+
24+
_max_pow_of_two_factor() {
25+
local part1=$1
26+
local part2=-$1
27+
local retval=$((part1 & part2))
28+
echo "$retval"
29+
}
30+
31+
# Find max atomic size given a boundary and chunk size
32+
# @unit is set if we want atomic write "unit" size, i.e power-of-2
33+
# @chunk must be > 0
34+
_md_atomics_boundaries_max() {
35+
local boundary=$1
36+
local chunk=$2
37+
local unit=$3
38+
local retval
39+
40+
if [ "$boundary" -eq 0 ]
41+
then
42+
if [ "$unit" -eq 1 ]
43+
then
44+
retval=$(_max_pow_of_two_factor "$chunk")
45+
echo "$retval"
46+
return
47+
fi
48+
49+
echo "$chunk"
50+
return
51+
fi
52+
53+
# boundary is always a power-of-2
54+
if [ "$boundary" -eq "$chunk" ]
55+
then
56+
echo "$boundary"
57+
return
58+
fi
59+
60+
if [ "$boundary" -gt "$chunk" ]
61+
then
62+
if (( boundary % chunk == 0))
63+
then
64+
if [ "$unit" -eq 1 ]
65+
then
66+
retval=$(_max_pow_of_two_factor "$chunk")
67+
echo "$retval"
68+
return
69+
fi
70+
echo "$chunk"
71+
return
72+
fi
73+
echo "0"
74+
return
75+
fi
76+
77+
if (( chunk % boundary == 0))
78+
then
79+
echo "$boundary"
80+
return
81+
fi
82+
83+
echo "0"
84+
}
85+
86+
declare -A MD_DEVICES
87+
88+
_md_atomics_test() {
89+
local md_sysfs_max_hw_sectors_kb
90+
local md_sysfs_max_hw
91+
local md_chunk_size
92+
local sysfs_logical_block_size
93+
local sysfs_atomic_write_max
94+
local sysfs_atomic_write_unit_min
95+
local sysfs_atomic_write_unit_max
96+
local bytes_to_write
97+
local bytes_written
98+
local test_desc
99+
local md_dev
100+
local md_dev_sysfs
101+
local raw_atomic_write_unit_min
102+
local raw_atomic_write_unit_max
103+
local raw_atomic_write_max
104+
local raw_atomic_write_boundary
105+
local raw_atomic_write_supported=1
106+
local dev0=$1
107+
local dev1=$2
108+
local dev2=$3
109+
local dev3=$4
110+
111+
unset MD_DEVICES
112+
MD_DEVICES=([0]=$dev0 [1]=$dev1 [2]=$dev2 [3]=$dev3);
113+
114+
# Calculate what we expect the atomic write limits to be
115+
# Don't consider any chunk size at this stage
116+
# Use the limits from the first device and then loop again to find
117+
# lowest common supported
118+
raw_atomic_write_unit_min=$(< /sys/block/"$dev0"/queue/atomic_write_unit_min_bytes);
119+
raw_atomic_write_unit_max=$(< /sys/block/"$dev0"/queue/atomic_write_unit_max_bytes);
120+
raw_atomic_write_max=$(< /sys/block/"$dev0"/queue/atomic_write_max_bytes);
121+
raw_atomic_write_boundary=$(< /sys/block/"$dev0"/queue/atomic_write_boundary_bytes);
122+
123+
for i in "${MD_DEVICES[@]}"; do
124+
if [[ $(< /sys/block/"$i"/queue/atomic_write_unit_min_bytes) -gt raw_atomic_write_unit_min ]]; then
125+
raw_atomic_write_unit_min=$(< /sys/block/"$i"/queue/atomic_write_unit_min_bytes)
126+
fi
127+
if [[ $(< /sys/block/"$i"/queue/atomic_write_unit_max_bytes) -lt raw_atomic_write_unit_max ]]; then
128+
raw_atomic_write_unit_max=$(< /sys/block/"$i"/queue/atomic_write_unit_max_bytes)
129+
fi
130+
if [[ $(< /sys/block/"$i"/queue/atomic_write_max_bytes) -lt raw_atomic_write_max ]]; then
131+
raw_atomic_write_max=$(< /sys/block/"$i"/queue/atomic_write_max_bytes)
132+
fi
133+
# The kernel only supports same boundary size for all devices in the array
134+
if [[ $(< /sys/block/"$i"/queue/atomic_write_boundary_bytes) -ne raw_atomic_write_boundary ]]; then
135+
raw_atomic_write_supported=0;
136+
fi
137+
done
138+
139+
# Check if we can support atomic writes for the array of devices given.
140+
# If we cannot, then it is still worth trying to test that atomic
141+
# writes don't work (as we would expect).
142+
143+
if [[ raw_atomic_write_supported -eq 0 ]]; then
144+
raw_atomic_write_unit_min=0;
145+
raw_atomic_write_unit_max=0;
146+
raw_atomic_write_max=0;
147+
raw_atomic_write_boundary=0;
148+
fi
149+
150+
for personality in raid0 raid1 raid10; do
151+
local step_limit
152+
if [ "$personality" = raid0 ] || [ "$personality" = raid10 ]
153+
then
154+
step_limit=4
155+
else
156+
step_limit=1
157+
fi
158+
chunk_gran=$(( "$raw_atomic_write_unit_max" / 2))
159+
if [ "$chunk_gran" -lt 4096 ]
160+
then
161+
chunk_gran=4096
162+
fi
163+
164+
local chunk_multiple=1
165+
for step in $(seq 1 $step_limit)
166+
do
167+
local expected_atomic_write_unit_min
168+
local expected_atomic_write_unit_max
169+
local expected_atomic_write_max
170+
local expected_atomic_write_boundary
171+
local atomics_boundaries_unit_max
172+
local atomics_boundaries_max
173+
174+
# only raid0 does not require a power-of-2 chunk size
175+
if [ "$personality" = raid0 ]
176+
then
177+
chunk_multiple=$step
178+
else
179+
chunk_multiple=$(( 2 * "$chunk_multiple"))
180+
fi
181+
md_chunk_size=$(( "$chunk_gran" * "$chunk_multiple"))
182+
md_chunk_size_kb=$(( "$md_chunk_size" / 1024))
183+
184+
# We may reassign these for RAID0/10
185+
expected_atomic_write_unit_min=$raw_atomic_write_unit_min
186+
expected_atomic_write_unit_max=$raw_atomic_write_unit_max
187+
expected_atomic_write_max=$raw_atomic_write_max
188+
expected_atomic_write_boundary=$raw_atomic_write_boundary
189+
190+
if [ "$personality" = raid0 ] || [ "$personality" = raid10 ]
191+
then
192+
echo y | mdadm --create /dev/md/blktests_md --level=$personality \
193+
--chunk="${md_chunk_size_kb}"K \
194+
--raid-devices=4 --force /dev/"${dev0}" /dev/"${dev1}" \
195+
/dev/"${dev2}" /dev/"${dev3}" 2> /dev/null 1>&2
196+
197+
atomics_boundaries_unit_max=$(_md_atomics_boundaries_max "$raw_atomic_write_boundary" $md_chunk_size "1")
198+
atomics_boundaries_max=$(_md_atomics_boundaries_max "$raw_atomic_write_boundary" "$md_chunk_size" "0")
199+
expected_atomic_write_unit_min=$(_min "$expected_atomic_write_unit_min" "$atomics_boundaries_unit_max")
200+
expected_atomic_write_unit_max=$(_min "$expected_atomic_write_unit_max" "$atomics_boundaries_unit_max")
201+
expected_atomic_write_max=$(_min "$expected_atomic_write_max" "$atomics_boundaries_max")
202+
if [ "$atomics_boundaries_max" -eq 0 ]
203+
then
204+
expected_atomic_write_boundary=0
205+
fi
206+
md_dev=$(readlink /dev/md/blktests_md | sed 's|\.\./||')
207+
fi
208+
209+
if [ "$personality" = raid1 ]
210+
then
211+
echo y | mdadm --create /dev/md/blktests_md --level=$personality \
212+
--raid-devices=4 --force /dev/"${dev0}" /dev/"${dev1}" \
213+
/dev/"${dev2}" /dev/"${dev3}" 2> /dev/null 1>&2
214+
215+
md_dev=$(readlink /dev/md/blktests_md | sed 's|\.\./||')
216+
fi
217+
218+
md_dev_sysfs="/sys/devices/virtual/block/${md_dev}"
219+
220+
sysfs_logical_block_size=$(< "${md_dev_sysfs}"/queue/logical_block_size)
221+
md_sysfs_max_hw_sectors_kb=$(< "${md_dev_sysfs}"/queue/max_hw_sectors_kb)
222+
md_sysfs_max_hw=$(( "$md_sysfs_max_hw_sectors_kb" * 1024 ))
223+
sysfs_atomic_write_max=$(< "${md_dev_sysfs}"/queue/atomic_write_max_bytes)
224+
sysfs_atomic_write_unit_max=$(< "${md_dev_sysfs}"/queue/atomic_write_unit_max_bytes)
225+
sysfs_atomic_write_unit_min=$(< "${md_dev_sysfs}"/queue/atomic_write_unit_min_bytes)
226+
sysfs_atomic_write_boundary=$(< "${md_dev_sysfs}"/queue/atomic_write_boundary_bytes)
227+
228+
test_desc="TEST 1 $personality step $step - Verify md sysfs atomic attributes matches"
229+
if [ "$sysfs_atomic_write_unit_min" = "$expected_atomic_write_unit_min" ] &&
230+
[ "$sysfs_atomic_write_unit_max" = "$expected_atomic_write_unit_max" ]
231+
then
232+
echo "$test_desc - pass"
233+
else
234+
echo "$test_desc - fail sysfs_atomic_write_unit_min=$sysfs_atomic_write_unit_min" \
235+
"expected_atomic_write_unit_min=$expected_atomic_write_unit_min" \
236+
"sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
237+
"expected_atomic_write_unit_max=$expected_atomic_write_unit_max" \
238+
"md_chunk_size=$md_chunk_size"
239+
fi
240+
241+
test_desc="TEST 2 $personality step $step - Verify sysfs atomic attributes"
242+
if [ "$md_sysfs_max_hw" -ge "$sysfs_atomic_write_max" ] &&
243+
[ "$sysfs_atomic_write_unit_max" -ge "$sysfs_atomic_write_unit_min" ] &&
244+
[ "$sysfs_atomic_write_max" -ge "$sysfs_atomic_write_unit_max" ]
245+
then
246+
echo "$test_desc - pass"
247+
else
248+
echo "$test_desc - fail md_sysfs_max_hw=$md_sysfs_max_hw" \
249+
"sysfs_atomic_write_max=$sysfs_atomic_write_max" \
250+
"sysfs_atomic_write_unit_min=$sysfs_atomic_write_unit_min" \
251+
"sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
252+
"md_chunk_size=$md_chunk_size"
253+
fi
254+
255+
test_desc="TEST 3 $personality step $step - Verify md sysfs_atomic_write_max is equal to "
256+
test_desc+="expected_atomic_write_max"
257+
if [ "$sysfs_atomic_write_max" -eq "$expected_atomic_write_max" ]
258+
then
259+
echo "$test_desc - pass"
260+
else
261+
echo "$test_desc - fail sysfs_atomic_write_max=$sysfs_atomic_write_max" \
262+
"expected_atomic_write_max=$expected_atomic_write_max" \
263+
"md_chunk_size=$md_chunk_size"
264+
fi
265+
266+
test_desc="TEST 4 $personality step $step - Verify sysfs atomic_write_unit_max_bytes = expected_atomic_write_unit_max"
267+
if [ "$sysfs_atomic_write_unit_max" = "$expected_atomic_write_unit_max" ]
268+
then
269+
echo "$test_desc - pass"
270+
else
271+
echo "$test_desc - fail sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
272+
"expected_atomic_write_unit_max=$expected_atomic_write_unit_max" \
273+
"md_chunk_size=$md_chunk_size"
274+
fi
275+
276+
test_desc="TEST 5 $personality step $step - Verify sysfs atomic_write_unit_boundary_bytes = expected atomic_write_unit_boundary_bytes"
277+
if [ "$sysfs_atomic_write_boundary" = "$expected_atomic_write_boundary" ]
278+
then
279+
echo "$test_desc - pass"
280+
else
281+
echo "$test_desc - fail sysfs_atomic_write_boundary=$sysfs_atomic_write_boundary" \
282+
"expected_atomic_write_boundary=$expected_atomic_write_boundary"
283+
fi
284+
285+
test_desc="TEST 6 $personality step $step - Verify statx stx_atomic_write_unit_min"
286+
statx_atomic_write_unit_min=$(run_xfs_io_xstat /dev/"$md_dev" "stat.atomic_write_unit_min")
287+
if [ "$statx_atomic_write_unit_min" = "$sysfs_atomic_write_unit_min" ]
288+
then
289+
echo "$test_desc - pass"
290+
else
291+
echo "$test_desc - fail statx_atomic_write_unit_min=$statx_atomic_write_unit_min" \
292+
"sysfs_atomic_write_unit_min=$sysfs_atomic_write_unit_min" \
293+
"md_chunk_size=$md_chunk_size"
294+
fi
295+
296+
test_desc="TEST 7 $personality step $step - Verify statx stx_atomic_write_unit_max"
297+
statx_atomic_write_unit_max=$(run_xfs_io_xstat /dev/"$md_dev" "stat.atomic_write_unit_max")
298+
if [ "$statx_atomic_write_unit_max" = "$sysfs_atomic_write_unit_max" ]
299+
then
300+
echo "$test_desc - pass"
301+
else
302+
echo "$test_desc - fail statx_atomic_write_unit_max=$statx_atomic_write_unit_max" \
303+
"sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
304+
"md_chunk_size=$md_chunk_size"
305+
fi
306+
307+
test_desc="TEST 8 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_max_bytes with "
308+
test_desc+="RWF_ATOMIC flag - pwritev2 should fail"
309+
if [ "$sysfs_atomic_write_unit_max" = 0 ]
310+
then
311+
echo "$test_desc - pass"
312+
else
313+
bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$sysfs_atomic_write_unit_max")
314+
if [ "$bytes_written" = "$sysfs_atomic_write_unit_max" ]
315+
then
316+
echo "$test_desc - pass"
317+
else
318+
echo "$test_desc - fail bytes_written=$bytes_written" \
319+
"sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
320+
"md_chunk_size=$md_chunk_size"
321+
fi
322+
fi
323+
324+
test_desc="TEST 9 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_max_bytes + LBS "
325+
test_desc+="bytes with RWF_ATOMIC flag - pwritev2 should not be succesful"
326+
if [ "$sysfs_atomic_write_unit_max" = 0 ]
327+
then
328+
echo "pwrite: Invalid argument"
329+
echo "$test_desc - pass"
330+
else
331+
bytes_to_write=$(( "${sysfs_atomic_write_unit_max}" + "${sysfs_logical_block_size}" ))
332+
bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$bytes_to_write")
333+
if [ "$bytes_written" = "" ]
334+
then
335+
echo "$test_desc - pass"
336+
else
337+
echo "$test_desc - fail bytes_written=$bytes_written" \
338+
"bytes_to_write=$bytes_to_write" \
339+
"sysfs_atomic_write_unit_max=$sysfs_atomic_write_unit_max" \
340+
"md_chunk_size=$md_chunk_size"
341+
fi
342+
fi
343+
344+
test_desc="TEST 10 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_min_bytes "
345+
test_desc+="with RWF_ATOMIC flag - pwritev2 should fail"
346+
if [ "$sysfs_atomic_write_unit_min" = 0 ]
347+
then
348+
echo "$test_desc - pass"
349+
else
350+
bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$sysfs_atomic_write_unit_min")
351+
if [ "$bytes_written" = "$sysfs_atomic_write_unit_min" ]
352+
then
353+
echo "$test_desc - pass"
354+
else
355+
echo "$test_desc - fail bytes_written=$bytes_written" \
356+
"sysfs_atomic_write_unit_min=$sysfs_atomic_write_unit_min" \
357+
"md_chunk_size=$md_chunk_size"
358+
fi
359+
fi
360+
361+
test_desc="TEST 11 $personality step $step - perform a pwritev2 with a size of sysfs_atomic_write_unit_max_bytes - LBS "
362+
test_desc+="bytes with RWF_ATOMIC flag - pwritev2 should fail"
363+
if [ "${sysfs_atomic_write_unit_max}" -le "${sysfs_logical_block_size}" ]
364+
then
365+
echo "pwrite: Invalid argument"
366+
echo "$test_desc - pass"
367+
else
368+
bytes_to_write=$(( "${sysfs_atomic_write_unit_max}" - "${sysfs_logical_block_size}" ))
369+
bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$bytes_to_write")
370+
if [ "$bytes_written" = "" ]
371+
then
372+
echo "$test_desc - pass"
373+
else
374+
echo "$test_desc - fail bytes_written=$bytes_written" \
375+
"bytes_to_write=$bytes_to_write" \
376+
"md_chunk_size=$md_chunk_size"
377+
fi
378+
fi
379+
380+
if [ "$personality" = raid0 ] || [ "$personality" = raid1 ] || [ "$personality" = raid10 ]
381+
then
382+
mdadm --stop /dev/md/blktests_md 2> /dev/null 1>&2
383+
384+
for i in "${MD_DEVICES[@]}"; do
385+
mdadm --zero-superblock /dev/"$i" 2> /dev/null 1>&2
386+
done
387+
fi
388+
done
389+
done
390+
}

0 commit comments

Comments
 (0)