hacks as of 9 mar

fsn1.disktest/.gitignore

@@ -1 +1 @@
-log.txt
+*.txt

fsn1.disktest/Makefile

@@ -7,10 +7,13 @@ runlog:
 	script ./log.$(WHEN).txt bash -c "make run"
 
 run: sync
-	ssh root@$(HOST) "bash /root/disktest.sh"
+	ssh root@$(HOST) "bash /root/disktest.sh 2>&1"
 
 sync:
 	rsync -avP ./disktest.sh root@$(HOST):/root/disktest.sh
 
 clean:
 	rm log.*.txt
+
+results:
+	grep -ie 'cpu\|encryption\|zfs\|ashift\|64kchunk\|read\|write\|samsung' *.txt  | grep -v groupid | grep -v '^readwrite' | grep -v echo | grep -v rm | grep -v Laying

fsn1.disktest/disktest.sh

@@ -1,12 +1,6 @@
 #!/bin/bash
 
 set -x
-set -e
-
-date -u
-zfs version
-killall fio || true
-sleep 1
 
 RAWDISKS=$(
     cd /dev/disk/by-id ;
@@ -18,54 +12,124 @@ for RAWDISK in $RAWDISKS ; do
     FULLPATHS+=" /dev/disk/by-id/$RAWDISK"
 done
 
-PN="tank"
+POOLNAME="tank"
 MPT="/srv/z"
+JOBFILE="/tmp/jobfile.fio"
+KEYLOC="/tmp/testkey.raw"
 
-if [[ -e $MPT/.zfs ]]; then
-    zpool destroy $PN
-fi
+function main() {
+    show_system_info
+    prep_system
+    do_direct_tests
+    do_zfs_tests
+}
 
-lsblk -t
+function do_direct_tests() {
+    trim_disks
+    for DISK in $FULLPATHS; do
+        print_divider
+        echo "testing $DISK"
+        for NJ in 1 16; do
+            for BS in 64k ; do
+cat > $JOBFILE <<EOF
+[test-${BS}chunk-${NJ}thread]
+bs=$BS
+filename=$DISK
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=$NJ
+size=$(expr 10000 / $NJ)M
+runtime=60
+time_based=1
+group_reporting=1
+EOF
+                cat $JOBFILE
+                fio $FO $JOBFILE 2>&1
+            done
+        done
+    done
+}
 
-for ASHIFTVAL in 12 13 14 15 16 ; do
+function print_divider() {
+    echo '##############################################################'
+    echo '##############################################################'
+}
 
+function trim_disks() {
     for DISK in $FULLPATHS ; do
         blkdiscard $DISK
-        nvme format $DISK
+        nvme format -r $DISK
     done
+    print_divider
+    echo "################## waiting 60s for nvme format to complete..."
+    sleep 60
+    nvme list
+}
 
-    zpool create $PN \
-        -o ashift=$ASHIFTVAL -O mountpoint=$MPT \
-        raidz1 $FULLPATHS
 
-    KEYLOC="/tmp/testkey.raw"
-    dd if=/dev/urandom of=$KEYLOC bs=32 count=1
+function show_system_info() {
+    print_divider
+    date -u
+    lsb_release -a
+    uname -a
+    lsblk -t
+    nvme list
+    modinfo zfs | grep -iw version
+    modinfo spl | grep -iw version
+    zfs version
+    print_divider
+}
 
-    COMP="-o compression=lz4"
-    ENC="-o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file://$KEYLOC"
+function prep_system() {
+    killall fio || true
+    sleep 1
 
-    # plain
-    zfs create $PN/none
-    # compressed
-    zfs create $COMP $PN/comp
-    # encrypted
-    zfs create $ENC $PN/enc
-    # compressed and encrypted
-    zfs create $COMP $ENC $PN/both
+    if [[ -e $MPT/.zfs ]]; then
+        zpool destroy $POOLNAME
+    fi
+}
 
-    FO="--eta-newline=10 --eta-interval=10 --eta=always"
+function do_zfs_tests() {
+    for ASHIFTVAL in 12 13 14 15 16 ; do
 
-    JF="/tmp/jobfile.fio"
-    rm -f $JF
+        trim_disks
 
-    for NJ in 1 8 32 ; do
-        for BS in 64k 1M ; do
-            for DIRNAME in none comp enc both ; do
+        print_divider
 
-# 400000M comes from ~1.5x machine's ram size: 256G
-cat >> $JF <<EOF
+        zpool create $POOLNAME \
+            -o ashift=$ASHIFTVAL -O mountpoint=$MPT \
+            raidz1 $FULLPATHS
 
-[test-$DIRNAME-${BS}chunk-${NJ}thread]
+        dd if=/dev/urandom of=$KEYLOC bs=32 count=1
+
+        COMP="
+            -o compression=lz4
+        "
+        ENC="
+            -o encryption=aes-256-gcm
+            -o keyformat=raw
+            -o keylocation=file://$KEYLOC
+        "
+
+        # plain
+        zfs create $POOLNAME/none
+        # compressed
+        zfs create $COMP $POOLNAME/comp
+        # encrypted
+        zfs create $ENC $POOLNAME/enc
+        # compressed and encrypted
+        zfs create $COMP $ENC $POOLNAME/both
+
+        FO="--eta-newline=10 --eta-interval=10 --eta=always"
+
+        for NJ in 1 16 ; do
+            for BS in 64k ; do
+                for DIRNAME in none comp enc both ; do
+
+cat > $JOBFILE <<EOF
+
+[test-$DIRNAME-${BS}chunk-${NJ}thread-ashift${ASHIFTVAL}]
 bs=$BS
 directory=$MPT/$DIRNAME/
 iodepth=64
@@ -73,18 +137,20 @@ ioengine=libaio
 readwrite=readwrite
 numjobs=$NJ
 nrfiles=$NJ
-size=$(expr 400000 / $NJ)M
-runtime=300
+size=10000M
+runtime=120
 time_based=1
 group_reporting=1
 EOF
+                cat $JOBFILE
+                fio $FO $JOBFILE 2>&1
+                rm -rf $MPT/$DIRNAME/*
+                done
             done
         done
+
+        zpool destroy $POOLNAME
     done
+}
 
-    cat $JF
-    fio $FO $JF
-    zpool destroy $PN
-done
-
-
+main