diff --git a/download-starred-github-repos/download-starred-github-repos.sh b/download-starred-github-repos/download-starred-github-repos.sh
new file mode 100644
index 0000000..dbb68b0
--- /dev/null
+++ b/download-starred-github-repos/download-starred-github-repos.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# thanks to @sebble for the basis for this script, he did the hard work:
+# https://gist.github.com/sebble/e5af3d03700bfd31c62054488bfe8d4f
+
+function list_starred_repos {
+    USER=${1:-sneak}
+
+    STARS=$(curl -sI https://api.github.com/users/$USER/starred?per_page=1|egrep '^Link'|egrep -o 'page=[0-9]+'|tail -1|cut -c6-)
+    PAGES=$((658/100+1))
+
+    echo You have $STARS starred repositories. > /dev/stderr
+
+    # cloning
+    # https://github.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
+    # breaks github's auth system
+    REPOS=""
+    for PAGE in `seq $PAGES`; do
+        REPOS+=" $(
+            curl -sH "Accept: application/vnd.github.v3.star+json" "https://api.github.com/users/$USER/starred?per_page=100&page=$PAGE" |
+            jq -r '.[]|[.repo.full_name][0]' |
+            grep -v eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
+        )"
+    done
+
+    for REPO in $REPOS ; do
+        echo "$REPO"
+    done
+}
+
+function fetch_starred_repos {
+    for SHORTNAME in $(list_starred_repos); do
+        UN="$(echo $SHORTNAME | cut -d'/' -f1)"
+        if [[ ! -d "$SHORTNAME" ]]; then
+            if [[ ! -d "$UN" ]]; then
+                mkdir -p $UN
+            fi
+            git clone https://github.com/$SHORTNAME $SHORTNAME
+        fi
+    done
+}
+
+fetch_starred_repos
diff --git a/fsn1.disktest/.gitignore b/fsn1.disktest/.gitignore
index 6bd6746..2211df6 100644
--- a/fsn1.disktest/.gitignore
+++ b/fsn1.disktest/.gitignore
@@ -1 +1 @@
-log.txt
+*.txt
diff --git a/fsn1.disktest/Makefile b/fsn1.disktest/Makefile
index 21cd095..1c18592 100644
--- a/fsn1.disktest/Makefile
+++ b/fsn1.disktest/Makefile
@@ -7,10 +7,13 @@ runlog:
 	script ./log.$(WHEN).txt bash -c "make run"
 
 run: sync
-	ssh root@$(HOST) "bash /root/disktest.sh"
+	ssh root@$(HOST) "bash /root/disktest.sh 2>&1"
 
 sync:
 	rsync -avP ./disktest.sh root@$(HOST):/root/disktest.sh
 
 clean:
 	rm log.*.txt
+
+results:
+	grep -ie 'cpu\|encryption\|zfs\|ashift\|64kchunk\|read\|write\|samsung' *.txt  | grep -v groupid | grep -v '^readwrite' | grep -v echo | grep -v rm | grep -v Laying
diff --git a/fsn1.disktest/disktest.sh b/fsn1.disktest/disktest.sh
index f4d4d15..8bb70d5 100644
--- a/fsn1.disktest/disktest.sh
+++ b/fsn1.disktest/disktest.sh
@@ -1,12 +1,6 @@
 #!/bin/bash
 
 set -x
-set -e
-
-date -u
-zfs version
-killall fio || true
-sleep 1
 
 RAWDISKS=$(
     cd /dev/disk/by-id ;
@@ -18,54 +12,124 @@ for RAWDISK in $RAWDISKS ; do
     FULLPATHS+=" /dev/disk/by-id/$RAWDISK"
 done
 
-PN="tank"
+POOLNAME="tank"
 MPT="/srv/z"
+JOBFILE="/tmp/jobfile.fio"
+KEYLOC="/tmp/testkey.raw"
 
-if [[ -e $MPT/.zfs ]]; then
-    zpool destroy $PN
-fi
+function main() {
+    show_system_info
+    prep_system
+    do_direct_tests
+    do_zfs_tests
+}
 
-lsblk -t
+function do_direct_tests() {
+    trim_disks
+    for DISK in $FULLPATHS; do
+        print_divider
+        echo "testing $DISK"
+        for NJ in 1 16; do
+            for BS in 64k ; do
+cat > $JOBFILE <<EOF
+[test-${BS}chunk-${NJ}thread]
+bs=$BS
+filename=$DISK
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=$NJ
+size=$(expr 10000 / $NJ)M
+runtime=60
+time_based=1
+group_reporting=1
+EOF
+                cat $JOBFILE
+                fio $FO $JOBFILE 2>&1
+            done
+        done
+    done
+}
 
-for ASHIFTVAL in 12 13 14 15 16 ; do
+function print_divider() {
+    echo '##############################################################'
+    echo '##############################################################'
+}
 
+function trim_disks() {
     for DISK in $FULLPATHS ; do
         blkdiscard $DISK
-        nvme format $DISK
+        nvme format -r $DISK
     done
+    print_divider
+    echo "################## waiting 60s for nvme format to complete..."
+    sleep 60
+    nvme list
+}
 
-    zpool create $PN \
-        -o ashift=$ASHIFTVAL -O mountpoint=$MPT \
-        raidz1 $FULLPATHS
 
-    KEYLOC="/tmp/testkey.raw"
-    dd if=/dev/urandom of=$KEYLOC bs=32 count=1
+function show_system_info() {
+    print_divider
+    date -u
+    lsb_release -a
+    uname -a
+    lsblk -t
+    nvme list
+    modinfo zfs | grep -iw version
+    modinfo spl | grep -iw version
+    zfs version
+    print_divider
+}
 
-    COMP="-o compression=lz4"
-    ENC="-o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file://$KEYLOC"
+function prep_system() {
+    killall fio || true
+    sleep 1
 
-    # plain
-    zfs create $PN/none
-    # compressed
-    zfs create $COMP $PN/comp
-    # encrypted
-    zfs create $ENC $PN/enc
-    # compressed and encrypted
-    zfs create $COMP $ENC $PN/both
+    if [[ -e $MPT/.zfs ]]; then
+        zpool destroy $POOLNAME
+    fi
+}
 
-    FO="--eta-newline=10 --eta-interval=10 --eta=always"
+function do_zfs_tests() {
+    for ASHIFTVAL in 12 13 14 15 16 ; do
 
-    JF="/tmp/jobfile.fio"
-    rm -f $JF
+        trim_disks
 
-    for NJ in 1 8 32 ; do
-        for BS in 64k 1M ; do
-            for DIRNAME in none comp enc both ; do
+        print_divider
 
-# 400000M comes from ~1.5x machine's ram size: 256G
-cat >> $JF <<EOF
+        zpool create $POOLNAME \
+            -o ashift=$ASHIFTVAL -O mountpoint=$MPT \
+            raidz1 $FULLPATHS
 
-[test-$DIRNAME-${BS}chunk-${NJ}thread]
+        dd if=/dev/urandom of=$KEYLOC bs=32 count=1
+
+        COMP="
+            -o compression=lz4
+        "
+        ENC="
+            -o encryption=aes-256-gcm
+            -o keyformat=raw
+            -o keylocation=file://$KEYLOC
+        "
+
+        # plain
+        zfs create $POOLNAME/none
+        # compressed
+        zfs create $COMP $POOLNAME/comp
+        # encrypted
+        zfs create $ENC $POOLNAME/enc
+        # compressed and encrypted
+        zfs create $COMP $ENC $POOLNAME/both
+
+        FO="--eta-newline=10 --eta-interval=10 --eta=always"
+
+        for NJ in 1 16 ; do
+            for BS in 64k ; do
+                for DIRNAME in none comp enc both ; do
+
+cat > $JOBFILE <<EOF
+
+[test-$DIRNAME-${BS}chunk-${NJ}thread-ashift${ASHIFTVAL}]
 bs=$BS
 directory=$MPT/$DIRNAME/
 iodepth=64
@@ -73,18 +137,20 @@ ioengine=libaio
 readwrite=readwrite
 numjobs=$NJ
 nrfiles=$NJ
-size=$(expr 400000 / $NJ)M
-runtime=300
+size=10000M
+runtime=120
 time_based=1
 group_reporting=1
 EOF
+                cat $JOBFILE
+                fio $FO $JOBFILE 2>&1
+                rm -rf $MPT/$DIRNAME/*
+                done
             done
         done
+
+        zpool destroy $POOLNAME
     done
+}
 
-    cat $JF
-    fio $FO $JF
-    zpool destroy $PN
-done
-
-
+main
diff --git a/nbg1.disktest/Makefile b/nbg1.disktest/Makefile
new file mode 100644
index 0000000..fde65e8
--- /dev/null
+++ b/nbg1.disktest/Makefile
@@ -0,0 +1,19 @@
+HOST := nbg1.datavibe.net
+WHEN := $(shell date -u +%Y-%m-%d.%H.%M.%S)
+
+default: runlog
+
+runlog:
+	script ./log.$(WHEN).txt bash -c "make run"
+
+run: sync
+	ssh root@$(HOST) "bash /root/disktest.sh 2>&1"
+
+sync:
+	rsync -avP ./disktest.sh root@$(HOST):/root/disktest.sh
+
+clean:
+	rm log.*.txt
+
+results:
+	grep -ie 'cpu\|encryption\|zfs\|ashift\|64kchunk\|read\|write\|samsung' *.txt  | grep -v groupid | grep -v '^readwrite' | grep -v echo | grep -v rm | grep -v Laying
diff --git a/nbg1.disktest/disktest.sh b/nbg1.disktest/disktest.sh
new file mode 100644
index 0000000..ed64cd9
--- /dev/null
+++ b/nbg1.disktest/disktest.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -x
+
+POOLNAME="tank"
+MPT="/srv/z"
+JOBFILE="/tmp/jobfile.fio"
+KEYLOC="/tmp/testkey.raw"
+
+function main() {
+    show_system_info
+    do_zfs_tests
+}
+
+function print_divider() {
+    echo '##############################################################'
+    echo '##############################################################'
+}
+
+function show_system_info() {
+    print_divider
+    date -u
+    lsb_release -a
+    uname -a
+    lsblk -t
+    nvme list
+    modinfo zfs | grep -iw version
+    modinfo spl | grep -iw version
+    zfs version
+    print_divider
+}
+
+function prep_system() {
+    killall fio || true
+    sleep 1
+
+}
+
+function do_zfs_tests() {
+        print_divider
+
+        FO="--eta-newline=10 --eta-interval=10 --eta=always"
+
+        for NJ in 1 16 ; do
+            for BS in 64k ; do
+                for DIRNAME in none comp enc both ; do
+
+cat > $JOBFILE <<EOF
+
+[test-$DIRNAME-${BS}chunk-${NJ}thread]
+bs=$BS
+directory=$MPT/$DIRNAME/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=$NJ
+nrfiles=$NJ
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
+EOF
+                cat $JOBFILE
+                fio $FO $JOBFILE 2>&1
+                rm -rf $MPT/$DIRNAME/*
+                done
+            done
+        done
+
+}
+
+main
diff --git a/nbg1.disktest/log.2020-02-28.21.46.16.txt b/nbg1.disktest/log.2020-02-28.21.46.16.txt
new file mode 100644
index 0000000..ddeebd5
--- /dev/null
+++ b/nbg1.disktest/log.2020-02-28.21.46.16.txt
@@ -0,0 +1,9 @@
+Script started on Fri Feb 28 13:46:16 2020
+command: bash -c make run
+rsync -avP ./disktest.sh root@nbg1.datavibe.net:/root/disktest.sh
+bash: rsync: command not found
+rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+rsync error: remote command not found (code 127) at io.c(226) [sender=3.1.3]
+make[1]: *** [sync] Error 127
+
+Script done on Fri Feb 28 13:46:16 2020
diff --git a/nbg1.disktest/log.2020-02-28.21.46.25.txt b/nbg1.disktest/log.2020-02-28.21.46.25.txt
new file mode 100644
index 0000000..2cd43db
--- /dev/null
+++ b/nbg1.disktest/log.2020-02-28.21.46.25.txt
@@ -0,0 +1,692 @@
+Script started on Fri Feb 28 13:46:25 2020
+command: bash -c make run
+rsync -avP ./disktest.sh root@nbg1.datavibe.net:/root/disktest.sh
+sending incremental file list
+disktest.sh
+
          1,247 100%    0.00kB/s    0:00:00  
          1,247 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)
+
+sent 1,356 bytes  received 35 bytes  927.33 bytes/sec
+total size is 1,247  speedup is 0.90
+ssh root@nbg1.datavibe.net "bash /root/disktest.sh 2>&1"
++ POOLNAME=tank
++ MPT=/srv/z
++ JOBFILE=/tmp/jobfile.fio
++ KEYLOC=/tmp/testkey.raw
++ main
++ show_system_info
++ print_divider
++ echo '##############################################################'
+##############################################################
++ echo '##############################################################'
+##############################################################
++ date -u
+Fri 28 Feb 2020 09:46:26 PM UTC
++ lsb_release -a
+No LSB modules are available.
+Distributor ID:	Ubuntu
+Description:	Ubuntu Focal Fossa (development branch)
+Release:	20.04
+Codename:	focal
++ uname -a
+Linux nbg1 5.4.0-14-generic #17-Ubuntu SMP Thu Feb 6 22:47:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
++ lsblk -t
+NAME    ALIGNMENT MIN-IO  OPT-IO PHY-SEC LOG-SEC ROTA SCHED       RQ-SIZE   RA WSAME
+sda             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sda1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sda2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sda3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+sdb             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sdb1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sdb2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sdb3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+sdc             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sdc1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sdc2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sdc3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+sdd             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sdd1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sdd2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sdd3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+sde             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sde1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sde2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sde3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+sdf             0   4096       0    4096     512    0 mq-deadline     256  128    0B
+├─sdf1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md0         0   4096       0    4096     512    0                 128  128    0B
+├─sdf2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
+└─sdf3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
++ nvme list
+/root/disktest.sh: line 26: nvme: command not found
++ modinfo zfs
++ grep -iw version
+version:        0.8.3-1ubuntu3
++ modinfo spl
++ grep -iw version
+version:        0.8.3-1ubuntu3
++ zfs version
+zfs-0.8.3-1ubuntu5
+zfs-kmod-0.8.3-1ubuntu3
++ print_divider
++ echo '##############################################################'
+##############################################################
++ echo '##############################################################'
+##############################################################
++ do_zfs_tests
++ print_divider
++ echo '##############################################################'
+##############################################################
++ echo '##############################################################'
+##############################################################
++ FO='--eta-newline=10 --eta-interval=10 --eta=always'
++ for NJ in 1 16
++ for BS in 64k
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-none-64kchunk-1thread]
+bs=64k
+directory=/srv/z/none/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=1
+nrfiles=1
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+fio-3.16
+Starting 1 process
+test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
+Jobs: 1 (f=1): [M(1)][8.3%][r=917MiB/s,w=919MiB/s][r=14.7k,w=14.7k IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.5%][r=953MiB/s,w=953MiB/s][r=15.3k,w=15.2k IOPS][eta 01m:41s]
Jobs: 1 (f=1): [M(1)][24.2%][r=897MiB/s,w=893MiB/s][r=14.3k,w=14.3k IOPS][eta 01m:31s]

+Jobs: 1 (f=1): [M(1)][32.8%][r=882MiB/s,w=880MiB/s][r=14.1k,w=14.1k IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.5%][r=907MiB/s,w=905MiB/s][r=14.5k,w=14.5k IOPS][eta 01m:12s] 

+Jobs: 1 (f=1): [M(1)][48.3%][r=880MiB/s,w=881MiB/s][r=14.1k,w=14.1k IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=892MiB/s,w=892MiB/s][r=14.3k,w=14.3k IOPS][eta 00m:52s] 

+Jobs: 1 (f=1): [M(1)][65.5%][r=902MiB/s,w=905MiB/s][r=14.4k,w=14.5k IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=906MiB/s,w=903MiB/s][r=14.5k,w=14.5k IOPS][eta 00m:33s] 

+Jobs: 1 (f=1): [M(1)][80.8%][r=886MiB/s,w=881MiB/s][r=14.2k,w=14.1k IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][88.4%][r=906MiB/s,w=905MiB/s][r=14.5k,w=14.5k IOPS][eta 00m:14s] 

+Jobs: 1 (f=1): [M(1)][96.7%][r=879MiB/s,w=883MiB/s][r=14.1k,w=14.1k IOPS][eta 00m:04s]

+test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=285277: Fri Feb 28 21:48:41 2020
+  read: IOPS=14.4k, BW=901MiB/s (945MB/s)(106GiB/120001msec)
+    slat (usec): min=6, max=100872, avg=36.06, stdev=136.62
+    clat (usec): min=41, max=118051, avg=2186.43, stdev=893.80
+     lat (usec): min=114, max=118339, avg=2222.59, stdev=909.16
+    clat percentiles (usec):
+     |  1.00th=[ 1418],  5.00th=[ 1663], 10.00th=[ 1762], 20.00th=[ 1827],
+     | 30.00th=[ 1876], 40.00th=[ 1926], 50.00th=[ 2040], 60.00th=[ 2245],
+     | 70.00th=[ 2376], 80.00th=[ 2507], 90.00th=[ 2704], 95.00th=[ 2999],
+     | 99.00th=[ 3720], 99.50th=[ 4113], 99.90th=[ 5538], 99.95th=[ 6652],
+     | 99.99th=[ 9765]
+   bw (  KiB/s): min=606080, max=1427200, per=99.97%, avg=922432.68, stdev=95336.08, samples=239
+   iops        : min= 9470, max=22300, avg=14412.97, stdev=1489.61, samples=239
+  write: IOPS=14.4k, BW=900MiB/s (944MB/s)(105GiB/120001msec); 0 zone resets
+    slat (usec): min=9, max=8839, avg=31.46, stdev=16.75
+    clat (nsec): min=1690, max=118306k, avg=2185831.96, stdev=903829.11
+     lat (usec): min=39, max=118336, avg=2217.42, stdev=907.24
+    clat percentiles (usec):
+     |  1.00th=[ 1418],  5.00th=[ 1663], 10.00th=[ 1762], 20.00th=[ 1827],
+     | 30.00th=[ 1876], 40.00th=[ 1926], 50.00th=[ 2040], 60.00th=[ 2245],
+     | 70.00th=[ 2376], 80.00th=[ 2507], 90.00th=[ 2704], 95.00th=[ 2999],
+     | 99.00th=[ 3720], 99.50th=[ 4113], 99.90th=[ 5538], 99.95th=[ 6783],
+     | 99.99th=[10290]
+   bw (  KiB/s): min=607360, max=1419776, per=99.97%, avg=921526.12, stdev=94577.93, samples=239
+   iops        : min= 9490, max=22184, avg=14398.77, stdev=1477.77, samples=239
+  lat (usec)   : 2=0.01%, 50=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
+  lat (usec)   : 1000=0.01%
+  lat (msec)   : 2=47.74%, 4=51.66%, 10=0.59%, 20=0.01%, 250=0.01%
+  cpu          : usr=6.98%, sys=91.83%, ctx=7753, majf=0, minf=13653
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=1730179,1728438,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=901MiB/s (945MB/s), 901MiB/s-901MiB/s (945MB/s-945MB/s), io=106GiB (113GB), run=120001-120001msec
+  WRITE: bw=900MiB/s (944MB/s), 900MiB/s-900MiB/s (944MB/s-944MB/s), io=105GiB (113GB), run=120001-120001msec
++ rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-comp-64kchunk-1thread]
+bs=64k
+directory=/srv/z/comp/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=1
+nrfiles=1
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+fio-3.16
+Starting 1 process
+test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
+Jobs: 1 (f=1): [M(1)][8.3%][r=886MiB/s,w=887MiB/s][r=14.2k,w=14.2k IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.7%][r=944MiB/s,w=945MiB/s][r=15.1k,w=15.1k IOPS][eta 01m:40s]
Jobs: 1 (f=1): [M(1)][24.2%][r=885MiB/s,w=880MiB/s][r=14.2k,w=14.1k IOPS][eta 01m:31s]

+Jobs: 1 (f=1): [M(1)][32.8%][r=834MiB/s,w=834MiB/s][r=13.3k,w=13.3k IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.8%][r=857MiB/s,w=854MiB/s][r=13.7k,w=13.7k IOPS][eta 01m:11s] 

+Jobs: 1 (f=1): [M(1)][48.3%][r=841MiB/s,w=840MiB/s][r=13.4k,w=13.4k IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=835MiB/s,w=834MiB/s][r=13.4k,w=13.3k IOPS][eta 00m:52s] 

+Jobs: 1 (f=1): [M(1)][65.5%][r=864MiB/s,w=868MiB/s][r=13.8k,w=13.9k IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=860MiB/s,w=859MiB/s][r=13.8k,w=13.7k IOPS][eta 00m:33s] 

+Jobs: 1 (f=1): [M(1)][80.8%][r=830MiB/s,w=826MiB/s][r=13.3k,w=13.2k IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=823MiB/s,w=823MiB/s][r=13.2k,w=13.2k IOPS][eta 00m:12s] 

+Jobs: 1 (f=1): [M(1)][96.7%][r=849MiB/s,w=848MiB/s][r=13.6k,w=13.6k IOPS][eta 00m:04s]

+test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1329996: Fri Feb 28 21:50:56 2020
+  read: IOPS=13.8k, BW=860MiB/s (902MB/s)(101GiB/120001msec)
+    slat (usec): min=6, max=106574, avg=38.02, stdev=278.16
+    clat (usec): min=49, max=170126, avg=2291.59, stdev=1934.03
+     lat (usec): min=125, max=170188, avg=2329.72, stdev=1963.60
+    clat percentiles (usec):
+     |  1.00th=[  1450],  5.00th=[  1696], 10.00th=[  1778], 20.00th=[  1844],
+     | 30.00th=[  1893], 40.00th=[  1942], 50.00th=[  2008], 60.00th=[  2311],
+     | 70.00th=[  2474], 80.00th=[  2638], 90.00th=[  2900], 95.00th=[  3097],
+     | 99.00th=[  3785], 99.50th=[  5080], 99.90th=[ 23200], 99.95th=[ 34341],
+     | 99.99th=[102237]
+   bw (  KiB/s): min=424448, max=1341696, per=99.89%, avg=879662.20, stdev=116352.77, samples=239
+   iops        : min= 6632, max=20964, avg=13744.67, stdev=1817.99, samples=239
+  write: IOPS=13.7k, BW=859MiB/s (901MB/s)(101GiB/120001msec); 0 zone resets
+    slat (usec): min=9, max=108207, avg=32.56, stdev=116.95
+    clat (usec): min=2, max=170135, avg=2289.69, stdev=1914.67
+     lat (usec): min=48, max=170173, avg=2322.38, stdev=1921.04
+    clat percentiles (usec):
+     |  1.00th=[  1450],  5.00th=[  1696], 10.00th=[  1778], 20.00th=[  1844],
+     | 30.00th=[  1893], 40.00th=[  1942], 50.00th=[  2008], 60.00th=[  2278],
+     | 70.00th=[  2474], 80.00th=[  2638], 90.00th=[  2900], 95.00th=[  3097],
+     | 99.00th=[  3752], 99.50th=[  5014], 99.90th=[ 21890], 99.95th=[ 34866],
+     | 99.99th=[102237]
+   bw (  KiB/s): min=416000, max=1341568, per=99.89%, avg=878853.61, stdev=116939.69, samples=239
+   iops        : min= 6500, max=20962, avg=13732.02, stdev=1827.17, samples=239
+  lat (usec)   : 4=0.01%, 50=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
+  lat (usec)   : 1000=0.01%
+  lat (msec)   : 2=48.94%, 4=50.22%, 10=0.63%, 20=0.09%, 50=0.08%
+  lat (msec)   : 100=0.02%, 250=0.01%
+  cpu          : usr=7.02%, sys=89.10%, ctx=8743, majf=0, minf=27256
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=1651235,1649716,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=860MiB/s (902MB/s), 860MiB/s-860MiB/s (902MB/s-902MB/s), io=101GiB (108GB), run=120001-120001msec
+  WRITE: bw=859MiB/s (901MB/s), 859MiB/s-859MiB/s (901MB/s-901MB/s), io=101GiB (108GB), run=120001-120001msec
++ rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-enc-64kchunk-1thread]
+bs=64k
+directory=/srv/z/enc/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=1
+nrfiles=1
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+fio-3.16
+Starting 1 process
+test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
+Jobs: 1 (f=1): [M(1)][8.3%][r=54.7MiB/s,w=54.8MiB/s][r=875,w=877 IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.7%][r=55.5MiB/s,w=55.3MiB/s][r=888,w=885 IOPS][eta 01m:40s]
Jobs: 1 (f=1): [M(1)][24.2%][r=55.4MiB/s,w=55.0MiB/s][r=886,w=880 IOPS][eta 01m:31s]

+Jobs: 1 (f=1): [M(1)][32.8%][r=55.6MiB/s,w=56.5MiB/s][r=890,w=903 IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.8%][r=56.0MiB/s,w=53.5MiB/s][r=896,w=855 IOPS][eta 01m:11s] 

+Jobs: 1 (f=1): [M(1)][48.3%][r=54.3MiB/s,w=55.2MiB/s][r=868,w=882 IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=54.3MiB/s,w=54.6MiB/s][r=869,w=874 IOPS][eta 00m:52s] 

+Jobs: 1 (f=1): [M(1)][65.5%][r=54.3MiB/s,w=52.9MiB/s][r=868,w=846 IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=55.9MiB/s,w=55.7MiB/s][r=894,w=890 IOPS][eta 00m:33s] 

+Jobs: 1 (f=1): [M(1)][80.8%][r=55.2MiB/s,w=55.4MiB/s][r=882,w=886 IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=54.0MiB/s,w=55.1MiB/s][r=879,w=881 IOPS][eta 00m:12s] 

+Jobs: 1 (f=1): [M(1)][96.7%][r=54.5MiB/s,w=55.4MiB/s][r=871,w=886 IOPS][eta 00m:04s]

+test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2235096: Fri Feb 28 21:53:11 2020
+  read: IOPS=882, BW=55.1MiB/s (57.8MB/s)(6615MiB/120001msec)
+    slat (usec): min=7, max=106999, avg=861.13, stdev=1127.17
+    clat (usec): min=2, max=151067, avg=35748.24, stdev=6537.32
+     lat (msec): min=2, max=153, avg=36.61, stdev= 6.65
+    clat percentiles (msec):
+     |  1.00th=[   26],  5.00th=[   28], 10.00th=[   30], 20.00th=[   32],
+     | 30.00th=[   33], 40.00th=[   34], 50.00th=[   35], 60.00th=[   36],
+     | 70.00th=[   38], 80.00th=[   40], 90.00th=[   43], 95.00th=[   47],
+     | 99.00th=[   61], 99.50th=[   64], 99.90th=[   71], 99.95th=[   74],
+     | 99.99th=[  150]
+   bw (  KiB/s): min=38144, max=67840, per=99.95%, avg=56421.40, stdev=4863.10, samples=240
+   iops        : min=  596, max= 1060, avg=881.56, stdev=76.01, samples=240
+  write: IOPS=880, BW=55.0MiB/s (57.7MB/s)(6605MiB/120001msec); 0 zone resets
+    slat (usec): min=10, max=10019, avg=270.92, stdev=690.41
+    clat (msec): min=4, max=151, avg=35.72, stdev= 6.50
+     lat (msec): min=4, max=151, avg=35.99, stdev= 6.52
+    clat percentiles (msec):
+     |  1.00th=[   26],  5.00th=[   28], 10.00th=[   30], 20.00th=[   32],
+     | 30.00th=[   33], 40.00th=[   34], 50.00th=[   35], 60.00th=[   36],
+     | 70.00th=[   38], 80.00th=[   40], 90.00th=[   42], 95.00th=[   46],
+     | 99.00th=[   61], 99.50th=[   64], 99.90th=[   71], 99.95th=[   75],
+     | 99.99th=[  150]
+   bw (  KiB/s): min=36352, max=68352, per=99.95%, avg=56338.82, stdev=5655.56, samples=240
+   iops        : min=  568, max= 1068, avg=880.26, stdev=88.39, samples=240
+  lat (usec)   : 4=0.01%
+  lat (msec)   : 4=0.01%, 10=0.01%, 20=0.03%, 50=96.32%, 100=3.61%
+  lat (msec)   : 250=0.03%
+  cpu          : usr=0.48%, sys=99.11%, ctx=1360, majf=0, minf=13645
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=105847,105686,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=55.1MiB/s (57.8MB/s), 55.1MiB/s-55.1MiB/s (57.8MB/s-57.8MB/s), io=6615MiB (6937MB), run=120001-120001msec
+  WRITE: bw=55.0MiB/s (57.7MB/s), 55.0MiB/s-55.0MiB/s (57.7MB/s-57.7MB/s), io=6605MiB (6926MB), run=120001-120001msec
++ rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-both-64kchunk-1thread]
+bs=64k
+directory=/srv/z/both/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=1
+nrfiles=1
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+fio-3.16
+Starting 1 process
+test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
+Jobs: 1 (f=1): [M(1)][8.3%][r=56.7MiB/s,w=56.7MiB/s][r=907,w=907 IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.5%][r=55.8MiB/s,w=55.7MiB/s][r=892,w=891 IOPS][eta 01m:41s]
Jobs: 1 (f=1): [M(1)][24.2%][r=55.6MiB/s,w=55.4MiB/s][r=889,w=886 IOPS][eta 01m:31s]

+Jobs: 1 (f=1): [M(1)][32.8%][r=56.0MiB/s,w=56.5MiB/s][r=896,w=903 IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.5%][r=57.1MiB/s,w=54.7MiB/s][r=913,w=875 IOPS][eta 01m:12s] 

+Jobs: 1 (f=1): [M(1)][48.3%][r=55.6MiB/s,w=56.4MiB/s][r=889,w=902 IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=54.8MiB/s,w=55.2MiB/s][r=877,w=883 IOPS][eta 00m:52s] 

+Jobs: 1 (f=1): [M(1)][65.0%][r=55.1MiB/s,w=53.5MiB/s][r=880,w=856 IOPS][eta 00m:42s]
Jobs: 1 (f=1): [M(1)][72.5%][r=56.7MiB/s,w=56.8MiB/s][r=906,w=909 IOPS][eta 00m:33s] 

+Jobs: 1 (f=1): [M(1)][80.8%][r=54.2MiB/s,w=54.3MiB/s][r=868,w=868 IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=54.7MiB/s,w=55.1MiB/s][r=875,w=882 IOPS][eta 00m:12s] 

+Jobs: 1 (f=1): [M(1)][96.7%][r=55.1MiB/s,w=56.0MiB/s][r=881,w=896 IOPS][eta 00m:04s]

+test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2366276: Fri Feb 28 21:55:26 2020
+  read: IOPS=890, BW=55.7MiB/s (58.4MB/s)(6680MiB/120001msec)
+    slat (usec): min=7, max=9126, avg=855.94, stdev=1070.39
+    clat (usec): min=2, max=89273, avg=35406.88, stdev=6185.51
+     lat (usec): min=2068, max=89307, avg=36262.95, stdev=6296.63
+    clat percentiles (usec):
+     |  1.00th=[24511],  5.00th=[27395], 10.00th=[28967], 20.00th=[31065],
+     | 30.00th=[32375], 40.00th=[33424], 50.00th=[34866], 60.00th=[35914],
+     | 70.00th=[36963], 80.00th=[39060], 90.00th=[41681], 95.00th=[46924],
+     | 99.00th=[58459], 99.50th=[61080], 99.90th=[67634], 99.95th=[71828],
+     | 99.99th=[84411]
+   bw (  KiB/s): min=37504, max=67840, per=99.93%, avg=56961.33, stdev=5137.37, samples=239
+   iops        : min=  586, max= 1060, avg=890.00, stdev=80.27, samples=239
+  write: IOPS=888, BW=55.5MiB/s (58.2MB/s)(6664MiB/120001msec); 0 zone resets
+    slat (usec): min=10, max=26367, avg=265.38, stdev=681.08
+    clat (usec): min=2081, max=89233, avg=35398.21, stdev=6203.43
+     lat (usec): min=2110, max=90922, avg=35663.72, stdev=6219.74
+    clat percentiles (usec):
+     |  1.00th=[24511],  5.00th=[27395], 10.00th=[28967], 20.00th=[30802],
+     | 30.00th=[32375], 40.00th=[33424], 50.00th=[34866], 60.00th=[35914],
+     | 70.00th=[36963], 80.00th=[39060], 90.00th=[41681], 95.00th=[46924],
+     | 99.00th=[58459], 99.50th=[61080], 99.90th=[67634], 99.95th=[70779],
+     | 99.99th=[84411]
+   bw (  KiB/s): min=36352, max=68608, per=99.97%, avg=56842.27, stdev=5917.48, samples=239
+   iops        : min=  568, max= 1072, avg=888.14, stdev=92.46, samples=239
+  lat (usec)   : 4=0.01%
+  lat (msec)   : 4=0.01%, 10=0.01%, 20=0.02%, 50=96.24%, 100=3.74%
+  cpu          : usr=0.51%, sys=99.08%, ctx=1576, majf=0, minf=14707
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=106885,106617,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=55.7MiB/s (58.4MB/s), 55.7MiB/s-55.7MiB/s (58.4MB/s-58.4MB/s), io=6680MiB (7005MB), run=120001-120001msec
+  WRITE: bw=55.5MiB/s (58.2MB/s), 55.5MiB/s-55.5MiB/s (58.2MB/s-58.2MB/s), io=6664MiB (6987MB), run=120001-120001msec
++ rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0
++ for NJ in 1 16
++ for BS in 64k
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-none-64kchunk-16thread]
+bs=64k
+directory=/srv/z/none/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=16
+nrfiles=16
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+...
+fio-3.16
+Starting 16 processes
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+Jobs: 16 (f=256): [M(16)][8.3%][r=1531MiB/s,w=1532MiB/s][r=24.5k,w=24.5k IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.5%][r=1323MiB/s,w=1324MiB/s][r=21.2k,w=21.2k IOPS][eta 01m:41s]
Jobs: 16 (f=256): [M(16)][24.2%][r=1338MiB/s,w=1335MiB/s][r=21.4k,w=21.4k IOPS][eta 01m:31s]

+Jobs: 16 (f=256): [M(16)][32.8%][r=1314MiB/s,w=1322MiB/s][r=21.0k,w=21.2k IOPS][eta 01m:20s]
Jobs: 16 (f=256): [M(16)][40.0%][r=1304MiB/s,w=1303MiB/s][r=20.9k,w=20.8k IOPS][eta 01m:12s] 

+Jobs: 16 (f=256): [M(16)][48.3%][r=1284MiB/s,w=1291MiB/s][r=20.5k,w=20.7k IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.2%][r=1281MiB/s,w=1280MiB/s][r=20.5k,w=20.5k IOPS][eta 00m:53s] 

+Jobs: 16 (f=256): [M(16)][64.7%][r=1288MiB/s,w=1290MiB/s][r=20.6k,w=20.6k IOPS][eta 00m:42s]
Jobs: 16 (f=256): [M(16)][72.5%][r=1276MiB/s,w=1277MiB/s][r=20.4k,w=20.4k IOPS][eta 00m:33s] 

+Jobs: 16 (f=256): [M(16)][80.0%][r=1231MiB/s,w=1230MiB/s][r=19.7k,w=19.7k IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=1096MiB/s,w=1099MiB/s][r=17.5k,w=17.6k IOPS][eta 00m:14s] 

+Jobs: 16 (f=256): [M(16)][95.9%][r=1129MiB/s,w=1122MiB/s][r=18.1k,w=17.0k IOPS][eta 00m:05s]

+test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3119643: Fri Feb 28 22:02:06 2020
+  read: IOPS=20.4k, BW=1278MiB/s (1340MB/s)(150GiB/120005msec)
+    slat (usec): min=16, max=121262, avg=422.11, stdev=709.84
+    clat (usec): min=2, max=307795, avg=24625.01, stdev=12814.99
+     lat (usec): min=152, max=308982, avg=25047.64, stdev=12992.67
+    clat percentiles (msec):
+     |  1.00th=[    6],  5.00th=[   12], 10.00th=[   16], 20.00th=[   17],
+     | 30.00th=[   18], 40.00th=[   19], 50.00th=[   20], 60.00th=[   22],
+     | 70.00th=[   28], 80.00th=[   34], 90.00th=[   41], 95.00th=[   48],
+     | 99.00th=[   69], 99.50th=[   80], 99.90th=[  124], 99.95th=[  144],
+     | 99.99th=[  169]
+   bw (  MiB/s): min=  515, max= 3432, per=99.95%, avg=1277.33, stdev=25.81, samples=3826
+   iops        : min= 8245, max=54925, avg=20436.82, stdev=412.91, samples=3826
+  write: IOPS=20.5k, BW=1279MiB/s (1341MB/s)(150GiB/120005msec); 0 zone resets
+    slat (usec): min=21, max=120141, avg=352.90, stdev=881.81
+    clat (usec): min=4, max=307438, avg=24641.29, stdev=12830.63
+     lat (usec): min=219, max=308785, avg=24994.66, stdev=13009.21
+    clat percentiles (msec):
+     |  1.00th=[    6],  5.00th=[   12], 10.00th=[   16], 20.00th=[   17],
+     | 30.00th=[   18], 40.00th=[   19], 50.00th=[   20], 60.00th=[   22],
+     | 70.00th=[   28], 80.00th=[   34], 90.00th=[   41], 95.00th=[   48],
+     | 99.00th=[   69], 99.50th=[   80], 99.90th=[  125], 99.95th=[  144],
+     | 99.99th=[  169]
+   bw (  MiB/s): min=  515, max= 3423, per=99.96%, avg=1278.68, stdev=25.77, samples=3826
+   iops        : min= 8246, max=54781, avg=20458.50, stdev=412.35, samples=3826
+  lat (usec)   : 4=0.01%, 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
+  lat (usec)   : 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.35%, 10=3.42%, 20=48.14%, 50=43.92%
+  lat (msec)   : 100=3.98%, 250=0.19%, 500=0.01%
+  cpu          : usr=1.67%, sys=22.82%, ctx=4244023, majf=0, minf=203278
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=2453778,2456080,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=1278MiB/s (1340MB/s), 1278MiB/s-1278MiB/s (1340MB/s-1340MB/s), io=150GiB (161GB), run=120005-120005msec
+  WRITE: bw=1279MiB/s (1341MB/s), 1279MiB/s-1279MiB/s (1341MB/s-1341MB/s), io=150GiB (161GB), run=120005-120005msec
++ rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-comp-64kchunk-16thread]
+bs=64k
+directory=/srv/z/comp/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=16
+nrfiles=16
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+...
+fio-3.16
+Starting 16 processes
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+Jobs: 16 (f=256): [M(16)][8.3%][r=1437MiB/s,w=1437MiB/s][r=22.0k,w=22.0k IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.5%][r=1311MiB/s,w=1313MiB/s][r=20.0k,w=21.0k IOPS][eta 01m:41s]
Jobs: 16 (f=256): [M(16)][24.2%][r=1288MiB/s,w=1287MiB/s][r=20.6k,w=20.6k IOPS][eta 01m:31s]

+Jobs: 16 (f=256): [M(16)][32.8%][r=1291MiB/s,w=1293MiB/s][r=20.7k,w=20.7k IOPS][eta 01m:20s]
Jobs: 16 (f=256): [M(16)][40.5%][r=1270MiB/s,w=1270MiB/s][r=20.3k,w=20.3k IOPS][eta 01m:12s] 

+Jobs: 16 (f=256): [M(16)][48.3%][r=1292MiB/s,w=1298MiB/s][r=20.7k,w=20.8k IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.7%][r=1286MiB/s,w=1286MiB/s][r=20.6k,w=20.6k IOPS][eta 00m:52s] 

+Jobs: 16 (f=256): [M(16)][65.0%][r=1279MiB/s,w=1281MiB/s][r=20.5k,w=20.5k IOPS][eta 00m:42s]
Jobs: 16 (f=256): [M(16)][72.5%][r=1268MiB/s,w=1273MiB/s][r=20.3k,w=20.4k IOPS][eta 00m:33s] 

+Jobs: 16 (f=256): [M(16)][80.2%][r=1273MiB/s,w=1267MiB/s][r=20.4k,w=20.3k IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=1164MiB/s,w=1167MiB/s][r=18.6k,w=18.7k IOPS][eta 00m:14s] 

+Jobs: 16 (f=256): [M(16)][96.7%][r=1144MiB/s,w=1142MiB/s][r=18.3k,w=18.3k IOPS][eta 00m:04s]

+test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=4154965: Fri Feb 28 22:08:43 2020
+  read: IOPS=20.4k, BW=1272MiB/s (1334MB/s)(149GiB/120007msec)
+    slat (usec): min=17, max=117295, avg=435.09, stdev=778.86
+    clat (usec): min=2, max=291935, avg=24736.95, stdev=12279.16
+     lat (usec): min=33, max=292132, avg=25172.63, stdev=12463.61
+    clat percentiles (msec):
+     |  1.00th=[    8],  5.00th=[   12], 10.00th=[   15], 20.00th=[   17],
+     | 30.00th=[   18], 40.00th=[   20], 50.00th=[   21], 60.00th=[   23],
+     | 70.00th=[   28], 80.00th=[   33], 90.00th=[   41], 95.00th=[   48],
+     | 99.00th=[   67], 99.50th=[   75], 99.90th=[  106], 99.95th=[  136],
+     | 99.99th=[  182]
+   bw (  MiB/s): min=  541, max= 2952, per=99.92%, avg=1271.17, stdev=23.49, samples=3836
+   iops        : min= 8666, max=47232, avg=20338.14, stdev=375.84, samples=3836
+  write: IOPS=20.4k, BW=1273MiB/s (1335MB/s)(149GiB/120007msec); 0 zone resets
+    slat (usec): min=23, max=129833, avg=343.66, stdev=813.37
+    clat (usec): min=3, max=291635, avg=24750.94, stdev=12282.40
+     lat (usec): min=157, max=292355, avg=25095.09, stdev=12437.16
+    clat percentiles (msec):
+     |  1.00th=[    8],  5.00th=[   12], 10.00th=[   15], 20.00th=[   17],
+     | 30.00th=[   18], 40.00th=[   20], 50.00th=[   21], 60.00th=[   23],
+     | 70.00th=[   28], 80.00th=[   33], 90.00th=[   41], 95.00th=[   48],
+     | 99.00th=[   67], 99.50th=[   75], 99.90th=[  106], 99.95th=[  134],
+     | 99.99th=[  180]
+   bw (  MiB/s): min=  539, max= 2955, per=99.92%, avg=1272.14, stdev=23.53, samples=3836
+   iops        : min= 8626, max=47283, avg=20353.57, stdev=376.46, samples=3836
+  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 250=0.01%
+  lat (usec)   : 500=0.01%, 750=0.01%, 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.07%, 10=2.47%, 20=44.98%, 50=48.31%
+  lat (msec)   : 100=4.05%, 250=0.12%, 500=0.01%
+  cpu          : usr=1.68%, sys=22.57%, ctx=4332673, majf=0, minf=217195
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=2442860,2444709,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=1272MiB/s (1334MB/s), 1272MiB/s-1272MiB/s (1334MB/s-1334MB/s), io=149GiB (160GB), run=120007-120007msec
+  WRITE: bw=1273MiB/s (1335MB/s), 1273MiB/s-1273MiB/s (1335MB/s-1335MB/s), io=149GiB (160GB), run=120007-120007msec
++ rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-enc-64kchunk-16thread]
+bs=64k
+directory=/srv/z/enc/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=16
+nrfiles=16
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+...
+fio-3.16
+Starting 16 processes
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+Jobs: 16 (f=256): [M(16)][8.3%][r=567MiB/s,w=569MiB/s][r=9064,w=9096 IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.7%][r=578MiB/s,w=581MiB/s][r=9241,w=9301 IOPS][eta 01m:40s]
Jobs: 16 (f=256): [M(16)][24.2%][r=575MiB/s,w=573MiB/s][r=9193,w=9169 IOPS][eta 01m:31s]

+Jobs: 16 (f=256): [M(16)][32.5%][r=555MiB/s,w=551MiB/s][r=8879,w=8819 IOPS][eta 01m:21s]
Jobs: 16 (f=256): [M(16)][40.0%][r=561MiB/s,w=562MiB/s][r=8981,w=8994 IOPS][eta 01m:12s] 

+Jobs: 16 (f=256): [M(16)][48.3%][r=535MiB/s,w=535MiB/s][r=8559,w=8566 IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.2%][r=508MiB/s,w=508MiB/s][r=8123,w=8124 IOPS][eta 00m:53s] 

+Jobs: 16 (f=256): [M(16)][64.2%][r=499MiB/s,w=495MiB/s][r=7991,w=7919 IOPS][eta 00m:43s]
Jobs: 16 (f=256): [M(16)][72.5%][r=483MiB/s,w=483MiB/s][r=7724,w=7734 IOPS][eta 00m:33s] 

+Jobs: 16 (f=256): [M(16)][80.2%][r=485MiB/s,w=485MiB/s][r=7756,w=7763 IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=478MiB/s,w=477MiB/s][r=7640,w=7626 IOPS][eta 00m:14s] 

+Jobs: 16 (f=256): [M(16)][96.7%][r=467MiB/s,w=469MiB/s][r=7476,w=7510 IOPS][eta 00m:04s]

+test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=2340655: Fri Feb 28 22:27:27 2020
+  read: IOPS=8361, BW=523MiB/s (548MB/s)(61.3GiB/120040msec)
+    slat (usec): min=14, max=256636, avg=922.71, stdev=4034.66
+    clat (usec): min=5, max=573459, avg=60227.28, stdev=53181.51
+     lat (msec): min=3, max=573, avg=61.15, stdev=53.89
+    clat percentiles (msec):
+     |  1.00th=[    5],  5.00th=[    6], 10.00th=[   11], 20.00th=[   20],
+     | 30.00th=[   31], 40.00th=[   41], 50.00th=[   49], 60.00th=[   56],
+     | 70.00th=[   66], 80.00th=[   84], 90.00th=[  131], 95.00th=[  176],
+     | 99.00th=[  257], 99.50th=[  288], 99.90th=[  359], 99.95th=[  393],
+     | 99.99th=[  477]
+   bw (  KiB/s): min=116536, max=1371827, per=99.57%, avg=532808.25, stdev=19629.32, samples=3840
+   iops        : min= 1816, max=21429, avg=8321.54, stdev=306.71, samples=3840
+  write: IOPS=8360, BW=523MiB/s (548MB/s)(61.3GiB/120040msec); 0 zone resets
+    slat (usec): min=18, max=180031, avg=983.49, stdev=4112.47
+    clat (usec): min=4, max=573443, avg=60227.90, stdev=53246.90
+     lat (msec): min=2, max=573, avg=61.21, stdev=53.89
+    clat percentiles (msec):
+     |  1.00th=[    5],  5.00th=[    6], 10.00th=[   11], 20.00th=[   20],
+     | 30.00th=[   31], 40.00th=[   41], 50.00th=[   49], 60.00th=[   56],
+     | 70.00th=[   66], 80.00th=[   84], 90.00th=[  131], 95.00th=[  176],
+     | 99.00th=[  257], 99.50th=[  288], 99.90th=[  359], 99.95th=[  397],
+     | 99.99th=[  481]
+   bw (  KiB/s): min=119117, max=1365805, per=99.56%, avg=532748.41, stdev=19715.12, samples=3840
+   iops        : min= 1859, max=21336, avg=8320.62, stdev=308.05, samples=3840
+  lat (usec)   : 10=0.01%
+  lat (msec)   : 4=0.19%, 10=9.44%, 20=11.37%, 50=30.99%, 100=32.35%
+  lat (msec)   : 250=14.53%, 500=1.12%, 750=0.01%
+  cpu          : usr=0.51%, sys=40.92%, ctx=239959, majf=0, minf=371051
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=1003709,1003604,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=523MiB/s (548MB/s), 523MiB/s-523MiB/s (548MB/s-548MB/s), io=61.3GiB (65.8GB), run=120040-120040msec
+  WRITE: bw=523MiB/s (548MB/s), 523MiB/s-523MiB/s (548MB/s-548MB/s), io=61.3GiB (65.8GB), run=120040-120040msec
++ rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9
++ for DIRNAME in none comp enc both
++ cat
++ cat /tmp/jobfile.fio
+
+[test-both-64kchunk-16thread]
+bs=64k
+directory=/srv/z/both/
+iodepth=64
+ioengine=libaio
+readwrite=readwrite
+numjobs=16
+nrfiles=16
+size=10000M
+runtime=120
+time_based=1
+group_reporting=1
++ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
+test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
+...
+fio-3.16
+Starting 16 processes
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
+Jobs: 16 (f=256): [M(16)][8.3%][r=557MiB/s,w=560MiB/s][r=8916,w=8954 IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.7%][r=539MiB/s,w=541MiB/s][r=8616,w=8651 IOPS][eta 01m:40s]
Jobs: 16 (f=256): [M(16)][24.2%][r=520MiB/s,w=520MiB/s][r=8317,w=8317 IOPS][eta 01m:31s]

+Jobs: 16 (f=256): [M(16)][32.5%][r=519MiB/s,w=516MiB/s][r=8303,w=8260 IOPS][eta 01m:21s]
Jobs: 16 (f=256): [M(16)][40.5%][r=477MiB/s,w=477MiB/s][r=7637,w=7636 IOPS][eta 01m:12s] 

+Jobs: 16 (f=256): [M(16)][48.3%][r=493MiB/s,w=494MiB/s][r=7884,w=7906 IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.7%][r=484MiB/s,w=482MiB/s][r=7738,w=7715 IOPS][eta 00m:52s] 

+Jobs: 16 (f=256): [M(16)][64.2%][r=462MiB/s,w=465MiB/s][r=7397,w=7435 IOPS][eta 00m:43s]
Jobs: 16 (f=256): [M(16)][72.5%][r=462MiB/s,w=463MiB/s][r=7398,w=7400 IOPS][eta 00m:33s] 

+Jobs: 16 (f=256): [M(16)][80.2%][r=469MiB/s,w=466MiB/s][r=7505,w=7449 IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=454MiB/s,w=454MiB/s][r=7264,w=7262 IOPS][eta 00m:14s] 

+Jobs: 16 (f=256): [M(16)][96.7%][r=450MiB/s,w=446MiB/s][r=7207,w=7141 IOPS][eta 00m:04s]

+test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=344989: Fri Feb 28 22:46:09 2020
+  read: IOPS=7822, BW=489MiB/s (513MB/s)(57.3GiB/120004msec)
+    slat (usec): min=14, max=211751, avg=969.49, stdev=4169.66
+    clat (usec): min=2, max=544781, avg=64436.11, stdev=57921.87
+     lat (usec): min=31, max=563219, avg=65406.05, stdev=58687.86
+    clat percentiles (msec):
+     |  1.00th=[    5],  5.00th=[    5], 10.00th=[    9], 20.00th=[   18],
+     | 30.00th=[   32], 40.00th=[   44], 50.00th=[   52], 60.00th=[   59],
+     | 70.00th=[   70], 80.00th=[   94], 90.00th=[  144], 95.00th=[  188],
+     | 99.00th=[  271], 99.50th=[  305], 99.90th=[  376], 99.95th=[  405],
+     | 99.99th=[  481]
+   bw (  KiB/s): min=102976, max=1560572, per=99.46%, avg=497942.69, stdev=23994.75, samples=3824
+   iops        : min= 1603, max=24383, avg=7777.17, stdev=374.92, samples=3824
+  write: IOPS=7818, BW=489MiB/s (512MB/s)(57.3GiB/120004msec); 0 zone resets
+    slat (usec): min=18, max=263163, avg=1069.56, stdev=4489.40
+    clat (usec): min=2, max=544758, avg=64374.60, stdev=57746.14
+     lat (usec): min=756, max=544822, avg=65444.58, stdev=58468.14
+    clat percentiles (msec):
+     |  1.00th=[    5],  5.00th=[    5], 10.00th=[    9], 20.00th=[   18],
+     | 30.00th=[   32], 40.00th=[   44], 50.00th=[   52], 60.00th=[   59],
+     | 70.00th=[   70], 80.00th=[   94], 90.00th=[  144], 95.00th=[  188],
+     | 99.00th=[  271], 99.50th=[  305], 99.90th=[  372], 99.95th=[  401],
+     | 99.99th=[  481]
+   bw (  KiB/s): min=106826, max=1534676, per=99.46%, avg=497682.75, stdev=23855.92, samples=3824
+   iops        : min= 1663, max=23976, avg=7773.21, stdev=372.75, samples=3824
+  lat (usec)   : 4=0.01%, 10=0.01%, 50=0.01%, 750=0.01%, 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.57%, 10=11.75%, 20=9.77%, 50=26.03%
+  lat (msec)   : 100=33.42%, 250=16.95%, 500=1.50%, 750=0.01%
+  cpu          : usr=0.49%, sys=38.25%, ctx=269609, majf=0, minf=282080
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=938705,938231,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=489MiB/s (513MB/s), 489MiB/s-489MiB/s (513MB/s-513MB/s), io=57.3GiB (61.5GB), run=120004-120004msec
+  WRITE: bw=489MiB/s (512MB/s), 489MiB/s-489MiB/s (512MB/s-512MB/s), io=57.3GiB (61.5GB), run=120004-120004msec
++ rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9
+
+Script done on Fri Feb 28 14:46:10 2020