Script started on Fri Feb 28 13:46:25 2020
command: bash -c make run
rsync -avP ./disktest.sh root@nbg1.datavibe.net:/root/disktest.sh
sending incremental file list
disktest.sh

          1,247 100%    0.00kB/s    0:00:00  
          1,247 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=0/1)

sent 1,356 bytes  received 35 bytes  927.33 bytes/sec
total size is 1,247  speedup is 0.90
ssh root@nbg1.datavibe.net "bash /root/disktest.sh 2>&1"
+ POOLNAME=tank
+ MPT=/srv/z
+ JOBFILE=/tmp/jobfile.fio
+ KEYLOC=/tmp/testkey.raw
+ main
+ show_system_info
+ print_divider
+ echo '##############################################################'
##############################################################
+ echo '##############################################################'
##############################################################
+ date -u
Fri 28 Feb 2020 09:46:26 PM UTC
+ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu Focal Fossa (development branch)
Release:	20.04
Codename:	focal
+ uname -a
Linux nbg1 5.4.0-14-generic #17-Ubuntu SMP Thu Feb 6 22:47:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
+ lsblk -t
NAME    ALIGNMENT MIN-IO  OPT-IO PHY-SEC LOG-SEC ROTA SCHED       RQ-SIZE   RA WSAME
sda             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sda1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sda2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sda3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
sdb             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sdb1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sdb2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sdb3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
sdc             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sdc1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sdc2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sdc3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
sdd             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sdd1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sdd2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sdd3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
sde             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sde1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sde2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sde3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
sdf             0   4096       0    4096     512    0 mq-deadline     256  128    0B
├─sdf1          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md0         0   4096       0    4096     512    0                 128  128    0B
├─sdf2          0   4096       0    4096     512    0 mq-deadline     256  128    0B
│ └─md1         0 524288 2097152    4096     512    0                 128 4096    0B
└─sdf3          0   4096       0    4096     512    0 mq-deadline     256  128    0B
+ nvme list
/root/disktest.sh: line 26: nvme: command not found
+ modinfo zfs
+ grep -iw version
version:        0.8.3-1ubuntu3
+ modinfo spl
+ grep -iw version
version:        0.8.3-1ubuntu3
+ zfs version
zfs-0.8.3-1ubuntu5
zfs-kmod-0.8.3-1ubuntu3
+ print_divider
+ echo '##############################################################'
##############################################################
+ echo '##############################################################'
##############################################################
+ do_zfs_tests
+ print_divider
+ echo '##############################################################'
##############################################################
+ echo '##############################################################'
##############################################################
+ FO='--eta-newline=10 --eta-interval=10 --eta=always'
+ for NJ in 1 16
+ for BS in 64k
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-none-64kchunk-1thread]
bs=64k
directory=/srv/z/none/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=1
nrfiles=1
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
Jobs: 1 (f=1): [M(1)][8.3%][r=917MiB/s,w=919MiB/s][r=14.7k,w=14.7k IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.5%][r=953MiB/s,w=953MiB/s][r=15.3k,w=15.2k IOPS][eta 01m:41s]
Jobs: 1 (f=1): [M(1)][24.2%][r=897MiB/s,w=893MiB/s][r=14.3k,w=14.3k IOPS][eta 01m:31s]

Jobs: 1 (f=1): [M(1)][32.8%][r=882MiB/s,w=880MiB/s][r=14.1k,w=14.1k IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.5%][r=907MiB/s,w=905MiB/s][r=14.5k,w=14.5k IOPS][eta 01m:12s] 

Jobs: 1 (f=1): [M(1)][48.3%][r=880MiB/s,w=881MiB/s][r=14.1k,w=14.1k IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=892MiB/s,w=892MiB/s][r=14.3k,w=14.3k IOPS][eta 00m:52s] 

Jobs: 1 (f=1): [M(1)][65.5%][r=902MiB/s,w=905MiB/s][r=14.4k,w=14.5k IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=906MiB/s,w=903MiB/s][r=14.5k,w=14.5k IOPS][eta 00m:33s] 

Jobs: 1 (f=1): [M(1)][80.8%][r=886MiB/s,w=881MiB/s][r=14.2k,w=14.1k IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][88.4%][r=906MiB/s,w=905MiB/s][r=14.5k,w=14.5k IOPS][eta 00m:14s] 

Jobs: 1 (f=1): [M(1)][96.7%][r=879MiB/s,w=883MiB/s][r=14.1k,w=14.1k IOPS][eta 00m:04s]

test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=285277: Fri Feb 28 21:48:41 2020
  read: IOPS=14.4k, BW=901MiB/s (945MB/s)(106GiB/120001msec)
    slat (usec): min=6, max=100872, avg=36.06, stdev=136.62
    clat (usec): min=41, max=118051, avg=2186.43, stdev=893.80
     lat (usec): min=114, max=118339, avg=2222.59, stdev=909.16
    clat percentiles (usec):
     |  1.00th=[ 1418],  5.00th=[ 1663], 10.00th=[ 1762], 20.00th=[ 1827],
     | 30.00th=[ 1876], 40.00th=[ 1926], 50.00th=[ 2040], 60.00th=[ 2245],
     | 70.00th=[ 2376], 80.00th=[ 2507], 90.00th=[ 2704], 95.00th=[ 2999],
     | 99.00th=[ 3720], 99.50th=[ 4113], 99.90th=[ 5538], 99.95th=[ 6652],
     | 99.99th=[ 9765]
   bw (  KiB/s): min=606080, max=1427200, per=99.97%, avg=922432.68, stdev=95336.08, samples=239
   iops        : min= 9470, max=22300, avg=14412.97, stdev=1489.61, samples=239
  write: IOPS=14.4k, BW=900MiB/s (944MB/s)(105GiB/120001msec); 0 zone resets
    slat (usec): min=9, max=8839, avg=31.46, stdev=16.75
    clat (nsec): min=1690, max=118306k, avg=2185831.96, stdev=903829.11
     lat (usec): min=39, max=118336, avg=2217.42, stdev=907.24
    clat percentiles (usec):
     |  1.00th=[ 1418],  5.00th=[ 1663], 10.00th=[ 1762], 20.00th=[ 1827],
     | 30.00th=[ 1876], 40.00th=[ 1926], 50.00th=[ 2040], 60.00th=[ 2245],
     | 70.00th=[ 2376], 80.00th=[ 2507], 90.00th=[ 2704], 95.00th=[ 2999],
     | 99.00th=[ 3720], 99.50th=[ 4113], 99.90th=[ 5538], 99.95th=[ 6783],
     | 99.99th=[10290]
   bw (  KiB/s): min=607360, max=1419776, per=99.97%, avg=921526.12, stdev=94577.93, samples=239
   iops        : min= 9490, max=22184, avg=14398.77, stdev=1477.77, samples=239
  lat (usec)   : 2=0.01%, 50=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
  lat (usec)   : 1000=0.01%
  lat (msec)   : 2=47.74%, 4=51.66%, 10=0.59%, 20=0.01%, 250=0.01%
  cpu          : usr=6.98%, sys=91.83%, ctx=7753, majf=0, minf=13653
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=1730179,1728438,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=901MiB/s (945MB/s), 901MiB/s-901MiB/s (945MB/s-945MB/s), io=106GiB (113GB), run=120001-120001msec
  WRITE: bw=900MiB/s (944MB/s), 900MiB/s-900MiB/s (944MB/s-944MB/s), io=105GiB (113GB), run=120001-120001msec
+ rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-comp-64kchunk-1thread]
bs=64k
directory=/srv/z/comp/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=1
nrfiles=1
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
Jobs: 1 (f=1): [M(1)][8.3%][r=886MiB/s,w=887MiB/s][r=14.2k,w=14.2k IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.7%][r=944MiB/s,w=945MiB/s][r=15.1k,w=15.1k IOPS][eta 01m:40s]
Jobs: 1 (f=1): [M(1)][24.2%][r=885MiB/s,w=880MiB/s][r=14.2k,w=14.1k IOPS][eta 01m:31s]

Jobs: 1 (f=1): [M(1)][32.8%][r=834MiB/s,w=834MiB/s][r=13.3k,w=13.3k IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.8%][r=857MiB/s,w=854MiB/s][r=13.7k,w=13.7k IOPS][eta 01m:11s] 

Jobs: 1 (f=1): [M(1)][48.3%][r=841MiB/s,w=840MiB/s][r=13.4k,w=13.4k IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=835MiB/s,w=834MiB/s][r=13.4k,w=13.3k IOPS][eta 00m:52s] 

Jobs: 1 (f=1): [M(1)][65.5%][r=864MiB/s,w=868MiB/s][r=13.8k,w=13.9k IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=860MiB/s,w=859MiB/s][r=13.8k,w=13.7k IOPS][eta 00m:33s] 

Jobs: 1 (f=1): [M(1)][80.8%][r=830MiB/s,w=826MiB/s][r=13.3k,w=13.2k IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=823MiB/s,w=823MiB/s][r=13.2k,w=13.2k IOPS][eta 00m:12s] 

Jobs: 1 (f=1): [M(1)][96.7%][r=849MiB/s,w=848MiB/s][r=13.6k,w=13.6k IOPS][eta 00m:04s]

test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1329996: Fri Feb 28 21:50:56 2020
  read: IOPS=13.8k, BW=860MiB/s (902MB/s)(101GiB/120001msec)
    slat (usec): min=6, max=106574, avg=38.02, stdev=278.16
    clat (usec): min=49, max=170126, avg=2291.59, stdev=1934.03
     lat (usec): min=125, max=170188, avg=2329.72, stdev=1963.60
    clat percentiles (usec):
     |  1.00th=[  1450],  5.00th=[  1696], 10.00th=[  1778], 20.00th=[  1844],
     | 30.00th=[  1893], 40.00th=[  1942], 50.00th=[  2008], 60.00th=[  2311],
     | 70.00th=[  2474], 80.00th=[  2638], 90.00th=[  2900], 95.00th=[  3097],
     | 99.00th=[  3785], 99.50th=[  5080], 99.90th=[ 23200], 99.95th=[ 34341],
     | 99.99th=[102237]
   bw (  KiB/s): min=424448, max=1341696, per=99.89%, avg=879662.20, stdev=116352.77, samples=239
   iops        : min= 6632, max=20964, avg=13744.67, stdev=1817.99, samples=239
  write: IOPS=13.7k, BW=859MiB/s (901MB/s)(101GiB/120001msec); 0 zone resets
    slat (usec): min=9, max=108207, avg=32.56, stdev=116.95
    clat (usec): min=2, max=170135, avg=2289.69, stdev=1914.67
     lat (usec): min=48, max=170173, avg=2322.38, stdev=1921.04
    clat percentiles (usec):
     |  1.00th=[  1450],  5.00th=[  1696], 10.00th=[  1778], 20.00th=[  1844],
     | 30.00th=[  1893], 40.00th=[  1942], 50.00th=[  2008], 60.00th=[  2278],
     | 70.00th=[  2474], 80.00th=[  2638], 90.00th=[  2900], 95.00th=[  3097],
     | 99.00th=[  3752], 99.50th=[  5014], 99.90th=[ 21890], 99.95th=[ 34866],
     | 99.99th=[102237]
   bw (  KiB/s): min=416000, max=1341568, per=99.89%, avg=878853.61, stdev=116939.69, samples=239
   iops        : min= 6500, max=20962, avg=13732.02, stdev=1827.17, samples=239
  lat (usec)   : 4=0.01%, 50=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
  lat (usec)   : 1000=0.01%
  lat (msec)   : 2=48.94%, 4=50.22%, 10=0.63%, 20=0.09%, 50=0.08%
  lat (msec)   : 100=0.02%, 250=0.01%
  cpu          : usr=7.02%, sys=89.10%, ctx=8743, majf=0, minf=27256
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=1651235,1649716,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=860MiB/s (902MB/s), 860MiB/s-860MiB/s (902MB/s-902MB/s), io=101GiB (108GB), run=120001-120001msec
  WRITE: bw=859MiB/s (901MB/s), 859MiB/s-859MiB/s (901MB/s-901MB/s), io=101GiB (108GB), run=120001-120001msec
+ rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-enc-64kchunk-1thread]
bs=64k
directory=/srv/z/enc/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=1
nrfiles=1
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
Jobs: 1 (f=1): [M(1)][8.3%][r=54.7MiB/s,w=54.8MiB/s][r=875,w=877 IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.7%][r=55.5MiB/s,w=55.3MiB/s][r=888,w=885 IOPS][eta 01m:40s]
Jobs: 1 (f=1): [M(1)][24.2%][r=55.4MiB/s,w=55.0MiB/s][r=886,w=880 IOPS][eta 01m:31s]

Jobs: 1 (f=1): [M(1)][32.8%][r=55.6MiB/s,w=56.5MiB/s][r=890,w=903 IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.8%][r=56.0MiB/s,w=53.5MiB/s][r=896,w=855 IOPS][eta 01m:11s] 

Jobs: 1 (f=1): [M(1)][48.3%][r=54.3MiB/s,w=55.2MiB/s][r=868,w=882 IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=54.3MiB/s,w=54.6MiB/s][r=869,w=874 IOPS][eta 00m:52s] 

Jobs: 1 (f=1): [M(1)][65.5%][r=54.3MiB/s,w=52.9MiB/s][r=868,w=846 IOPS][eta 00m:41s]
Jobs: 1 (f=1): [M(1)][72.5%][r=55.9MiB/s,w=55.7MiB/s][r=894,w=890 IOPS][eta 00m:33s] 

Jobs: 1 (f=1): [M(1)][80.8%][r=55.2MiB/s,w=55.4MiB/s][r=882,w=886 IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=54.0MiB/s,w=55.1MiB/s][r=879,w=881 IOPS][eta 00m:12s] 

Jobs: 1 (f=1): [M(1)][96.7%][r=54.5MiB/s,w=55.4MiB/s][r=871,w=886 IOPS][eta 00m:04s]

test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2235096: Fri Feb 28 21:53:11 2020
  read: IOPS=882, BW=55.1MiB/s (57.8MB/s)(6615MiB/120001msec)
    slat (usec): min=7, max=106999, avg=861.13, stdev=1127.17
    clat (usec): min=2, max=151067, avg=35748.24, stdev=6537.32
     lat (msec): min=2, max=153, avg=36.61, stdev= 6.65
    clat percentiles (msec):
     |  1.00th=[   26],  5.00th=[   28], 10.00th=[   30], 20.00th=[   32],
     | 30.00th=[   33], 40.00th=[   34], 50.00th=[   35], 60.00th=[   36],
     | 70.00th=[   38], 80.00th=[   40], 90.00th=[   43], 95.00th=[   47],
     | 99.00th=[   61], 99.50th=[   64], 99.90th=[   71], 99.95th=[   74],
     | 99.99th=[  150]
   bw (  KiB/s): min=38144, max=67840, per=99.95%, avg=56421.40, stdev=4863.10, samples=240
   iops        : min=  596, max= 1060, avg=881.56, stdev=76.01, samples=240
  write: IOPS=880, BW=55.0MiB/s (57.7MB/s)(6605MiB/120001msec); 0 zone resets
    slat (usec): min=10, max=10019, avg=270.92, stdev=690.41
    clat (msec): min=4, max=151, avg=35.72, stdev= 6.50
     lat (msec): min=4, max=151, avg=35.99, stdev= 6.52
    clat percentiles (msec):
     |  1.00th=[   26],  5.00th=[   28], 10.00th=[   30], 20.00th=[   32],
     | 30.00th=[   33], 40.00th=[   34], 50.00th=[   35], 60.00th=[   36],
     | 70.00th=[   38], 80.00th=[   40], 90.00th=[   42], 95.00th=[   46],
     | 99.00th=[   61], 99.50th=[   64], 99.90th=[   71], 99.95th=[   75],
     | 99.99th=[  150]
   bw (  KiB/s): min=36352, max=68352, per=99.95%, avg=56338.82, stdev=5655.56, samples=240
   iops        : min=  568, max= 1068, avg=880.26, stdev=88.39, samples=240
  lat (usec)   : 4=0.01%
  lat (msec)   : 4=0.01%, 10=0.01%, 20=0.03%, 50=96.32%, 100=3.61%
  lat (msec)   : 250=0.03%
  cpu          : usr=0.48%, sys=99.11%, ctx=1360, majf=0, minf=13645
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=105847,105686,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=55.1MiB/s (57.8MB/s), 55.1MiB/s-55.1MiB/s (57.8MB/s-57.8MB/s), io=6615MiB (6937MB), run=120001-120001msec
  WRITE: bw=55.0MiB/s (57.7MB/s), 55.0MiB/s-55.0MiB/s (57.7MB/s-57.7MB/s), io=6605MiB (6926MB), run=120001-120001msec
+ rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-both-64kchunk-1thread]
bs=64k
directory=/srv/z/both/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=1
nrfiles=1
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB)
Jobs: 1 (f=1): [M(1)][8.3%][r=56.7MiB/s,w=56.7MiB/s][r=907,w=907 IOPS][eta 01m:50s]
Jobs: 1 (f=1): [M(1)][16.5%][r=55.8MiB/s,w=55.7MiB/s][r=892,w=891 IOPS][eta 01m:41s]
Jobs: 1 (f=1): [M(1)][24.2%][r=55.6MiB/s,w=55.4MiB/s][r=889,w=886 IOPS][eta 01m:31s]

Jobs: 1 (f=1): [M(1)][32.8%][r=56.0MiB/s,w=56.5MiB/s][r=896,w=903 IOPS][eta 01m:20s]
Jobs: 1 (f=1): [M(1)][40.5%][r=57.1MiB/s,w=54.7MiB/s][r=913,w=875 IOPS][eta 01m:12s] 

Jobs: 1 (f=1): [M(1)][48.3%][r=55.6MiB/s,w=56.4MiB/s][r=889,w=902 IOPS][eta 01m:02s]
Jobs: 1 (f=1): [M(1)][56.7%][r=54.8MiB/s,w=55.2MiB/s][r=877,w=883 IOPS][eta 00m:52s] 

Jobs: 1 (f=1): [M(1)][65.0%][r=55.1MiB/s,w=53.5MiB/s][r=880,w=856 IOPS][eta 00m:42s]
Jobs: 1 (f=1): [M(1)][72.5%][r=56.7MiB/s,w=56.8MiB/s][r=906,w=909 IOPS][eta 00m:33s] 

Jobs: 1 (f=1): [M(1)][80.8%][r=54.2MiB/s,w=54.3MiB/s][r=868,w=868 IOPS][eta 00m:23s]
Jobs: 1 (f=1): [M(1)][89.9%][r=54.7MiB/s,w=55.1MiB/s][r=875,w=882 IOPS][eta 00m:12s] 

Jobs: 1 (f=1): [M(1)][96.7%][r=55.1MiB/s,w=56.0MiB/s][r=881,w=896 IOPS][eta 00m:04s]

test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2366276: Fri Feb 28 21:55:26 2020
  read: IOPS=890, BW=55.7MiB/s (58.4MB/s)(6680MiB/120001msec)
    slat (usec): min=7, max=9126, avg=855.94, stdev=1070.39
    clat (usec): min=2, max=89273, avg=35406.88, stdev=6185.51
     lat (usec): min=2068, max=89307, avg=36262.95, stdev=6296.63
    clat percentiles (usec):
     |  1.00th=[24511],  5.00th=[27395], 10.00th=[28967], 20.00th=[31065],
     | 30.00th=[32375], 40.00th=[33424], 50.00th=[34866], 60.00th=[35914],
     | 70.00th=[36963], 80.00th=[39060], 90.00th=[41681], 95.00th=[46924],
     | 99.00th=[58459], 99.50th=[61080], 99.90th=[67634], 99.95th=[71828],
     | 99.99th=[84411]
   bw (  KiB/s): min=37504, max=67840, per=99.93%, avg=56961.33, stdev=5137.37, samples=239
   iops        : min=  586, max= 1060, avg=890.00, stdev=80.27, samples=239
  write: IOPS=888, BW=55.5MiB/s (58.2MB/s)(6664MiB/120001msec); 0 zone resets
    slat (usec): min=10, max=26367, avg=265.38, stdev=681.08
    clat (usec): min=2081, max=89233, avg=35398.21, stdev=6203.43
     lat (usec): min=2110, max=90922, avg=35663.72, stdev=6219.74
    clat percentiles (usec):
     |  1.00th=[24511],  5.00th=[27395], 10.00th=[28967], 20.00th=[30802],
     | 30.00th=[32375], 40.00th=[33424], 50.00th=[34866], 60.00th=[35914],
     | 70.00th=[36963], 80.00th=[39060], 90.00th=[41681], 95.00th=[46924],
     | 99.00th=[58459], 99.50th=[61080], 99.90th=[67634], 99.95th=[70779],
     | 99.99th=[84411]
   bw (  KiB/s): min=36352, max=68608, per=99.97%, avg=56842.27, stdev=5917.48, samples=239
   iops        : min=  568, max= 1072, avg=888.14, stdev=92.46, samples=239
  lat (usec)   : 4=0.01%
  lat (msec)   : 4=0.01%, 10=0.01%, 20=0.02%, 50=96.24%, 100=3.74%
  cpu          : usr=0.51%, sys=99.08%, ctx=1576, majf=0, minf=14707
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=106885,106617,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=55.7MiB/s (58.4MB/s), 55.7MiB/s-55.7MiB/s (58.4MB/s-58.4MB/s), io=6680MiB (7005MB), run=120001-120001msec
  WRITE: bw=55.5MiB/s (58.2MB/s), 55.5MiB/s-55.5MiB/s (58.2MB/s-58.2MB/s), io=6664MiB (6987MB), run=120001-120001msec
+ rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0
+ for NJ in 1 16
+ for BS in 64k
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-none-64kchunk-16thread]
bs=64k
directory=/srv/z/none/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=16
nrfiles=16
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
...
fio-3.16
Starting 16 processes
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
Jobs: 16 (f=256): [M(16)][8.3%][r=1531MiB/s,w=1532MiB/s][r=24.5k,w=24.5k IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.5%][r=1323MiB/s,w=1324MiB/s][r=21.2k,w=21.2k IOPS][eta 01m:41s]
Jobs: 16 (f=256): [M(16)][24.2%][r=1338MiB/s,w=1335MiB/s][r=21.4k,w=21.4k IOPS][eta 01m:31s]

Jobs: 16 (f=256): [M(16)][32.8%][r=1314MiB/s,w=1322MiB/s][r=21.0k,w=21.2k IOPS][eta 01m:20s]
Jobs: 16 (f=256): [M(16)][40.0%][r=1304MiB/s,w=1303MiB/s][r=20.9k,w=20.8k IOPS][eta 01m:12s] 

Jobs: 16 (f=256): [M(16)][48.3%][r=1284MiB/s,w=1291MiB/s][r=20.5k,w=20.7k IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.2%][r=1281MiB/s,w=1280MiB/s][r=20.5k,w=20.5k IOPS][eta 00m:53s] 

Jobs: 16 (f=256): [M(16)][64.7%][r=1288MiB/s,w=1290MiB/s][r=20.6k,w=20.6k IOPS][eta 00m:42s]
Jobs: 16 (f=256): [M(16)][72.5%][r=1276MiB/s,w=1277MiB/s][r=20.4k,w=20.4k IOPS][eta 00m:33s] 

Jobs: 16 (f=256): [M(16)][80.0%][r=1231MiB/s,w=1230MiB/s][r=19.7k,w=19.7k IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=1096MiB/s,w=1099MiB/s][r=17.5k,w=17.6k IOPS][eta 00m:14s] 

Jobs: 16 (f=256): [M(16)][95.9%][r=1129MiB/s,w=1122MiB/s][r=18.1k,w=17.0k IOPS][eta 00m:05s]

test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3119643: Fri Feb 28 22:02:06 2020
  read: IOPS=20.4k, BW=1278MiB/s (1340MB/s)(150GiB/120005msec)
    slat (usec): min=16, max=121262, avg=422.11, stdev=709.84
    clat (usec): min=2, max=307795, avg=24625.01, stdev=12814.99
     lat (usec): min=152, max=308982, avg=25047.64, stdev=12992.67
    clat percentiles (msec):
     |  1.00th=[    6],  5.00th=[   12], 10.00th=[   16], 20.00th=[   17],
     | 30.00th=[   18], 40.00th=[   19], 50.00th=[   20], 60.00th=[   22],
     | 70.00th=[   28], 80.00th=[   34], 90.00th=[   41], 95.00th=[   48],
     | 99.00th=[   69], 99.50th=[   80], 99.90th=[  124], 99.95th=[  144],
     | 99.99th=[  169]
   bw (  MiB/s): min=  515, max= 3432, per=99.95%, avg=1277.33, stdev=25.81, samples=3826
   iops        : min= 8245, max=54925, avg=20436.82, stdev=412.91, samples=3826
  write: IOPS=20.5k, BW=1279MiB/s (1341MB/s)(150GiB/120005msec); 0 zone resets
    slat (usec): min=21, max=120141, avg=352.90, stdev=881.81
    clat (usec): min=4, max=307438, avg=24641.29, stdev=12830.63
     lat (usec): min=219, max=308785, avg=24994.66, stdev=13009.21
    clat percentiles (msec):
     |  1.00th=[    6],  5.00th=[   12], 10.00th=[   16], 20.00th=[   17],
     | 30.00th=[   18], 40.00th=[   19], 50.00th=[   20], 60.00th=[   22],
     | 70.00th=[   28], 80.00th=[   34], 90.00th=[   41], 95.00th=[   48],
     | 99.00th=[   69], 99.50th=[   80], 99.90th=[  125], 99.95th=[  144],
     | 99.99th=[  169]
   bw (  MiB/s): min=  515, max= 3423, per=99.96%, avg=1278.68, stdev=25.77, samples=3826
   iops        : min= 8246, max=54781, avg=20458.50, stdev=412.35, samples=3826
  lat (usec)   : 4=0.01%, 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
  lat (usec)   : 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.35%, 10=3.42%, 20=48.14%, 50=43.92%
  lat (msec)   : 100=3.98%, 250=0.19%, 500=0.01%
  cpu          : usr=1.67%, sys=22.82%, ctx=4244023, majf=0, minf=203278
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=2453778,2456080,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=1278MiB/s (1340MB/s), 1278MiB/s-1278MiB/s (1340MB/s-1340MB/s), io=150GiB (161GB), run=120005-120005msec
  WRITE: bw=1279MiB/s (1341MB/s), 1279MiB/s-1279MiB/s (1341MB/s-1341MB/s), io=150GiB (161GB), run=120005-120005msec
+ rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-comp-64kchunk-16thread]
bs=64k
directory=/srv/z/comp/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=16
nrfiles=16
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
...
fio-3.16
Starting 16 processes
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
Jobs: 16 (f=256): [M(16)][8.3%][r=1437MiB/s,w=1437MiB/s][r=22.0k,w=22.0k IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.5%][r=1311MiB/s,w=1313MiB/s][r=20.0k,w=21.0k IOPS][eta 01m:41s]
Jobs: 16 (f=256): [M(16)][24.2%][r=1288MiB/s,w=1287MiB/s][r=20.6k,w=20.6k IOPS][eta 01m:31s]

Jobs: 16 (f=256): [M(16)][32.8%][r=1291MiB/s,w=1293MiB/s][r=20.7k,w=20.7k IOPS][eta 01m:20s]
Jobs: 16 (f=256): [M(16)][40.5%][r=1270MiB/s,w=1270MiB/s][r=20.3k,w=20.3k IOPS][eta 01m:12s] 

Jobs: 16 (f=256): [M(16)][48.3%][r=1292MiB/s,w=1298MiB/s][r=20.7k,w=20.8k IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.7%][r=1286MiB/s,w=1286MiB/s][r=20.6k,w=20.6k IOPS][eta 00m:52s] 

Jobs: 16 (f=256): [M(16)][65.0%][r=1279MiB/s,w=1281MiB/s][r=20.5k,w=20.5k IOPS][eta 00m:42s]
Jobs: 16 (f=256): [M(16)][72.5%][r=1268MiB/s,w=1273MiB/s][r=20.3k,w=20.4k IOPS][eta 00m:33s] 

Jobs: 16 (f=256): [M(16)][80.2%][r=1273MiB/s,w=1267MiB/s][r=20.4k,w=20.3k IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=1164MiB/s,w=1167MiB/s][r=18.6k,w=18.7k IOPS][eta 00m:14s] 

Jobs: 16 (f=256): [M(16)][96.7%][r=1144MiB/s,w=1142MiB/s][r=18.3k,w=18.3k IOPS][eta 00m:04s]

test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=4154965: Fri Feb 28 22:08:43 2020
  read: IOPS=20.4k, BW=1272MiB/s (1334MB/s)(149GiB/120007msec)
    slat (usec): min=17, max=117295, avg=435.09, stdev=778.86
    clat (usec): min=2, max=291935, avg=24736.95, stdev=12279.16
     lat (usec): min=33, max=292132, avg=25172.63, stdev=12463.61
    clat percentiles (msec):
     |  1.00th=[    8],  5.00th=[   12], 10.00th=[   15], 20.00th=[   17],
     | 30.00th=[   18], 40.00th=[   20], 50.00th=[   21], 60.00th=[   23],
     | 70.00th=[   28], 80.00th=[   33], 90.00th=[   41], 95.00th=[   48],
     | 99.00th=[   67], 99.50th=[   75], 99.90th=[  106], 99.95th=[  136],
     | 99.99th=[  182]
   bw (  MiB/s): min=  541, max= 2952, per=99.92%, avg=1271.17, stdev=23.49, samples=3836
   iops        : min= 8666, max=47232, avg=20338.14, stdev=375.84, samples=3836
  write: IOPS=20.4k, BW=1273MiB/s (1335MB/s)(149GiB/120007msec); 0 zone resets
    slat (usec): min=23, max=129833, avg=343.66, stdev=813.37
    clat (usec): min=3, max=291635, avg=24750.94, stdev=12282.40
     lat (usec): min=157, max=292355, avg=25095.09, stdev=12437.16
    clat percentiles (msec):
     |  1.00th=[    8],  5.00th=[   12], 10.00th=[   15], 20.00th=[   17],
     | 30.00th=[   18], 40.00th=[   20], 50.00th=[   21], 60.00th=[   23],
     | 70.00th=[   28], 80.00th=[   33], 90.00th=[   41], 95.00th=[   48],
     | 99.00th=[   67], 99.50th=[   75], 99.90th=[  106], 99.95th=[  134],
     | 99.99th=[  180]
   bw (  MiB/s): min=  539, max= 2955, per=99.92%, avg=1272.14, stdev=23.53, samples=3836
   iops        : min= 8626, max=47283, avg=20353.57, stdev=376.46, samples=3836
  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 250=0.01%
  lat (usec)   : 500=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.07%, 10=2.47%, 20=44.98%, 50=48.31%
  lat (msec)   : 100=4.05%, 250=0.12%, 500=0.01%
  cpu          : usr=1.68%, sys=22.57%, ctx=4332673, majf=0, minf=217195
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=2442860,2444709,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=1272MiB/s (1334MB/s), 1272MiB/s-1272MiB/s (1334MB/s-1334MB/s), io=149GiB (160GB), run=120007-120007msec
  WRITE: bw=1273MiB/s (1335MB/s), 1273MiB/s-1273MiB/s (1335MB/s-1335MB/s), io=149GiB (160GB), run=120007-120007msec
+ rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-enc-64kchunk-16thread]
bs=64k
directory=/srv/z/enc/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=16
nrfiles=16
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
...
fio-3.16
Starting 16 processes
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
Jobs: 16 (f=256): [M(16)][8.3%][r=567MiB/s,w=569MiB/s][r=9064,w=9096 IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.7%][r=578MiB/s,w=581MiB/s][r=9241,w=9301 IOPS][eta 01m:40s]
Jobs: 16 (f=256): [M(16)][24.2%][r=575MiB/s,w=573MiB/s][r=9193,w=9169 IOPS][eta 01m:31s]

Jobs: 16 (f=256): [M(16)][32.5%][r=555MiB/s,w=551MiB/s][r=8879,w=8819 IOPS][eta 01m:21s]
Jobs: 16 (f=256): [M(16)][40.0%][r=561MiB/s,w=562MiB/s][r=8981,w=8994 IOPS][eta 01m:12s] 

Jobs: 16 (f=256): [M(16)][48.3%][r=535MiB/s,w=535MiB/s][r=8559,w=8566 IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.2%][r=508MiB/s,w=508MiB/s][r=8123,w=8124 IOPS][eta 00m:53s] 

Jobs: 16 (f=256): [M(16)][64.2%][r=499MiB/s,w=495MiB/s][r=7991,w=7919 IOPS][eta 00m:43s]
Jobs: 16 (f=256): [M(16)][72.5%][r=483MiB/s,w=483MiB/s][r=7724,w=7734 IOPS][eta 00m:33s] 

Jobs: 16 (f=256): [M(16)][80.2%][r=485MiB/s,w=485MiB/s][r=7756,w=7763 IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=478MiB/s,w=477MiB/s][r=7640,w=7626 IOPS][eta 00m:14s] 

Jobs: 16 (f=256): [M(16)][96.7%][r=467MiB/s,w=469MiB/s][r=7476,w=7510 IOPS][eta 00m:04s]

test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=2340655: Fri Feb 28 22:27:27 2020
  read: IOPS=8361, BW=523MiB/s (548MB/s)(61.3GiB/120040msec)
    slat (usec): min=14, max=256636, avg=922.71, stdev=4034.66
    clat (usec): min=5, max=573459, avg=60227.28, stdev=53181.51
     lat (msec): min=3, max=573, avg=61.15, stdev=53.89
    clat percentiles (msec):
     |  1.00th=[    5],  5.00th=[    6], 10.00th=[   11], 20.00th=[   20],
     | 30.00th=[   31], 40.00th=[   41], 50.00th=[   49], 60.00th=[   56],
     | 70.00th=[   66], 80.00th=[   84], 90.00th=[  131], 95.00th=[  176],
     | 99.00th=[  257], 99.50th=[  288], 99.90th=[  359], 99.95th=[  393],
     | 99.99th=[  477]
   bw (  KiB/s): min=116536, max=1371827, per=99.57%, avg=532808.25, stdev=19629.32, samples=3840
   iops        : min= 1816, max=21429, avg=8321.54, stdev=306.71, samples=3840
  write: IOPS=8360, BW=523MiB/s (548MB/s)(61.3GiB/120040msec); 0 zone resets
    slat (usec): min=18, max=180031, avg=983.49, stdev=4112.47
    clat (usec): min=4, max=573443, avg=60227.90, stdev=53246.90
     lat (msec): min=2, max=573, avg=61.21, stdev=53.89
    clat percentiles (msec):
     |  1.00th=[    5],  5.00th=[    6], 10.00th=[   11], 20.00th=[   20],
     | 30.00th=[   31], 40.00th=[   41], 50.00th=[   49], 60.00th=[   56],
     | 70.00th=[   66], 80.00th=[   84], 90.00th=[  131], 95.00th=[  176],
     | 99.00th=[  257], 99.50th=[  288], 99.90th=[  359], 99.95th=[  397],
     | 99.99th=[  481]
   bw (  KiB/s): min=119117, max=1365805, per=99.56%, avg=532748.41, stdev=19715.12, samples=3840
   iops        : min= 1859, max=21336, avg=8320.62, stdev=308.05, samples=3840
  lat (usec)   : 10=0.01%
  lat (msec)   : 4=0.19%, 10=9.44%, 20=11.37%, 50=30.99%, 100=32.35%
  lat (msec)   : 250=14.53%, 500=1.12%, 750=0.01%
  cpu          : usr=0.51%, sys=40.92%, ctx=239959, majf=0, minf=371051
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=1003709,1003604,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=523MiB/s (548MB/s), 523MiB/s-523MiB/s (548MB/s-548MB/s), io=61.3GiB (65.8GB), run=120040-120040msec
  WRITE: bw=523MiB/s (548MB/s), 523MiB/s-523MiB/s (548MB/s-548MB/s), io=61.3GiB (65.8GB), run=120040-120040msec
+ rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9
+ for DIRNAME in none comp enc both
+ cat
+ cat /tmp/jobfile.fio

[test-both-64kchunk-16thread]
bs=64k
directory=/srv/z/both/
iodepth=64
ioengine=libaio
readwrite=readwrite
numjobs=16
nrfiles=16
size=10000M
runtime=120
time_based=1
group_reporting=1
+ fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio
test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64
...
fio-3.16
Starting 16 processes
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB)
Jobs: 16 (f=256): [M(16)][8.3%][r=557MiB/s,w=560MiB/s][r=8916,w=8954 IOPS][eta 01m:50s]
Jobs: 16 (f=256): [M(16)][16.7%][r=539MiB/s,w=541MiB/s][r=8616,w=8651 IOPS][eta 01m:40s]
Jobs: 16 (f=256): [M(16)][24.2%][r=520MiB/s,w=520MiB/s][r=8317,w=8317 IOPS][eta 01m:31s]

Jobs: 16 (f=256): [M(16)][32.5%][r=519MiB/s,w=516MiB/s][r=8303,w=8260 IOPS][eta 01m:21s]
Jobs: 16 (f=256): [M(16)][40.5%][r=477MiB/s,w=477MiB/s][r=7637,w=7636 IOPS][eta 01m:12s] 

Jobs: 16 (f=256): [M(16)][48.3%][r=493MiB/s,w=494MiB/s][r=7884,w=7906 IOPS][eta 01m:02s]
Jobs: 16 (f=256): [M(16)][56.7%][r=484MiB/s,w=482MiB/s][r=7738,w=7715 IOPS][eta 00m:52s] 

Jobs: 16 (f=256): [M(16)][64.2%][r=462MiB/s,w=465MiB/s][r=7397,w=7435 IOPS][eta 00m:43s]
Jobs: 16 (f=256): [M(16)][72.5%][r=462MiB/s,w=463MiB/s][r=7398,w=7400 IOPS][eta 00m:33s] 

Jobs: 16 (f=256): [M(16)][80.2%][r=469MiB/s,w=466MiB/s][r=7505,w=7449 IOPS][eta 00m:24s]
Jobs: 16 (f=256): [M(16)][88.3%][r=454MiB/s,w=454MiB/s][r=7264,w=7262 IOPS][eta 00m:14s] 

Jobs: 16 (f=256): [M(16)][96.7%][r=450MiB/s,w=446MiB/s][r=7207,w=7141 IOPS][eta 00m:04s]

test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=344989: Fri Feb 28 22:46:09 2020
  read: IOPS=7822, BW=489MiB/s (513MB/s)(57.3GiB/120004msec)
    slat (usec): min=14, max=211751, avg=969.49, stdev=4169.66
    clat (usec): min=2, max=544781, avg=64436.11, stdev=57921.87
     lat (usec): min=31, max=563219, avg=65406.05, stdev=58687.86
    clat percentiles (msec):
     |  1.00th=[    5],  5.00th=[    5], 10.00th=[    9], 20.00th=[   18],
     | 30.00th=[   32], 40.00th=[   44], 50.00th=[   52], 60.00th=[   59],
     | 70.00th=[   70], 80.00th=[   94], 90.00th=[  144], 95.00th=[  188],
     | 99.00th=[  271], 99.50th=[  305], 99.90th=[  376], 99.95th=[  405],
     | 99.99th=[  481]
   bw (  KiB/s): min=102976, max=1560572, per=99.46%, avg=497942.69, stdev=23994.75, samples=3824
   iops        : min= 1603, max=24383, avg=7777.17, stdev=374.92, samples=3824
  write: IOPS=7818, BW=489MiB/s (512MB/s)(57.3GiB/120004msec); 0 zone resets
    slat (usec): min=18, max=263163, avg=1069.56, stdev=4489.40
    clat (usec): min=2, max=544758, avg=64374.60, stdev=57746.14
     lat (usec): min=756, max=544822, avg=65444.58, stdev=58468.14
    clat percentiles (msec):
     |  1.00th=[    5],  5.00th=[    5], 10.00th=[    9], 20.00th=[   18],
     | 30.00th=[   32], 40.00th=[   44], 50.00th=[   52], 60.00th=[   59],
     | 70.00th=[   70], 80.00th=[   94], 90.00th=[  144], 95.00th=[  188],
     | 99.00th=[  271], 99.50th=[  305], 99.90th=[  372], 99.95th=[  401],
     | 99.99th=[  481]
   bw (  KiB/s): min=106826, max=1534676, per=99.46%, avg=497682.75, stdev=23855.92, samples=3824
   iops        : min= 1663, max=23976, avg=7773.21, stdev=372.75, samples=3824
  lat (usec)   : 4=0.01%, 10=0.01%, 50=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.57%, 10=11.75%, 20=9.77%, 50=26.03%
  lat (msec)   : 100=33.42%, 250=16.95%, 500=1.50%, 750=0.01%
  cpu          : usr=0.49%, sys=38.25%, ctx=269609, majf=0, minf=282080
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=938705,938231,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=489MiB/s (513MB/s), 489MiB/s-489MiB/s (513MB/s-513MB/s), io=57.3GiB (61.5GB), run=120004-120004msec
  WRITE: bw=489MiB/s (512MB/s), 489MiB/s-489MiB/s (512MB/s-512MB/s), io=57.3GiB (61.5GB), run=120004-120004msec
+ rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9

Script done on Fri Feb 28 14:46:10 2020
