Nested Virtualization Benchmarks
20250811 -
CPU: 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
Disk: 2 x 256GB
Filesystem: ZFS Raid-1 Mirror
fio simple write test
fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1fio random read job file
[global]
ioengine=libaio
direct=1
group_reporting
runtime=60
time_based
[random_read_job]
rw=randread
bs=4k
iodepth=64
filename=/root/testfile
size=10GHardware: DELL Latitude 5520 (Laptop)
PVE benchmark
# Disk information
root@pve:~# smartctl -a /dev/nvme0n1
Model Number: Micron 2450 NVMe 256GB
Temperature: 43 Celsius
Percentage Used: 15%
root@pve:~# smartctl -a /dev/nvme1n1
Model Number: PC SN520 NVMe WDC 256GB
Temperature: 45 Celsius
Percentage Used: 28%
# CPU Information
root@pve:~# cat /proc/cpuinfo | grep "model name"
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
# Benchmark - write
root@pve:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.39
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][100.0%][w=60.1MiB/s][w=60 IOPS][eta 00m:00s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1838716: Mon Aug 11 13:47:12 2025
write: IOPS=64, BW=64.1MiB/s (67.2MB/s)(10.0GiB/159737msec); 0 zone resets
clat (usec): min=1521, max=2857.4k, avg=15554.55, stdev=30350.48
lat (usec): min=1564, max=2857.4k, avg=15595.47, stdev=30349.98
clat percentiles (msec):
| 1.00th=[ 3], 5.00th=[ 3], 10.00th=[ 3], 20.00th=[ 15],
| 30.00th=[ 15], 40.00th=[ 16], 50.00th=[ 17], 60.00th=[ 17],
| 70.00th=[ 18], 80.00th=[ 18], 90.00th=[ 20], 95.00th=[ 22],
| 99.00th=[ 32], 99.50th=[ 37], 99.90th=[ 110], 99.95th=[ 226],
| 99.99th=[ 760]
bw ( KiB/s): min= 4096, max=448512, per=100.00%, avg=66923.14, stdev=44636.99, samples=313
iops : min= 4, max= 438, avg=65.35, stdev=43.59, samples=313
lat (msec) : 2=0.50%, 4=12.31%, 10=1.23%, 20=77.88%, 50=7.84%
lat (msec) : 100=0.13%, 250=0.07%, 500=0.02%, 1000=0.01%, >=2000=0.01%
cpu : usr=0.34%, sys=13.49%, ctx=11028, majf=5, minf=10
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=64.1MiB/s (67.2MB/s), 64.1MiB/s-64.1MiB/s (67.2MB/s-67.2MB/s), io=10.0GiB (10.7GB), run=159737-159737msec
# Benchmark - read
root@pve:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.39
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=10.3MiB/s][r=2626 IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=1841119: Mon Aug 11 13:54:44 2025
read: IOPS=2638, BW=10.3MiB/s (10.8MB/s)(618MiB/60001msec)
slat (usec): min=167, max=6793, avg=377.03, stdev=155.55
clat (nsec): min=1728, max=36276k, avg=23873354.46, stdev=1833473.80
lat (usec): min=458, max=36588, avg=24250.38, stdev=1850.88
clat percentiles (usec):
| 1.00th=[20579], 5.00th=[21627], 10.00th=[21890], 20.00th=[22414],
| 30.00th=[22938], 40.00th=[23200], 50.00th=[23725], 60.00th=[23987],
| 70.00th=[24511], 80.00th=[25035], 90.00th=[26084], 95.00th=[27132],
| 99.00th=[29754], 99.50th=[31065], 99.90th=[33162], 99.95th=[33817],
| 99.99th=[34866]
bw ( KiB/s): min= 7944, max=11024, per=100.00%, avg=10553.40, stdev=363.11, samples=120
iops : min= 1986, max= 2756, avg=2638.35, stdev=90.78, samples=120
lat (usec) : 2=0.01%, 500=0.01%, 750=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.65%, 50=99.33%
cpu : usr=0.64%, sys=6.66%, ctx=158463, majf=0, minf=72
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=158301,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=10.3MiB/s (10.8MB/s), 10.3MiB/s-10.3MiB/s (10.8MB/s-10.8MB/s), io=618MiB (648MB), run=60001-60001msecNested VM Benchmark
Ubuntu 24.04
root@nested-vm-l2:~$ cat /proc/cpuinfo | grep "model name"
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
root@nested-vm-l2:~$ fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.36
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][97.1%][w=178MiB/s][w=177 IOPS][eta 00m:08s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1820: Mon Aug 11 18:43:38 2025
write: IOPS=37, BW=37.9MiB/s (39.8MB/s)(10.0GiB/269886msec); 0 zone resets
clat (nsec): min=0, max=44001M, avg=26294437.47, stdev=800866646.25
lat (nsec): min=0, max=44001M, avg=26331483.73, stdev=800866317.34
clat percentiles (usec):
| 1.00th=[ 1270], 5.00th=[ 1385], 10.00th=[ 1434],
| 20.00th=[ 1516], 30.00th=[ 1582], 40.00th=[ 1647],
| 50.00th=[ 1745], 60.00th=[ 1860], 70.00th=[ 2057],
| 80.00th=[ 2606], 90.00th=[ 4293], 95.00th=[ 5997],
| 99.00th=[ 11600], 99.50th=[ 13960], 99.90th=[ 826278],
| 99.95th=[17112761], 99.99th=[17112761]
bw ( KiB/s): min= 2043, max=610304, per=100.00%, avg=332192.42, stdev=168523.35, samples=62
iops : min= 1, max= 596, avg=324.08, stdev=164.57, samples=62
lat (nsec) : 2=0.26%
lat (usec) : 20=0.01%, 100=0.01%, 500=0.03%, 750=0.03%, 1000=0.02%
lat (msec) : 2=67.67%, 4=20.74%, 10=9.56%, 20=1.42%, 50=0.10%
lat (msec) : 100=0.04%, 250=0.01%, 1000=0.01%, >=2000=0.10%
cpu : usr=0.23%, sys=1.18%, ctx=10245, majf=0, minf=16
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=37.9MiB/s (39.8MB/s), 37.9MiB/s-37.9MiB/s (39.8MB/s-39.8MB/s), io=10.0GiB (10.7GB), run=269886-269886msec
Disk stats (read/write):
sda: ios=3017/10737, sectors=75792/21008312, merge=253/833, ticks=3237/526767, in_queue=686667, util=96.37%
# Benchmark - read
root@nested-vm-l2:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.36
Starting 1 process
random_read_job: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [r(1)][100.0%][r=20.0MiB/s][r=5112 IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=3300: Mon Aug 11 19:02:19 2025
read: IOPS=4748, BW=18.5MiB/s (19.4MB/s)(1113MiB/60004msec)
slat (nsec): min=0, max=13840k, avg=116055.33, stdev=350395.40
clat (usec): min=2917, max=48588, avg=13307.65, stdev=5035.71
lat (usec): min=2958, max=48739, avg=13423.70, stdev=5063.84
clat percentiles (usec):
| 1.00th=[ 5735], 5.00th=[ 6980], 10.00th=[ 7832], 20.00th=[ 8848],
| 30.00th=[ 9896], 40.00th=[10945], 50.00th=[12125], 60.00th=[13566],
| 70.00th=[15533], 80.00th=[17695], 90.00th=[20317], 95.00th=[22938],
| 99.00th=[27657], 99.50th=[29754], 99.90th=[33817], 99.95th=[35390],
| 99.99th=[39584]
bw ( KiB/s): min=12645, max=34142, per=100.00%, avg=19000.53, stdev=4050.47, samples=119
iops : min= 3161, max= 8535, avg=4749.84, stdev=1012.64, samples=119
lat (msec) : 4=0.01%, 10=30.78%, 20=58.31%, 50=10.90%
cpu : usr=3.98%, sys=90.23%, ctx=2410, majf=0, minf=74
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=284908,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=18.5MiB/s (19.4MB/s), 18.5MiB/s-18.5MiB/s (19.4MB/s-19.4MB/s), io=1113MiB (1167MB), run=60004-60004msec
Disk stats (read/write):
sda: ios=283830/8, sectors=2270640/72, merge=0/1, ticks=1580398/232, in_queue=1580794, util=49.28%Hardware: HPE
PVE benchmark - Mirrored 15K SAS
# CPU
root@btr-pve-c9ne1:~# grep "model name" /proc/cpuinfo
model name : Intel(R) Xeon(R) CPU E5-2687W v4 @ 3.00GHz
# 2 Sockets, 24 Cores per socket, 48 total cores
# DISK: ZFS Mirrored HP EH0300JDYTH 15k SAS
# Benchmark - write
root@pve-hpe:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.39
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][100.0%][w=100MiB/s][w=100 IOPS][eta 00m:00s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1156898: Mon Aug 11 14:02:36 2025
write: IOPS=93, BW=93.5MiB/s (98.0MB/s)(10.0GiB/109530msec); 0 zone resets
clat (usec): min=8364, max=80908, avg=10618.01, stdev=3390.34
lat (usec): min=8507, max=80994, avg=10691.62, stdev=3390.64
clat percentiles (usec):
| 1.00th=[ 8717], 5.00th=[ 8848], 10.00th=[ 8848], 20.00th=[ 8979],
| 30.00th=[ 9110], 40.00th=[ 9503], 50.00th=[10028], 60.00th=[10552],
| 70.00th=[11076], 80.00th=[11469], 90.00th=[11863], 95.00th=[13042],
| 99.00th=[25035], 99.50th=[32113], 99.90th=[53216], 99.95th=[57934],
| 99.99th=[80217]
bw ( KiB/s): min=51200, max=104448, per=100.00%, avg=95758.09, stdev=11129.23, samples=218
iops : min= 50, max= 102, avg=93.51, stdev=10.87, samples=218
lat (msec) : 10=50.61%, 20=47.46%, 50=1.81%, 100=0.13%
cpu : usr=0.74%, sys=65.95%, ctx=10420, majf=5, minf=8
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=93.5MiB/s (98.0MB/s), 93.5MiB/s-93.5MiB/s (98.0MB/s-98.0MB/s), io=10.0GiB (10.7GB), run=109530-109530msec
# Benchmark - read
root@pve-hpe:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.39
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=820KiB/s][r=205 IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=1159109: Mon Aug 11 14:09:42 2025
read: IOPS=217, BW=871KiB/s (892kB/s)(51.0MiB/60001msec)
slat (usec): min=462, max=40890, avg=4580.11, stdev=1975.13
clat (usec): min=13, max=531922, avg=288716.77, stdev=40807.24
lat (msec): min=6, max=535, avg=293.30, stdev=41.16
clat percentiles (msec):
| 1.00th=[ 251], 5.00th=[ 262], 10.00th=[ 266], 20.00th=[ 271],
| 30.00th=[ 271], 40.00th=[ 275], 50.00th=[ 279], 60.00th=[ 284],
| 70.00th=[ 288], 80.00th=[ 292], 90.00th=[ 313], 95.00th=[ 393],
| 99.00th=[ 460], 99.50th=[ 481], 99.90th=[ 527], 99.95th=[ 531],
| 99.99th=[ 531]
bw ( KiB/s): min= 376, max= 1112, per=99.92%, avg=870.73, stdev=99.13, samples=120
iops : min= 94, max= 278, avg=217.68, stdev=24.78, samples=120
lat (usec) : 20=0.01%
lat (msec) : 10=0.01%, 20=0.02%, 50=0.05%, 100=0.09%, 250=0.54%
lat (msec) : 500=98.97%, 750=0.32%
cpu : usr=0.33%, sys=3.76%, ctx=13070, majf=0, minf=408
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.2%, >=64=99.5%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=13061,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=871KiB/s (892kB/s), 871KiB/s-871KiB/s (892kB/s-892kB/s), io=51.0MiB (53.5MB), run=60001-60001msecL1 VM - ZFS Mirrored SSD
# Benchmark - write
root@us-pve-101-testing:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.36
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][96.6%][eta 00m:06s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=7040: Mon Aug 11 22:41:53 2025
write: IOPS=58, BW=59.0MiB/s (61.8MB/s)(10.0GiB/173676msec); 0 zone resets
clat (usec): min=647, max=45665k, avg=16898.79, stdev=707938.73
lat (usec): min=656, max=45665k, avg=16955.22, stdev=707938.94
clat percentiles (usec):
| 1.00th=[ 988], 5.00th=[ 1352], 10.00th=[ 1450],
| 20.00th=[ 1582], 30.00th=[ 1647], 40.00th=[ 1713],
| 50.00th=[ 1762], 60.00th=[ 1795], 70.00th=[ 1827],
| 80.00th=[ 1860], 90.00th=[ 1909], 95.00th=[ 1958],
| 99.00th=[ 2278], 99.50th=[ 2409], 99.90th=[ 4359],
| 99.95th=[11475616], 99.99th=[17112761]
bw ( KiB/s): min=26677, max=819200, per=100.00%, avg=486353.63, stdev=170810.86, samples=43
iops : min= 26, max= 800, avg=474.84, stdev=166.83, samples=43
lat (usec) : 750=0.17%, 1000=0.89%
lat (msec) : 2=95.51%, 4=3.31%, 10=0.06%, 750=0.01%, >=2000=0.06%
cpu : usr=0.51%, sys=0.95%, ctx=10241, majf=0, minf=13
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=59.0MiB/s (61.8MB/s), 59.0MiB/s-59.0MiB/s (61.8MB/s-61.8MB/s), io=10.0GiB (10.7GB), run=173676-173676msec
Disk stats (read/write):
sda: ios=0/10537, sectors=0/21021040, merge=0/500, ticks=0/4724709, in_queue=4862347, util=97.87%
# Benchmark - read
root@us-pve-101-testing:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=308MiB/s][r=78.9k IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=7047: Mon Aug 11 22:43:10 2025
read: IOPS=76.1k, BW=297MiB/s (312MB/s)(17.4GiB/60001msec)
slat (usec): min=2, max=5361, avg= 8.95, stdev= 8.40
clat (usec): min=115, max=8190, avg=831.02, stdev=330.81
lat (usec): min=158, max=8199, avg=839.97, stdev=334.39
clat percentiles (usec):
| 1.00th=[ 351], 5.00th=[ 388], 10.00th=[ 416], 20.00th=[ 486],
| 30.00th=[ 562], 40.00th=[ 619], 50.00th=[ 898], 60.00th=[ 1057],
| 70.00th=[ 1090], 80.00th=[ 1123], 90.00th=[ 1172], 95.00th=[ 1221],
| 99.00th=[ 1663], 99.50th=[ 1745], 99.90th=[ 1876], 99.95th=[ 2008],
| 99.99th=[ 2540]
bw ( KiB/s): min=213812, max=539568, per=100.00%, avg=305124.23, stdev=72008.27, samples=119
iops : min=53453, max=134892, avg=76281.10, stdev=18002.05, samples=119
lat (usec) : 250=0.01%, 500=21.93%, 750=26.28%, 1000=4.38%
lat (msec) : 2=47.36%, 4=0.05%, 10=0.01%
cpu : usr=32.95%, sys=65.00%, ctx=11406, majf=0, minf=77
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=4564316,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=297MiB/s (312MB/s), 297MiB/s-297MiB/s (312MB/s-312MB/s), io=17.4GiB (18.7GB), run=60001-60001msec
Disk stats (read/write):
sda: ios=4557508/19, sectors=36460064/176, merge=0/5, ticks=758830/37038, in_queue=832901, util=68.19%L1 VM - ZFS Mirrored 10K SAS + SLOG + Cache
root@us-pve-101-testing:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [W(1)][100.0%][w=185MiB/s][w=185 IOPS][eta 00m:00s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=997: Tue Aug 12 14:53:40 2025
write: IOPS=253, BW=253MiB/s (265MB/s)(10.0GiB/40449msec); 0 zone resets
clat (usec): min=1113, max=188603, avg=3871.65, stdev=2642.15
lat (usec): min=1149, max=188642, avg=3943.94, stdev=2641.43
clat percentiles (usec):
| 1.00th=[ 1500], 5.00th=[ 1631], 10.00th=[ 1745], 20.00th=[ 1827],
| 30.00th=[ 1942], 40.00th=[ 2311], 50.00th=[ 4817], 60.00th=[ 5211],
| 70.00th=[ 5407], 80.00th=[ 5604], 90.00th=[ 5932], 95.00th=[ 6128],
| 99.00th=[ 6652], 99.50th=[ 6849], 99.90th=[ 7177], 99.95th=[ 7898],
| 99.99th=[64750]
bw ( KiB/s): min=149504, max=610304, per=100.00%, avg=260124.17, stdev=140385.29, samples=80
iops : min= 146, max= 596, avg=253.86, stdev=137.09, samples=80
lat (msec) : 2=32.85%, 4=15.03%, 10=52.07%, 20=0.03%, 100=0.01%
lat (msec) : 250=0.01%
cpu : usr=3.08%, sys=3.15%, ctx=10243, majf=0, minf=13
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=253MiB/s (265MB/s), 253MiB/s-253MiB/s (265MB/s-265MB/s), io=10.0GiB (10.7GB), run=40449-40449msec
Disk stats (read/write):
sda: ios=1/10259, sectors=8/20959881, merge=0/12, ticks=3/58119, in_queue=77002, util=93.49%
root@us-pve-101-testing:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=486MiB/s][r=124k IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=1000: Tue Aug 12 14:54:52 2025
read: IOPS=93.0k, BW=363MiB/s (381MB/s)(21.3GiB/60001msec)
slat (usec): min=2, max=525, avg= 7.23, stdev= 7.05
clat (usec): min=106, max=4144, avg=679.83, stdev=343.27
lat (usec): min=166, max=4152, avg=687.06, stdev=346.99
clat percentiles (usec):
| 1.00th=[ 355], 5.00th=[ 383], 10.00th=[ 400], 20.00th=[ 420],
| 30.00th=[ 437], 40.00th=[ 457], 50.00th=[ 486], 60.00th=[ 537],
| 70.00th=[ 832], 80.00th=[ 1139], 90.00th=[ 1188], 95.00th=[ 1254],
| 99.00th=[ 1532], 99.50th=[ 1762], 99.90th=[ 1876], 99.95th=[ 1909],
| 99.99th=[ 2089]
bw ( KiB/s): min=211328, max=580672, per=99.82%, avg=371243.65, stdev=97421.62, samples=119
iops : min=52832, max=145168, avg=92810.87, stdev=24355.37, samples=119
lat (usec) : 250=0.01%, 500=54.35%, 750=14.73%, 1000=2.17%
lat (msec) : 2=28.73%, 4=0.02%, 10=0.01%
cpu : usr=35.72%, sys=62.92%, ctx=11510, majf=0, minf=76
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=5578560,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=363MiB/s (381MB/s), 363MiB/s-363MiB/s (381MB/s-381MB/s), io=21.3GiB (22.8GB), run=60001-60001msec
Disk stats (read/write):
sda: ios=5562235/9, sectors=44497880/80, merge=0/2, ticks=1046679/49508, in_queue=1124064, util=53.76%
Nested VM Benchmark - ZFS Mirrored SSD
# Benchmark - Write
root@us-hv-nested:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.36
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][99.1%][w=165MiB/s][w=165 IOPS][eta 00m:02s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1439: Mon Aug 11 22:26:29 2025
write: IOPS=44, BW=44.5MiB/s (46.6MB/s)(10.0GiB/230323msec); 0 zone resets
clat (usec): min=9, max=28370k, avg=22023.80, stdev=535433.32
lat (usec): min=9, max=28370k, avg=22285.21, stdev=535432.32
clat percentiles (msec):
| 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 6],
| 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 7], 60.00th=[ 7],
| 70.00th=[ 7], 80.00th=[ 8], 90.00th=[ 9], 95.00th=[ 10],
| 99.00th=[ 16], 99.50th=[ 25], 99.90th=[ 1838], 99.95th=[14429],
| 99.99th=[17113]
bw ( KiB/s): min= 4079, max=176761, per=100.00%, avg=132012.25, stdev=31904.93, samples=157
iops : min= 3, max= 172, avg=128.38, stdev=31.21, samples=157
lat (usec) : 10=0.01%, 500=0.01%
lat (msec) : 2=0.06%, 4=0.99%, 10=94.77%, 20=3.56%, 50=0.33%
lat (msec) : 100=0.06%, 250=0.08%, 500=0.01%, 750=0.01%, 2000=0.02%
lat (msec) : >=2000=0.10%
cpu : usr=0.91%, sys=8.89%, ctx=12455, majf=0, minf=16
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=44.5MiB/s (46.6MB/s), 44.5MiB/s-44.5MiB/s (46.6MB/s-46.6MB/s), io=10.0GiB (10.7GB), run=230323-230323msec
Disk stats (read/write):
sda: ios=111/10313, sectors=3728/20900664, merge=90/235, ticks=2312/790448, in_queue=937669, util=89.85%
# Benchmark - Read
root@us-hv-nested:~# fio random_read_test.fio
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.36
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=3498KiB/s][r=874 IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=1454: Mon Aug 11 22:27:43 2025
read: IOPS=1151, BW=4608KiB/s (4718kB/s)(270MiB/60009msec)
slat (nsec): min=1200, max=30439k, avg=506941.31, stdev=1047300.59
clat (msec): min=2, max=335, avg=54.75, stdev=34.49
lat (msec): min=3, max=335, avg=55.26, stdev=34.75
clat percentiles (msec):
| 1.00th=[ 15], 5.00th=[ 16], 10.00th=[ 17], 20.00th=[ 19],
| 30.00th=[ 24], 40.00th=[ 37], 50.00th=[ 61], 60.00th=[ 68],
| 70.00th=[ 73], 80.00th=[ 80], 90.00th=[ 93], 95.00th=[ 111],
| 99.00th=[ 157], 99.50th=[ 176], 99.90th=[ 215], 99.95th=[ 288],
| 99.99th=[ 326]
bw ( KiB/s): min= 2003, max=11768, per=100.00%, avg=4613.21, stdev=1889.89, samples=118
iops : min= 500, max= 2942, avg=1152.94, stdev=472.50, samples=118
lat (msec) : 4=0.01%, 10=0.01%, 20=24.47%, 50=21.04%, 100=46.64%
lat (msec) : 250=7.75%, 500=0.08%
cpu : usr=5.15%, sys=81.55%, ctx=7548, majf=0, minf=76
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=69124,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=4608KiB/s (4718kB/s), 4608KiB/s-4608KiB/s (4718kB/s-4718kB/s), io=270MiB (283MB), run=60009-60009msec
Disk stats (read/write):
sda: ios=69121/20, sectors=552968/288, merge=0/5, ticks=583586/133, in_queue=583767, util=67.38%