Skip to main content

Nested Virtualization Benchmarks

20250811 - 

CPU: 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
Disk: 2 x 256GB 
Filesystem: ZFS Raid-1 Mirror

fio simple write test

fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=/root/testfile --direct=1

fio random read job file

[global]
ioengine=libaio
direct=1
group_reporting
runtime=60
time_based

[random_read_job]
rw=randread
bs=4k
iodepth=64
filename=/root/testfile
size=10G

 

PVE benchmark

# Disk information
root@pve:~# smartctl -a /dev/nvme0n1
Model Number:                       Micron 2450 NVMe 256GB
Temperature:                        43 Celsius
Percentage Used:                    15%

root@pve:~# smartctl -a /dev/nvme1n1
Model Number:                       PC SN520 NVMe WDC 256GB
Temperature:                        45 Celsius
Percentage Used:                    28%



# CPU Information
root@pve:~# cat /proc/cpuinfo | grep "model name"
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz



# Benchmark - write
root@pve:~# fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.39
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][100.0%][w=60.1MiB/s][w=60 IOPS][eta 00m:00s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1838716: Mon Aug 11 13:47:12 2025
  write: IOPS=64, BW=64.1MiB/s (67.2MB/s)(10.0GiB/159737msec); 0 zone resets
    clat (usec): min=1521, max=2857.4k, avg=15554.55, stdev=30350.48
     lat (usec): min=1564, max=2857.4k, avg=15595.47, stdev=30349.98
    clat percentiles (msec):
     |  1.00th=[    3],  5.00th=[    3], 10.00th=[    3], 20.00th=[   15],
     | 30.00th=[   15], 40.00th=[   16], 50.00th=[   17], 60.00th=[   17],
     | 70.00th=[   18], 80.00th=[   18], 90.00th=[   20], 95.00th=[   22],
     | 99.00th=[   32], 99.50th=[   37], 99.90th=[  110], 99.95th=[  226],
     | 99.99th=[  760]
   bw (  KiB/s): min= 4096, max=448512, per=100.00%, avg=66923.14, stdev=44636.99, samples=313
   iops        : min=    4, max=  438, avg=65.35, stdev=43.59, samples=313
  lat (msec)   : 2=0.50%, 4=12.31%, 10=1.23%, 20=77.88%, 50=7.84%
  lat (msec)   : 100=0.13%, 250=0.07%, 500=0.02%, 1000=0.01%, >=2000=0.01%
  cpu          : usr=0.34%, sys=13.49%, ctx=11028, majf=5, minf=10
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=64.1MiB/s (67.2MB/s), 64.1MiB/s-64.1MiB/s (67.2MB/s-67.2MB/s), io=10.0GiB (10.7GB), run=159737-159737msec



# Benchmark - read
root@pve:~# fio random_read_test.fio 
random_read_job: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.39
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=10.3MiB/s][r=2626 IOPS][eta 00m:00s]
random_read_job: (groupid=0, jobs=1): err= 0: pid=1841119: Mon Aug 11 13:54:44 2025
  read: IOPS=2638, BW=10.3MiB/s (10.8MB/s)(618MiB/60001msec)
    slat (usec): min=167, max=6793, avg=377.03, stdev=155.55
    clat (nsec): min=1728, max=36276k, avg=23873354.46, stdev=1833473.80
     lat (usec): min=458, max=36588, avg=24250.38, stdev=1850.88
    clat percentiles (usec):
     |  1.00th=[20579],  5.00th=[21627], 10.00th=[21890], 20.00th=[22414],
     | 30.00th=[22938], 40.00th=[23200], 50.00th=[23725], 60.00th=[23987],
     | 70.00th=[24511], 80.00th=[25035], 90.00th=[26084], 95.00th=[27132],
     | 99.00th=[29754], 99.50th=[31065], 99.90th=[33162], 99.95th=[33817],
     | 99.99th=[34866]
   bw (  KiB/s): min= 7944, max=11024, per=100.00%, avg=10553.40, stdev=363.11, samples=120
   iops        : min= 1986, max= 2756, avg=2638.35, stdev=90.78, samples=120
  lat (usec)   : 2=0.01%, 500=0.01%, 750=0.01%
  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.65%, 50=99.33%
  cpu          : usr=0.64%, sys=6.66%, ctx=158463, majf=0, minf=72
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=158301,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=10.3MiB/s (10.8MB/s), 10.3MiB/s-10.3MiB/s (10.8MB/s-10.8MB/s), io=618MiB (648MB), run=60001-60001msec

Nested VM Benchmark

Ubuntu 24.04

root@nested-vm-l2:~$ cat /proc/cpuinfo | grep "model name"
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz
model name      : 11th Gen Intel(R) Core(TM) i5-1145G7 @ 2.60GHz

root@nested-vm-l2:~$ fio --name=sequential_write --rw=write --bs=1M --size=10G --filename=testfile --direct=1
sequential_write: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=psync, iodepth=1
fio-3.36
Starting 1 process
sequential_write: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][97.1%][w=178MiB/s][w=177 IOPS][eta 00m:08s]
sequential_write: (groupid=0, jobs=1): err= 0: pid=1820: Mon Aug 11 18:43:38 2025
  write: IOPS=37, BW=37.9MiB/s (39.8MB/s)(10.0GiB/269886msec); 0 zone resets
    clat (nsec): min=0, max=44001M, avg=26294437.47, stdev=800866646.25
     lat (nsec): min=0, max=44001M, avg=26331483.73, stdev=800866317.34
    clat percentiles (usec):
     |  1.00th=[    1270],  5.00th=[    1385], 10.00th=[    1434],
     | 20.00th=[    1516], 30.00th=[    1582], 40.00th=[    1647],
     | 50.00th=[    1745], 60.00th=[    1860], 70.00th=[    2057],
     | 80.00th=[    2606], 90.00th=[    4293], 95.00th=[    5997],
     | 99.00th=[   11600], 99.50th=[   13960], 99.90th=[  826278],
     | 99.95th=[17112761], 99.99th=[17112761]
   bw (  KiB/s): min= 2043, max=610304, per=100.00%, avg=332192.42, stdev=168523.35, samples=62
   iops        : min=    1, max=  596, avg=324.08, stdev=164.57, samples=62
  lat (nsec)   : 2=0.26%
  lat (usec)   : 20=0.01%, 100=0.01%, 500=0.03%, 750=0.03%, 1000=0.02%
  lat (msec)   : 2=67.67%, 4=20.74%, 10=9.56%, 20=1.42%, 50=0.10%
  lat (msec)   : 100=0.04%, 250=0.01%, 1000=0.01%, >=2000=0.10%
  cpu          : usr=0.23%, sys=1.18%, ctx=10245, majf=0, minf=16
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,10240,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=37.9MiB/s (39.8MB/s), 37.9MiB/s-37.9MiB/s (39.8MB/s-39.8MB/s), io=10.0GiB (10.7GB), run=269886-269886msec

Disk stats (read/write):
  sda: ios=3017/10737, sectors=75792/21008312, merge=253/833, ticks=3237/526767, in_queue=686667, util=96.37%

 

Back to top