illumos gate

with boot/NMI set:

Configuring devices.
pseudo-device: ucode0
ucode0 is /pseudo/ucode@0
pseudo-device: devinfo0
devinfo0 is /pseudo/devinfo@0
hyperv_init: Checkin

ppaanniicc[[ccppuu10]]//tthhrreeaadd==ffffffffffee00b0d04fc6164e8c4200:: BBAADD TTRRAAPP:: ttyyppee==dd ((##ggpp GGeenneerraall pprrootteeccttiioonn)) rrpp==ffffffffffee0000100f544ffe488e00 aaddddrr==ffffffffffee00bbdd2244111133ed88

H
  ypdere-Vv sfupspoardtm..:.
 #gp General protection
addr=0xfffffe0bd24113e8
pid=50, pc=0xfffffffffb883932, sp=0xfffffe001054f578, eflags=0x206
cr0: 80050033<pg,wp,ne,et,mp,pe>  cr4: 3606f8<smap,smep,osxsav,pcide,xmme,fxsr,pge,mce,pae,pse,de>
cr2: fed2918f  cr3: 1eb449000  cr8: c

rdi:         40000083 rsi:                0 rdx:                0
        rcx:         40000083  r8:                1  r9: fffffe0bd245d000
        rax:               c0 rbx: fffffe0bd2eb2000 rbp: fffffe001054f5a0
        r10: fffffe0bd2eb2000 r11: fffffffffbc4bb98 r12:                3
        r13:                1 r14:                0 r15: fffffe0bd24113e8
        fsb:                0 gsb: fffffe0bd245d000  ds:               4b
         es:               4b  fs:                0  gs:              1c3
        trp:                d err:                0 rip: fffffffffb883932
         cs:               30 rfl:              206 rsp: fffffe001054f578
         ss:               38

Warning - stack not written to the dump buffer
fffffe001054f390 unix:real_mode_stop_cpu_stage2_end+c14a ()
fffffe001054f470 unix:trap+855 ()
fffffe001054f480 unix:cmntrap+e9 ()
fffffe001054f5a0 unix:rdmsr+2 ()
fffffe001054f600 unix:xc_serv+2f3 ()
fffffe001054f690 unix:real_mode_stop_cpu_stage2_end+12e28 ()
fffffe001054f6e0 unix:xc_sync+3d ()
fffffe001054f750 hv_vmbus:vmbus_xcall+53 ()
fffffe001054f790 hv_vmbus:vmbus_doattach+fb ()
fffffe001054f7c0 hv_vmbus:vmbus_attach+ad ()
gsfkfifpfpei0n0g1 0s5y4sft8e3m0 dguemnpu n-i xn:od edvuim_pa tdteavcihc+eb 1c o(n)f
eufrfefdf
001054f870 genunix:attach_node+8b ()
fffffe001054f8c0 genunix:i_ndi_config_node+118 ()
fffffe001054f8f0 genunix:i_ddi_attachchild+3e ()
fffffe001054f930 genunix:devi_attach_node+64 ()
fffffe001054f9b0 genunix:config_immediate_children+c0 ()
fffffe001054fa00 genunix:devi_config_common+6a ()
fffffe001054fa20 genunix:ndi_devi_config+1a ()
fffffe001054fa80 devinfo:di_copytree+e8 ()
fffffe001054fb60 devinfo:di_snapshot+1dc ()
fffffe001054fb90 devinfo:di_snapshot_and_clean+24 ()
fffffe001054fbc0 devinfo:di_cache_update+2c ()
fffffe001054fc70 devinfo:di_ioctl+555 ()
fffffe001054fcb0 genunix:cdev_ioctl+3f ()
fffffe001054fd00 specfs:spec_ioctl+55 ()
fffffe001054fd90 genunix:fop_ioctl+40 ()
fffffe001054feb0 genunix:ioctl+144 ()
fffffe001054ff10 unix:brand_sys_sysenter+1dc ()

panic: entering debugger (no dump device, continue to reboot)
Loaded modules: [ scsi_vhci stmf_sbd mac uppc neti ufs zfs s1394 apix uhci hook lofs ip usba xhci specfs pcplusmp cpu.generic arp stmf sockfs ]
kmdb: target stopped at:
kmdb_enter+0xb: movq %rax,%rdi
[1]> $c
kmdb_enter+0xb()
debug_enter+0x75()
panicsys+0x616()
vpanic+0x15c()
0xfffffffffb8a8911()
0xfffffffffb8681f3()
trap+0x855()
cmntrap_pushed+0x3c()
rdmsr+2()
xc_serv+0x2f3()
0xfffffffffb86eed1()
xc_sync+0x3d()
hv_vmbus`vmbus_xcall+0x53(fffffffff7fc8ff0, fffffe0bd2eb2000)
hv_vmbus`vmbus_doattach+0xfb(fffffe0bd2eb2000)
hv_vmbus`vmbus_attach+0xad(fffffe0bd16880f8, 0)
devi_attach+0xb1(fffffe0bd16880f8, 0)
attach_node+0x8b(fffffe0bd16880f8)
i_ndi_config_node+0x118(fffffe0bd16880f8, 6, 0)
i_ddi_attachchild+0x3e(fffffe0bd16880f8)
devi_attach_node+0x64(fffffe0bd16880f8, 4004048)
config_immediate_children+0xc0(fffffe0bd0255c28, 4004048, ffffffff)
devi_config_common+0x6a(fffffe0bd0255c28, 4004048, ffffffff)
ndi_devi_config+0x1a(fffffe0bd0255c28, 4004048)
devinfo`di_copytree+0xe8(fffffe0bd0255c28, fffffe0bd4d26020, fffffe0bd31e43c0)
devinfo`di_snapshot+0x1dc(fffffe0bd31e43c0)
devinfo`di_snapshot_and_clean+0x24(fffffe0bd31e43c0)
devinfo`di_cache_update+0x2c(fffffe0bd31e43c0)
devinfo`di_ioctl+0x555(5800000002, df2f, 8047778, 100001, fffffe0bd17ba3a0, fffffe001054fdc8)
cdev_ioctl+0x3f(5800000002, df2f, 8047778, 100001, fffffe0bd17ba3a0, fffffe001054fdc8)
specfs`spec_ioctl+0x55(fffffe0bd4b84980, df2f, 8047778, 100001, fffffe0bd17ba3a0, fffffe001054fdc8)
fop_ioctl+0x40(fffffe0bd4b84980, df2f, 8047778, 100001, fffffe0bd17ba3a0, fffffe001054fdc8)
ioctl+0x144(5, df2f, 8047778)
_sys_sysenter_post_swapgs+0x159()
[1]> *nvme_state::walk softstate | ::print nvme_t n_idctl->id_vwc.vwc_present
kmdb: failed to dereference symbol: unknown symbol name

I tried this on a cluster based on dell hardware and got the nvme problem again. I think it was that cluster that first displayed the problem. The cluster with the details in the initial report here is based on SuperMicro hardware.

TSC calibrated using PIT; freq is 2394 MHz
WARNING: illegal PCI request: offset = 100, size = 4
Using default device instance data
SMBIOS v2.8 loaded (426 bytes)initialized model-specific module 'cpu_ms.GenuineIntel' on chip 0 core 0 strand 0
root nexus = i86pc
pseudo0 at root
pseudo0 is /pseudo
scsi_vhci0 at root
scsi_vhci0 is /scsi_vhci
Reading Intel IOMMU boot options
npe0 at root: space 0 offset 0
npe0 is /pci@0,0
PCI Express-device: isa@1, isa0
pseudo-device: acpippm0
acpippm0 is /pseudo/acpippm@0
pseudo-device: ppm0
ppm0 is /pseudo/ppm@0
ramdisk0 at root
ramdisk0 is /ramdisk
root on /ramdisk:a fstype ufs
acpinex0 at root
acpinex0 is /fw
acpinex: sb@0, acpinex1
acpinex1 is /fw/sb@0
pseudo-device: dld0
dld0 is /pseudo/dld@0
ISA-device: asy0
asy0 is /pci@0,0/isa@1/asy@1,3f8
PCI Express-device: pci1af4,1100@1,2, uhci0
uhci0 is /pci@0,0/pci1af4,1100@1,2
8042 device: keyboard@0, kb8042 # 0
kb80420 is /pci@0,0/isa@1/i8042@1,60/keyboard@0
8042 device: mouse@1, mouse8042 # 0
mouse80420 is /pci@0,0/isa@1/i8042@1,60/mouse@1
cpu0: x86 (GenuineIntel 50657 family 6 model 85 step 7 clock 2400 MHz)
cpu0: Intel(r) Xeon(r) Silver 4210R CPU 2.40GHz
KPTI enabled (PCID in use, INVPCID in use)
initialized cpu module 'cpu.generic' on chip 1 core 0 strand 0
initialized model-specific module 'cpu_ms.GenuineIntel' on chip 1 core 0 strand 0
cpu1: x86 (GenuineIntel 50657 family 6 model 85 step 7 clock 2400 MHz)
cpu1: Intel(r) Xeon(r) Silver 4210R CPU 2.40GHz
cpu1 initialization complete - online
pseudo-device: audio0
audio0 is /pseudo/audio@0
USB 2.0 device (usb627,1) operating at full speed (USB 1.x) on USB 1.10 root hub: input@2, hid0 at bus address 2
QEMU QEMU USB Tablet 42
hid0 is /pci@0,0/pci1af4,1100@1,2/input@2
/pci@0,0/pci1af4,1100@1,2/input@2 (hid0) online
Configuring devices.
pseudo-device: ucode0
ucode0 is /pseudo/ucode@0
pseudo-device: devinfo0
devinfo0 is /pseudo/devinfo@0
hyperv_init: Checking Hyper-V support...
hyperv_identify: NOT Hyper-V environment: 0x4hyperv_init: Hyper-V not supported on this environmentpseudo-device: pseudo1
pseudo1 is /pseudo/zconsnex@1
pseudo-device: pseudo2
pseudo2 is /pseudo/zfdnex@2
pseudo-device: bpf0
bpf0 is /pseudo/bpf@0
pseudo-device: fcp0
fcp0 is /pseudo/fcp@0
pseudo-device: fcsm0
fcsm0 is /pseudo/fcsm@0
pseudo-device: fct0
fct0 is /pseudo/fct@0
pseudo-device: fssnap0
fssnap0 is /pseudo/fssnap@0
pseudo-device: llc10
llc10 is /pseudo/llc1@0
pseudo-device: lofi0
lofi0 is /pseudo/lofi@0
pseudo-device: nsmb0
nsmb0 is /pseudo/nsmb@0
pseudo-device: pm0
pm0 is /pseudo/pm@0
pseudo-device: pool0
pool0 is /pseudo/pool@0
pseudo-device: power0
power0 is /pseudo/power@0
pseudo-device: ramdisk1024
ramdisk1024 is /pseudo/ramdisk@1024
pseudo-device: stmf0
stmf0 is /pseudo/stmf@0
pseudo-device: stmf_sbd0
stmf_sbd0 is /pseudo/stmf_sbd@0
pseudo-device: zfs0
zfs0 is /pseudo/zfs@0
nvme0: NVMe spec version 1.3
panic[cpu0]/thread=fffffe001224ec20: programming error: invalid NS/format in cmd fffffe0bd3a9cd40

Warning - stack not written to the dump buffer
fffffe001224e700 genunix:dev_err+81 ()
fffffe001224e740 nvme:nvme_check_generic_cmd_status+236 ()
fffffe001224e7c0 nvme:nvme_set_features+e5 ()
fffffe001224e7f0 nvme:nvme_write_cache_set+47 ()
fffffe001224e890 nvme:nvme_init+9f0 ()
fffffe001224e960 nvme:nvme_attach+565 ()
fffffe001224e9d0 genunix:devi_attach+b1 ()
fffffe001224ea10 genunix:attach_node+8b ()
fffffe001224ea60 genunix:i_ndi_config_node+118 ()
fffffe001224ea90 genunix:i_ddi_attachchild+3e ()
fffffe001224ead0 genunix:devi_attach_node+64 ()
fffffe001224eb50 genunix:config_immediate_children+c0 ()
fffffe001224eba0 genunix:devi_config_common+6a ()
fffffe001224ec00 genunix:mt_config_thread+10b ()
fffffe001224ec10 unix:thread_start+b ()

panic: entering debugger (no dump device, continue to reboot)

Welcome to kmdb
kmdb: unable to determine terminal type: assuming `vt100'
Loaded modules: [ scsi_vhci stmf_sbd crypto mac uppc neti ptm ufs unix mpt zfs krtld s1394 sata apix uhci pmcs hook lofs genunix ip logindmux usba xhci specfs pcplusmp nfs random cpu.generic arp mpt_sas stmf emlxs sockfs ]
[0]> $c
kmdb_enter+0xb()
debug_enter+0x75()
panicsys+0x616()
vpanic+0x15c()
0xfffffffffba4babd()
dev_err+0x81(fffffe0bcec6f690, 3, fffffffff83ed788)
nvme`nvme_check_generic_cmd_status+0x236(fffffe0bd3a9cd40)
nvme`nvme_set_features+0xe5(fffffe0bd39fe940, 0, 0, 6, 1, fffffe001224e7d4)
nvme`nvme_write_cache_set+0x47(fffffe0bd39fe940, 1)
nvme`nvme_init+0x9f0(fffffe0bd39fe940)
nvme`nvme_attach+0x565(fffffe0bcec6f690, 0)
devi_attach+0xb1(fffffe0bcec6f690, 0)
attach_node+0x8b(fffffe0bcec6f690)
i_ndi_config_node+0x118(fffffe0bcec6f690, 6, 0)
i_ddi_attachchild+0x3e(fffffe0bcec6f690)
devi_attach_node+0x64(fffffe0bcec6f690, 4004048)
config_immediate_children+0xc0(fffffe0bcec73688, 4004048, ffffffff)
devi_config_common+0x6a(fffffe0bcec73688, 4004048, ffffffff)
mt_config_thread+0x10b(fffffe0bd3776648)
thread_start+0xb()
[0]> *nvme_state::walk softstate | ::print nvme_t n_idctl->id_vwc.vwc_present
n_idctl->id_vwc.vwc_present = 0x1

using
boot -B disable-nvme=true,disable-hv_vmbus=true
from the bootloader gets a running installer on both the dell and SuperMicro hypervisors.

OmniOS r151046 Version omnios-r151046-82ebda23c9 64-bit
Copyright (c) 2012-2017 OmniTI Computer Consulting, Inc.
Copyright (c) 2017-2023 OmniOS Community Edition (OmniOSce) Association.
NOTICE: driver hv_vmbus disabled
NOTICE: driver nvme disabled
WARNING: illegal PCI request: offset = 100, size = 4
Configuring devices.
Applying initial boot settings...

Scanning for media...
Found r151046 media at /devices/pci@0,0/pci-ide@1,1/ide@1/sd@1,0:q

Press return to start the OmniOS installer.
To use the old text installer, press T within 6 seconds...

For the record, the dell hypervisors only need
boot -B disable-nvme

The supermicro hypervisors need both.

Per request, this should be the full sqe/cqe.

nvme0: NVMe spec version 1.3
panic[cpu0]/thread=fffffe00122e2c20: programming error: invalid NS/format in cmd fffffe0bd3a9bd40

Warning - stack not written to the dump buffer
fffffe00122e2700 genunix:dev_err+81 ()
fffffe00122e2740 nvme:nvme_check_generic_cmd_status+236 ()
fffffe00122e27c0 nvme:nvme_set_features+e5 ()
fffffe00122e27f0 nvme:nvme_write_cache_set+47 ()
fffffe00122e2890 nvme:nvme_init+9f0 ()
fffffe00122e2960 nvme:nvme_attach+565 ()
fffffe00122e29d0 genunix:devi_attach+b1 ()
fffffe00122e2a10 genunix:attach_node+8b ()
fffffe00122e2a60 genunix:i_ndi_config_node+118 ()
fffffe00122e2a90 genunix:i_ddi_attachchild+3e ()
fffffe00122e2ad0 genunix:devi_attach_node+64 ()
fffffe00122e2b50 genunix:config_immediate_children+c0 ()
fffffe00122e2ba0 genunix:devi_config_common+6a ()
fffffe00122e2c00 genunix:mt_config_thread+10b ()
fffffe00122e2c10 unix:thread_start+b ()

panic: entering debugger (no dump device, continue to reboot)

Welcome to kmdb
kmdb: unable to determine terminal type: assuming `vt100'
Loaded modules: [ scsi_vhci stmf_sbd crypto mac uppc neti ptm ufs unix mpt zfs krtld s1394 sata apix uhci pmcs hook lofs genunix ip logindmux usba xhci specfs pcplusmp nfs random cpu.generic arp mpt_sas stmf emlxs sockfs ]
[0]> $C
fffffffffbc913a0 kmdb_enter+0xb()
fffffffffbc913d0 debug_enter+0x75()
fffffffffbc914c0 panicsys+0x616()
fffffe00122e25f0 vpanic+0x15c()
fffffe00122e2660 0xfffffffffba4babd()
fffffe00122e2700 dev_err+0x81(fffffe0bcec6f690, 3, fffffffff83ed788)
fffffe00122e2740 nvme`nvme_check_generic_cmd_status+0x236(fffffe0bd3a9bd40)
fffffe00122e27c0 nvme`nvme_set_features+0xe5(fffffe0bd16ddc00, 0, 0, 6, 1, fffffe00122e27d4)
fffffe00122e27f0 nvme`nvme_write_cache_set+0x47(fffffe0bd16ddc00, 1)
fffffe00122e2890 nvme`nvme_init+0x9f0(fffffe0bd16ddc00)
fffffe00122e2960 nvme`nvme_attach+0x565(fffffe0bcec6f690, 0)
fffffe00122e29d0 devi_attach+0xb1(fffffe0bcec6f690, 0)
fffffe00122e2a10 attach_node+0x8b(fffffe0bcec6f690)
fffffe00122e2a60 i_ndi_config_node+0x118(fffffe0bcec6f690, 6, 0)
fffffe00122e2a90 i_ddi_attachchild+0x3e(fffffe0bcec6f690)
fffffe00122e2ad0 devi_attach_node+0x64(fffffe0bcec6f690, 4004048)
fffffe00122e2b50 config_immediate_children+0xc0(fffffe0bcec73688, 4004048, ffffffff)
fffffe00122e2ba0 devi_config_common+0x6a(fffffe0bcec73688, 4004048, ffffffff)
fffffe00122e2c00 mt_config_thread+0x10b(fffffe0bd0eb1c18)
fffffe00122e2c10 thread_start+0xb()
[0]> fffffe0bd3a9bd40::print nvme_cmd_t nc_sqe nc_cqe
nc_sqe = {
nc_sqe.sqe_opc = 0x9
nc_sqe.sqe_fuse = 0
nc_sqe.sqe_rsvd = 0
nc_sqe.sqe_psdt = 0
nc_sqe.sqe_cid = 0x2
nc_sqe.sqe_nsid = 0
nc_sqe.sqe_rsvd1 = 0
nc_sqe.sqe_m = {
m_ptr = 0
m_sglp = 0
}
nc_sqe.sqe_dptr = {
d_prp = [ 0, 0 ]
d_sgl = {
sgl_addr = 0
sgl_len = 0
sgl_rsvd = [ 0, 0, 0 ]
sgl_zero = 0
sgl_type = 0
}
}
nc_sqe.sqe_cdw10 = 0x6
nc_sqe.sqe_cdw11 = 0x1
nc_sqe.sqe_cdw12 = 0
nc_sqe.sqe_cdw13 = 0
nc_sqe.sqe_cdw14 = 0
nc_sqe.sqe_cdw15 = 0
}
nc_cqe = {
nc_cqe.cqe_dw0 = 0
nc_cqe.cqe_rsvd1 = 0
nc_cqe.cqe_sqhd = 0x3
nc_cqe.cqe_sqid = 0
nc_cqe.cqe_cid = 0x2
nc_cqe.cqe_sf = {
sf_p = 0x1
sf_sc = 0xb
sf_sct = 0
sf_rsvd2 = 0
sf_m = 0
sf_dnr = 0x1
}
}