illumos gate

One of the systems running end of December 2014 based bits of Illumos panicked with assertion failure:

ht->ht_lock_cnt == 0, file: ../../i86pc/vm/htable.c, line: 1185

Panic stack:

ffffff01ef440580 vpanic()
ffffff01ef4405b0 0xfffffffffbdf4ba8()
ffffff01ef440610 htable_release+0x1c0(ffffff4f14652280)
ffffff01ef4406b0 htable_walk+0x41a(ffffff4aac31d370, ffffff01ef4406c0, ffffff01ef4406c8, 8047000)
ffffff01ef440710 hat_unlock+0xee(ffffff4aac31d370, 8046000, 1000)
ffffff01ef4407b0 segvn_softunlock+0x3a3(ffffff4e5831d000, 8046000, 1000, 1)
ffffff01ef440990 segvn_fault+0xaa5(ffffff4aac31d370, ffffff4e5831d000, 8046000, 1000, 3, 1)
ffffff01ef440ad0 urw+0x410(ffffffd2f0d1b0b0, 0, ffffff01ef440b30, 1, 8046ea4)
ffffff01ef440b00 uread+0x28(ffffffd2f0d1b0b0, ffffff01ef440b30, 1, 8046ea4)
ffffff01ef440bc0 prusrio+0xb9(ffffffd2f0d1b0b0, 0, ffffff01ef440dd0, 0)
ffffff01ef440c10 pr_read_as+0xc6(ffffff5d6a082ba8, ffffff01ef440dd0)
ffffff01ef440c60 prread+0x60(ffffffe91ba91a00, ffffff01ef440dd0, 0, ffffff494f16dc58, 0)
ffffff01ef440d10 fop_read+0xa9(ffffffe91ba91a00, ffffff01ef440dd0, 0, ffffff494f16dc58, 0)
ffffff01ef440e70 read+0x2b8(120, b51a58e4, 1)
ffffff01ef440ea0 read32+0x1e(120, b51a58e4, 1)
ffffff01ef440f00 _sys_sysenter_post_swapgs+0x241()

The ht_lock_cnt value is 1. There are two more threads in the vicinity:

ffffff01e8017c40 fffffffffbc3e340                0   0  60                0
  PC: panic_idle+0x20    TASKQ: kmem_taskq
  stack pointer for thread ffffff01e8017c40: ffffff01e8017580
    0xff4567a8c580()
    apix_hilevel_intr_epilog+0x35()
    apix_hilevel_intr_prolog+0x3e()
    0xffffff4567a8c580()
    default_lock_delay+0x4e()
    mutex_vector_enter+0xc5()
    htable_steal_active+0x81()
    htable_steal+0x218()
    htable_reap+0x7f()
    kmem_cache_reap+0x54()
    taskq_thread+0x318()
    thread_start+8()

and

ffffff45cb6cf800 ffffff9b3522f0b0 ffffffb68e3ffe40   2   2                0
  PC: panic_idle+0x20    CMD: /opt/local/bin/dmake install
  stack pointer for thread ffffff45cb6cf800: ffffff01ef5ec0e0
    mutex_enter+0x10()
    page_get_mnode_freelist+0x612()
    page_get_freelist+0x1a8()
    page_create_va+0x57e()
    page_get_physical+0xb0()
    ptable_alloc+0xb5()
    htable_alloc+0x32b()
    htable_create+0x20a()
    hati_load_common+0x268()
    hat_memload+0x132()
    hat_memload_region+0x25()
    segvn_faultpage+0x5ff()
    segvn_fault+0xcc6()
    as_fault+0x3a2()
    pagefault+0x96()
    trap+0xdc3()
    0xfffffffffb8002ba()