Project

General

Profile

Bug #7395

mutex not owner panic in dnode_dest() -> mutex_destroy()

Added by Youzhong Yang about 3 years ago. Updated over 2 years ago.

Status:
New
Priority:
Normal
Assignee:
-
Category:
zfs - Zettabyte File System
Start date:
2016-09-21
Due date:
% Done:

0%

Estimated time:
Difficulty:
Medium
Tags:
needs-triage

Description

> ::status
debugging crash dump vmcore.0 (64-bit) from batfs0505
operating system: 5.11 joyent_20160525T044136Z (i86pc)
image uuid: (not set)
panic message: mutex_destroy: not owner, lp=ffffff6e2d59f5f0 owner=ffffff449396f400 thread=ffffff01e8062c40
dump content: kernel pages only

> $C
ffffff01e8062a50 vpanic()
ffffff01e8062a70 mutex_panic+0x58(fffffffffb94c94b, ffffff6e2d59f5f0)
ffffff01e8062aa0 mutex_destroy+0x60(ffffff6e2d59f5f0)
ffffff01e8062ad0 dnode_dest+0x28(ffffff6e2d59f510, 0)
ffffff01e8062b10 kmem_slab_free_constructed+0xcc(ffffff42edff0008, ffffff6e2d59f510, 0)
ffffff01e8062b60 kmem_move_buffer+0x26a(ffffff55a02b4370)
ffffff01e8062c20 taskq_thread+0x2d0(ffffff42e2ce2e40)
ffffff01e8062c30 thread_start+8()

> ffffff6e2d59f5f0::mutex
            ADDR  TYPE             HELD MINSPL OLDSPL WAITERS
ffffff6e2d59f5f0 adapt               no      -      -      no

> ffffff449396f400::findstack -v
stack pointer for thread ffffff449396f400: ffffff020a0fe5a0
  ffffff020a0fe5e0 dbuf_rele+0x30(ffffff504697cae0, 0)
  ffffff020a0fe600 dmu_buf_rele+0x15(ffffff504697cae0, 0)
  ffffff020a0fe620 sa_buf_rele+0x15(ffffff504697cae0, 0)
  ffffff020a0fe6c0 zfs_zget+0x169(fffffffff791191e, ffffff4f917b2298, fffffffff7d05168)
  ffffff020a0fe700 0xffffff020a0fe790()
  ffffff020a0fe8c0 mutex_enter()
  ffffff020a0fea50 dbuf_prefetch+0x68(ffffff4e2196cc70, 0, 69504, 0, 0)
  ffffff020a0feae0 dmu_prefetch+0x136(ffffff48d6995400, d2a088, 0, 0, 0, 0)
  ffffff020a0fed60 zfs_readdir+0x659(ffffff44bef53300, ffffff020a0fee10, ffffff44bc540bc0, ffffff020a0fee5c, 0, 0)
  ffffff020a0fedf0 fop_readdir+0x6b(ffffff44bef53300, ffffff020a0fee10, ffffff44bc540bc0, ffffff020a0fee5c, 0, 0)
  ffffff020a0feeb0 getdents64+0xf5(22, feda8000, 2000)
  ffffff020a0fef10 _sys_sysenter_post_swapgs+0x153()

History

#1

Updated by Youzhong Yang over 2 years ago

A recent crash:

> ::status
debugging crash dump vmcore.1 (64-bit) from batfs0555
operating system: 5.11 joyent_20160525T044136Z (i86pc)
image uuid: (not set)
panic message: mutex_destroy: not owner, lp=ffffff4c1fb16870 owner=ffffff4ceb135460 thread=ffffff01e7e62c40
dump content: kernel pages only

> ffffff4ceb135460::findstack -v
stack pointer for thread ffffff4ceb135460: ffffff020de6a680
[ ffffff020de6a680 _resume_from_idle+0x112() ]
  ffffff020de6a7b0 atomic_dec_64_nv+0xb()
  ffffff020de6a7f0 dnode_rele+0x33(ffffff4c1fb16790, fffffffff7dcdf68)
  ffffff020de6a850 dmu_bonus_hold+0x99(ffffff8bc9d42c80, 4, 0, ffffff020de6a8c8)
  ffffff020de6a880 sa_buf_hold+0x1d(ffffff8bc9d42c80, 4, 0, ffffff020de6a8c8)
  ffffff020de6a920 zfs_zget+0x64(ffffff4389996000, 4, ffffff020de6a948)
  ffffff020de6a980 zfs_root+0x78(ffffff44cceeb4e8, ffffff020de6a9d8)
  ffffff020de6a9c0 fsop_root+0x2d(ffffff44cceeb4e8, ffffff020de6a9d8)
  ffffff020de6aa10 traverse+0x87(ffffff020de6ab08)
  ffffff020de6ac80 lookuppnvp+0x262(ffffff020de6ad60, 0, 1, 0, ffffff020de6aea0, ffffff47fba89700, ffffff4fc7d94b40, ffffff53342460e0)
  ffffff020de6ad20 lookuppnatcred+0x15e(ffffff020de6ad60, 0, 1, 0, ffffff020de6aea0, 0, ffffff53342460e0)
  ffffff020de6ae30 lookupnameatcred+0x119(197ab50, 0, 1, 0, ffffff020de6aea0, 0, ffffff53342460e0)
  ffffff020de6ae80 lookupname+0x38(197ab50, 0, 1, 0, ffffff020de6aea0)
  ffffff020de6af00 acl+0x8c(197ab50, 6, 0, 0)
  ffffff020de6af10 sys_syscall+0x1bd()

> ffffff01e7e62c40::findstack -v
stack pointer for thread ffffff01e7e62c40: ffffff01e7e62690
  ffffff01e7e62730 0xffffff42eba1bac0()
  ffffff01e7e62c40 0xc6955646c7b3a7()

> $C
ffffff01e7e62a50 vpanic()
ffffff01e7e62a70 mutex_panic+0x58(fffffffffb94c94b, ffffff4c1fb16870)
ffffff01e7e62aa0 mutex_destroy+0x60(ffffff4c1fb16870)
ffffff01e7e62ad0 dnode_dest+0x28(ffffff4c1fb16790, 0)
ffffff01e7e62b10 kmem_slab_free_constructed+0xcc(ffffff42ec349008, ffffff4c1fb16790, 0)
ffffff01e7e62b60 kmem_move_buffer+0x26a(ffffffc170351788)
ffffff01e7e62c20 taskq_thread+0x2d0(ffffff42e0c89e40)
ffffff01e7e62c30 thread_start+8()

> ffffff4c1fb16870::mutex
            ADDR  TYPE             HELD MINSPL OLDSPL WAITERS
ffffff4c1fb16870 adapt ffffff4ceb135460      -      -      no

> ffffff4c1fb16790::print -ta dnode_t dn_mtx
ffffff4c1fb16870 kmutex_t dn_mtx = {
    ffffff4c1fb16870 void *[1] dn_mtx._opaque = [ 0xffffff4ceb135460 ]
}

> ffffff020de6a680,2e/PN
0xffffff020de6a680:             0xffffff020de6a7b0
                atomic_dec_64_nv+0xb
                0xffffff4c1fb16948
                __func__.35286
                0xffffff4ceb135460
                0xffffff4ceb135460
                0xd7f81f65c9b60
                __func__.27114
                3
                0xffffff492deb8640
                0xffffff020de6a7b0
                0xffffff020de6ab08
                0
                0xffffff4c1fb16790
                0xffffffc14a11b268
                0xffffff020de6a948
                0
                0xffffff492deb8640
                0xffffff020de6a7c0
                0x4b
                0x4b
                0
                0
                0x20
                0
                atomic_dec_64_nv+0xb
                0x30
                0x213
                0xffffff020de6a778
                0x38
                0xffffff4389996000
                dnode_rele_and_unlock+0x37
                0xffffff4c1fb16790
                0
                0xffffff5039ab5c00
                0xffffff020de6a948
                __func__.35286
                0xffffff4c1fb16790
                0xffffff020de6a7f0
                dnode_rele+0x33
                0xffffff020de6a7f0
                __func__.35286
                0xffffff020de6a808
                0xffffff020de6a8c8
                __func__.35286
                0xffffff4c1fb16790

Also available in: Atom PDF