Actions
Bug #7395
closed
mutex not owner panic in dnode_dest() -> mutex_destroy()
Status:
Closed
Priority:
Normal
Assignee:
-
Category:
zfs - Zettabyte File System
Start date:
2016-09-21
Due date:
% Done:
0%
Estimated time:
Difficulty:
Medium
Tags:
needs-triage
Gerrit CR:
External Bug:
Description
> ::status
debugging crash dump vmcore.0 (64-bit) from batfs0505
operating system: 5.11 joyent_20160525T044136Z (i86pc)
image uuid: (not set)
panic message: mutex_destroy: not owner, lp=ffffff6e2d59f5f0 owner=ffffff449396f400 thread=ffffff01e8062c40
dump content: kernel pages only
> $C
ffffff01e8062a50 vpanic()
ffffff01e8062a70 mutex_panic+0x58(fffffffffb94c94b, ffffff6e2d59f5f0)
ffffff01e8062aa0 mutex_destroy+0x60(ffffff6e2d59f5f0)
ffffff01e8062ad0 dnode_dest+0x28(ffffff6e2d59f510, 0)
ffffff01e8062b10 kmem_slab_free_constructed+0xcc(ffffff42edff0008, ffffff6e2d59f510, 0)
ffffff01e8062b60 kmem_move_buffer+0x26a(ffffff55a02b4370)
ffffff01e8062c20 taskq_thread+0x2d0(ffffff42e2ce2e40)
ffffff01e8062c30 thread_start+8()
> ffffff6e2d59f5f0::mutex
ADDR TYPE HELD MINSPL OLDSPL WAITERS
ffffff6e2d59f5f0 adapt no - - no
> ffffff449396f400::findstack -v
stack pointer for thread ffffff449396f400: ffffff020a0fe5a0
ffffff020a0fe5e0 dbuf_rele+0x30(ffffff504697cae0, 0)
ffffff020a0fe600 dmu_buf_rele+0x15(ffffff504697cae0, 0)
ffffff020a0fe620 sa_buf_rele+0x15(ffffff504697cae0, 0)
ffffff020a0fe6c0 zfs_zget+0x169(fffffffff791191e, ffffff4f917b2298, fffffffff7d05168)
ffffff020a0fe700 0xffffff020a0fe790()
ffffff020a0fe8c0 mutex_enter()
ffffff020a0fea50 dbuf_prefetch+0x68(ffffff4e2196cc70, 0, 69504, 0, 0)
ffffff020a0feae0 dmu_prefetch+0x136(ffffff48d6995400, d2a088, 0, 0, 0, 0)
ffffff020a0fed60 zfs_readdir+0x659(ffffff44bef53300, ffffff020a0fee10, ffffff44bc540bc0, ffffff020a0fee5c, 0, 0)
ffffff020a0fedf0 fop_readdir+0x6b(ffffff44bef53300, ffffff020a0fee10, ffffff44bc540bc0, ffffff020a0fee5c, 0, 0)
ffffff020a0feeb0 getdents64+0xf5(22, feda8000, 2000)
ffffff020a0fef10 _sys_sysenter_post_swapgs+0x153()
Related issues
Updated by Youzhong Yang over 6 years ago
A recent crash:
> ::status
debugging crash dump vmcore.1 (64-bit) from batfs0555
operating system: 5.11 joyent_20160525T044136Z (i86pc)
image uuid: (not set)
panic message: mutex_destroy: not owner, lp=ffffff4c1fb16870 owner=ffffff4ceb135460 thread=ffffff01e7e62c40
dump content: kernel pages only
> ffffff4ceb135460::findstack -v
stack pointer for thread ffffff4ceb135460: ffffff020de6a680
[ ffffff020de6a680 _resume_from_idle+0x112() ]
ffffff020de6a7b0 atomic_dec_64_nv+0xb()
ffffff020de6a7f0 dnode_rele+0x33(ffffff4c1fb16790, fffffffff7dcdf68)
ffffff020de6a850 dmu_bonus_hold+0x99(ffffff8bc9d42c80, 4, 0, ffffff020de6a8c8)
ffffff020de6a880 sa_buf_hold+0x1d(ffffff8bc9d42c80, 4, 0, ffffff020de6a8c8)
ffffff020de6a920 zfs_zget+0x64(ffffff4389996000, 4, ffffff020de6a948)
ffffff020de6a980 zfs_root+0x78(ffffff44cceeb4e8, ffffff020de6a9d8)
ffffff020de6a9c0 fsop_root+0x2d(ffffff44cceeb4e8, ffffff020de6a9d8)
ffffff020de6aa10 traverse+0x87(ffffff020de6ab08)
ffffff020de6ac80 lookuppnvp+0x262(ffffff020de6ad60, 0, 1, 0, ffffff020de6aea0, ffffff47fba89700, ffffff4fc7d94b40, ffffff53342460e0)
ffffff020de6ad20 lookuppnatcred+0x15e(ffffff020de6ad60, 0, 1, 0, ffffff020de6aea0, 0, ffffff53342460e0)
ffffff020de6ae30 lookupnameatcred+0x119(197ab50, 0, 1, 0, ffffff020de6aea0, 0, ffffff53342460e0)
ffffff020de6ae80 lookupname+0x38(197ab50, 0, 1, 0, ffffff020de6aea0)
ffffff020de6af00 acl+0x8c(197ab50, 6, 0, 0)
ffffff020de6af10 sys_syscall+0x1bd()
> ffffff01e7e62c40::findstack -v
stack pointer for thread ffffff01e7e62c40: ffffff01e7e62690
ffffff01e7e62730 0xffffff42eba1bac0()
ffffff01e7e62c40 0xc6955646c7b3a7()
> $C
ffffff01e7e62a50 vpanic()
ffffff01e7e62a70 mutex_panic+0x58(fffffffffb94c94b, ffffff4c1fb16870)
ffffff01e7e62aa0 mutex_destroy+0x60(ffffff4c1fb16870)
ffffff01e7e62ad0 dnode_dest+0x28(ffffff4c1fb16790, 0)
ffffff01e7e62b10 kmem_slab_free_constructed+0xcc(ffffff42ec349008, ffffff4c1fb16790, 0)
ffffff01e7e62b60 kmem_move_buffer+0x26a(ffffffc170351788)
ffffff01e7e62c20 taskq_thread+0x2d0(ffffff42e0c89e40)
ffffff01e7e62c30 thread_start+8()
> ffffff4c1fb16870::mutex
ADDR TYPE HELD MINSPL OLDSPL WAITERS
ffffff4c1fb16870 adapt ffffff4ceb135460 - - no
> ffffff4c1fb16790::print -ta dnode_t dn_mtx
ffffff4c1fb16870 kmutex_t dn_mtx = {
ffffff4c1fb16870 void *[1] dn_mtx._opaque = [ 0xffffff4ceb135460 ]
}
> ffffff020de6a680,2e/PN
0xffffff020de6a680: 0xffffff020de6a7b0
atomic_dec_64_nv+0xb
0xffffff4c1fb16948
__func__.35286
0xffffff4ceb135460
0xffffff4ceb135460
0xd7f81f65c9b60
__func__.27114
3
0xffffff492deb8640
0xffffff020de6a7b0
0xffffff020de6ab08
0
0xffffff4c1fb16790
0xffffffc14a11b268
0xffffff020de6a948
0
0xffffff492deb8640
0xffffff020de6a7c0
0x4b
0x4b
0
0
0x20
0
atomic_dec_64_nv+0xb
0x30
0x213
0xffffff020de6a778
0x38
0xffffff4389996000
dnode_rele_and_unlock+0x37
0xffffff4c1fb16790
0
0xffffff5039ab5c00
0xffffff020de6a948
__func__.35286
0xffffff4c1fb16790
0xffffff020de6a7f0
dnode_rele+0x33
0xffffff020de6a7f0
__func__.35286
0xffffff020de6a808
0xffffff020de6a8c8
__func__.35286
0xffffff4c1fb16790
Updated by Jason King about 3 years ago
- Is duplicate of Bug #13234: Prevent race condition in dnode_dest added
Updated by Jason King about 3 years ago
Updated by Electric Monk about 3 years ago
- Status changed from New to Closed
git commit b390f3a9a166c4ae80f78a79961396a4847672b7
commit b390f3a9a166c4ae80f78a79961396a4847672b7
Author: John Poduska <jpoduska@datto.com>
Date: 2020-10-22T21:47:39.000Z
13234 Prevent race condition in dnode_dest
Portions contributed by: Jason King <jason.king@joyent.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Approved by: Dan McDonald <danmcd@joyent.com>
Actions