Project

General

Profile

Bug #3746 » zrlock-issue.txt

one of our servers had lots of threads blocked due to a zrlock in LOCKED state - Youzhong Yang, 2016-09-23 03:07 PM

 
1
--------------------------------------------------------------------------
2
> ::stacks -c nlm_host_findcreate
3
THREAD           STATE    SOBJ                COUNT
4
ffffff44730f53a0 SLEEP    MUTEX                1024
5
                 swtch+0x141
6
                 turnstile_block+0x21a
7
                 mutex_vector_enter+0x3a3
8
                 nlm_host_findcreate+0x4b
9
                 nlm_do_lock+0xd7
10
                 nlm4_lock_msg_4_svc+0x53
11
                 nlm_dispatch+0xfe
12
                 nlm_prog_4+0x34
13
                 svc_getreq+0x1c1
14
                 svc_run+0xe0
15
                 svc_do_run+0x8e
16
                 nfssys+0x111
17
                 _sys_sysenter_post_swapgs+0x153
18

    
19
> ffffff44730f53a0::findstack -v
20
stack pointer for thread ffffff44730f53a0: ffffff020f619740
21
[ ffffff020f619740 _resume_from_idle+0x112() ]
22
  ffffff020f619770 swtch+0x141()
23
  ffffff020f619810 turnstile_block+0x21a(0, 0, ffffff4320151900, fffffffffbc07ac0, 0, 0)
24
  ffffff020f619880 mutex_vector_enter+0x3a3(ffffff4320151900)
25
  ffffff020f619950 nlm_host_findcreate+0x4b(ffffff4320151900, ffffff452ebef850, ffffff44518133c8, ffffff44e5d7b428, ffffff44e5d7b438)
26
  ffffff020f619a70 nlm_do_lock+0xd7(ffffff020f619b50, ffffff020f619a90, ffffff020f619c90, 0, fffffffff81a5dd0, fffffffff81a5d30)
27
  ffffff020f619ae0 nlm4_lock_msg_4_svc+0x53(ffffff020f619b50, ffffff020f619b00, ffffff020f619c90)
28
  ffffff020f619c10 nlm_dispatch+0xfe(ffffff020f619c90, ffffff44e5d7b400, fffffffff81a86d8)
29
  ffffff020f619c30 nlm_prog_4+0x34(ffffff020f619c90, ffffff44e5d7b400)
30
  ffffff020f619d10 svc_getreq+0x1c1(ffffff44e5d7b400, ffffff43385cb760)
31
  ffffff020f619d80 svc_run+0xe0(ffffff44500b01a8)
32
  ffffff020f619dc0 svc_do_run+0x8e(2)
33
  ffffff020f619eb0 nfssys+0x111(e, febd0fbc)
34
  ffffff020f619f10 _sys_sysenter_post_swapgs+0x153()
35
				 
36
> ffffff4985822400::findstack -v
37
stack pointer for thread ffffff4985822400: ffffff0206c2b740
38
[ ffffff0206c2b740 _resume_from_idle+0x112() ]
39
  ffffff0206c2b770 swtch+0x141()
40
  ffffff0206c2b810 turnstile_block+0x21a(ffffff431ecb5918, 0, ffffff4320151900, fffffffffbc07ac0, 0, 0)
41
  ffffff0206c2b880 mutex_vector_enter+0x3a3(ffffff4320151900)
42
  ffffff0206c2b950 nlm_host_findcreate+0x4b(ffffff4320151900, ffffff49b151b8a8, ffffff44518133c8, ffffff57df02f028, ffffff57df02f038)
43
  ffffff0206c2ba70 nlm_do_lock+0xd7(ffffff0206c2bb50, ffffff0206c2ba90, ffffff0206c2bc90, 0, fffffffff81a5dd0, fffffffff81a5d30)
44
  ffffff0206c2bae0 nlm4_lock_msg_4_svc+0x53(ffffff0206c2bb50, ffffff0206c2bb00, ffffff0206c2bc90)
45
  ffffff0206c2bc10 nlm_dispatch+0xfe(ffffff0206c2bc90, ffffff57df02f000, fffffffff81a86d8)
46
  ffffff0206c2bc30 nlm_prog_4+0x34(ffffff0206c2bc90, ffffff57df02f000)
47
  ffffff0206c2bd10 svc_getreq+0x1c1(ffffff57df02f000, ffffff4705746100)
48
  ffffff0206c2bd80 svc_run+0xe0(ffffff44500b01a8)
49
  ffffff0206c2bdc0 svc_do_run+0x8e(2)
50
  ffffff0206c2beb0 nfssys+0x111(e, fa790fbc)
51
  ffffff0206c2bf10 _sys_sysenter_post_swapgs+0x153()
52
  
53
> ffffff4320151900::mutex
54
            ADDR  TYPE             HELD MINSPL OLDSPL WAITERS
55
ffffff4320151900 adapt ffffff01fd664c40      -      -     yes
56

    
57
> ffffff01fd664c40::findstack -v
58
stack pointer for thread ffffff01fd664c40: ffffff01fd6647a0
59
[ ffffff01fd6647a0 _resume_from_idle+0x112() ]
60
  ffffff01fd6647d0 swtch+0x141()
61
  ffffff01fd664810 cv_wait+0x70(ffffff76c8b0d79c, ffffff76c8b0d790)
62
  ffffff01fd664840 zrl_add_impl+0x83(ffffff76c8b0d790, fffffffff7de2590)
63
  ffffff01fd6648f0 dnode_hold_impl+0x155(ffffff51d1045180, 3, 1, ffffff49cb8c2800, ffffff01fd664948)
64
  ffffff01fd664920 dnode_hold+0x28(ffffff51d1045180, 3, ffffff49cb8c2800, ffffff01fd664948)
65
  ffffff01fd664990 dmu_tx_hold_object_impl+0x47(ffffff49cb8c2800, ffffff51d1045180, 3, 4, 0, 0)
66
  ffffff01fd6649f0 dmu_tx_hold_zap+0x4f(ffffff49cb8c2800, 3, 0, 0)
67
  ffffff01fd664a40 zfs_rmnode+0xc8(ffffff46457c4410)
68
  ffffff01fd664a80 zfs_zinactive+0xe8(ffffff46457c4410)
69
  ffffff01fd664ae0 zfs_inactive+0x75(ffffff471c310400, ffffff42e2c33e18, 0)
70
  ffffff01fd664b40 fop_inactive+0x76(ffffff471c310400, ffffff42e2c33e18, 0)
71
  ffffff01fd664b70 vn_rele+0x82(ffffff471c310400)
72
  ffffff01fd664bb0 nlm_vhold_destroy+0x62(ffffff4574589080, ffffff44e81bb7e8)
73
  ffffff01fd664c20 nlm_gc+0x1f3(ffffff4320151900)
74
  ffffff01fd664c30 thread_start+8()
75

    
76
> ffffff76c8b0d79c::wchaninfo -v
77
ADDR             TYPE NWAITERS   THREAD           PROC
78
ffffff76c8b0d79c cond      131:  ffffff43406d1c40 rm
79
                                 ffffff431fc27ae0 rm
80
                                 ffffff444994dc60 rm
81
                                 ffffff431fe3c780 rm
82
                                 ffffff5993c40b20 nfsd
83
                                 ffffff4a6cfab0e0 nfsd
84
                                 ffffff493a4ea460 nfsd
85
                                 ffffff466eb5c0c0 nfsd
86
                                 ffffff44515b1be0 nfsd
87
                                 ffffff43406d54e0 smbd
88
                                 ffffff48f1b65880 nfsd
89
                                 ffffff4321b53780 nfsd
90
                                 ffffff01fd664c40 sched
91
                                 ffffff489b17e7c0 nfsd
92
                                 ffffff44518a8be0 nfsd
93
                                 ffffff46718df4e0 nfsd
94
                                 ffffff4ee8671ba0 nfsd
95
                                 ffffff4d3c981460 nfsd
96
                                 ffffffe4f9460880 nfsd
97
                                 ffffff01f4be0c40 sched
98
                                 ffffff9e848e8c40 nfsd
99
                                 ffffff466c534c40 nfsd
100
                                 ffffff4673d18480 nfsd
101
                                 ffffff447db1ec20 nfsd
102
                                 ffffff4ebb5d6420 nfsd
103
                                 ffffff9fe761fbe0 nfsd
104
                                 ffffff444f1b9b40 nfsd
105
                                 ffffff4eb94c1080 nfsd
106
                                 ffffff4cf4682100 nfsd
107
                                 ffffff4a8eeb47c0 nfsd
108
                                 ffffff4eb95aeba0 nfsd
109
                                 ffffff53b4f23400 nfsd
110
                                 ffffff44516a57e0 nfsd
111
                                 ffffff4eb9517440 nfsd
112
                                 ffffff444f1c0780 nfsd
113
                                 ffffff444f222880 nfsd
114
                                 ffffff48b580a800 nfsd
115
                                 ffffff8d6be0b080 nfsd
116
                                 ffffff4321b65140 nfsd
117
                                 ffffff578d2813e0 nfsd
118
                                 ffffff46b8a79ae0 nfsd
119
                                 ffffff45abd1c080 nfsd
120
                                 ffffff444f3df8c0 nfsd
121
                                 fffffffa550a1b40 nfsd
122
                                 ffffff49dd5094c0 nfsd
123
                                 ffffffa232661060 nfsd
124
                                 ffffff4e4bdb2060 nfsd
125
                                 ffffff5787c7b400 nfsd
126
                                 ffffff4878e09be0 nfsd
127
                                 ffffff44516553a0 nfsd
128
                                 ffffffee503533c0 nfsd
129
                                 ffffff50904f5b40 nfsd
130
                                 ffffff4eb71b80c0 nfsd
131
                                 ffffff46c4a8e420 nfsd
132
                                 ffffff45abd1cb60 rm
133
                                 ffffff431fa75160 nfsd
134
                                 ffffff8da4c5e7c0 nfsd
135
                                 ffffff47f95a5c60 nfsd
136
                                 ffffff4669feaba0 nfsd
137
                                 ffffff482da36020 nfsd
138
                                 ffffff473906f860 nfsd
139
                                 ffffffa231a614c0 nfsd
140
                                 ffffff4614b198a0 nfsd
141
                                 ffffff4658e033c0 nfsd
142
                                 ffffff46600ae3a0 nfsd
143
                                 ffffff49c1eb8840 nfsd
144
                                 ffffff4ece130b40 nfsd
145
                                 ffffffee7054e3c0 nfsd
146
                                 ffffff447da39b60 nfsd
147
                                 ffffff4ec9f7f820 nfsd
148
                                 ffffff4449418b60 nfsd
149
                                 ffffff48f1b654e0 nfsd
150
                                 ffffff498ff4c4c0 nfsd
151
                                 ffffff48841a0140 nfsd
152
                                 ffffff447f279100 nfsd
153
                                 ffffff4674f98c00 nfsd
154
                                 ffffff48d1497bc0 nfsd
155
                                 ffffff48803cac00 nfsd
156
                                 ffffff49164d8b20 nfsd
157
                                 ffffff4ed00bcbc0 nfsd
158
                                 ffffff4ee90b8bc0 nfsd
159
                                 ffffff4a6cfabbc0 nfsd
160
                                 ffffff494fb79820 nfsd
161
                                 ffffff4476b8f040 nfsd
162
                                 ffffff9e848e88a0 nfsd
163
                                 ffffff4451664c00 nfsd
164
                                 ffffff475e856ae0 nfsd
165
                                 ffffff4ebb2fa760 nfsd
166
                                 ffffff4678a43100 nfsd
167
                                 ffffff5260962760 nfsd
168
                                 ffffffee4fe3db40 nfsd
169
                                 ffffff47c628a760 nfsd
170
                                 ffffff467684a3c0 nfsd
171
                                 ffffff47edad9c40 nfsd
172
                                 ffffff4665a220c0 nfsd
173
                                 ffffffee4fe3d060 nfsd
174
                                 ffffff44730f98a0 nfsd
175
                                 ffffff4eb28d7180 nfsd
176
                                 ffffff430e5983e0 nfsd
177
                                 ffffff444940e8a0 nfsd
178
                                 ffffff46c1eddb20 nfsd
179
                                 ffffffa2d7fde7e0 nfsd
180
                                 ffffff4340f6ec20 nfsd
181
                                 ffffff4451776400 nfsd
182
                                 ffffff4764e80480 nfsd
183
                                 ffffff4ece541080 nfsd
184
                                 ffffff4340f92be0 nfsd
185
                                 ffffff44515bc760 nfsd
186
                                 ffffffe6908e3120 nfsd
187
                                 ffffff476a901b20 nfsd
188
                                 ffffff44493c67e0 nfsd
189
                                 ffffff4661f72c20 nfsd
190
                                 ffffff4eb845ec00 nfsd
191
                                 ffffff4925b61820 nfsd
192
                                 ffffff4340f924a0 nfsd
193
                                 ffffff444941f500 nfsd
194
                                 ffffff4a3dc7bb80 nfsd
195
                                 ffffff8d6be0bb60 nfsd
196
                                 ffffff431ee61ba0 rm
197
                                 ffffff49929a18c0 nfsd
198
                                 ffffff492ef5f040 nfsd
199
                                 ffffff444940f4e0 nfsd
200
                                 ffffff4a74cec3a0 nfsd
201
                                 ffffff431e7e8820 nfsd
202
                                 ffffffe6b5f2c3a0 rm
203
                                 ffffff4451659520 rm
204
                                 ffffff4ebbd1cc20 rm
205
                                 ffffff47c7086c20 nfsd
206
                                 ffffff4eb71b8800 rm
207
                                 ffffff51c3f13740 nfsd
208
                                 ffffff46718e28a0 nfsd
209

    
210
> ffffff76c8b0d790::print -ta zrlock_t
211
ffffff76c8b0d790 zrlock_t {
212
    ffffff76c8b0d790 kmutex_t zr_mtx = {
213
        ffffff76c8b0d790 void *[1] _opaque = [ 0 ]
214
    }
215
    ffffff76c8b0d798 volatile int32_t zr_refcount = 0xffffffff
216
    ffffff76c8b0d79c kcondvar_t zr_cv = {
217
        ffffff76c8b0d79c ushort_t _opaque = 0x83
218
    }
219
    ffffff76c8b0d79e uint16_t zr_pad = 0
220
}
221
--------------------------------------------------------------------------------------
(1-1/2)