Project

General

Profile

Bug #7351 » zfs-deadman.txt

Youzhong Yang, 2016-09-01 05:30 PM

 
1
======================= Case 1 =======================
2
> $C
3
ffffd005c79769d0 vpanic()
4
ffffd005c7976a20 vdev_deadman+0x10b(ffffd127f47716c0)
5
ffffd005c7976a70 vdev_deadman+0x4a(ffffd127f4771d00)
6
ffffd005c7976ac0 vdev_deadman+0x4a(ffffd127ee1bc680)
7
ffffd005c7976af0 spa_deadman+0xad(ffffd127f48b8000)
8
ffffd005c7976b90 cyclic_softint+0xfd(ffffd127eb5d6ac0, 0)
9
ffffd005c7976ba0 cbe_low_level+0x14()
10
ffffd005c7976bf0 av_dispatch_softvect+0x78(2)
11
ffffd005c7976c20 apix_dispatch_softint+0x35(0, 0)
12
ffffd005cafb9a20 switch_sp_and_call+0x13()
13
ffffd005cafb9a70 apix_do_softint+0x6c(ffffd005cafb9ae0)
14
ffffd005cafb9ad0 apix_do_interrupt+0x34a(ffffd005cafb9ae0, 2e)
15
ffffd005cafb9ae0 _interrupt+0xba()
16
ffffd005cafb9bd0 mach_cpu_idle+6()
17
ffffd005cafb9c00 cpu_idle+0x11a()
18
ffffd005cafb9c20 idle+0xa7()
19
ffffd005cafb9c30 thread_start+8()
20

    
21
> ffffd005d8a4ac40::findstack -v
22
stack pointer for thread ffffd005d8a4ac40: ffffd005d8a4a920
23
[ ffffd005d8a4a920 _resume_from_idle+0x112() ]
24
  ffffd005d8a4a950 swtch+0x141()
25
  ffffd005d8a4a990 cv_wait+0x70(ffffd128cf271bf8, ffffd128cf271bf0)
26
  ffffd005d8a4a9d0 zio_wait+0x6b(ffffd128cf2718a8)
27
  ffffd005d8a4aa10 dsl_pool_sync_mos+0x4a(ffffd127f48bd040, ffffd128ea995680)
28
  ffffd005d8a4aaa0 dsl_pool_sync+0x25e(ffffd127f48bd040, 57c15)
29
  ffffd005d8a4ab70 spa_sync+0x386(ffffd127f48b8000, 57c15)
30
  ffffd005d8a4ac20 txg_sync_thread+0x227(ffffd127f48bd040)
31
  ffffd005d8a4ac30 thread_start+8()
32

    
33
> ffffd128cf2718a8::zio -c -r
34
ADDRESS                 TYPE  STAGE            WAITER           TIME_ELAPSED
35
ffffd128cf2718a8        NULL  CHECKSUM_VERIFY  ffffd005d8a4ac40 -
36
 ffffd12883d62000       WRITE CHECKSUM_VERIFY  -                -
37
  ffffd129880c8480      WRITE CHECKSUM_VERIFY  -                -
38
   ffffd12956f2f498     WRITE VDEV_IO_START    -                -
39
    ffffd128d3a7cb80    WRITE VDEV_IO_START    -                -
40
     ffffd128c088c3f0   WRITE VDEV_IO_START    -                1015634ms
41

    
42
> ffffd128c088c3f0::print -t zio_t io_offset io_size io_vd
43
uint64_t io_offset = 0x3d90f5000
44
uint64_t io_size = 0x1000
45
vdev_t *io_vd = 0xffffd127f47716c0
46

    
47
> 0xffffd127f47716c0::print -t vdev_t vdev_path vdev_devid vdev_physpath
48
char *vdev_path = 0xffffd127eecccd78 "/dev/dsk/c5t1d0s0"
49
char *vdev_devid = 0xffffd127f58fd4f0 "id1,kdev@E8086-INTEL_SSDPE2ME400G4_____________________-CVMD50920035400FGN__-1/a"
50
char *vdev_physpath = 0xffffd127f58fd490 "/pci@0,0/pci8086,6f08@3/pci10b5,9765@0/pci10b5,9765@9/pci8086,370a@0/blkdev@1,0:a"
51

    
52
> *nvme_state::print -t i_ddi_soft_state
53
i_ddi_soft_state {
54
    void **array = 0xffffd127ed5b1100
55
    kmutex_t lock = {
56
        void *[1] _opaque = [ 0 ]
57
    }
58
    size_t size = 0x290
59
    size_t n_items = 0x20
60
    struct i_ddi_soft_state *next = 0xffffd127ed20fdd8
61
}
62

    
63
> 0xffffd127ed5b1100,100::dump -e -g 8
64
ffffd127ed5b1100:  ffffd1272aa193c0 ffffd127edca0040
65
ffffd127ed5b1110:  ffffd127edca0340 ffffd127ec259640
66
ffffd127ed5b1120:  ffffd1272aae7d00 ffffd127ecb07c00
67
ffffd127ed5b1130:  ffffd127ecb07000 ffffd1272a7abc40
68
ffffd127ed5b1140:  ffffd127edc92980 ffffd127edca0940
69
ffffd127ed5b1150:  ffffd127edc92c80 ffffd127edca0640
70
ffffd127ed5b1160:  ffffd127f14006c0 ffffd127f0219c80
71
ffffd127ed5b1170:  ffffd127ec167300 ffffd127f0219080
72
ffffd127ed5b1180:  ffffd127f0219380 ffffd127ee91bc40
73
ffffd127ed5b1190:  ffffd127f1400cc0 ffffd127f14009c0
74
ffffd127ed5b11a0:  ffffd127f14003c0 ffffd1272a0c7100
75
ffffd127ed5b11b0:  ffffd127f14000c0 ffffd127ec9c6a00
76
ffffd127ed5b11c0:  0000000000000000 0000000000000000
77
ffffd127ed5b11d0:  0000000000000000 0000000000000000
78
ffffd127ed5b11e0:  0000000000000000 0000000000000000
79
ffffd127ed5b11f0:  0000000000000000 0000000000000000
80

    
81
> ffffd127ee91bc40::print -t nvme_t n_ns | ::print -t nvme_namespace_t
82
nvme_namespace_t {
83
    nvme_t *ns_nvme = 0xffffd127ee91bc40
84
    bd_handle_t ns_bd_hdl = 0xffffd127ed219580
85
    uint32_t ns_id = 0x1
86
    size_t ns_block_count = 0x2e9390b0
87
    size_t ns_block_size = 0x200
88
    size_t ns_best_block_size = 0x1000
89
    boolean_t ns_ignore = 0 (0)
90
    nvme_identify_nsid_t *ns_idns = 0xffffd127f15a4000
91
    char [76] ns_devid = [ "8086-INTEL SSDPE2ME400G4                     -CVMD50920035400FGN  -1" ]
92
}
93

    
94
> ffffd127ee91bc40::print -t nvme_t n_adminq n_ioq
95
nvme_qpair_t *n_adminq = 0xffffd127ec8616b8
96
nvme_qpair_t **n_ioq = 0xffffd127f168a2e8
97

    
98
> 0xffffd127f168a2e8,48::dump -e -g 8
99
ffffd127f168a2e8:  ffffd127ec8616b8 ffffd127f167e868
100
ffffd127f168a2f8:  ffffd127f167e018 ffffd127f167f4e0
101
ffffd127f168a308:  ffffd127f19bdb80 ffffd127f19bd4f0
102
ffffd127f168a318:  ffffd127f19bdc60 ffffd127f19b9728
103
ffffd127f168a328:  ffffd127f19b92c8 0000000000000000
104

    
105
> ffffd127f19b9728::print -t nvme_qpair_t
106
nvme_qpair_t {
107
    size_t nq_nentry = 0x400
108
    nvme_dma_t *nq_sqdma = 0xffffd127f16762c0
109
    nvme_sqe_t *nq_sq = 0xffffd127f1b40000
110
    uint_t nq_sqhead = 0x21e
111
    uint_t nq_sqtail = 0x21f
112
    uintptr_t nq_sqtdbl = 0x1038
113
    nvme_dma_t *nq_cqdma = 0xffffd127f1676280
114
    nvme_cqe_t *nq_cq = 0xffffd127f1b28000
115
    uint_t nq_cqhead = 0x21e
116
    uint_t nq_cqtail = 0
117
    uintptr_t nq_cqhdbl = 0x103c
118
    nvme_cmd_t **nq_cmd = 0xffffd127f1ab8000
119
    uint16_t nq_next_cmd = 0x21f
120
    uint_t nq_active_cmds = 0x1
121
    int nq_phase = 0
122
    kmutex_t nq_mutex = {
123
        void *[1] _opaque = [ 0 ]
124
    }
125
}
126

    
127
> 0xffffd127f1ab8000+21e*8::dump -e -g 8
128
ffffd127f1ab90f0:  ffffd1289b3eb540 0000000000000000
129

    
130
> ffffd1289b3eb540::print -t nvme_cmd_t
131
nvme_cmd_t {
132
    nvme_sqe_t nc_sqe = {
133
        uint8_t sqe_opc = 0x1
134
        unsigned char:2 sqe_fuse :2 = 0
135
        unsigned char:5 sqe_rsvd :5 = 0
136
        unsigned char:1 sqe_psdt :1 = 0
137
        uint16_t sqe_cid = 0x21e
138
        uint32_t sqe_nsid = 0x1
139
        uint64_t sqe_rsvd1 = 0
140
        union  sqe_m = {
141
            uint64_t m_ptr = 0
142
            uint64_t m_sglp = 0
143
        }
144
        union  sqe_dptr = {
145
            uint64_t [2] d_prp = [ 0xb863326000, 0 ]
146
            nvme_sgl_t d_sgl = {
147
                uint64_t sgl_addr = 0xb863326000
148
                uint32_t sgl_len = 0
149
                uint8_t [3] sgl_rsvd = [ 0, 0, 0 ]
150
                unsigned char:4 sgl_zero :4 = 0
151
                unsigned char:4 sgl_type :4 = 0
152
            }
153
        }
154
        uint32_t sqe_cdw10 = 0x1ec88a8
155
        uint32_t sqe_cdw11 = 0
156
        uint32_t sqe_cdw12 = 0x7
157
        uint32_t sqe_cdw13 = 0
158
        uint32_t sqe_cdw14 = 0
159
        uint32_t sqe_cdw15 = 0
160
    }
161
    nvme_cqe_t nc_cqe = {
162
        uint32_t cqe_dw0 = 0
163
        uint32_t cqe_rsvd1 = 0
164
        uint16_t cqe_sqhd = 0
165
        uint16_t cqe_sqid = 0
166
        uint16_t cqe_cid = 0
167
        nvme_cqe_sf_t cqe_sf = {
168
            unsigned short:1 sf_p :1 = 0
169
            unsigned short:8 sf_sc :8 = 0
170
            unsigned short:3 sf_sct :3 = 0
171
            unsigned short:2 sf_rsvd2 :2 = 0
172
            unsigned short:1 sf_m :1 = 0
173
            unsigned short:1 sf_dnr :1 = 0
174
        }
175
    }
176
    void (*)() nc_callback = nvme_bd_xfer_done
177
    bd_xfer_t *nc_xfer = 0xffffd128008f6d10
178
    boolean_t nc_completed = 0 (0)
179
    uint16_t nc_sqid = 0x7
180
    nvme_dma_t *nc_dma = 0
181
    kmutex_t nc_mutex = {
182
        void *[1] _opaque = [ 0 ]
183
    }
184
    kcondvar_t nc_cv = {
185
        ushort_t _opaque = 0
186
    }
187
    taskq_ent_t nc_tqent = {
188
        struct taskq_ent *tqent_next = 0
189
        struct taskq_ent *tqent_prev = 0
190
        task_func_t *tqent_func = 0
191
        void *tqent_arg = 0
192
        union  tqent_un = {
193
            taskq_bucket_t *tqent_bucket = 0
194
            uintptr_t tqent_flags = 0
195
        }
196
        kthread_t *tqent_thread = 0
197
        kcondvar_t tqent_cv = {
198
            ushort_t _opaque = 0
199
        }
200
    }
201
    nvme_t *nc_nvme = 0xffffd127ee91bc40
202
}
203

    
204
> 0xffffd128008f6d10::print -t bd_xfer_t
205
bd_xfer_t {
206
    diskaddr_t x_blkno = 0x1ec88a8
207
    size_t x_nblks = 0x8
208
    ddi_dma_handle_t x_dmah = 0xffffd12800febb80
209
    ddi_dma_cookie_t x_dmac = {
210
        union  _dmu = {
211
            uint64_t _dmac_ll = 0x5ae798d000
212
            uint32_t [2] _dmac_la = [ 0xe798d000, 0x5a ]
213
        }
214
        size_t dmac_size = 0x1000
215
        uint_t dmac_type = 0
216
    }
217
    unsigned int x_ndmac = 0x1
218
    caddr_t x_kaddr = 0
219
    unsigned int x_flags = 0
220
}
221

    
222
> 0xffffd127f1b28000+21c*10::array nvme_cqe_t 4 | ::print -t nvme_cqe_t
223
nvme_cqe_t {
224
    uint32_t cqe_dw0 = 0
225
    uint32_t cqe_rsvd1 = 0
226
    uint16_t cqe_sqhd = 0x21d
227
    uint16_t cqe_sqid = 0x7
228
    uint16_t cqe_cid = 0x21c
229
    nvme_cqe_sf_t cqe_sf = {
230
        unsigned short:1 sf_p :1 = 0x1
231
        unsigned short:8 sf_sc :8 = 0x1
232
        unsigned short:3 sf_sct :3 = 0
233
        unsigned short:2 sf_rsvd2 :2 = 0
234
        unsigned short:1 sf_m :1 = 0
235
        unsigned short:1 sf_dnr :1 = 0
236
    }
237
}
238
nvme_cqe_t {
239
    uint32_t cqe_dw0 = 0
240
    uint32_t cqe_rsvd1 = 0
241
    uint16_t cqe_sqhd = 0x21e
242
    uint16_t cqe_sqid = 0x7
243
    uint16_t cqe_cid = 0x21d
244
    nvme_cqe_sf_t cqe_sf = {
245
        unsigned short:1 sf_p :1 = 0x1
246
        unsigned short:8 sf_sc :8 = 0x1
247
        unsigned short:3 sf_sct :3 = 0
248
        unsigned short:2 sf_rsvd2 :2 = 0
249
        unsigned short:1 sf_m :1 = 0
250
        unsigned short:1 sf_dnr :1 = 0
251
    }
252
}
253
nvme_cqe_t {
254
    uint32_t cqe_dw0 = 0
255
    uint32_t cqe_rsvd1 = 0
256
    uint16_t cqe_sqhd = 0x21f
257
    uint16_t cqe_sqid = 0x7
258
    uint16_t cqe_cid = 0x21e
259
    nvme_cqe_sf_t cqe_sf = {
260
        unsigned short:1 sf_p :1 = 0x1
261
        unsigned short:8 sf_sc :8 = 0x1
262
        unsigned short:3 sf_sct :3 = 0
263
        unsigned short:2 sf_rsvd2 :2 = 0
264
        unsigned short:1 sf_m :1 = 0
265
        unsigned short:1 sf_dnr :1 = 0
266
    }
267
}
268
nvme_cqe_t {
269
    uint32_t cqe_dw0 = 0
270
    uint32_t cqe_rsvd1 = 0
271
    uint16_t cqe_sqhd = 0x220
272
    uint16_t cqe_sqid = 0x7
273
    uint16_t cqe_cid = 0x21f
274
    nvme_cqe_sf_t cqe_sf = {
275
        unsigned short:1 sf_p :1 = 0
276
        unsigned short:8 sf_sc :8 = 0
277
        unsigned short:3 sf_sct :3 = 0
278
        unsigned short:2 sf_rsvd2 :2 = 0
279
        unsigned short:1 sf_m :1 = 0
280
        unsigned short:1 sf_dnr :1 = 0
281
    }
282
}
283

    
284
> 0xffffd127f1b40000+21c*40::array nvme_sqe_t 4 | ::print -t nvme_sqe_t
285
nvme_sqe_t {
286
    uint8_t sqe_opc = 0x1
287
    unsigned char:2 sqe_fuse :2 = 0
288
    unsigned char:5 sqe_rsvd :5 = 0
289
    unsigned char:1 sqe_psdt :1 = 0
290
    uint16_t sqe_cid = 0x21c
291
    uint32_t sqe_nsid = 0x1
292
    uint64_t sqe_rsvd1 = 0
293
    union  sqe_m = {
294
        uint64_t m_ptr = 0
295
        uint64_t m_sglp = 0
296
    }
297
    union  sqe_dptr = {
298
        uint64_t [2] d_prp = [ 0x46a74c0000, 0 ]
299
        nvme_sgl_t d_sgl = {
300
            uint64_t sgl_addr = 0x46a74c0000
301
            uint32_t sgl_len = 0
302
            uint8_t [3] sgl_rsvd = [ 0, 0, 0 ]
303
            unsigned char:4 sgl_zero :4 = 0
304
            unsigned char:4 sgl_type :4 = 0
305
        }
306
    }
307
    uint32_t sqe_cdw10 = 0x1ec8878
308
    uint32_t sqe_cdw11 = 0
309
    uint32_t sqe_cdw12 = 0x7
310
    uint32_t sqe_cdw13 = 0
311
    uint32_t sqe_cdw14 = 0
312
    uint32_t sqe_cdw15 = 0
313
}
314
nvme_sqe_t {
315
    uint8_t sqe_opc = 0x1
316
    unsigned char:2 sqe_fuse :2 = 0
317
    unsigned char:5 sqe_rsvd :5 = 0
318
    unsigned char:1 sqe_psdt :1 = 0
319
    uint16_t sqe_cid = 0x21d
320
    uint32_t sqe_nsid = 0x1
321
    uint64_t sqe_rsvd1 = 0
322
    union  sqe_m = {
323
        uint64_t m_ptr = 0
324
        uint64_t m_sglp = 0
325
    }
326
    union  sqe_dptr = {
327
        uint64_t [2] d_prp = [ 0xb28bf08000, 0x9e8a970000 ]
328
        nvme_sgl_t d_sgl = {
329
            uint64_t sgl_addr = 0xb28bf08000
330
            uint32_t sgl_len = 0x8a970000
331
            uint8_t [3] sgl_rsvd = [ 0x9e, 0, 0 ]
332
            unsigned char:4 sgl_zero :4 = 0
333
            unsigned char:4 sgl_type :4 = 0
334
        }
335
    }
336
    uint32_t sqe_cdw10 = 0x1ec8880
337
    uint32_t sqe_cdw11 = 0
338
    uint32_t sqe_cdw12 = 0x17
339
    uint32_t sqe_cdw13 = 0
340
    uint32_t sqe_cdw14 = 0
341
    uint32_t sqe_cdw15 = 0
342
}
343
nvme_sqe_t {
344
    uint8_t sqe_opc = 0x1
345
    unsigned char:2 sqe_fuse :2 = 0
346
    unsigned char:5 sqe_rsvd :5 = 0
347
    unsigned char:1 sqe_psdt :1 = 0
348
    uint16_t sqe_cid = 0x21e
349
    uint32_t sqe_nsid = 0x1
350
    uint64_t sqe_rsvd1 = 0
351
    union  sqe_m = {
352
        uint64_t m_ptr = 0
353
        uint64_t m_sglp = 0
354
    }
355
    union  sqe_dptr = {
356
        uint64_t [2] d_prp = [ 0xb863326000, 0 ]
357
        nvme_sgl_t d_sgl = {
358
            uint64_t sgl_addr = 0xb863326000
359
            uint32_t sgl_len = 0
360
            uint8_t [3] sgl_rsvd = [ 0, 0, 0 ]
361
            unsigned char:4 sgl_zero :4 = 0
362
            unsigned char:4 sgl_type :4 = 0
363
        }
364
    }
365
    uint32_t sqe_cdw10 = 0x1ec88a8
366
    uint32_t sqe_cdw11 = 0
367
    uint32_t sqe_cdw12 = 0x7
368
    uint32_t sqe_cdw13 = 0
369
    uint32_t sqe_cdw14 = 0
370
    uint32_t sqe_cdw15 = 0
371
}
372
nvme_sqe_t {
373
    uint8_t sqe_opc = 0x1
374
    unsigned char:2 sqe_fuse :2 = 0
375
    unsigned char:5 sqe_rsvd :5 = 0
376
    unsigned char:1 sqe_psdt :1 = 0
377
    uint16_t sqe_cid = 0x21f
378
    uint32_t sqe_nsid = 0x1
379
    uint64_t sqe_rsvd1 = 0
380
    union  sqe_m = {
381
        uint64_t m_ptr = 0
382
        uint64_t m_sglp = 0
383
    }
384
    union  sqe_dptr = {
385
        uint64_t [2] d_prp = [ 0x5e8fa3d000, 0x46cadd3000 ]
386
        nvme_sgl_t d_sgl = {
387
            uint64_t sgl_addr = 0x5e8fa3d000
388
            uint32_t sgl_len = 0xcadd3000
389
            uint8_t [3] sgl_rsvd = [ 0x46, 0, 0 ]
390
            unsigned char:4 sgl_zero :4 = 0
391
            unsigned char:4 sgl_type :4 = 0
392
        }
393
    }
394
    uint32_t sqe_cdw10 = 0x1e78b38
395
    uint32_t sqe_cdw11 = 0
396
    uint32_t sqe_cdw12 = 0x27
397
    uint32_t sqe_cdw13 = 0
398
    uint32_t sqe_cdw14 = 0
399
    uint32_t sqe_cdw15 = 0
400
}
401

    
402
======================= Case 2 =======================
403
> ffffd005cfbecc40::findstack -v
404
stack pointer for thread ffffd005cfbecc40: ffffd005cfbec960
405
[ ffffd005cfbec960 _resume_from_idle+0x112() ]
406
  ffffd005cfbec990 swtch+0x141()
407
  ffffd005cfbec9d0 cv_wait+0x70(ffffd12d2e9783c8, ffffd12d2e9783c0)
408
  ffffd005cfbeca10 zio_wait+0x6b(ffffd12d2e978078)
409
  ffffd005cfbecaa0 dsl_pool_sync+0xb3(ffffd127f480a100, b1db0)
410
  ffffd005cfbecb70 spa_sync+0x386(ffffd127f443c000, b1db0)
411
  ffffd005cfbecc20 txg_sync_thread+0x227(ffffd127f480a100)
412
  ffffd005cfbecc30 thread_start+8()
413

    
414
> ffffd12d2e978078::zio -c -r
415
ADDRESS                 TYPE  STAGE            WAITER           TIME_ELAPSED
416
ffffd12d2e978078        NULL  CHECKSUM_VERIFY  ffffd005cfbecc40 -
417
 ffffd12d3ed21508       WRITE CHECKSUM_VERIFY  -                -
418
  ffffd12d2ed02b38      WRITE CHECKSUM_VERIFY  -                -
419
   ffffd12d7c96fba0     WRITE CHECKSUM_VERIFY  -                -
420
    ffffd12d3ec62b28    WRITE CHECKSUM_VERIFY  -                -
421
     ffffd12d3f703480   WRITE CHECKSUM_VERIFY  -                -
422
      ffffd12d3ea46838  WRITE CHECKSUM_VERIFY  -                -
423
       ffffd12d3ea84b08 WRITE CHECKSUM_VERIFY  -                -
424
       ffffd12d3f1808b0 WRITE VDEV_IO_START    -                -
425
       ffffd12d450b3bd8 WRITE VDEV_IO_START    -                -
426
       ffffd12878c50480 WRITE VDEV_IO_DONE     -                1014581ms
427
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
428
       ffffd12d3eea4ba8 WRITE VDEV_IO_START    -                -
429
       ffffd128ed2b9b98 WRITE VDEV_IO_START    -                -
430
       ffffd12d7bb0e4a8 WRITE VDEV_IO_DONE     -                1014581ms
431
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
432
       ffffd1284c8137a0 WRITE VDEV_IO_START    -                -
433
       ffffd12d7be2e840 WRITE VDEV_IO_START    -                -
434
       ffffd12d3ea5eba8 WRITE VDEV_IO_DONE     -                1014581ms
435
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
436
       ffffd12d3f571078 WRITE VDEV_IO_START    -                -
437
       ffffd12d7c0c33a8 WRITE VDEV_IO_START    -                -
438
       ffffd1287916bc18 WRITE VDEV_IO_DONE     -                1014580ms
439
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
440
       ffffd12d7bfa2bf8 WRITE VDEV_IO_START    -                -
441
       ffffd12d44b8b7a8 WRITE VDEV_IO_START    -                -
442
       ffffd12d3f0bf438 WRITE VDEV_IO_DONE     -                1014581ms
443
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
444
       ffffd12d7c8758a0 WRITE VDEV_IO_START    -                -
445
       ffffd1287e067010 WRITE VDEV_IO_START    -                -
446
       ffffd12d3f1f5b90 WRITE VDEV_IO_DONE     -                1014581ms
447
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
448
       ffffd12d3f5f7b20 WRITE VDEV_IO_START    -                -
449
       ffffd12d2e766b98 WRITE VDEV_IO_START    -                -
450
       ffffd12d3ee4d830 WRITE VDEV_IO_DONE     -                1014581ms
451
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
452
       ffffd128a3597c40 WRITE VDEV_IO_START    -                -
453
       ffffd12d3ed077a0 WRITE VDEV_IO_START    -                -
454
       ffffd12d7c82c870 WRITE VDEV_IO_DONE     -                1014581ms
455
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
456
       ffffd1287fba3028 WRITE VDEV_IO_START    -                -
457
       ffffd12d7bc8f130 WRITE VDEV_IO_START    -                -
458
       ffffd12d7c770ba0 WRITE VDEV_IO_DONE     -                1014581ms
459
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
460
       ffffd12d7c31e4b8 WRITE CHECKSUM_VERIFY  -                -
461
       ffffd12d2e443120 WRITE CHECKSUM_VERIFY  -                -
462
       ffffd12d3ed627a8 WRITE VDEV_IO_START    -                -
463
       ffffd1287d791bc0 WRITE VDEV_IO_START    -                -
464
       ffffd12d7befc7a0 WRITE VDEV_IO_DONE     -                1014581ms
465
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
466
       ffffd12d44fd1058 WRITE CHECKSUM_VERIFY  -                -
467
       ffffd12d7c6a0bc8 WRITE VDEV_IO_START    -                -
468
       ffffd12d2e6b94d8 WRITE VDEV_IO_START    -                -
469
       ffffd12d7c282010 WRITE VDEV_IO_DONE     -                1014580ms
470
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
471
       ffffd12d3f41b7c8 WRITE VDEV_IO_START    -                -
472
       ffffd1289f5a8b48 WRITE VDEV_IO_START    -                -
473
       ffffd12d451c67f0 WRITE VDEV_IO_DONE     -                1014580ms
474
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
475
       ffffd12d7c13f0f0 WRITE VDEV_IO_START    -                -
476
       ffffd12d3f3534d0 WRITE VDEV_IO_START    -                -
477
       ffffd12d3ef7f4e8 WRITE VDEV_IO_DONE     -                1014581ms
478
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
479
       ffffd12874958c30 WRITE CHECKSUM_VERIFY  -                -
480
       ffffd12d3f571420 WRITE VDEV_IO_START    -                -
481
       ffffd1287da90440 WRITE VDEV_IO_START    -                -
482
       ffffd1287df9d848 WRITE VDEV_IO_DONE     -                1014580ms
483
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
484
       ffffd12d450af850 WRITE CHECKSUM_VERIFY  -                -
485
       ffffd12d7bf9bc28 WRITE VDEV_IO_START    -                -
486
       ffffd12d3ec49838 WRITE VDEV_IO_START    -                -
487
       ffffd12d2e7ce438 WRITE VDEV_IO_DONE     -                1014579ms
488
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
489
       ffffd12d7c4620c0 WRITE CHECKSUM_VERIFY  -                -
490
       ffffd12d3ef2e148 WRITE VDEV_IO_START    -                -
491
       ffffd12d7c957110 WRITE VDEV_IO_START    -                -
492
       ffffd12d7c640838 WRITE VDEV_IO_DONE     -                1014581ms
493
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
494
 ffffd128a34c6808       WRITE CHECKSUM_VERIFY  -                -
495
  ffffd12d7c1b0478      WRITE CHECKSUM_VERIFY  -                -
496
   ffffd12d44d080d8     WRITE CHECKSUM_VERIFY  -                -
497
    ffffd12d44cc0020    WRITE CHECKSUM_VERIFY  -                -
498
     ffffd128f4e9ec10   WRITE CHECKSUM_VERIFY  -                -
499
      ffffd12878dadb88  WRITE CHECKSUM_VERIFY  -                -
500
       ffffd12d7c1cd3f8 WRITE CHECKSUM_VERIFY  -                -
501
       ffffd12d7bda4000 WRITE CHECKSUM_VERIFY  -                -
502
       ffffd12d7c845410 WRITE VDEV_IO_START    -                -
503
       ffffd12d7bd19840 WRITE VDEV_IO_START    -                -
504
       ffffd12d2e4b3c20 WRITE VDEV_IO_DONE     -                1014580ms
505
       ffffd12d2ea71118 WRITE VDEV_IO_START    -                1014581ms
506

    
507
> ffffd12d2ea71118::print -t zio_t io_size io_offset io_vd
508
uint64_t io_size = 0x20000
509
uint64_t io_offset = 0x13a2ca9000
510
vdev_t *io_vd = 0xffffd127f2a9dd40
511

    
512
> 0xffffd127f2a9dd40::print -t vdev_t vdev_path vdev_devid vdev_physpath
513
char *vdev_path = 0xffffd127f26aea50 "/dev/dsk/c8t1d0s0"
514
char *vdev_devid = 0xffffd127f679d258 "id1,kdev@E8086-INTEL_SSDPE2ME400G4_____________________-CVMD44500088400FGN__-1/a"
515
char *vdev_physpath = 0xffffd127f679d1f8 "/pci@0,0/pci8086,6f08@3/pci10b5,9765@0/pci10b5,9765@c/pci8086,370a@0/blkdev@1,0:a"
516

    
517
> *nvme_state::print -t i_ddi_soft_state
518
i_ddi_soft_state {
519
    void **array = 0xffffd127ed039e40
520
    kmutex_t lock = {
521
        void *[1] _opaque = [ 0 ]
522
    }
523
    size_t size = 0x290
524
    size_t n_items = 0x20
525
    struct i_ddi_soft_state *next = 0xffffd127f0f2bc88
526
}
527

    
528
> 0xffffd127ed039e40,100::dump -e -g 8
529
ffffd127ed039e40:  ffffd127ed848080 ffffd127ece0aa00
530
ffffd127ed039e50:  ffffd127edbbdcc0 ffffd127ec43d080
531
ffffd127ed039e60:  ffffd127ece0a100 ffffd1272aa840c0
532
ffffd127ed039e70:  ffffd127edbbd9c0 ffffd127ed848380
533
ffffd127ed039e80:  ffffd127ec1d4c00 ffffd1272ae5d600
534
ffffd127ed039e90:  ffffd127ed063940 ffffd127ec28a940
535
ffffd127ed039ea0:  ffffd127ece0a400 ffffd127ed063040
536
ffffd127ed039eb0:  ffffd127f13e3cc0 ffffd127ef0af040
537
ffffd127ed039ec0:  ffffd127f0ea0080 ffffd127ec28ac40
538
ffffd127ed039ed0:  ffffd127ec43d380 ffffd127f13e39c0
539
ffffd127ed039ee0:  ffffd127f13e36c0 ffffd127edbbd3c0
540
ffffd127ed039ef0:  ffffd127f13e30c0 ffffd127f13e33c0
541
ffffd127ed039f00:  0000000000000000 0000000000000000
542
ffffd127ed039f10:  0000000000000000 0000000000000000
543
ffffd127ed039f20:  0000000000000000 0000000000000000
544
ffffd127ed039f30:  0000000000000000 0000000000000000
545

    
546
> ffffd127f13e36c0::print -t nvme_t n_ns | ::print -t nvme_namespace_t ns_devid
547
char [76] ns_devid = [ "8086-INTEL SSDPE2ME400G4                     -CVMD44500088400FGN  -1" ]
548

    
549
> ffffd127f13e36c0::print -t nvme_t n_ioq n_adminq
550
nvme_qpair_t **n_ioq = 0xffffd127f150c3d8
551
nvme_qpair_t *n_adminq = 0xffffd127ecebfb18
552
> 0xffffd127f150c3d8,48::dump -e -g 8
553
ffffd127f150c3d8:  ffffd127ecebfb18 ffffd127ed1015d0
554
ffffd127f150c3e8:  ffffd127ed101480 ffffd127ed101330
555
ffffd127f150c3f8:  ffffd127f1535190 ffffd127f1532208
556
ffffd127f150c408:  ffffd127f1afa750 ffffd127f1532e48
557
ffffd127f150c418:  ffffd127f1aecc28 ffffd127f150c3d0
558

    
559
> ffffd127f1afa750::print -t nvme_qpair_t
560
nvme_qpair_t {
561
    size_t nq_nentry = 0x400
562
    nvme_dma_t *nq_sqdma = 0xffffd127f1afbc00
563
    nvme_sqe_t *nq_sq = 0xffffd127f1bf0000
564
    uint_t nq_sqhead = 0x307
565
    uint_t nq_sqtail = 0x308
566
    uintptr_t nq_sqtdbl = 0x1030
567
    nvme_dma_t *nq_cqdma = 0xffffd127f1afbb80
568
    nvme_cqe_t *nq_cq = 0xffffd127f1bdc000
569
    uint_t nq_cqhead = 0x307
570
    uint_t nq_cqtail = 0
571
    uintptr_t nq_cqhdbl = 0x1034
572
    nvme_cmd_t **nq_cmd = 0xffffd127f1b68000
573
    uint16_t nq_next_cmd = 0x308
574
    uint_t nq_active_cmds = 0x1
575
    int nq_phase = 0x1
576
    kmutex_t nq_mutex = {
577
        void *[1] _opaque = [ 0 ]
578
    }
579
}
580

    
581
> 0xffffd127f1bdc000+305*10::print -t nvme_cqe_t
582
nvme_cqe_t {
583
    uint32_t cqe_dw0 = 0
584
    uint32_t cqe_rsvd1 = 0
585
    uint16_t cqe_sqhd = 0x306
586
    uint16_t cqe_sqid = 0x6
587
    uint16_t cqe_cid = 0x305
588
    nvme_cqe_sf_t cqe_sf = {
589
        unsigned short:1 sf_p :1 = 0
590
        unsigned short:8 sf_sc :8 = 0
591
        unsigned short:3 sf_sct :3 = 0
592
        unsigned short:2 sf_rsvd2 :2 = 0
593
        unsigned short:1 sf_m :1 = 0
594
        unsigned short:1 sf_dnr :1 = 0
595
    }
596
}
597

    
598
> 0xffffd127f1bdc000+306*10::print -t nvme_cqe_t
599
nvme_cqe_t {
600
    uint32_t cqe_dw0 = 0
601
    uint32_t cqe_rsvd1 = 0
602
    uint16_t cqe_sqhd = 0x307
603
    uint16_t cqe_sqid = 0x6
604
    uint16_t cqe_cid = 0x306
605
    nvme_cqe_sf_t cqe_sf = {
606
        unsigned short:1 sf_p :1 = 0
607
        unsigned short:8 sf_sc :8 = 0
608
        unsigned short:3 sf_sct :3 = 0
609
        unsigned short:2 sf_rsvd2 :2 = 0
610
        unsigned short:1 sf_m :1 = 0
611
        unsigned short:1 sf_dnr :1 = 0
612
    }
613
}
614

    
615
> 0xffffd127f1bdc000+307*10::print -t nvme_cqe_t
616
nvme_cqe_t {
617
    uint32_t cqe_dw0 = 0
618
    uint32_t cqe_rsvd1 = 0
619
    uint16_t cqe_sqhd = 0x308
620
    uint16_t cqe_sqid = 0x6
621
    uint16_t cqe_cid = 0x307
622
    nvme_cqe_sf_t cqe_sf = {
623
        unsigned short:1 sf_p :1 = 0
624
        unsigned short:8 sf_sc :8 = 0
625
        unsigned short:3 sf_sct :3 = 0
626
        unsigned short:2 sf_rsvd2 :2 = 0
627
        unsigned short:1 sf_m :1 = 0
628
        unsigned short:1 sf_dnr :1 = 0
629
    }
630
}
631

    
632
> 0xffffd127f1bdc000+308*10::print -t nvme_cqe_t
633
nvme_cqe_t {
634
    uint32_t cqe_dw0 = 0
635
    uint32_t cqe_rsvd1 = 0
636
    uint16_t cqe_sqhd = 0x309
637
    uint16_t cqe_sqid = 0x6
638
    uint16_t cqe_cid = 0x308
639
    nvme_cqe_sf_t cqe_sf = {
640
        unsigned short:1 sf_p :1 = 0x1
641
        unsigned short:8 sf_sc :8 = 0x1
642
        unsigned short:3 sf_sct :3 = 0
643
        unsigned short:2 sf_rsvd2 :2 = 0
644
        unsigned short:1 sf_m :1 = 0
645
        unsigned short:1 sf_dnr :1 = 0
646
    }
647
}
648

    
649
> 0xffffd127f1bf0000+307*40::print -t nvme_sqe_t
650
nvme_sqe_t {
651
    uint8_t sqe_opc = 0x1
652
    unsigned char:2 sqe_fuse :2 = 0
653
    unsigned char:5 sqe_rsvd :5 = 0
654
    unsigned char:1 sqe_psdt :1 = 0
655
    uint16_t sqe_cid = 0x307
656
    uint32_t sqe_nsid = 0x1
657
    uint64_t sqe_rsvd1 = 0
658
    union  sqe_m = {
659
        uint64_t m_ptr = 0
660
        uint64_t m_sglp = 0
661
    }
662
    union  sqe_dptr = {
663
        uint64_t [2] d_prp = [ 0x5efbaab000, 0x34dbed9000 ]
664
        nvme_sgl_t d_sgl = {
665
            uint64_t sgl_addr = 0x5efbaab000
666
            uint32_t sgl_len = 0xdbed9000
667
            uint8_t [3] sgl_rsvd = [ 0x34, 0, 0 ]
668
            unsigned char:4 sgl_zero :4 = 0
669
            unsigned char:4 sgl_type :4 = 0
670
        }
671
    }
672
    uint32_t sqe_cdw10 = 0x9d16648
673
    uint32_t sqe_cdw11 = 0
674
    uint32_t sqe_cdw12 = 0xff
675
    uint32_t sqe_cdw13 = 0
676
    uint32_t sqe_cdw14 = 0
677
    uint32_t sqe_cdw15 = 0
678
}
679

    
680
> 0xffffd127f1b68000+307*8=J
681
                ffffd127f1b69838
682
				
683
> 0xffffd127f1b68000+307*8::dump -e -g 8
684
ffffd127f1b69838:  ffffd1290deddc40 0000000000000000
685

    
686
> ffffd1290deddc40::print -t nvme_cmd_t
687
nvme_cmd_t {
688
    nvme_sqe_t nc_sqe = {
689
        uint8_t sqe_opc = 0x1
690
        unsigned char:2 sqe_fuse :2 = 0
691
        unsigned char:5 sqe_rsvd :5 = 0
692
        unsigned char:1 sqe_psdt :1 = 0
693
        uint16_t sqe_cid = 0x307
694
        uint32_t sqe_nsid = 0x1
695
        uint64_t sqe_rsvd1 = 0
696
        union  sqe_m = {
697
            uint64_t m_ptr = 0
698
            uint64_t m_sglp = 0
699
        }
700
        union  sqe_dptr = {
701
            uint64_t [2] d_prp = [ 0x5efbaab000, 0x34dbed9000 ]
702
            nvme_sgl_t d_sgl = {
703
                uint64_t sgl_addr = 0x5efbaab000
704
                uint32_t sgl_len = 0xdbed9000
705
                uint8_t [3] sgl_rsvd = [ 0x34, 0, 0 ]
706
                unsigned char:4 sgl_zero :4 = 0
707
                unsigned char:4 sgl_type :4 = 0
708
            }
709
        }
710
        uint32_t sqe_cdw10 = 0x9d16648
711
        uint32_t sqe_cdw11 = 0
712
        uint32_t sqe_cdw12 = 0xff
713
        uint32_t sqe_cdw13 = 0
714
        uint32_t sqe_cdw14 = 0
715
        uint32_t sqe_cdw15 = 0
716
    }
717
    nvme_cqe_t nc_cqe = {
718
        uint32_t cqe_dw0 = 0
719
        uint32_t cqe_rsvd1 = 0
720
        uint16_t cqe_sqhd = 0
721
        uint16_t cqe_sqid = 0
722
        uint16_t cqe_cid = 0
723
        nvme_cqe_sf_t cqe_sf = {
724
            unsigned short:1 sf_p :1 = 0
725
            unsigned short:8 sf_sc :8 = 0
726
            unsigned short:3 sf_sct :3 = 0
727
            unsigned short:2 sf_rsvd2 :2 = 0
728
            unsigned short:1 sf_m :1 = 0
729
            unsigned short:1 sf_dnr :1 = 0
730
        }
731
    }
732
    void (*)() nc_callback = nvme_bd_xfer_done
733
    bd_xfer_t *nc_xfer = 0xffffd12800f7fa58
734
    boolean_t nc_completed = 0 (0)
735
    uint16_t nc_sqid = 0x6
736
    nvme_dma_t *nc_dma = 0xffffd127f644e280
737
    kmutex_t nc_mutex = {
738
        void *[1] _opaque = [ 0 ]
739
    }
740
    kcondvar_t nc_cv = {
741
        ushort_t _opaque = 0
742
    }
743
    taskq_ent_t nc_tqent = {
744
        struct taskq_ent *tqent_next = 0
745
        struct taskq_ent *tqent_prev = 0
746
        task_func_t *tqent_func = 0
747
        void *tqent_arg = 0
748
        union  tqent_un = {
749
            taskq_bucket_t *tqent_bucket = 0
750
            uintptr_t tqent_flags = 0
751
        }
752
        kthread_t *tqent_thread = 0
753
        kcondvar_t tqent_cv = {
754
            ushort_t _opaque = 0
755
        }
756
    }
757
    nvme_t *nc_nvme = 0xffffd127f13e36c0
758
}
759

    
760
> 0xffffd12800f7fa58::print -t bd_xfer_t
761
bd_xfer_t {
762
    diskaddr_t x_blkno = 0x9d16648
763
    size_t x_nblks = 0x100
764
    ddi_dma_handle_t x_dmah = 0xffffd12801155080
765
    ddi_dma_cookie_t x_dmac = {
766
        union  _dmu = {
767
            uint64_t _dmac_ll = 0x2000000000
768
            uint32_t [2] _dmac_la = [ 0, 0x20 ]
769
        }
770
        size_t dmac_size = 0
771
        uint_t dmac_type = 0x20000
772
    }
773
    unsigned int x_ndmac = 0
774
    caddr_t x_kaddr = 0
775
    unsigned int x_flags = 0
776
}
(1-1/2)