Project

General

Profile

Bug #1333 » oi_151_released.101hours.txt

oi_151a released - analysis of dtrace and other scripts - Jon Strabala, 2011-09-19 06:33 PM

 
1
This time unix`acpi_cpu_cstate actually decreased across 101 hours until a kernel crash at 101.5 hours where we had a panic.
2

    
3
What I now see now is a corrlation in growth in 'psm_get_cpu_id', 'cpu_idle_exit', 'apix_intr_enter' entry events.  
4

    
5
The system (which is completely idle crashed on a dtrace script after the kernel CPU crept to about 1.6% for no known reason.
6

    
7
panic[cpu0]/thread ffffff001e8cbc40: deadman: timed out after 50 seconds of clock inactivity
8

    
9
      *** skip stack dump, but see image "oi_151_released.console.png" for full details  ***
10
      *** this thread seems to be a 'highly' called thread whatever it is, see '6958602..d' below ***
11

    
12
syncing to file systems... done
13
dumping to /dev/zvol/dsk/rpool/dump, offset 65536, content: kernel
14
 2:23 100% done
15

    
16
Unfortunately no dump exists in the file system after a reboot or I didn't know how to generate the dump.
17

    
18
root@lab10:/# dumpadm
19
      Dump content: kernel pages
20
       Dump device: /dev/zvol/dsk/rpool/dump (dedicated)
21
Savecore directory: /var/crash/lab10
22
  Savecore enabled: no
23
   Save compressed: on
24

    
25
root@lab10:/# savecore
26
savecore: bad magic number 0
27

    
28
root@lab10:/# ls -ltrLa /var/crash
29
total 5
30
drwxr-xr-x 39 root sys  39 2011-09-13 08:07 ..
31
drwxr-xr-x  2 root root  2 2011-09-13 08:07 lab10
32
drwxr-xr-x  3 root root  3 2011-09-13 08:07 .
33
root@lab10:/#
34

    
35

    
36

    
37
#------------------------------------------------------------------------------
38
looking at dtrace script '6958602..d' we see thread ffffff001e8cbc40: seems to dominate abot 50% of all data lines.  I am pretty sure the other treads are changing and only come to a fraction of this thread, NOTE this is what caused the panic/crash dump
39

    
40

    
41
#!/usr/sbin/dtrace -s
42
#pragma D option bufpolicy=ring
43
mutex_vector_enter:entry
44
/ (uchar_t)*((uchar_t*)(arg0)) != 0xff/
45
{
46
        trace (timestamp);
47
        printf (" t=%p lp=%p owner=%p", curthread, arg0, (uint64_t)*((uint64_t*)(arg0)));
48
}
49
adaptive-acquire
50
{
51
        trace (timestamp);
52
        printf (" t=%p lp=%p", curthread, arg0);
53
}
54
adaptive-release
55
{
56
        trace (timestamp);
57
        printf (" t=%p lp=%p", curthread, arg0);
58
}
59
tick-1s
60
{
61
        exit(0);
62
}
63

    
64

    
65

    
66
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# ls -ltr adap* | head -1
67
-rw-r--r-- 1 root root   924798 2011-09-14 12:43 adaptive.2011-09-14_1240.out
68

    
69
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# ls -ltr adap* | tail -2
70
-rw-r--r-- 1 root root  1050302 2011-09-18 17:07 adaptive.2011-09-18_1700.out
71
-rw-r--r-- 1 root root   926103 2011-09-18 17:37 adaptive.2011-09-18_1730.out
72

    
73
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# grep ffffff001e8cbc40 adaptive.2011-09-14_1240.out | head -4
74
  0  10984     mutex_enter:adaptive-acquire     1980370006895 t=ffffff001e8cbc40 lp=ffffff04e624d880
75
  0  10987      mutex_exit:adaptive-release     1980370007742 t=ffffff001e8cbc40 lp=ffffff04e624d880
76
  0  10984     mutex_enter:adaptive-acquire     1980370008409 t=ffffff001e8cbc40 lp=ffffff04e624cfc0
77
  0  10987      mutex_exit:adaptive-release     1980370008958 t=ffffff001e8cbc40 lp=ffffff04e624cfc0
78

    
79
root@lab10:~/hotkernel_30/data.oi_151_released.101hours#  grep ffffff001e8cbc40  adaptive.2011-09-18_1700.out | head -4
80
  0  10984     mutex_enter:adaptive-acquire   363435570004828 t=ffffff001e8cbc40 lp=ffffff04e6083538
81
  0  10987      mutex_exit:adaptive-release   363435570005900 t=ffffff001e8cbc40 lp=ffffff04e6083538
82
  0  10984     mutex_enter:adaptive-acquire   363435570006339 t=ffffff001e8cbc40 lp=ffffff04e6083560
83
  0  10987      mutex_exit:adaptive-release   363435570006511 t=ffffff001e8cbc40 lp=ffffff04e6083560
84

    
85
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# grep ffffff001e8cbc40 adaptive.2011-09-18_1730.out | head -4
86
  0  10984     mutex_enter:adaptive-acquire   365240440003656 t=ffffff001e8cbc40 lp=ffffff04e6083df8
87
  0  10987      mutex_exit:adaptive-release   365240440004201 t=ffffff001e8cbc40 lp=ffffff04e6083df8
88
  0  10984     mutex_enter:adaptive-acquire   365240440004557 t=ffffff001e8cbc40 lp=ffffff04e6083e20
89
  0  10987      mutex_exit:adaptive-release   365240440004714 t=ffffff001e8cbc40 lp=ffffff04e6083e20
90

    
91
root@lab10:~/hotkernel_30/data.oi_151_released.101hours#  wc -l adaptive.2011-09-14_1240.out adaptive.2011-09-18_1700.out adaptive.2011-09-18_1730.out
92
   9160 adaptive.2011-09-14_1240.out
93
  10403 adaptive.2011-09-18_1700.out
94
   9173 adaptive.2011-09-18_1730.out
95
  28736 total
96

    
97
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# grep ffffff001e8cbc40 adaptive.2011-09-14_1240.out | wc -l
98
5152
99

    
100
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# grep ffffff001e8cbc40  adaptive.2011-09-18_1700.out | wc -l
101
4894
102

    
103
root@lab10:~/hotkernel_30/data.oi_151_released.101hours# grep ffffff001e8cbc40 adaptive.2011-09-18_1730.out | wc -l
104
4898
105

    
106
root@lab10:~/hotkernel_30/data.oi_151_released.101hours#
107

    
108
----------------------------------------------------------------------------------
109
Looking at an early and final sample of the dtrce script we see a dramtic increase in apix_intr_enter calls.
110

    
111

    
112
BEGIN
113
{
114
        printf("\n100 ms sample of all kernel entry and returns\n");
115
        start = timestamp;
116
}
117

    
118
*:*:*intr*:entry { @e[probefunc]=count() }
119
*:*:*intr*:return { @r[probefunc]=count() }
120
END {
121
        printf("\n%20s %10s %10s\n", "PROGRAM", "entries", "returns");
122
        printa("%20s %10@d %10@d\n", @e, @r);
123
}
124
tick-100ms
125
{
126
        exit(0);
127
}
128

    
129

    
130
cat intr_100ms_ent_ret.d.2011-09-15_1820.out
131
dtrace: script '/root/hotkernel_30/intr_100ms_ent_ret.d' matched 664 probes
132
CPU     ID                    FUNCTION:NAME
133
  4      1                           :BEGIN
134
100 ms sample of all kernel entry and returns
135

    
136
  4  73969                      :tick-100ms
137
  4      2                             :END
138
             PROGRAM    entries    returns
139
      timer_softintr          7          7
140
      apix_intr_exit         11         11
141
apix_hilevel_intr_prolog        105        105
142
apix_hilevel_intr_epilog        111        111
143
     apix_intr_enter       1144       1144
144

    
145

    
146
cat intr_100ms_ent_ret.d.2011-09-18_1800.out
147
dtrace: script '/root/hotkernel_30/intr_100ms_ent_ret.d' matched 664 probes
148
CPU     ID                    FUNCTION:NAME
149
  5      1                           :BEGIN
150
100 ms sample of all kernel entry and returns
151

    
152
  5  73969                      :tick-100ms
153
  5      2                             :END
154
             PROGRAM    entries    returns
155
      apix_intr_exit          5          5
156
      timer_softintr          8          8
157
apix_hilevel_intr_prolog         99         99
158
apix_hilevel_intr_epilog        105        105
159
     apix_intr_enter       7158       7158
160

    
161

    
162
root@lab10:~/hotkernel_30/data.oi_151_released.101hours#
163

    
164

    
165
----------------------------------------------------------------------------------
166
Info from last availabel sample prior to crash file: info.2011-09-18_1800.out
167

    
168
     date
169
     Sun Sep 18 18:00:19 PDT 2011
170

    
171
     uname -a
172
     SunOS lab10 5.11 oi_151a i86pc i386 i86pc
173

    
174
     rup lab10
175
               lab10    up  4 days,  5:39,    load average: 0.00, 0.00, 0.02
176

    
177
     beadm list
178
     BE                        Active Mountpoint Space Policy Created
179
     151a                      -      -          16.4M static 2011-09-07 13:41
180
     acpica-nightly-2011-09-09 -      -          56.0M static 2011-09-09 13:22
181
     acpica-nightly-2011-09-10 -      -          24.1G static 2011-09-10 16:48
182
     nightly-2011-09-08        -      -          370M  static 2011-09-08 16:03
183
     oi_148b                   -      -          994M  static 2011-05-05 10:02
184
     oi_148b_backup            -      -          63.0K static 2011-09-07 10:46
185
     oi_151a                   NR     /          57.6G static 2011-09-14 09:01
186

    
187
     /etc/system
188
     set ip:ip_squeue_worker_wait=0
189
     set snooping=1
190
     set pcplusmp:apic_panic_on_nmi=1
191

    
192
     /etc/power.conf
193
     cpu_deep_idle enable
194
     cpu-threshold 10s
195
     cpupm enable poll-mode
196

    
197

    
198
     echo hz/D | sudo mdb -k
199
     hz:
200
     hz:             100
201

    
202
     echo apic_timer::print apic_timer_t | sudo mdb -k
203
     {
204
         mode = 0x2
205
         apic_timer_enable_ops = deadline_timer_enable
206
         apic_timer_disable_ops = deadline_timer_disable
207
         apic_timer_reprogram_ops = deadline_timer_reprogram
208
     }
209

    
210

    
211

    
212
----------------------------------------------------------------------------------
213
Looking at top_10sec.\* output spaced by 10 hour samples with last three samples
214
at bottom with crash during 2011-09-18_1800
215

    
216
displaying first sample of the command 'top -s 10 -n 15 -d 2 -b'
217

    
218
top_10sec.2011-09-14_1240.out: CPU states:  100% idle,  0.0% user,  0.0% kernel,  0.0% iowait,  0.0% swap
219
top_10sec.2011-09-14_1620.out: CPU states: 99.9% idle,  0.0% user,  0.1% kernel,  0.0% iowait,  0.0% swap
220
top_10sec.2011-09-14_2000.out: CPU states: 99.9% idle,  0.0% user,  0.1% kernel,  0.0% iowait,  0.0% swap
221
top_10sec.2011-09-14_2340.out: CPU states: 99.8% idle,  0.0% user,  0.2% kernel,  0.0% iowait,  0.0% swap
222
top_10sec.2011-09-15_0320.out: CPU states: 99.8% idle,  0.0% user,  0.2% kernel,  0.0% iowait,  0.0% swap
223
top_10sec.2011-09-15_0700.out: CPU states: 99.8% idle,  0.0% user,  0.2% kernel,  0.0% iowait,  0.0% swap
224
top_10sec.2011-09-15_1040.out: CPU states: 99.9% idle,  0.0% user,  0.1% kernel,  0.0% iowait,  0.0% swap
225
top_10sec.2011-09-15_1420.out: CPU states: 99.5% idle,  0.0% user,  0.5% kernel,  0.0% iowait,  0.0% swap
226
top_10sec.2011-09-15_1800.out: CPU states: 99.7% idle,  0.0% user,  0.3% kernel,  0.0% iowait,  0.0% swap
227
top_10sec.2011-09-15_2140.out: CPU states: 99.1% idle,  0.0% user,  0.9% kernel,  0.0% iowait,  0.0% swap
228
top_10sec.2011-09-16_0120.out: CPU states: 99.5% idle,  0.0% user,  0.5% kernel,  0.0% iowait,  0.0% swap
229
top_10sec.2011-09-16_0500.out: CPU states: 99.1% idle,  0.0% user,  0.9% kernel,  0.0% iowait,  0.0% swap
230
top_10sec.2011-09-16_0840.out: CPU states: 99.6% idle,  0.0% user,  0.4% kernel,  0.0% iowait,  0.0% swap
231
top_10sec.2011-09-16_1220.out: CPU states: 99.5% idle,  0.0% user,  0.5% kernel,  0.0% iowait,  0.0% swap
232
top_10sec.2011-09-16_1600.out: CPU states: 99.1% idle,  0.0% user,  0.9% kernel,  0.0% iowait,  0.0% swap
233
top_10sec.2011-09-16_2100.out: CPU states: 99.1% idle,  0.0% user,  0.9% kernel,  0.0% iowait,  0.0% swap
234
top_10sec.2011-09-17_0230.out: CPU states: 99.0% idle,  0.0% user,  1.0% kernel,  0.0% iowait,  0.0% swap
235
top_10sec.2011-09-17_0800.out: CPU states: 98.8% idle,  0.0% user,  1.2% kernel,  0.0% iowait,  0.0% swap
236
top_10sec.2011-09-17_1330.out: CPU states: 99.0% idle,  0.0% user,  1.0% kernel,  0.0% iowait,  0.0% swap
237
top_10sec.2011-09-17_1900.out: CPU states: 98.6% idle,  0.0% user,  1.4% kernel,  0.0% iowait,  0.0% swap
238
top_10sec.2011-09-18_0030.out: CPU states: 98.6% idle,  0.0% user,  1.4% kernel,  0.0% iowait,  0.0% swap
239
top_10sec.2011-09-18_0600.out: CPU states: 98.9% idle,  0.0% user,  1.1% kernel,  0.0% iowait,  0.0% swap
240
top_10sec.2011-09-18_1130.out: CPU states: 99.2% idle,  0.0% user,  0.8% kernel,  0.0% iowait,  0.0% swap
241
top_10sec.2011-09-18_1700.out: CPU states: 98.4% idle,  0.0% user,  1.6% kernel,  0.0% iowait,  0.0% swap
242
top_10sec.2011-09-18_1730.out: CPU states: 98.8% idle,  0.0% user,  1.2% kernel,  0.0% iowait,  0.0% swap
243
top_10sec.2011-09-18_1800.out: CPU states: 98.4% idle,  0.0% user,  1.6% kernel,  0.0% iowait,  0.0% swap
244

    
245
----------------------------------------------------------------------------------
246
The wierd thing is prstat, prstat -L -s cpu -m -T -c 2 4, shows nothing WRT the > 0.0% kernel CPU
247

    
248
cat prstat-L-s-cpu-m-T-c_2_4.2011-09-18_1730.out
249
   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID
250
  4878 root      31  68 0.0 0.0 0.0 0.0 0.0 0.6   0   0 63K   0 prstat/1
251
  4846 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   3   0  69   0 runh30.csh/1
252
  4860 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0  75   1 rpc.rstatd/1
253
  1077 admin    0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0  23   0 sshd/1
254
   760 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 automountd/4
255
  1020 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   1   0 intrd/1
256
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/146
257
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/144
258
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/143
259
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/141
260
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/139
261
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/137
262
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0   0   0 zpool-rpool/26
263
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0   0   0 zpool-rpool/25
264
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0   0   0 zpool-rpool/24
265
TASKID     NLWP  SWAP   RSS MEMORY      TIME  CPU PROJECT
266
   668        2 2720K 6364K   0.0%   0:00:00 0.0% user.root
267
     0      159    0K    0K   0.0%   0:03:07 0.0% system
268
   669        1  664K 2584K   0.0%   0:00:00 0.0% system
269
    92        2 2000K 7716K   0.0%   0:00:41 0.0% group.staff
270
    69        6 1244K 3372K   0.0%   0:00:23 0.0% system
271
Total: 49 processes, 363 lwps, load averages: 0.00, 0.00, 0.01
272
   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID
273
  4878 root     0.1 0.5 0.0 0.0 0.0 0.0  99 0.0  21   1 402   0 prstat/1
274
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   4   0   0   0 zpool-rpool/148
275
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  14   0   0   0 zpool-rpool/19
276
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   3   0   0   0 zpool-rpool/146
277
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   4   0   0   0 zpool-rpool/143
278
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   3   0   0   0 zpool-rpool/13
279
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  13   0   0   0 zpool-rpool/25
280
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  12   0   0   0 zpool-rpool/26
281
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  13   0   0   0 zpool-rpool/24
282
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  13   0   0   0 zpool-rpool/23
283
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  14   0   0   0 zpool-rpool/21
284
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  16   0   0   0 zpool-rpool/20
285
  4860 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0  58   2 rpc.rstatd/1
286
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   5   0   0   0 zpool-rpool/137
287
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  13   0   0   0 zpool-rpool/22
288
TASKID     NLWP  SWAP   RSS MEMORY      TIME  CPU PROJECT
289
   668        2 2808K 6480K   0.0%   0:00:00 0.0% user.root
290
     0      159    0K    0K   0.0%   0:03:07 0.0% system
291
   669        1  664K 2584K   0.0%   0:00:00 0.0% system
292
    45       29 7824K 6804K   0.0%   0:00:03 0.0% system
293
    69        6 1244K 3372K   0.0%   0:00:23 0.0% system
294
Total: 49 processes, 363 lwps, load averages: 0.00, 0.00, 0.01
295
   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID
296
  4878 root     0.0 0.4 0.0 0.0 0.0 0.0 100 0.0  21   0 258   0 prstat/1
297
  4860 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   2   0  58   2 rpc.rstatd/1
298
   355 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  20   0 120   0 nscd/6
299
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/6
300
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/5
301
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/4
302
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/3
303
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/2
304
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 zpool-rpool/1
305
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/9
306
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/8
307
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/7
308
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/6
309
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/5
310
     4 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   0   0   0   0 kcfpoold/4
311
TASKID     NLWP  SWAP   RSS MEMORY      TIME  CPU PROJECT
312
   668        2 2808K 6480K   0.0%   0:00:00 0.0% user.root
313
   669        1  664K 2584K   0.0%   0:00:00 0.0% system
314
    45       29 7824K 6804K   0.0%   0:00:03 0.0% system
315
    69        6 1244K 3372K   0.0%   0:00:23 0.0% system
316
    18        3  604K 2508K   0.0%   0:00:00 0.0% system
317
Total: 49 processes, 363 lwps, load averages: 0.00, 0.00, 0.01
318
   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID
319
  4878 root     0.0 0.4 0.0 0.0 0.0 0.0 100 0.0  21   0 258   0 prstat/1
320
     6 root     0.0 0.1 0.0 0.0 0.0 0.0 100 0.0   8   0   0   0 zpool-rpool/148
321
     6 root     0.0 0.1 0.0 0.0 0.0 0.0 100 0.0  29   1   0   0 zpool-rpool/23
322
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  28   0   0   0 zpool-rpool/19
323
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   8   0   0   0 zpool-rpool/143
324
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  27   0   0   0 zpool-rpool/22
325
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  29   1   0   0 zpool-rpool/24
326
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  24   0   0   0 zpool-rpool/21
327
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  26   0   0   0 zpool-rpool/25
328
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   8   0   0   0 zpool-rpool/144
329
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  26   0   0   0 zpool-rpool/26
330
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0  25   0   0   0 zpool-rpool/20
331
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   7   0   0   0 zpool-rpool/137
332
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   7   0   0   0 zpool-rpool/141
333
     6 root     0.0 0.0 0.0 0.0 0.0 0.0 100 0.0   7   0   0   0 zpool-rpool/139
334
TASKID     NLWP  SWAP   RSS MEMORY      TIME  CPU PROJECT
335
     0      159    0K    0K   0.0%   0:03:07 0.0% system
336
   668        2 2808K 6480K   0.0%   0:00:00 0.0% user.root
337
   669        1  664K 2584K   0.0%   0:00:00 0.0% system
338
    45       29 7824K 6804K   0.0%   0:00:03 0.0% system
339
    69        6 1244K 3372K   0.0%   0:00:23 0.0% system
340
Total: 49 processes, 363 lwps, load averages: 0.00, 0.00, 0.01
341

    
342

    
343

    
344
----------------------------------------------------------------------------------
345
Daily vmstat outputs, vmstat 3 3, followed by the last three samples before the crash
346

    
347
cat vmstat_3_3.2011-09-14_1400.out
348
 kthr      memory            page            disk          faults      cpu
349
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
350
 1 0 0 21713856 15444760 44 523 0 0 0 0 195 7  6  0  0 2416  548  477  0  0 100
351
 1 0 0 21742852 15455576 4 20 0 0  0  0  0  7  6  0  0  758  144  432  0  0 100
352
 1 0 0 21742740 15455492 0 3 0  0  0  0  0  0  0  0  0  652  155  314  0  0 100
353

    
354
cat vmstat_3_3.2011-09-15_1400.out
355
 kthr      memory            page            disk          faults      cpu
356
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
357
 1 0 0 21522420 15239876 16 603 0 0 0 0 13  6  6  0  0 3403  382  367  0  1 99
358
 1 0 0 21351596 15065688 4 20 0 0  0  0  0  7  7  0  0 1123  144  438  0  0 100
359
 2 0 0 21351484 15065604 0 3 0  0  0  0  0 14 14  0  0 1108  143  419  0  0 100
360

    
361
cat vmstat_3_3.2011-09-16_1400.out
362
 kthr      memory            page            disk          faults      cpu
363
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
364
 1 0 0 21031820 14749324 18 663 0 0 0 0  7  7  6  0  0 4110  418  368  0  2 98
365
 2 0 0 20282432 13996496 4 22 0 0  0  0  0  0  0  0  0 1045  147  317  0  1 99
366
 1 0 0 20282320 13997044 0 1 0  0  0  0  0 14 15  0  0 1016  139  425  0  1 99
367

    
368
cat vmstat_3_3.2011-09-17_1400.out
369
 kthr      memory            page            disk          faults      cpu
370
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
371
 1 0 0 20727688 14445240 17 632 0 0 0 0  4  6  6  0  0 3977  414  363  0  2 98
372
 1 0 0 19976288 13691748 4 20 0 0  0  0  0  7  8  0  0 1097  145  429  0  1 99
373
 1 0 0 19976172 13691660 0 3 0  0  0  0  0 15 13  0  0 1053  141  427  0  1 99
374

    
375
cat vmstat_3_3.2011-09-18_1400.out
376
 kthr      memory            page            disk          faults      cpu
377
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
378
 1 0 0 20499888 14217468 16 615 0 0 0 0  3  6  6  0  0 3916  399  360  0  3 97
379
 1 0 0 19724148 13439548 4 20 0 0  0  0  0  7  7  0  0 1131  144  428  0  1 99
380
 1 0 0 19724036 13439464 0 3 0  0  0  0  0  0  0  0  0 1006  135  314  0  1 99
381

    
382
Last three samples
383

    
384

    
385
cat vmstat_3_3.2011-09-18_1700.out
386
 kthr      memory            page            disk          faults      cpu
387
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
388
 1 0 0 20474748 14192332 16 614 0 0 0 0  3  6  6  0  0 3912  398  360  0  3 97
389
 0 0 0 19695376 13410640 4 20 0 0  0  0  0  0  0  0  0 1092  153  321  0  2 98
390
 2 0 0 19695264 13410556 0 3 0  0  0  0  0  8  7  0  0 1042  141  436  0  1 99
391

    
392
cat vmstat_3_3.2011-09-18_1730.out
393
 kthr      memory            page            disk          faults      cpu
394
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
395
 1 0 0 20470620 14188204 16 613 0 0 0 0  3  6  6  0  0 3911  397  360  0  3 97
396
 0 0 0 19691020 13406268 4 20 0 0  0  0  0  0  0  0  0 1091  144  312  0  1 99
397
 0 0 0 19690908 13406184 0 3 0  0  0  0  0  6  7  0  0 1128  139  430  0  1 99
398

    
399
cat vmstat_3_3.2011-09-18_1800.out
400
 kthr      memory            page            disk          faults      cpu
401
 r b w   swap  free  re  mf pi po fr de sr s2 s3 -- --   in   sy   cs us sy id
402
 1 0 0 20466520 14184100 16 613 0 0 0 0  3  6  6  0  0 3911  397  360  0  3 97
403
 0 0 0 19687720 13402828 4 20 0 0  0  0  0  0  0  0  0 1057  145  319  0  1 99
404
 0 0 0 19687608 13402744 0 3 0  0  0  0  0 13 13  0  0 1059  135  415  0  1 99
405

    
406

    
407
----------------------------------------------------------------------------------
408

    
409
An early powertop run, powertop -d 3 -t 10
410

    
411

    
412
cat powertop-d_3-t_10.2011-09-14_2320.out
413
powertop: battery kstat not found (-1)
414
powertop: battery kstat not found (-1)
415
powertop: battery kstat not found (-1)
416
OpenIndiana PowerTOP version 1.2   (C) 2009 Intel Corporation
417

    
418
Collecting data for 10.00 second(s)
419
C-states (idle power)   Avg     Residency
420
C0 (cpu running)                (0.0%)
421
C1                      3.7ms   (0.2%)
422
C2                      0.0ms   (0.0%)
423
C3                      4.3ms   (99.8%)
424
P-states (frequencies)
425
1600 Mhz        0.0%
426
1800 Mhz        0.0%
427
2000 Mhz        0.0%
428
2200 Mhz        0.0%
429
2400 Mhz        0.0%
430
2600 Mhz        0.0%
431
2800 Mhz        0.0%
432
3000 Mhz        0.0%
433
3200 Mhz        0.0%
434
3400 Mhz        36.8%
435
3411 Mhz(turbo) 63.2%
436
Wakeups-from-idle per second: 1918.7    interval: 10.0s
437
Top causes for wakeups:
438
71.0% (1361.6)               sched :<xcalls> unix`dtrace_xcall_func
439
 5.2% (100.0)            <kernel> :genunix`clock
440
 3.9% ( 75.2)            <kernel> :genunix`cv_wakeup
441
 2.6% ( 50.0)            <kernel> :SDC`sysdc_update
442
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
443
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
444
 0.4% (  7.6)            <kernel> :ehci`ehci_handle_root_hub_status_change
445
 0.2% (  4.0)            <kernel> :genunix`schedpaging
446
 0.1% (  1.4)               sched :<xcalls> unix`speedstep_pstate_transition
447
 0.1% (  1.1)         <interrupt> :e1000g#0
448
 0.1% (  1.0)            <kernel> :TS`ts_update
449
 0.1% (  1.0)            <kernel> :sd`sd_pm_idletimeout_handler
450
 0.1% (  1.0)            <kernel> :e1000g`e1000g_local_timer
451
 0.1% (  1.0)         <interrupt> :ehci#0
452
 0.1% (  1.0)         <interrupt> :ehci#1
453
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
454
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
455
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
456
 0.0% (  0.1)            <kernel> :ibcm`ibcm_path_cache_timeout_cb
457
 0.0% (  0.1)            <kernel> :kcf`rnd_handler
458
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
459
 0.0% (  0.1)            <kernel> :genunix`vmem_update
460
 0.0% (  0.1)            <kernel> :genunix`kmem_update
461
no ACPI power usage estimate available
462
C-states (idle power)   Avg     Residency
463
C0 (cpu running)                (1.0%)
464
C1                      0.0ms   (0.0%)
465
C2                      0.0ms   (0.0%)
466
C3                      4.3ms   (99.0%)
467
P-states (frequencies)
468
1600 Mhz        0.0%
469
1800 Mhz        0.0%
470
2000 Mhz        0.0%
471
2200 Mhz        0.0%
472
2400 Mhz        0.0%
473
2600 Mhz        0.0%
474
2800 Mhz        0.0%
475
3000 Mhz        20.0%
476
3200 Mhz        80.0%
477
3400 Mhz        0.0%
478
3401 Mhz(turbo) 0.0%
479
Wakeups-from-idle per second: 1822.2    interval: 10.0s
480
Top causes for wakeups:
481
73.9% (1347.0)               sched :<xcalls> unix`dtrace_xcall_func
482
 5.5% (100.1)            <kernel> :genunix`clock
483
 4.1% ( 74.7)            <kernel> :genunix`cv_wakeup
484
 2.7% ( 50.1)            <kernel> :SDC`sysdc_update
485
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
486
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
487
 0.4% (  7.8)            <kernel> :ehci`ehci_handle_root_hub_status_change
488
 0.2% (  4.0)            <kernel> :genunix`schedpaging
489
 0.2% (  3.0)         <interrupt> :ahci#0
490
 0.1% (  1.2)            <kernel> :sd`sd_pm_idletimeout_handler
491
 0.1% (  1.1)         <interrupt> :e1000g#0
492
 0.1% (  1.0)            <kernel> :TS`ts_update
493
 0.1% (  1.0)            <kernel> :e1000g`e1000g_local_timer
494
 0.0% (  0.9)         <interrupt> :ehci#1
495
 0.0% (  0.9)         <interrupt> :ehci#0
496
 0.0% (  0.7)               sched :<xcalls> unix`speedstep_pstate_transition
497
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
498
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
499
 0.0% (  0.1)            <kernel> :genunix`vmem_update
500
 0.0% (  0.1)            <kernel> :kcf`rnd_handler
501
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
502
 0.0% (  0.1)            <kernel> :genunix`kmem_update
503
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
504
no ACPI power usage estimate available
505
C-states (idle power)   Avg     Residency
506
C0 (cpu running)                (1.0%)
507
C1                      0.0ms   (0.0%)
508
C2                      0.0ms   (0.0%)
509
C3                      4.2ms   (99.0%)
510
P-states (frequencies)
511
1600 Mhz        0.0%
512
1800 Mhz        0.0%
513
2000 Mhz        0.0%
514
2200 Mhz        0.0%
515
2400 Mhz        0.0%
516
2600 Mhz        0.0%
517
2800 Mhz        0.0%
518
3000 Mhz        100.0%
519
3200 Mhz        0.0%
520
3400 Mhz        0.0%
521
3401 Mhz(turbo) 0.0%
522
Wakeups-from-idle per second: 1873.3    interval: 10.0s
523
Top causes for wakeups:
524
72.1% (1350.0)               sched :<xcalls> unix`dtrace_xcall_func
525
 5.3% (100.0)            <kernel> :genunix`clock
526
 4.0% ( 74.6)            <kernel> :genunix`cv_wakeup
527
 2.7% ( 50.0)            <kernel> :SDC`sysdc_update
528
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
529
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
530
 0.4% (  7.6)            <kernel> :ehci`ehci_handle_root_hub_status_change
531
 0.4% (  7.6)         <interrupt> :ahci#0
532
 0.2% (  4.0)            <kernel> :genunix`schedpaging
533
 0.1% (  1.2)            <kernel> :sd`sd_pm_idletimeout_handler
534
 0.1% (  1.1)         <interrupt> :e1000g#0
535
 0.1% (  1.0)            <kernel> :TS`ts_update
536
 0.1% (  1.0)            <kernel> :e1000g`e1000g_local_timer
537
 0.1% (  1.0)         <interrupt> :ehci#0
538
 0.1% (  1.0)         <interrupt> :ehci#1
539
 0.0% (  0.2)            <kernel> :kcf`rnd_handler
540
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
541
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
542
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
543
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
544
no ACPI power usage estimate available
545

    
546

    
547

    
548
Last successful powertop run, powertop -d 3 -t 10
549

    
550

    
551
cat powertop-d_3-t_10.2011-09-18_1730.out
552
powertop: battery kstat not found (-1)
553
powertop: battery kstat not found (-1)
554
powertop: battery kstat not found (-1)
555
OpenIndiana PowerTOP version 1.2   (C) 2009 Intel Corporation
556

    
557
Collecting data for 10.00 second(s)
558
C-states (idle power)   Avg     Residency
559
C0 (cpu running)                (0.0%)
560
C1                      4.4ms   (0.2%)
561
C2                      0.0ms   (0.0%)
562
C3                      3.9ms   (99.8%)
563
P-states (frequencies)
564
1600 Mhz        0.0%
565
1800 Mhz        0.0%
566
2000 Mhz        0.0%
567
2200 Mhz        0.0%
568
2400 Mhz        0.0%
569
2600 Mhz        0.0%
570
2800 Mhz        0.0%
571
3000 Mhz        0.0%
572
3200 Mhz        1.1%
573
3400 Mhz        28.9%
574
3425 Mhz(turbo) 70.0%
575
Wakeups-from-idle per second: 2072.1    interval: 10.0s
576
Top causes for wakeups:
577
77.9% (1613.4)               sched :<xcalls> unix`dtrace_xcall_func
578
 4.8% (100.0)            <kernel> :genunix`clock
579
 3.6% ( 74.9)            <kernel> :genunix`cv_wakeup
580
 2.4% ( 50.0)            <kernel> :SDC`sysdc_update
581
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
582
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
583
 0.4% (  7.8)            <kernel> :ehci`ehci_handle_root_hub_status_change
584
 0.2% (  4.0)            <kernel> :genunix`schedpaging
585
 0.1% (  2.8)         <interrupt> :ahci#0
586
 0.1% (  2.3)         <interrupt> :e1000g#0
587
 0.0% (  1.0)            <kernel> :TS`ts_update
588
 0.0% (  1.0)            <kernel> :e1000g`e1000g_local_timer
589
 0.0% (  1.0)         <interrupt> :ehci#0
590
 0.0% (  1.0)         <interrupt> :ehci#1
591
 0.0% (  0.8)            <kernel> :sd`sd_pm_idletimeout_handler
592
 0.0% (  0.7)               sched :<xcalls> unix`speedstep_pstate_transition
593
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
594
 0.0% (  0.2)            <kernel> :kcf`rnd_handler
595
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
596
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
597
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
598
no ACPI power usage estimate available
599
C-states (idle power)   Avg     Residency
600
C0 (cpu running)                (3.5%)
601
C1                      0.0ms   (0.0%)
602
C2                      0.0ms   (0.0%)
603
C3                      3.9ms   (96.5%)
604
P-states (frequencies)
605
1600 Mhz        0.0%
606
1800 Mhz        0.0%
607
2000 Mhz        0.0%
608
2200 Mhz        0.0%
609
2400 Mhz        0.0%
610
2600 Mhz        0.0%
611
2800 Mhz        0.0%
612
3000 Mhz        0.0%
613
3200 Mhz        18.9%
614
3400 Mhz        81.1%
615
3401 Mhz(turbo) 0.0%
616
Wakeups-from-idle per second: 1960.5    interval: 10.0s
617
Top causes for wakeups:
618
81.6% (1599.6)               sched :<xcalls> unix`dtrace_xcall_func
619
 5.1% (100.0)            <kernel> :genunix`clock
620
 3.8% ( 74.5)            <kernel> :genunix`cv_wakeup
621
 2.6% ( 50.0)            <kernel> :SDC`sysdc_update
622
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
623
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
624
 0.4% (  7.6)            <kernel> :ehci`ehci_handle_root_hub_status_change
625
 0.2% (  4.0)            <kernel> :genunix`schedpaging
626
 0.2% (  3.0)         <interrupt> :ahci#0
627
 0.1% (  1.1)         <interrupt> :e1000g#0
628
 0.1% (  1.0)            <kernel> :TS`ts_update
629
 0.1% (  1.0)            <kernel> :e1000g`e1000g_local_timer
630
 0.1% (  1.0)         <interrupt> :ehci#0
631
 0.0% (  0.9)         <interrupt> :ehci#1
632
 0.0% (  0.8)            <kernel> :sd`sd_pm_idletimeout_handler
633
 0.0% (  0.7)               sched :<xcalls> unix`speedstep_pstate_transition
634
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
635
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
636
 0.0% (  0.1)            <kernel> :genunix`kmem_update
637
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
638
 0.0% (  0.1)            <kernel> :genunix`vmem_update
639
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
640
 0.0% (  0.1)            <kernel> :kcf`rnd_handler
641
no ACPI power usage estimate available
642
C-states (idle power)   Avg     Residency
643
C0 (cpu running)                (3.5%)
644
C1                      0.0ms   (0.0%)
645
C2                      0.0ms   (0.0%)
646
C3                      4.0ms   (96.5%)
647
P-states (frequencies)
648
1600 Mhz        0.0%
649
1800 Mhz        0.0%
650
2000 Mhz        0.0%
651
2200 Mhz        0.0%
652
2400 Mhz        0.0%
653
2600 Mhz        0.0%
654
2800 Mhz        0.0%
655
3000 Mhz        0.0%
656
3200 Mhz        100.0%
657
3400 Mhz        0.0%
658
3401 Mhz(turbo) 0.0%
659
Wakeups-from-idle per second: 1953.4    interval: 10.0s
660
Top causes for wakeups:
661
81.6% (1593.7)               sched :<xcalls> unix`dtrace_xcall_func
662
 5.1% (100.0)            <kernel> :genunix`clock
663
 3.8% ( 74.6)            <kernel> :genunix`cv_wakeup
664
 2.6% ( 50.0)            <kernel> :SDC`sysdc_update
665
 0.4% (  8.0)            <kernel> :genunix`pm_rescan
666
 0.4% (  8.0)            <kernel> :cpudrv`cpudrv_monitor_disp
667
 0.4% (  7.8)            <kernel> :ehci`ehci_handle_root_hub_status_change
668
 0.2% (  4.0)            <kernel> :genunix`schedpaging
669
 0.1% (  2.5)         <interrupt> :ahci#0
670
 0.1% (  1.2)         <interrupt> :e1000g#0
671
 0.1% (  1.0)            <kernel> :TS`ts_update
672
 0.1% (  1.0)            <kernel> :e1000g`e1000g_local_timer
673
 0.1% (  1.0)         <interrupt> :ehci#1
674
 0.0% (  0.9)         <interrupt> :ehci#0
675
 0.0% (  0.8)            <kernel> :sd`sd_pm_idletimeout_handler
676
 0.0% (  0.2)            <kernel> :ahci`ahci_watchdog_handler
677
 0.0% (  0.2)            <kernel> :swrand`rnd_handler
678
 0.0% (  0.1)            <kernel> :genunix`kmem_update
679
 0.0% (  0.1)            <kernel> :ip`igmp_slowtimo
680
 0.0% (  0.1)            <kernel> :genunix`vmem_update
681
 0.0% (  0.1)            <kernel> :kcf`rnd_handler
682
 0.0% (  0.1)            <kernel> :ip`mld_slowtimo
683
no ACPI power usage estimate available
684

    
685

    
686
----------------------------------------------------------------------------------
687
Looking at i86_cpu_wait_idle_power_thread.d.\* output spaced by 10 hour samples with last three samples
688
at bottom with crash during 2011-09-18_1800
689

    
690
displaying entry and return of psm_get_cpu_id
691

    
692
i86_cpu_wait_idle_power_thread.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
693
i86_cpu_wait_idle_power_thread.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
694
i86_cpu_wait_idle_power_thread.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
695
i86_cpu_wait_idle_power_thread.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
696
i86_cpu_wait_idle_power_thread.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
697
i86_cpu_wait_idle_power_thread.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
698
i86_cpu_wait_idle_power_thread.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
699
i86_cpu_wait_idle_power_thread.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
700
i86_cpu_wait_idle_power_thread.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
701
i86_cpu_wait_idle_power_thread.d.2011-09-15_2140.out:       psm_get_cpu_id       1860       1860
702
i86_cpu_wait_idle_power_thread.d.2011-09-16_0120.out:       psm_get_cpu_id       2110       2110
703
i86_cpu_wait_idle_power_thread.d.2011-09-16_0500.out:       psm_get_cpu_id       2273       2273
704
i86_cpu_wait_idle_power_thread.d.2011-09-16_0840.out:       psm_get_cpu_id       2858       2858
705
i86_cpu_wait_idle_power_thread.d.2011-09-16_1220.out:       psm_get_cpu_id       2467       2467
706
i86_cpu_wait_idle_power_thread.d.2011-09-16_1600.out:       psm_get_cpu_id       2744       2744
707
i86_cpu_wait_idle_power_thread.d.2011-09-16_2100.out:       psm_get_cpu_id       2837       2837
708
i86_cpu_wait_idle_power_thread.d.2011-09-17_0230.out:       psm_get_cpu_id       2680       2680
709
i86_cpu_wait_idle_power_thread.d.2011-09-17_0800.out:       psm_get_cpu_id       3048       3048
710
i86_cpu_wait_idle_power_thread.d.2011-09-17_1330.out:       psm_get_cpu_id       3410       3410
711
i86_cpu_wait_idle_power_thread.d.2011-09-17_1900.out:       psm_get_cpu_id       4484       4484
712
i86_cpu_wait_idle_power_thread.d.2011-09-18_0030.out:       psm_get_cpu_id       3785       3785
713
i86_cpu_wait_idle_power_thread.d.2011-09-18_0600.out:       psm_get_cpu_id       4716       4716
714
i86_cpu_wait_idle_power_thread.d.2011-09-18_1130.out:       psm_get_cpu_id       4894       4894
715
i86_cpu_wait_idle_power_thread.d.2011-09-18_1700.out:       psm_get_cpu_id       4240       4240
716
i86_cpu_wait_idle_power_thread.d.2011-09-18_1730.out:       psm_get_cpu_id       4943       4943
717
i86_cpu_wait_idle_power_thread.d.2011-09-18_1800.out:       psm_get_cpu_id       4636       4636
718

    
719
----------------------------------------------------------------------------------
720
Looking at i86_cpu_wait_idle_power_thread.d.\* output spaced by 10 hour samples with last three samples
721
at bottom with crash during 2011-09-18_1800
722

    
723
displaying entry and return of cpu_idle_exit
724

    
725
i86_cpu_wait_idle_power_thread.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
726
i86_cpu_wait_idle_power_thread.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
727
i86_cpu_wait_idle_power_thread.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
728
i86_cpu_wait_idle_power_thread.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
729
i86_cpu_wait_idle_power_thread.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
730
i86_cpu_wait_idle_power_thread.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
731
i86_cpu_wait_idle_power_thread.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
732
i86_cpu_wait_idle_power_thread.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
733
i86_cpu_wait_idle_power_thread.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
734
i86_cpu_wait_idle_power_thread.d.2011-09-15_2140.out:        cpu_idle_exit       1284       1284
735
i86_cpu_wait_idle_power_thread.d.2011-09-16_0120.out:        cpu_idle_exit       1814       1814
736
i86_cpu_wait_idle_power_thread.d.2011-09-16_0500.out:        cpu_idle_exit       2088       2088
737
i86_cpu_wait_idle_power_thread.d.2011-09-16_0840.out:        cpu_idle_exit       2318       2318
738
i86_cpu_wait_idle_power_thread.d.2011-09-16_1220.out:        cpu_idle_exit       2536       2536
739
i86_cpu_wait_idle_power_thread.d.2011-09-16_1600.out:        cpu_idle_exit       2972       2972
740
i86_cpu_wait_idle_power_thread.d.2011-09-16_2100.out:        cpu_idle_exit       3226       3226
741
i86_cpu_wait_idle_power_thread.d.2011-09-17_0230.out:        cpu_idle_exit       3054       3054
742
i86_cpu_wait_idle_power_thread.d.2011-09-17_0800.out:        cpu_idle_exit       3668       3668
743
i86_cpu_wait_idle_power_thread.d.2011-09-17_1330.out:        cpu_idle_exit       4382       4382
744
i86_cpu_wait_idle_power_thread.d.2011-09-17_1900.out:        cpu_idle_exit       6546       6546
745
i86_cpu_wait_idle_power_thread.d.2011-09-18_0030.out:        cpu_idle_exit       5114       5114
746
i86_cpu_wait_idle_power_thread.d.2011-09-18_0600.out:        cpu_idle_exit       5972       5972
747
i86_cpu_wait_idle_power_thread.d.2011-09-18_1130.out:        cpu_idle_exit       6340       6340
748
i86_cpu_wait_idle_power_thread.d.2011-09-18_1700.out:        cpu_idle_exit       6048       6048
749
i86_cpu_wait_idle_power_thread.d.2011-09-18_1730.out:        cpu_idle_exit       7118       7118
750
i86_cpu_wait_idle_power_thread.d.2011-09-18_1800.out:        cpu_idle_exit       6782       6782
751

    
752
----------------------------------------------------------------------------------
753
Looking at hotkernel_30sec.\* output spaced by 10 hour samples with last three samples
754
at bottom with crash during 2011-09-18_1800 - this doesnt' seem to indicate any issue.
755

    
756
displaying tail -5 of output
757

    
758
hotkernel_30sec.2011-09-14_1240.out:
759
      apix`apix_dispatch_softint                                  6   0.0%
760
      unix`do_splx                                               11   0.0%
761
      genunix`fsflush_do_pages                                   16   0.0%
762
      unix`acpi_cpu_cstate                                      835   0.3%
763
      unix`i86_mwait                                         247124  99.6%
764
hotkernel_30sec.2011-09-14_1620.out:
765
      unix`atomic_and_64                                          6   0.0%
766
      unix`mutex_enter                                            9   0.0%
767
      unix`cpu_idle_exit                                         45   0.0%
768
      unix`acpi_cpu_cstate                                      725   0.3%
769
      unix`i86_mwait                                         247212  99.7%
770
hotkernel_30sec.2011-09-14_2000.out:
771
      genunix`fsflush_do_pages                                    9   0.0%
772
      unix`cpu_idle_enter                                        26   0.0%
773
      unix`i86_monitor                                           32   0.0%
774
      unix`acpi_cpu_cstate                                      702   0.3%
775
      unix`i86_mwait                                         247206  99.7%
776
hotkernel_30sec.2011-09-14_2340.out:
777
      unix`page_nextn                                             4   0.0%
778
      genunix`fsflush_do_pages                                    4   0.0%
779
      unix`splr                                                   7   0.0%
780
      unix`acpi_cpu_cstate                                      835   0.3%
781
      unix`i86_mwait                                         247163  99.6%
782
hotkernel_30sec.2011-09-15_0320.out:
783
      unix`do_splx                                                4   0.0%
784
      zfs`lzjb_compress                                           4   0.0%
785
      unix`page_nextn                                             6   0.0%
786
      unix`acpi_cpu_cstate                                      624   0.3%
787
      unix`i86_mwait                                         247333  99.7%
788
hotkernel_30sec.2011-09-15_0700.out:
789
      genunix`fsflush                                            11   0.0%
790
      unix`do_splx                                               13   0.0%
791
      genunix`fsflush_do_pages                                   20   0.0%
792
      unix`acpi_cpu_cstate                                      658   0.3%
793
      unix`i86_mwait                                         247281  99.7%
794
hotkernel_30sec.2011-09-15_1040.out:
795
      unix`tsc_read                                               6   0.0%
796
      unix`do_splx                                                6   0.0%
797
      genunix`fsflush_do_pages                                    7   0.0%
798
      unix`acpi_cpu_cstate                                      625   0.3%
799
      unix`i86_mwait                                         247336  99.7%
800
hotkernel_30sec.2011-09-15_1420.out:
801
      unix`mutex_enter                                            2   0.0%
802
      unix`i86_monitor                                            2   0.0%
803
      genunix`fsflush_do_pages                                    5   0.0%
804
      unix`acpi_cpu_cstate                                      711   0.3%
805
      unix`i86_mwait                                         247299  99.7%
806
hotkernel_30sec.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
807
hotkernel_30sec.2011-09-15_2140.out:
808
      apix`apix_dispatch_softint                                  4   0.0%
809
      unix`page_nextn                                             6   0.0%
810
      genunix`dnlc_purge_vfsp                                     7   0.0%
811
      unix`acpi_cpu_cstate                                      794   0.3%
812
      unix`i86_mwait                                         247167  99.7%
813
hotkernel_30sec.2011-09-16_0120.out:
814
      unix`splr                                                   7   0.0%
815
      unix`page_nextn                                             8   0.0%
816
      genunix`dnlc_purge_vfsp                                     9   0.0%
817
      unix`acpi_cpu_cstate                                      860   0.3%
818
      unix`i86_mwait                                         247069  99.6%
819
hotkernel_30sec.2011-09-16_0500.out:
820
      unix`tsc_gethrtime                                          5   0.0%
821
      genunix`fsflush_do_pages                                    5   0.0%
822
      genunix`dnlc_purge_vfsp                                     8   0.0%
823
      unix`acpi_cpu_cstate                                      708   0.3%
824
      unix`i86_mwait                                         247254  99.7%
825
hotkernel_30sec.2011-09-16_0840.out:
826
      unix`do_splx                                                5   0.0%
827
      genunix`dnlc_purge_vfsp                                     8   0.0%
828
      unix`page_nextn                                            10   0.0%
829
      unix`acpi_cpu_cstate                                      739   0.3%
830
      unix`i86_mwait                                         247191  99.7%
831
hotkernel_30sec.2011-09-16_1220.out:
832
      unix`page_nextn                                             5   0.0%
833
      genunix`fsflush_do_pages                                    7   0.0%
834
      genunix`dnlc_purge_vfsp                                     9   0.0%
835
      unix`acpi_cpu_cstate                                      843   0.3%
836
      unix`i86_mwait                                         247108  99.6%
837
hotkernel_30sec.2011-09-16_1600.out:
838
      genunix`dnlc_purge_vfsp                                     8   0.0%
839
      genunix`fsflush_do_pages                                    9   0.0%
840
      unix`0xfffffffffb85                                        16   0.0%
841
      unix`acpi_cpu_cstate                                      697   0.3%
842
      unix`i86_mwait                                         245509  99.7%
843
hotkernel_30sec.2011-09-16_2100.out:
844
      zfs`lzjb_compress                                           6   0.0%
845
      genunix`fsflush_do_pages                                    9   0.0%
846
      unix`do_splx                                                9   0.0%
847
      unix`acpi_cpu_cstate                                      625   0.3%
848
      unix`i86_mwait                                         247299  99.7%
849
hotkernel_30sec.2011-09-17_0230.out:
850
      unix`page_nextn                                             7   0.0%
851
      genunix`fsflush                                             9   0.0%
852
      genunix`fsflush_do_pages                                   22   0.0%
853
      unix`acpi_cpu_cstate                                      643   0.3%
854
      unix`i86_mwait                                         247242  99.7%
855
hotkernel_30sec.2011-09-17_0800.out:
856
      unix`mutex_enter                                            6   0.0%
857
      unix`page_nextn                                            10   0.0%
858
      genunix`fsflush_do_pages                                   11   0.0%
859
      unix`acpi_cpu_cstate                                      600   0.2%
860
      unix`i86_mwait                                         247284  99.7%
861
hotkernel_30sec.2011-09-17_1330.out:
862
      unix`mutex_enter                                            3   0.0%
863
      unix`page_nextn                                             6   0.0%
864
      genunix`fsflush_do_pages                                    8   0.0%
865
      unix`acpi_cpu_cstate                                      796   0.3%
866
      unix`i86_mwait                                         247120  99.7%
867
hotkernel_30sec.2011-09-17_1900.out:
868
      unix`tsc_read                                               8   0.0%
869
      genunix`fsflush_do_pages                                    8   0.0%
870
      unix`page_nextn                                            10   0.0%
871
      unix`acpi_cpu_cstate                                      839   0.3%
872
      unix`i86_mwait                                         247018  99.6%
873
hotkernel_30sec.2011-09-18_0030.out:
874
      unix`page_nextn                                             5   0.0%
875
      unix`mutex_enter                                            6   0.0%
876
      genunix`fsflush_do_pages                                   13   0.0%
877
      unix`acpi_cpu_cstate                                      475   0.2%
878
      unix`i86_mwait                                         247364  99.8%
879
hotkernel_30sec.2011-09-18_0600.out:
880
      unix`do_splx                                                6   0.0%
881
      unix`page_nextn                                             7   0.0%
882
      genunix`fsflush_do_pages                                   13   0.0%
883
      unix`acpi_cpu_cstate                                      708   0.3%
884
      unix`i86_mwait                                         247120  99.7%
885
hotkernel_30sec.2011-09-18_1130.out:
886
      genunix`fsflush_do_pages                                    7   0.0%
887
      unix`do_splx                                                9   0.0%
888
      unix`page_nextn                                            11   0.0%
889
      unix`acpi_cpu_cstate                                      674   0.3%
890
      unix`i86_mwait                                         247138  99.7%
891
hotkernel_30sec.2011-09-18_1700.out:
892
      genunix`fsflush                                            10   0.0%
893
      genunix`fsflush_do_pages                                   13   0.0%
894
      unix`todpc_rtcget                                          19   0.0%
895
      unix`acpi_cpu_cstate                                      771   0.3%
896
      unix`i86_mwait                                         246948  99.6%
897
hotkernel_30sec.2011-09-18_1730.out:
898
      unix`i86_monitor                                            6   0.0%
899
      genunix`fsflush_do_pages                                    9   0.0%
900
      unix`page_nextn                                            11   0.0%
901
      unix`acpi_cpu_cstate                                      496   0.2%
902
      unix`i86_mwait                                         247319  99.8%
903
hotkernel_30sec.2011-09-18_1800.out:       <<not found script was not active yet, or system crashed>>
904

    
905
----------------------------------------------------------------------------------
906
Looking at cpu_idle_exit_1ms.d.\* output spaced by 10 hour samples with last three samples
907
at bottom with crash during 2011-09-18_1800
908

    
909
displaying counts of stack traces across 1ms for cpu_idle_exit:entry
910

    
911
typical output similar to consol information when we panic/crash dumped
912

    
913

    
914
                  samples below  from : cpu_idle_exit_1ms.d.2011-09-18_1800.out
915

    
916
                  5  13355              cpu_idle_exit:entry
917
                              apix`apix_do_interrupt+0x3e
918
                              unix`_interrupt+0xba
919
                              unix`wrmsr+0xd
920
                              apix`apic_timer_reprogram+0x19
921
                              unix`cbe_reprogram+0x25
922
                              genunix`cyclic_fire+0xd5
923
                              unix`cbe_fire+0x5a
924
                              apix`apix_dispatch_pending_autovect+0x12c
925
                              apix`apix_dispatch_pending_hilevel+0x15
926
                              unix`switch_sp_and_call+0x13
927
                              apix`apix_do_interrupt+0x31b
928
                              unix`_interrupt+0xba
929
                              genunix`fop_ioctl+0x7b
930
                              genunix`ioctl+0x18e
931
                              unix`sys_syscall+0x17a
932

    
933
                  5  13355              cpu_idle_exit:entry
934
                              apix`apix_do_interrupt+0x3e
935
                              unix`_interrupt+0xba
936
                              apix`deadline_timer_reprogram+0x34
937
                              apix`apic_timer_reprogram+0x19
938
                              unix`cbe_reprogram+0x25
939
                              genunix`cyclic_fire+0xd5
940
                              unix`cbe_fire+0x5a
941
                              apix`apix_dispatch_pending_autovect+0x12c
942
                              apix`apix_dispatch_pending_hilevel+0x15
943
                              unix`switch_sp_and_call+0x13
944
                              apix`apix_do_interrupt+0x31b
945
                              unix`_interrupt+0xba
946
                              genunix`fop_ioctl+0x7b
947
                              genunix`ioctl+0x18e
948
                              unix`sys_syscall+0x17a
949

    
950

    
951
cpu_idle_exit_1ms.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
952
cpu_idle_exit_1ms.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
953
cpu_idle_exit_1ms.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
954
cpu_idle_exit_1ms.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
955
cpu_idle_exit_1ms.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
956
cpu_idle_exit_1ms.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
957
cpu_idle_exit_1ms.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
958
cpu_idle_exit_1ms.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
959
cpu_idle_exit_1ms.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
960
cpu_idle_exit_1ms.d.2011-09-15_2140.out:       <<not found script was not active yet, or system crashed>>
961
cpu_idle_exit_1ms.d.2011-09-16_0120.out:       <<not found script was not active yet, or system crashed>>
962
cpu_idle_exit_1ms.d.2011-09-16_0500.out:       <<not found script was not active yet, or system crashed>>
963
cpu_idle_exit_1ms.d.2011-09-16_0840.out:       <<not found script was not active yet, or system crashed>>
964
cpu_idle_exit_1ms.d.2011-09-16_1220.out:       <<not found script was not active yet, or system crashed>>
965
cpu_idle_exit_1ms.d.2011-09-16_1600.out:       <<not found script was not active yet, or system crashed>>
966
cpu_idle_exit_1ms.d.2011-09-16_2100.out: 421
967
cpu_idle_exit_1ms.d.2011-09-17_0230.out: 519
968
cpu_idle_exit_1ms.d.2011-09-17_0800.out: 544
969
cpu_idle_exit_1ms.d.2011-09-17_1330.out: 531
970
cpu_idle_exit_1ms.d.2011-09-17_1900.out: 651
971
cpu_idle_exit_1ms.d.2011-09-18_0030.out: 660
972
cpu_idle_exit_1ms.d.2011-09-18_0600.out: 738
973
cpu_idle_exit_1ms.d.2011-09-18_1130.out: 780
974
cpu_idle_exit_1ms.d.2011-09-18_1700.out: 797
975
cpu_idle_exit_1ms.d.2011-09-18_1730.out: 758
976
cpu_idle_exit_1ms.d.2011-09-18_1800.out: 797
977

    
978
----------------------------------------------------------------------------------
979
Looking at cpu_idle_exit_1ms.d.\* output spaced by 10 hour samples with last three samples
980
at bottom with crash during 2011-09-18_1800
981

    
982
displaying counts of stack traces across 1ms for cpu_idle_exit:entry that contain cbe_fire
983

    
984
cpu_idle_exit_1ms.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
985
cpu_idle_exit_1ms.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
986
cpu_idle_exit_1ms.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
987
cpu_idle_exit_1ms.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
988
cpu_idle_exit_1ms.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
989
cpu_idle_exit_1ms.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
990
cpu_idle_exit_1ms.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
991
cpu_idle_exit_1ms.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
992
cpu_idle_exit_1ms.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
993
cpu_idle_exit_1ms.d.2011-09-15_2140.out:       <<not found script was not active yet, or system crashed>>
994
cpu_idle_exit_1ms.d.2011-09-16_0120.out:       <<not found script was not active yet, or system crashed>>
995
cpu_idle_exit_1ms.d.2011-09-16_0500.out:       <<not found script was not active yet, or system crashed>>
996
cpu_idle_exit_1ms.d.2011-09-16_0840.out:       <<not found script was not active yet, or system crashed>>
997
cpu_idle_exit_1ms.d.2011-09-16_1220.out:       <<not found script was not active yet, or system crashed>>
998
cpu_idle_exit_1ms.d.2011-09-16_1600.out:       <<not found script was not active yet, or system crashed>>
999
cpu_idle_exit_1ms.d.2011-09-16_2100.out: 412
1000
cpu_idle_exit_1ms.d.2011-09-17_0230.out: 504
1001
cpu_idle_exit_1ms.d.2011-09-17_0800.out: 543
1002
cpu_idle_exit_1ms.d.2011-09-17_1330.out: 522
1003
cpu_idle_exit_1ms.d.2011-09-17_1900.out: 639
1004
cpu_idle_exit_1ms.d.2011-09-18_0030.out: 653
1005
cpu_idle_exit_1ms.d.2011-09-18_0600.out: 737
1006
cpu_idle_exit_1ms.d.2011-09-18_1130.out: 772
1007
cpu_idle_exit_1ms.d.2011-09-18_1700.out: 792
1008
cpu_idle_exit_1ms.d.2011-09-18_1730.out: 757
1009
cpu_idle_exit_1ms.d.2011-09-18_1800.out: 796
1010

    
1011
----------------------------------------------------------------------------------
1012
Looking at cpu_idle_exit_1ms.d.\* output spaced by 10 hour samples with last three samples
1013
at bottom with crash during 2011-09-18_1800
1014

    
1015
displaying counts of stack traces across 1ms for cpu_idle_exit:entry that contain switch_sp_and_call
1016

    
1017
cpu_idle_exit_1ms.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
1018
cpu_idle_exit_1ms.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
1019
cpu_idle_exit_1ms.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
1020
cpu_idle_exit_1ms.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
1021
cpu_idle_exit_1ms.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
1022
cpu_idle_exit_1ms.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
1023
cpu_idle_exit_1ms.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
1024
cpu_idle_exit_1ms.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
1025
cpu_idle_exit_1ms.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
1026
cpu_idle_exit_1ms.d.2011-09-15_2140.out:       <<not found script was not active yet, or system crashed>>
1027
cpu_idle_exit_1ms.d.2011-09-16_0120.out:       <<not found script was not active yet, or system crashed>>
1028
cpu_idle_exit_1ms.d.2011-09-16_0500.out:       <<not found script was not active yet, or system crashed>>
1029
cpu_idle_exit_1ms.d.2011-09-16_0840.out:       <<not found script was not active yet, or system crashed>>
1030
cpu_idle_exit_1ms.d.2011-09-16_1220.out:       <<not found script was not active yet, or system crashed>>
1031
cpu_idle_exit_1ms.d.2011-09-16_1600.out:       <<not found script was not active yet, or system crashed>>
1032
cpu_idle_exit_1ms.d.2011-09-16_2100.out: 412
1033
cpu_idle_exit_1ms.d.2011-09-17_0230.out: 504
1034
cpu_idle_exit_1ms.d.2011-09-17_0800.out: 543
1035
cpu_idle_exit_1ms.d.2011-09-17_1330.out: 522
1036
cpu_idle_exit_1ms.d.2011-09-17_1900.out: 639
1037
cpu_idle_exit_1ms.d.2011-09-18_0030.out: 653
1038
cpu_idle_exit_1ms.d.2011-09-18_0600.out: 737
1039
cpu_idle_exit_1ms.d.2011-09-18_1130.out: 772
1040
cpu_idle_exit_1ms.d.2011-09-18_1700.out: 792
1041
cpu_idle_exit_1ms.d.2011-09-18_1730.out: 757
1042
cpu_idle_exit_1ms.d.2011-09-18_1800.out: 796
1043

    
1044
----------------------------------------------------------------------------------
1045
Looking at cpu_idle_exit_1ms.d.\* output spaced by 10 hour samples with last three samples
1046
at bottom with crash during 2011-09-18_1800
1047

    
1048
displaying counts of stack traces across 1ms for cpu_idle_exit:entry that contain deadline_timer_reprogram
1049
typically we see either deadline_timer_reprogram or wrmsr, this counts deadline_timer_reprogram
1050

    
1051
cpu_idle_exit_1ms.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
1052
cpu_idle_exit_1ms.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
1053
cpu_idle_exit_1ms.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
1054
cpu_idle_exit_1ms.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
1055
cpu_idle_exit_1ms.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
1056
cpu_idle_exit_1ms.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
1057
cpu_idle_exit_1ms.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
1058
cpu_idle_exit_1ms.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
1059
cpu_idle_exit_1ms.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
1060
cpu_idle_exit_1ms.d.2011-09-15_2140.out:       <<not found script was not active yet, or system crashed>>
1061
cpu_idle_exit_1ms.d.2011-09-16_0120.out:       <<not found script was not active yet, or system crashed>>
1062
cpu_idle_exit_1ms.d.2011-09-16_0500.out:       <<not found script was not active yet, or system crashed>>
1063
cpu_idle_exit_1ms.d.2011-09-16_0840.out:       <<not found script was not active yet, or system crashed>>
1064
cpu_idle_exit_1ms.d.2011-09-16_1220.out:       <<not found script was not active yet, or system crashed>>
1065
cpu_idle_exit_1ms.d.2011-09-16_1600.out:       <<not found script was not active yet, or system crashed>>
1066
cpu_idle_exit_1ms.d.2011-09-16_2100.out: 57
1067
cpu_idle_exit_1ms.d.2011-09-17_0230.out: 62
1068
cpu_idle_exit_1ms.d.2011-09-17_0800.out: 21
1069
cpu_idle_exit_1ms.d.2011-09-17_1330.out: 61
1070
cpu_idle_exit_1ms.d.2011-09-17_1900.out: 73
1071
cpu_idle_exit_1ms.d.2011-09-18_0030.out: 100
1072
cpu_idle_exit_1ms.d.2011-09-18_0600.out: 90
1073
cpu_idle_exit_1ms.d.2011-09-18_1130.out: 84
1074
cpu_idle_exit_1ms.d.2011-09-18_1700.out: 114
1075
cpu_idle_exit_1ms.d.2011-09-18_1730.out: 92
1076
cpu_idle_exit_1ms.d.2011-09-18_1800.out: 114
1077

    
1078
----------------------------------------------------------------------------------
1079
Looking at cpu_idle_exit_1ms.d.\* output spaced by 10 hour samples with last three samples
1080
at bottom with crash during 2011-09-18_1800
1081

    
1082
displaying counts of stack traces across 1ms for cpu_idle_exit:entry that contain deadline_timer_reprogram
1083
typically we see either deadline_timer_reprogram or wrmsr, this counts wrmsr
1084

    
1085
cpu_idle_exit_1ms.d.2011-09-14_1240.out:       <<not found script was not active yet, or system crashed>>
1086
cpu_idle_exit_1ms.d.2011-09-14_1620.out:       <<not found script was not active yet, or system crashed>>
1087
cpu_idle_exit_1ms.d.2011-09-14_2000.out:       <<not found script was not active yet, or system crashed>>
1088
cpu_idle_exit_1ms.d.2011-09-14_2340.out:       <<not found script was not active yet, or system crashed>>
1089
cpu_idle_exit_1ms.d.2011-09-15_0320.out:       <<not found script was not active yet, or system crashed>>
1090
cpu_idle_exit_1ms.d.2011-09-15_0700.out:       <<not found script was not active yet, or system crashed>>
1091
cpu_idle_exit_1ms.d.2011-09-15_1040.out:       <<not found script was not active yet, or system crashed>>
1092
cpu_idle_exit_1ms.d.2011-09-15_1420.out:       <<not found script was not active yet, or system crashed>>
1093
cpu_idle_exit_1ms.d.2011-09-15_1800.out:       <<not found script was not active yet, or system crashed>>
1094
cpu_idle_exit_1ms.d.2011-09-15_2140.out:       <<not found script was not active yet, or system crashed>>
1095
cpu_idle_exit_1ms.d.2011-09-16_0120.out:       <<not found script was not active yet, or system crashed>>
1096
cpu_idle_exit_1ms.d.2011-09-16_0500.out:       <<not found script was not active yet, or system crashed>>
1097
cpu_idle_exit_1ms.d.2011-09-16_0840.out:       <<not found script was not active yet, or system crashed>>
1098
cpu_idle_exit_1ms.d.2011-09-16_1220.out:       <<not found script was not active yet, or system crashed>>
1099
cpu_idle_exit_1ms.d.2011-09-16_1600.out:       <<not found script was not active yet, or system crashed>>
1100
cpu_idle_exit_1ms.d.2011-09-16_2100.out: 355
1101
cpu_idle_exit_1ms.d.2011-09-17_0230.out: 442
1102
cpu_idle_exit_1ms.d.2011-09-17_0800.out: 522
1103
cpu_idle_exit_1ms.d.2011-09-17_1330.out: 461
1104
cpu_idle_exit_1ms.d.2011-09-17_1900.out: 566
1105
cpu_idle_exit_1ms.d.2011-09-18_0030.out: 553
1106
cpu_idle_exit_1ms.d.2011-09-18_0600.out: 647
1107
cpu_idle_exit_1ms.d.2011-09-18_1130.out: 688
1108
cpu_idle_exit_1ms.d.2011-09-18_1700.out: 678
1109
cpu_idle_exit_1ms.d.2011-09-18_1730.out: 665
1110
cpu_idle_exit_1ms.d.2011-09-18_1800.out: 682
1111

    
1112
----------------------------------------------------------------------------------
1113
First sample of available script outputs - there are many items THAT may provide
1114
more information, but samples like adaptive.* and cpu_idle_exit_1ms.d.* are BIG
1115

    
1116
     -rw-r--r-- 1 root root   2881 2011-09-14 12:40 top_10sec.2011-09-14_1240.out
1117
     -rw-r--r-- 1 root root    765 2011-09-14 12:40 info.2011-09-14_1240.out
1118
     -rw-r--r-- 1 root root   1386 2011-09-14 12:40 mpstat_10_2.2011-09-14_1240.out
1119
     -rw-r--r-- 1 root root    394 2011-09-14 12:40 vmstat_3_3.2011-09-14_1240.out
1120
     -rw-r--r-- 1 root root   4756 2011-09-14 12:40 prstat-a-m_5_3.2011-09-14_1240.out
1121
     -rw-r--r-- 1 root root   2416 2011-09-14 12:40 prstat-L-c_4_2.2011-09-14_1240.out
1122
     -rw-r--r-- 1 root root    631 2011-09-14 12:40 profile-1001.kernel.2011-09-14_1240.out
1123
     -rw-r--r-- 1 root root 204235 2011-09-14 12:41 more.2011-09-14_1240.out
1124
     -rw-r--r-- 1 root root   2731 2011-09-14 12:41 whatfor.d.2011-09-14_1240.out
1125
     -rw-r--r-- 1 root root   3010 2011-09-14 12:41 whosteal.d.2011-09-14_1240.out
1126
     -rw-r--r-- 1 root root   5069 2011-09-14 12:41 qtime.d.2011-09-14_1240.out
1127
     -rw-r--r-- 1 root root   3274 2011-09-14 12:41 qlen.d.2011-09-14_1240.out
1128
     -rw-r--r-- 1 root root    270 2011-09-14 12:42 whoqueue.d.2011-09-14_1240.out
1129
     -rw-r--r-- 1 root root    919 2011-09-14 12:42 whofor.d.2011-09-14_1240.out
1130
     -rw-r--r-- 1 root root    152 2011-09-14 12:42 doors.d.2011-09-14_1240.out
1131
     -rw-r--r-- 1 root root     77 2011-09-14 12:42 whopreempt.d.2011-09-14_1240.out
1132
     -rw-r--r-- 1 root root    154 2011-09-14 12:42 capswait.d.2011-09-14_1240.out
1133
     -rw-r--r-- 1 root root    154 2011-09-14 12:43 whatlock.d.2011-09-14_1240.out
1134
     -rw-r--r-- 1 root root   5009 2011-09-14 12:43 profile-1971_5sec.2011-09-14_1240.out
1135
     -rw-r--r-- 1 root root 924798 2011-09-14 12:43 adaptive.2011-09-14_1240.out
1136
     -rw-r--r-- 1 root root    848 2011-09-14 12:43 syscall_cnt_by_syscall.2011-09-14_1240.out
1137
     -rw-r--r-- 1 root root   4275 2011-09-14 12:43 hotkernel_30sec.2011-09-14_1240.out
1138
     -rw-r--r-- 1 root root    618 2011-09-14 12:44 hotkernel_30sec_m.2011-09-14_1240.out
1139
     -rw-r--r-- 1 root root    843 2011-09-14 12:44 cpuwalk.d_10.2011-09-14_1240.out
1140
     -rw-r--r-- 1 root root   1855 2011-09-14 12:44 intoncpu_10sec.2011-09-14_1240.out
1141
     -rw-r--r-- 1 root root    187 2011-09-14 12:45 inttimes_10sec.2011-09-14_1240.out
1142
     -rw-r--r-- 1 root root    154 2011-09-14 12:45 xcallsbypid_10sec.2011-09-14_1240.out
1143
     -rw-r--r-- 1 root root   1133 2011-09-14 12:45 sysbypid_10sec.2011-09-14_1240.out
1144
     -rw-r--r-- 1 root root   1557 2011-09-14 12:45 topsyscall-C-s_5_2.2011-09-14_1240.out
1145
     -rw-r--r-- 1 root root   1193 2011-09-14 12:46 iotop-C-t_10_5_4.2011-09-14_1240.out
1146
     -rw-r--r-- 1 root root    685 2011-09-14 12:46 cswstat_10sec.2011-09-14_1240.out
1147
     -rw-r--r-- 1 root root    273 2011-09-14 12:46 idle-state_10sec.2011-09-14_1240.out
1148
     -rw-r--r-- 1 root root    934 2011-09-14 12:46 watch_cpu_acpi_read_port.2011-09-14_1240.out
1149
     -rw-r--r-- 1 root root   3713 2011-09-14 12:47 watch_cstate.2011-09-14_1240.out
1150
     -rw-r--r-- 1 root root      1 2011-09-14 12:47 speedstep.2011-09-14_1240.out
1151
     -rw-r--r-- 1 root root    136 2011-09-14 12:48 watch_acpi_cpu_cstate_60.2011-09-14_1240.out
1152
     -rw-r--r-- 1 root root    142 2011-09-14 12:49 watch_cpu_acpi_read_port_60.2011-09-14_1240.out
1153
     -rw-r--r-- 1 root root   2702 2011-09-14 12:49 cpudists.2011-09-14_1240.out
1154
     -rw-r--r-- 1 root root    159 2011-09-14 12:49 cputimes.2011-09-14_1240.out
1155
     -rw-r--r-- 1 root root    213 2011-09-14 12:49 dladm_show-link-s.2011-09-14_1240.out
1156
     -rw-r--r-- 1 root root    380 2011-09-14 12:49 dlstat_show-link-r.2011-09-14_1240.out
1157
     -rw-r--r-- 1 root root    104 2011-09-14 12:49 dlstat.2011-09-14_1240.out
1158
     -rw-r--r-- 1 root root   2706 2011-09-14 12:49 intrstat_5_3.2011-09-14_1240.out
1159
     -rw-r--r-- 1 root root   8275 2011-09-14 12:50 powertop-d_3-t_10.2011-09-14_1240.out
1160
     -rw-r--r-- 1 root root   1245 2011-09-14 12:50 lockstat-kIW-D_20_sleep_30.2011-09-14_1240.out
1161
     -rw-r--r-- 1 root root    193 2011-09-14 12:50 intrtime.pl.2011-09-14_1240.out
1162
     -rw-r--r-- 1 root root    553 2011-09-15 10:23 syscall_ent_ret.d.2011-09-15_1020.out
1163
     -rw-r--r-- 1 root root    294 2011-09-15 12:23 kern_ent_ret.d.2011-09-15_1220.out
1164
     -rw-r--r-- 1 root root  19015 2011-09-15 12:43 kern_100ms_ent_ret.d.2011-09-15_1240.out
1165
     -rw-r--r-- 1 root root   2816 2011-09-15 17:42 i86_cpu_wait_idle_power_thread.d.2011-09-15_1740.out
1166
     -rw-r--r-- 1 root root    478 2011-09-15 18:23 acpi_10s_ent_ret.d.2011-09-15_1820.out
1167
     -rw-r--r-- 1 root root    430 2011-09-15 18:24 i86_10s_ent_ret.d.2011-09-15_1820.out
1168
     -rw-r--r-- 1 root root   1295 2011-09-15 18:24 power_10s_ent_ret.d.2011-09-15_1820.out
1169
     -rw-r--r-- 1 root root    568 2011-09-15 18:24 acpi_100ms_ent_ret.d.2011-09-15_1820.out
1170
     -rw-r--r-- 1 root root    570 2011-09-15 18:24 intr_100ms_ent_ret.d.2011-09-15_1820.out
1171
     -rw-r--r-- 1 root root  18762 2011-09-15 19:04 kstat_n_sys.2011-09-15_1900.out
1172
     -rw-r--r-- 1 root root    352 2011-09-15 19:12 kstat_m_cpu_info_curfrq.2011-09-15_1900.out
1173
     -rw-r--r-- 1 root root   2877 2011-09-15 19:12 warmredo_top_10sec.2011-09-15_1900.out
1174
     -rw-r--r-- 1 root root    352 2011-09-16 07:20 beg_kstat_m_cpu_info_curfrq.2011-09-16_0720.out
1175
     -rw-r--r-- 1 root root    546 2011-09-16 07:46 watch_acpi_cpu_cstate_120.d.2011-09-16_0740.out
1176
     -rw-r--r-- 1 root root    186 2011-09-16 08:06 idle-state-transition.d.2011-09-16_0800.out
1177
     -rw-r--r-- 1 root root   1214 2011-09-16 09:14 mdb_k_interrupts.2011-09-16_0900.out
1178
     -rw-r--r-- 1 root root    627 2011-09-16 09:14 mdb_k_memstat.2011-09-16_0900.out
1179
     -rw-r--r-- 1 root root  39296 2011-09-16 09:14 mdb_k_kmastat.2011-09-16_0900.out
1180
     -rw-r--r-- 1 root root  23745 2011-09-16 09:34 mdb_k_stacks.2011-09-16_0920.out
1181
     -rw-r--r-- 1 root root    643 2011-09-16 16:46 putnext.d.2011-09-16_1640.out
1182
     -rw-r--r-- 1 root root   3430 2011-09-16 16:46 acpi_cpu_cstate_10ms.d.2011-09-16_1640.out
1183
     -rw-r--r-- 1 root root 312039 2011-09-16 16:46 psm_get_cpu_id_1ms.d.2011-09-16_1640.out
1184
     -rw-r--r-- 1 root root  12866 2011-09-16 16:46 new_cpu_mstate_10ms.d.2011-09-16_1640.out
1185
     -rw-r--r-- 1 root root 344312 2011-09-16 16:46 cpu_idle_exit_1ms.d.2011-09-16_1640.out
1186
     -rw-r--r-- 1 root root  38139 2011-09-16 17:06 apix_intr_enter_first50.d.2011-09-16_1700.out
1187
     -rw-r--r-- 1 root root   7078 2011-09-18 07:31 prstat-L-s-cpu-m-T-c_2_4.2011-09-18_0730.out
1188

    
1189
----------------------------------------------------------------------------------
1190
Last sample of available FULL script outputs (1/2 hour before crash) - again there are many 
1191
items THAT may provide more information, but samples like adaptive.* and cpu_idle_exit_1ms.d.* 
1192
are BIG.  Not I also created more sampeling scripts during the run.  Some of the new scripts
1193
also create big outputs like psm_get_cpu_id_1ms.d.* and pu_idle_exit_1ms.d.*
1194

    
1195
     -rw-r--r-- 1 root root    352 2011-09-18 17:30 beg_kstat_m_cpu_info_curfrq.2011-09-18_1730.out
1196
     -rw-r--r-- 1 root root   2906 2011-09-18 17:30 top_10sec.2011-09-18_1730.out
1197
     -rw-r--r-- 1 root root    936 2011-09-18 17:30 info.2011-09-18_1730.out
1198
     -rw-r--r-- 1 root root   1386 2011-09-18 17:30 mpstat_10_2.2011-09-18_1730.out
1199
     -rw-r--r-- 1 root root    391 2011-09-18 17:30 vmstat_3_3.2011-09-18_1730.out
1200
     -rw-r--r-- 1 root root   4767 2011-09-18 17:30 prstat-a-m_5_3.2011-09-18_1730.out
1201
     -rw-r--r-- 1 root root   2418 2011-09-18 17:30 prstat-L-c_4_2.2011-09-18_1730.out
1202
     -rw-r--r-- 1 root root   7133 2011-09-18 17:31 prstat-L-s-cpu-m-T-c_2_4.2011-09-18_1730.out
1203
     -rw-r--r-- 1 root root    631 2011-09-18 17:31 profile-1001.kernel.2011-09-18_1730.out
1204
     -rw-r--r-- 1 root root    188 2011-09-18 17:31 profile-1001.user.2011-09-18_1730.out
1205
     -rw-r--r-- 1 root root 211725 2011-09-18 17:31 more.2011-09-18_1730.out
1206
     -rw-r--r-- 1 root root   2593 2011-09-18 17:31 whatfor.d.2011-09-18_1730.out
1207
     -rw-r--r-- 1 root root   4354 2011-09-18 17:31 whosteal.d.2011-09-18_1730.out
1208
     -rw-r--r-- 1 root root   5690 2011-09-18 17:31 qtime.d.2011-09-18_1730.out
1209
     -rw-r--r-- 1 root root   3343 2011-09-18 17:32 qlen.d.2011-09-18_1730.out
1210
     -rw-r--r-- 1 root root    191 2011-09-18 17:32 whoqueue.d.2011-09-18_1730.out
1211
     -rw-r--r-- 1 root root    919 2011-09-18 17:32 whofor.d.2011-09-18_1730.out
1212
     -rw-r--r-- 1 root root    152 2011-09-18 17:32 doors.d.2011-09-18_1730.out
1213
     -rw-r--r-- 1 root root    153 2011-09-18 17:32 whopreempt.d.2011-09-18_1730.out
1214
     -rw-r--r-- 1 root root    154 2011-09-18 17:33 capswait.d.2011-09-18_1730.out
1215
     -rw-r--r-- 1 root root    154 2011-09-18 17:33 whatlock.d.2011-09-18_1730.out
1216
     -rw-r--r-- 1 root root    510 2011-09-18 17:33 syscall_ent_ret.d.2011-09-18_1730.out
1217
     -rw-r--r-- 1 root root    294 2011-09-18 17:33 kern_ent_ret.d.2011-09-18_1730.out
1218
     -rw-r--r-- 1 root root  20054 2011-09-18 17:33 kern_100ms_ent_ret.d.2011-09-18_1730.out
1219
     -rw-r--r-- 1 root root    478 2011-09-18 17:34 acpi_10s_ent_ret.d.2011-09-18_1730.out
1220
     -rw-r--r-- 1 root root    430 2011-09-18 17:34 i86_10s_ent_ret.d.2011-09-18_1730.out
1221
     -rw-r--r-- 1 root root   1295 2011-09-18 17:34 power_10s_ent_ret.d.2011-09-18_1730.out
1222
     -rw-r--r-- 1 root root    568 2011-09-18 17:34 acpi_100ms_ent_ret.d.2011-09-18_1730.out
1223
     -rw-r--r-- 1 root root    843 2011-09-18 17:34 intr_100ms_ent_ret.d.2011-09-18_1730.out
1224
     -rw-r--r-- 1 root root   4003 2011-09-18 17:34 i86_cpu_wait_idle_power_thread.d.2011-09-18_1730.out
1225
     -rw-r--r-- 1 root root    546 2011-09-18 17:36 watch_acpi_cpu_cstate_120.d.2011-09-18_1730.out
1226
     -rw-r--r-- 1 root root    186 2011-09-18 17:36 idle-state-transition.d.2011-09-18_1730.out
1227
     -rw-r--r-- 1 root root    223 2011-09-18 17:36 putnext.d.2011-09-18_1730.out
1228
     -rw-r--r-- 1 root root   2854 2011-09-18 17:36 acpi_cpu_cstate_10ms.d.2011-09-18_1730.out
1229
     -rw-r--r-- 1 root root 493210 2011-09-18 17:36 psm_get_cpu_id_1ms.d.2011-09-18_1730.out
1230
     -rw-r--r-- 1 root root  12268 2011-09-18 17:37 new_cpu_mstate_10ms.d.2011-09-18_1730.out
1231
     -rw-r--r-- 1 root root 632336 2011-09-18 17:37 cpu_idle_exit_1ms.d.2011-09-18_1730.out
1232
     -rw-r--r-- 1 root root  38786 2011-09-18 17:37 apix_intr_enter_first50.d.2011-09-18_1730.out
1233
     -rw-r--r-- 1 root root  18903 2011-09-18 17:37 kstat_n_sys.2011-09-18_1730.out
1234
     -rw-r--r-- 1 root root   2379 2011-09-18 17:37 profile-1971_5sec.2011-09-18_1730.out
1235
     -rw-r--r-- 1 root root 926103 2011-09-18 17:37 adaptive.2011-09-18_1730.out
1236
     -rw-r--r-- 1 root root    778 2011-09-18 17:37 syscall_cnt_by_syscall.2011-09-18_1730.out
1237
     -rw-r--r-- 1 root root   2136 2011-09-18 17:37 hotkernel_30sec.2011-09-18_1730.out
1238
     -rw-r--r-- 1 root root    480 2011-09-18 17:38 hotkernel_30sec_m.2011-09-18_1730.out
1239
     -rw-r--r-- 1 root root   1429 2011-09-18 17:38 cpuwalk.d_10.2011-09-18_1730.out
1240
     -rw-r--r-- 1 root root   1786 2011-09-18 17:38 intoncpu_10sec.2011-09-18_1730.out
1241
     -rw-r--r-- 1 root root    187 2011-09-18 17:39 inttimes_10sec.2011-09-18_1730.out
1242
     -rw-r--r-- 1 root root    154 2011-09-18 17:39 xcallsbypid_10sec.2011-09-18_1730.out
1243
     -rw-r--r-- 1 root root   1423 2011-09-18 17:39 sysbypid_10sec.2011-09-18_1730.out
1244
     -rw-r--r-- 1 root root   1305 2011-09-18 17:39 topsyscall-C-s_5_2.2011-09-18_1730.out
1245
     -rw-r--r-- 1 root root   1193 2011-09-18 17:40 iotop-C-t_10_5_4.2011-09-18_1730.out
1246
     -rw-r--r-- 1 root root    685 2011-09-18 17:40 cswstat_10sec.2011-09-18_1730.out
1247
     -rw-r--r-- 1 root root    273 2011-09-18 17:40 idle-state_10sec.2011-09-18_1730.out
1248
     -rw-r--r-- 1 root root    934 2011-09-18 17:40 watch_cpu_acpi_read_port.2011-09-18_1730.out
1249
     -rw-r--r-- 1 root root   3713 2011-09-18 17:41 watch_cstate.2011-09-18_1730.out
1250
     -rw-r--r-- 1 root root   2729 2011-09-18 17:41 speedstep.2011-09-18_1730.out
1251
     -rw-r--r-- 1 root root    136 2011-09-18 17:42 watch_acpi_cpu_cstate_60.2011-09-18_1730.out
1252
     -rw-r--r-- 1 root root    142 2011-09-18 17:43 watch_cpu_acpi_read_port_60.2011-09-18_1730.out
1253
     -rw-r--r-- 1 root root   1943 2011-09-18 17:43 cpudists.2011-09-18_1730.out
1254
     -rw-r--r-- 1 root root    159 2011-09-18 17:43 cputimes.2011-09-18_1730.out
1255
     -rw-r--r-- 1 root root    213 2011-09-18 17:43 dladm_show-link-s.2011-09-18_1730.out
1256
     -rw-r--r-- 1 root root    380 2011-09-18 17:43 dlstat_show-link-r.2011-09-18_1730.out
1257
     -rw-r--r-- 1 root root    104 2011-09-18 17:43 dlstat.2011-09-18_1730.out
1258
     -rw-r--r-- 1 root root   2706 2011-09-18 17:44 intrstat_5_3.2011-09-18_1730.out
1259
     -rw-r--r-- 1 root root   8074 2011-09-18 17:44 powertop-d_3-t_10.2011-09-18_1730.out
1260
     -rw-r--r-- 1 root root   1089 2011-09-18 17:45 lockstat-kIW-D_20_sleep_30.2011-09-18_1730.out
1261
     -rw-r--r-- 1 root root    195 2011-09-18 17:45 intrtime.pl.2011-09-18_1730.out
1262
     -rw-r--r-- 1 root root   1214 2011-09-18 17:45 mdb_k_interrupts.2011-09-18_1730.out
1263
     -rw-r--r-- 1 root root    627 2011-09-18 17:45 mdb_k_memstat.2011-09-18_1730.out
1264
     -rw-r--r-- 1 root root  39300 2011-09-18 17:45 mdb_k_kmastat.2011-09-18_1730.out
1265
     -rw-r--r-- 1 root root  23986 2011-09-18 17:45 mdb_k_stacks.2011-09-18_1730.out
1266
     -rw-r--r-- 1 root root    352 2011-09-18 17:45 kstat_m_cpu_info_curfrq.2011-09-18_1730.out
1267
     -rw-r--r-- 1 root root   2905 2011-09-18 17:45 warmredo_top_10sec.2011-09-18_1730.out
1268

    
1269

    
1270

    
1271

    
1272

    
1273

    
1274

    
1275

    
1276

    
1277

    
1278

    
1279

    
1280

    
1281

    
1282

    
1283

    
1284

    
(11-11/18)