Project

General

Profile

Bug #1333 » power_cpu_issue_PLUS_159ahours.txt

Jon Strabala, 2011-09-06 03:30 PM

 
1
root@lab10:~# date
2
September  6, 2011 07:21:51 AM PDT
3

    
4

    
5
root@lab10:~# tail -4 /etc/system
6
* August 30, 2011 02:26:39 PM PDT
7
* try to solve CPU creep issue
8
* https://defect.opensolaris.org/bz/show_bug.cgi?id=12799
9
set cpupm_tw_gov_interval=800000
10

    
11

    
12
########################################################
13
# Comment - the above 'cpupm_tw_gov_interval' value
14
# doesn't seem to stop the problem, but it might slow 
15
# the progresssion down
16
########################################################
17

    
18

    
19
root@lab10:~# rup lab10
20
          lab10    up  6 days, 15:41,    load average: 0.00, 0.00, 0.00
21

    
22

    
23

    
24
root@lab10:~# uname -a
25
SunOS lab10 5.11 oi_148b i86pc i386 i86pc Solaris
26

    
27

    
28

    
29
root@lab10:~# prstat -s cpu -n 5 > /tmp/prstat.s.cpu.n.5.out
30
^Croot@lab10:~# cat /tmp/prstat.s.cpu.n.5.out
31
   PID USERNAME  SIZE   RSS STATE  PRI NICE      TIME  CPU PROCESS/NLWP
32
  1204 root       59M   36M sleep   59    0   0:31:48 0.0% Xorg/3
33
  1657 gdm        87M   25M sleep   59    0   0:03:12 0.0% gdm-simple-gree/1
34
    10 root       16M   13M sleep   59    0   0:00:02 0.0% svc.startd/16
35
   894 root     6968K 5852K sleep   59    0   0:00:15 0.0% intrd/1
36
     1 root     2728K 1860K sleep   59    0   0:00:00 0.0% init/1
37
Total: 65 processes, 393 lwps, load averages: 0.00, 0.00, 0.00
38
   PID USERNAME  SIZE   RSS STATE  PRI NICE      TIME  CPU PROCESS/NLWP
39
  1204 root       59M   36M sleep   59    0   0:31:48 0.1% Xorg/3
40
  1657 gdm        87M   25M sleep   59    0   0:03:12 0.0% gdm-simple-gree/1
41
    10 root       16M   13M sleep   59    0   0:00:02 0.0% svc.startd/16
42
   894 root     6968K 5852K sleep   59    0   0:00:15 0.0% intrd/1
43
     1 root     2728K 1860K sleep   59    0   0:00:00 0.0% init/1
44
Total: 65 processes, 393 lwps, load averages: 0.00, 0.00, 0.00
45

    
46

    
47

    
48
root@lab10:~# mpstat 5 2
49
CPU minf mjf xcal  intr ithr  csw icsw migr smtx  srw syscl  usr sys  wt idl
50
  0    0   0    1   514  101   90    0    1    0    0     9    0  13   0  87
51
  1    0   0    0    81    9   38    0    1    0    0     7    0   2   0  98
52
  2    0   0    0    73    8   35    0    1    0    0     6    0   2   0  98
53
  3    0   0    0    67    7   32    0    1    0    0     5    0   2   0  98
54
  4    0   0    0    75    8   37    0    1    0    0     6    0   2   0  98
55
  5    0   0    0    64    6   31    0    1    0    0     6    0   2   0  98
56
  6    0   0    0    65    6   31    0    1    0    0     6    0   2   0  98
57
  7    0   0    0    65    6   31    0    1    0    0     5    0   2   0  98
58
CPU minf mjf xcal  intr ithr  csw icsw migr smtx  srw syscl  usr sys  wt idl
59
  0    0   0    0   513  101  128    0    1    1    0    36    0  26   0  73
60
  1    0   0    0    56    2   29    0    0    0    0     6    0   3   0  97
61
  2    0   0    0    34    3   15    0    0    0    0     6    0   2   0  98
62
  3    0   0    0    80    6   35    0    0    1    0     0    0   4   0  96
63
  4    5   0    0    59    1   30    0    1    1    0    61    0   3   0  97
64
  5    0   0    0    72    4   33    0    1    0    0     0    0   4   0  96
65
  6    0   0    0    82   14   40    0    0    0    0     0    0   5   0  95
66
  7    0   0    0    32    3   16    0    0    0    0    12    0   2   0  98
67

    
68

    
69

    
70
root@lab10:~#
71

    
72

    
73
root@lab10:~# powertop
74
OpenIndiana PowerTOP version 1.2   (C) 2009 Intel Corporation
75

    
76
Collecting data for 5.00 second(s)
77
root@lab10:~# vi /tmp/powertop.5.1
78
root@lab10:~# cat /tmp/powertop.5.1
79
                                                            OpenIndiana PowerTOP version 1.2
80

    
81
C-states (idle power)   Avg     Residency                                           P-states (frequencies)
82
C0 (cpu running)                (9.9%)                                              1600 Mhz        0.0%
83
C1                      0.0ms   (0.0%)                                              1800 Mhz        0.0%
84
C2                      0.0ms   (0.0%)                                              2000 Mhz        0.0%
85
C3                      2.4ms   (90.1%)                                             2200 Mhz        0.0%
86
                                                                                    2400 Mhz        0.0%
87
                                                                                    2600 Mhz        0.0%
88
                                                                                    2800 Mhz        0.0%
89
                                                                                    3000 Mhz        0.0%
90
                                                                                    3200 Mhz        0.0%
91
                                                                                    3400 Mhz        0.0%
92
                                                                                    3800 Mhz(turbo) 100.0%
93

    
94
Wakeups-from-idle per second: 3103.9    interval: 5.0s
95
no ACPI power usage estimate available
96

    
97
Top causes for wakeups:
98
91.8% (2848.9)                 sched :  <xcalls> unix`dtrace_xcall_func
99
 3.2% (100.2)               <kernel> :  genunix`clock
100
 2.5% ( 76.2)               <kernel> :  genunix`cv_wakeup
101
 1.6% ( 50.0)               <kernel> :  SDC`sysdc_update
102
 0.3% (  8.0)               <kernel> :  ehci`ehci_handle_root_hub_status_change
103
 0.3% (  8.0)               <kernel> :  cpudrv`cpudrv_monitor_disp
104
 0.1% (  4.0)               <kernel> :  genunix`schedpaging
105
 0.1% (  2.2)            <interrupt> :  e1000g#0
106
 0.0% (  1.0)               <kernel> :  TS`ts_update
107
 0.0% (  1.0)               <kernel> :  e1000g`e1000g_local_timer
108
 0.0% (  1.0)               <kernel> :  genunix`sigalarm2proc
109
 0.0% (  1.0)            <interrupt> :  ehci#0
110
 0.0% (  1.0)            <interrupt> :  ehci#1
111
 0.0% (  0.2)               <kernel> :  genunix`vmem_update
112
 0.0% (  0.2)               <kernel> :  ip`tcp_timer_callback
113
 0.0% (  0.2)               <kernel> :  swrand`rnd_handler
114
 0.0% (  0.2)                  sched :  <xcalls> unix`hati_demap_func
115
 0.0% (  0.2)               <kernel> :  genunix`kmem_update
116
 0.0% (  0.2)               <kernel> :  kcf`rnd_handler
117
 0.0% (  0.2)               <kernel> :  ahci`ahci_watchdog_handler
118

    
119
Suggestion: enable CPU power management by pressing the P key
120

    
121

    
122

    
123

    
124
root@lab10:~#
125

    
126

    
127
root@lab10:~# prtdiag
128
System Configuration: Supermicro X9SCI/X9SCA
129
BIOS Configuration: American Megatrends Inc. 4.6.4 02/24/2011
130
BMC Configuration: IPMI 2.0 (KCS: Keyboard Controller Style)
131

    
132
==== Processor Sockets ====================================
133

    
134
Version                          Location Tag
135
-------------------------------- --------------------------
136
Intel(R) Xeon(R) CPU E31270 @ 3.40GHz SOCKET 0
137

    
138
==== Memory Device Sockets ================================
139

    
140
Type        Status Set Device Locator      Bank Locator
141
----------- ------ --- ------------------- ----------------
142
Unknown     in use 0   DIMM_1A             BANK0
143
Unknown     in use 0   DIMM_2A             BANK0
144
Unknown     in use 0   DIMM_1B             BANK0
145
Unknown     in use 0   DIMM_2B             BANK0
146

    
147
==== On-Board Devices =====================================
148
   To Be Filled By O.E.M.
149

    
150
==== Upgradeable Slots ====================================
151

    
152
ID  Status    Type             Description
153
--- --------- ---------------- ----------------------------
154
0   in use    PCI Express      J6B2
155
1   in use    PCI Express      J6B1
156
2   in use    PCI Express      J6D1
157
3   in use    PCI Express      J7B1
158
4   in use    PCI Express      J8B4
159

    
160

    
161

    
162
root@lab10:~# cat /etc/power.conf
163
#
164
# Copyright 1996-2002 Sun Microsystems, Inc.  All rights reserved.
165
# Use is subject to license terms.
166
#
167
#pragma ident   "@(#)power.conf 2.1     02/03/04 SMI"
168
#
169
# Power Management Configuration File
170
#
171

    
172
cpu_deep_idle enable
173
cpu-threshold 10s
174
cpupm disable
175
cpupm disable
176

    
177

    
178

    
179
root@lab10:~# dtrace -n 'profile-10ms{@[stack()] = count()}'
180
dtrace: description 'profile-10ms' matched 1 probe
181
^C
182

    
183

    
184
              unix`mutex_enter+0x10
185
              unix`hment_mapcnt+0x1d
186
              unix`hat_page_getshare+0x16
187
              genunix`fsflush_do_pages+0x236
188
              genunix`fsflush+0x39a
189
              unix`thread_start+0x8
190
                1
191

    
192
              unix`x86pte_mapin+0x47
193
              unix`x86pte_access_pagetable+0x3d
194
              unix`x86pte_set+0x8d
195
              unix`hati_pte_map+0x187
196
              unix`hati_load_common+0x15d
197
              unix`hat_memload+0x81
198
              unix`segkmem_xalloc+0x13b
199
              unix`segkmem_alloc_vn+0xcd
200
              unix`segkmem_alloc+0x24
201
              genunix`vmem_xalloc+0x546
202
              genunix`vmem_alloc+0x161
203
              genunix`kmem_alloc+0x64
204
              genunix`kmem_zalloc+0x3b
205
              autofs`auto_calldaemon+0x22d
206
              autofs`auto_send_unmount_request+0x6b
207
              autofs`unmount_node+0x181
208
              autofs`try_unmount_node+0x13b
209
              autofs`unmount_subtree+0x79
210
              autofs`unmount_tree+0xcf
211
              autofs`unmount_zone_tree+0x1e
212
                1
213

    
214
              genunix`fsflush_do_pages+0x1da
215
              genunix`fsflush+0x39a
216
              unix`thread_start+0x8
217
                1
218

    
219
              unix`page_pptonum+0xc
220
              genunix`fsflush_do_pages+0x35d
221
              genunix`fsflush+0x39a
222
              unix`thread_start+0x8
223
                1
224

    
225
                7
226

    
227
              unix`i86_mwait+0xd
228
              unix`cpu_idle_mwait+0xf1
229
              unix`cpu_acpi_idle+0x8d
230
              unix`cpu_idle_adaptive+0x19
231
              unix`idle+0x114
232
              unix`thread_start+0x8
233
               10
234

    
235
              unix`acpi_cpu_cstate+0x2ae
236
              unix`cpu_acpi_idle+0x82
237
              unix`cpu_idle_adaptive+0x19
238
              unix`idle+0x114
239
              unix`thread_start+0x8
240
            10027
241

    
242

    
243

    
244
root@lab10:~#
245

    
246

    
247

    
248

    
249
root@lab10:~# echo ::interrupts | mdb -k
250
CPU/Vect  IRQ IPL Bus    Trg Type   Share APIC/INT# ISR
251
0/0x25    16  9   PCI    Lvl Fixed  1     0x0/0x10  ehci_intr
252
1/0x20    9   9   PCI    Lvl Fixed  1     0x0/0x9   acpi_wrapper_isr
253
1/0x21    23  9   PCI    Lvl Fixed  1     0x0/0x17  ehci_intr
254
1/0x22    4   12  ISA    Edg Fixed  1     0x0/0x4   asyintr
255
2/0x20    11  14  PCI    Lvl Fixed  1     0x0/0xb   hpet_isr
256
2/0x21    -   6   PCI    Edg MSI    1     -         e1000g_intr_pciexpress
257
3/0x20    -   5   PCI    Edg MSI    1     -         ahci_intr
258
3/0x21    3   12  ISA    Edg Fixed  1     0x0/0x3   asyintr
259
5/0x20    10  12  ISA    Edg Fixed  1     0x0/0xa   asyintr
260
6/0x20    1   5   ISA    Edg Fixed  1     0x0/0x1   i8042_intr
261
6/0x21    -   6   PCI    Edg MSI    1     -         e1000g_intr_pciexpress
262
7/0x20    12  5   ISA    Edg Fixed  1     0x0/0xc   i8042_intr
263
all/0xf0  -   15  -      Edg IPI    1     -         xc_serv
264
all/0xf1  -   11  -      Edg IPI    0     -         poke_cpu
265
all/0xf2  -   14  -      Edg IPI    1     -         kcpc_hw_overflow_intr
266
all/0xf3  -   15  -      Edg IPI    1     -         apic_error_intr
267
all/0xf4  -   2   -      Edg IPI    1     -         cmi_cmci_trap
268
all/0xf5  -   14  -      Edg IPI    1     -         cbe_fire
269
all/0xf6  -   14  -      Edg IPI    1     -         cbe_fire
270

    
271

    
272

    
273
root@lab10:~# echo ::interrupts -d | mdb -k
274
CPU/Vect  IRQ IPL Bus    Trg Type   Share APIC/INT# Driver Name(s)
275
0/0x25    16  9   PCI    Lvl Fixed  1     0x0/0x10  ehci#0
276
1/0x20    9   9   PCI    Lvl Fixed  1     0x0/0x9   acpi_wrapper_isr
277
1/0x21    23  9   PCI    Lvl Fixed  1     0x0/0x17  ehci#1
278
1/0x22    4   12  ISA    Edg Fixed  1     0x0/0x4   asy#0
279
2/0x20    11  14  PCI    Lvl Fixed  1     0x0/0xb   hpet_isr
280
2/0x21    -   6   PCI    Edg MSI    1     -         e1000g#0
281
3/0x20    -   5   PCI    Edg MSI    1     -         ahci#0
282
3/0x21    3   12  ISA    Edg Fixed  1     0x0/0x3   asy#1
283
5/0x20    10  12  ISA    Edg Fixed  1     0x0/0xa   asy#2
284
6/0x20    1   5   ISA    Edg Fixed  1     0x0/0x1   i8042#0
285
6/0x21    -   6   PCI    Edg MSI    1     -         e1000g#1
286
7/0x20    12  5   ISA    Edg Fixed  1     0x0/0xc   i8042#0
287
all/0xf0  -   15  -      Edg IPI    1     -         xc_serv
288
all/0xf1  -   11  -      Edg IPI    0     -         poke_cpu
289
all/0xf2  -   14  -      Edg IPI    1     -         kcpc_hw_overflow_intr
290
all/0xf3  -   15  -      Edg IPI    1     -         apic_error_intr
291
all/0xf4  -   2   -      Edg IPI    1     -         cmi_cmci_trap
292
all/0xf5  -   14  -      Edg IPI    1     -         cbe_fire
293
all/0xf6  -   14  -      Edg IPI    1     -         cbe_fire
294

    
295

    
296

    
297
root@lab10:~#
298

    
299

    
300
root@lab10:~# intrstat 30
301

    
302
      device |      cpu0 %tim      cpu1 %tim      cpu2 %tim      cpu3 %tim      cpu4 %tim      cpu5 %tim      cpu6 %tim      cpu7 %tim
303
-------------+------------------------------------------------------------------------------------------------------------------------
304
    e1000g#0 |         0  0.0         0  0.0         1  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0
305
      ehci#0 |         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0
306
      ehci#1 |         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0         0  0.0
307
^C
308

    
309

    
310

    
311
root@lab10:~#
312

    
313
root@lab10:~# powertop -t 30
314
OpenIndiana PowerTOP version 1.2   (C) 2009 Intel Corporation
315

    
316
Collecting data for 30.00 second(s)
317
root@lab10:~# vi /tmp/powertop.30.1
318
root@lab10:~# cat /tmp/powertop.30.1
319
                                                            OpenIndiana PowerTOP version 1.2
320

    
321
C-states (idle power)   Avg     Residency                                           P-states (frequencies)
322
C0 (cpu running)                (10.4%)                                             1600 Mhz        0.0%
323
C1                      0.0ms   (0.0%)                                              1800 Mhz        0.0%
324
C2                      0.0ms   (0.0%)                                              2000 Mhz        0.0%
325
C3                      2.4ms   (89.6%)                                             2200 Mhz        0.0%
326
                                                                                    2400 Mhz        0.0%
327
                                                                                    2600 Mhz        0.0%
328
                                                                                    2800 Mhz        0.0%
329
                                                                                    3000 Mhz        0.0%
330
                                                                                    3200 Mhz        0.0%
331
                                                                                    3400 Mhz        0.0%
332
                                                                                    3711 Mhz(turbo) 100.0%
333

    
334
Wakeups-from-idle per second: 3102.0    interval: 30.0s
335
no ACPI power usage estimate available
336

    
337
Top causes for wakeups:
338
91.8% (2848.9)                 sched :  <xcalls> unix`dtrace_xcall_func
339
 3.2% (100.0)               <kernel> :  genunix`clock
340
 2.5% ( 77.2)               <kernel> :  genunix`cv_wakeup
341
 1.6% ( 50.0)               <kernel> :  SDC`sysdc_update
342
 0.3% (  8.0)               <kernel> :  cpudrv`cpudrv_monitor_disp
343
 0.2% (  7.7)               <kernel> :  ehci`ehci_handle_root_hub_status_change
344
 0.1% (  4.0)               <kernel> :  genunix`schedpaging
345
 0.0% (  1.3)            <interrupt> :  e1000g#0
346
 0.0% (  1.0)               <kernel> :  TS`ts_update
347
 0.0% (  1.0)               <kernel> :  e1000g`e1000g_local_timer
348
 0.0% (  1.0)            <interrupt> :  ehci#1
349
 0.0% (  1.0)            <interrupt> :  ehci#0
350
 0.0% (  0.2)               <kernel> :  kcf`rnd_handler
351
 0.0% (  0.2)               <kernel> :  ahci`ahci_watchdog_handler
352
 0.0% (  0.2)               <kernel> :  swrand`rnd_handler
353
 0.0% (  0.1)                  sched :  <xcalls> unix`hati_demap_func
354
 0.0% (  0.1)               <kernel> :  ip`igmp_slowtimo
355
 0.0% (  0.1)               <kernel> :  genunix`vmem_update
356
 0.0% (  0.1)               <kernel> :  genunix`kmem_update
357
 0.0% (  0.0)               <kernel> :  ip`tcp_timer_callback
358

    
359
Suggestion: enable CPU power management by pressing the P key
360

    
361
root@lab10:~#
362

    
363

    
364
root@lab10:~# top -s 10
365
last pid:  3376;  load avg:  0.01,  0.01,  0.00;  up 6+15:48:02                                                                                 07:28:25
366
62 processes: 61 sleeping, 1 on cpu
367
CPU states: 93.4% idle,  0.1% user,  6.5% kernel,  0.0% iowait,  0.0% swap
368
Kernel: 319 ctxsw, 3 trap, 963 intr, 61 syscall, 2 flt
369
Memory: 16G phys mem, 14G free mem, 8187M total swap, 8187M free swap
370

    
371
   PID USERNAME NLWP PRI NICE  SIZE   RES STATE    TIME    CPU COMMAND
372
  1204 root        3  58    0   59M   36M sleep   31:49  0.04% Xorg
373
  1657 gdm         1  59    0   87M   25M sleep    3:12  0.00% gdm-simple-gree
374
  3375 root        1  59    0 3852K 2492K cpu/3    0:00  0.00% top
375
  1713 root       21  59    0 8428K 5332K sleep    0:04  0.00% nscd
376
  1733 admin       1  59    0 7776K 5188K sleep    0:01  0.00% sshd
377
  1642 gdm         1  59    0   82M   18M sleep    0:04  0.00% gnome-power-man
378
   264 root        6  59    0 4356K 3156K sleep    0:02  0.00% devfsadm
379
  1057 root        6  59    0   26M   10M sleep    0:02  0.00% fbserver
380
   761 root       26  59    0   18M   12M sleep    0:02  0.00% fmd
381
    10 root       16  59    0   16M   13M sleep    0:02  0.00% svc.startd
382
    12 root       22  59    0   16M   14M sleep    0:07  0.00% svc.configd
383
   755 root        4  59    0 6728K 3872K sleep    0:00  0.00% inetd
384
    45 netcfg      5  59    0 4660K 3660K sleep    0:01  0.00% netcfgd
385
   751 root        1  59    0 1620K  916K sleep    0:00  0.00% utmpd
386
   894 root        1  59    0 6968K 5852K sleep    0:15  0.00% intrd
387
  1726 root        4  59    0 3068K 1948K sleep    0:08  0.00% automountd
388
  1732 root        1  59    0 7152K 4536K sleep    0:04  0.00% sshd
389
   372 root        4  59    0 6912K 5548K sleep    0:03  0.00% hald
390
   401 root        1  59    0 3496K 2092K sleep    0:03  0.00% hald-addon-acpi
391
  1611 gdm         1  59    0   75M   13M sleep    0:01  0.00% metacity
392
  1596 gdm         1  59    0 7828K 6020K sleep    0:01  0.00% at-spi-registry
393
  1594 gdm         1  59    0 7152K 5948K sleep    0:01  0.00% gconfd-2
394
   753 root       11  59    0 4104K 2096K sleep    0:01  0.00% syslogd
395
   238 root        5  60  -20 2548K 1512K sleep    0:00  0.00% zonestatd
396
  1597 gdm         1  59    0   99M   35M sleep    0:00  0.00% gnome-settings-
397
  1581 gdm         2  59    0   15M   10M sleep    0:00  0.00% gnome-session
398
   885 root        4  59    0   10M 6652K sleep    0:00  0.00% rad
399
   289 root        1  59    0   11M 5684K sleep    0:00  0.00% cupsd
400
  1599 gdm         2  59    0 7140K 5120K sleep    0:00  0.00% bonobo-activati
401
    94 root        8  59    0 6320K 5108K sleep    0:00  0.00% nwamd
402

    
(3-3/18)