Project

General

Profile

Bug #4196

panic in mac_reserve_rx_groups() with two vnics, same mac address, different vlans

Added by Rafael Vanoni about 7 years ago. Updated about 7 years ago.

Status:
In Progress
Priority:
Normal
Assignee:
Category:
networking
Start date:
2013-10-10
Due date:
% Done:

0%

Estimated time:
Difficulty:
Medium
Tags:
needs-triage
Gerrit CR:

Description

We hit the following panic on reboot after creating two vnics with the same mac address on separate vlans. This happened before the dump device was configured, so no dump is available.

mac_reserve_rx_group+0x2eb(ffffff03e4256958, ffffff03e4287f94, 0)
mac_datapath_setup+0x1a6(ffffff03e4256958, ffffff03e4281b78, 1)
mac_client_datapath_setup+0x2cc(ffffff03e4256958, 0, ffffff03e4041590, ffffff03e1c32000, 1, ffffff0475a259d0)
i_mac_unicast_add+0x5ba(ffffff03e4256958, 0, 52, ffffff03e4237d38, 0, ffffff001716da8c)
mac_unicast_add+0x68(ffffff03e4256958, 0, 52, ffffff03e4237d38, 0, ffffff001716da8c)
dls_mac_active_set+0x45(ffffff03e4237c20)
dls_active_set+0x34(ffffff03e435bbe8)
proto_bind_req+0x73(ffffff03e435bbe8, ffffff047a7f9ac0)
dld_proto+0x90(ffffff03e435bbe8, ffffff047a7f9ac0)
dld_wput_nondata_task+0x8b(ffffff03e435bbe8)
taskq_d_thread+0xb1(ffffff047ed44cb8)
thread_start+8()

We were able to workaround the problem by moving datalink.conf aside from a previous boot environment.

#1

Updated by Rafael Vanoni about 7 years ago

  • Status changed from New to In Progress

The NULL ptr dereference happened right after a call to mac_get_grp_primary():

> mac_reserve_rx_group+0x2eb::dis
mac_reserve_rx_group+0x2da: movq 0x8(%r8),%rax
mac_reserve_rx_group+0x2de: testq %rax,%rax
mac_reserve_rx_group+0x2e1: jne +0x8 <mac_reserve_rx_group+0x2eb>
mac_reserve_rx_group+0x2e3: movq %r14,%rdi
mac_reserve_rx_group+0x2e6: call 0xcf3 <mac_get_grp_primary>
mac_reserve_rx_group+0x2eb: movq 0x108(%rax),%r8 <---

mac_reserve_rx_group+0x2f2: testq %r8,%r8
mac_reserve_rx_group+0x2f5: je +0x6 &lt;mac_reserve_rx_group+0x2fd&gt;
mac_reserve_rx_group+0x2f7: addq $0xc,%r8
> $r
%rax = 0x0000000000000000 %r9 = 0x0000000000000000 <----
%rbx = 0x0000000000000000 %r10 = 0xffffffffffffffea
%rcx = 0x0000000000000010 %r11 = 0x0000000000000000
%rdx = 0x0000000000000003 %r12 = 0xffffff03e40414f8
%rsi = 0x0000000000000001 %r13 = 0x0000000000000001
%rdi = 0xffffff03e40ac0d0 %r14 = 0xffffff03e40ac0d0
%r8 = 0xffffff03e425a008 %r15 = 0x0000000000000090
%rip = 0xfffffffff7809233 mac_reserve_rx_group+0x2eb
%rbp = 0xffffff001716d7f0
%rsp = 0xffffff001716d760
%rflags = 0x00010246
id=0 vip=0 vif=0 ac=0 vm=0 rf=1 nt=0 iopl=0x0
status=&lt;of,df,IF,tf,sf,ZF,af,PF,cf&gt;
%cs = 0x0030           %ds = 0x004b    %es = 0x004b
%trapno = 0xe %fs = 0x0000 %gs = 0x01c3
%err = 0x0

There are two call sites for this routine in mac_reserve_rx_group(), we're looking at the first one:

> mac_reserve_rx_group::dis ! egrep mac_get_grp_primary
mac_reserve_rx_group+0x2e6: call -0xcf3 &lt;mac_get_grp_primary&gt;
mac_reserve_rx_group+0x3f9: call -0xe06 &lt;mac_get_grp_primary&gt;

Which maps to the following portion of the source:

usr/src/uts/common/io/mac/mac.c: mac_reserve_rx_group(), starting around line 6277

for (i = 1; i < mip->mi_rx_group_count; i++) {
grp = &mip->mi_rx_groups[i];
DTRACE_PROBE3(rx__group__trying, char *, mip->mi_name,
int, grp->mrg_index, mac_group_state_t, grp->mrg_state);
/*
 * Check if this group could be a candidate group for
 * eviction if we need a group for this MAC client,
 * but there aren't any. A candidate group is one
 * that didn't ask for an exclusive group, but got
 * one and it has enough rings (combined with what
 * the donor group can donate) for the new MAC
 * client
/
if (grp->mrg_state >= MAC_GROUP_STATE_RESERVED) {
/
* If the primary/donor group is not the default * group, don't bother looking for a candidate group. * If we don't have enough rings we will check * if the primary group can be vacated.
*/
if (candidate_grp NULL &&
donorgrp MAC_DEFAULT_RX_GROUP(mip)) {
ASSERT(!MAC_GROUP_NO_CLIENT(grp));
gclient = MAC_GROUP_ONLY_CLIENT(grp);
if (gclient == NULL)
gclient = mac_get_grp_primary(grp);
ASSERT(gclient != NULL);
gmrp = MCIP_RESOURCE_PROPS(gclient); <--- NULL ptr deref
if (gclient->mci_share NULL &&

The 0x108 offset in the assembly is for the 'mci_flent' field of the mac_impl_t type, so we were in

#define MCIP_RESOURCE_PROPS(mcip)               \\ 
((mcip)->mci_flent NULL ? NULL : \\
&(mcip)->mci_flent->fe_resource_props)
> ::print -a mac_client_impl_t
0 {
0 mci_client_next
8 mci_name
108 mci_flent

Looking at the assembly just before the call, the 'grp' variable had been stored in %r14 and %rdi, so we know that the following macro returned NULL:

#define MAC_GROUP_ONLY_CLIENT(g)                        \\
((((g)->mrg_clients != NULL) && \\
((g)->mrg_clients->mgc_next == NULL)) ? \\
(g)->mrg_clients->mgc_client : NULL)
> 0xffffff03e40ac0d0::print -t mac_group_t
mac_group_t {
int mrg_index = 0x1
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 3 (MAC_GROUP_STATE_SHARED)
mac_group_t *mrg_next = 0xffffff03e40ac160
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cda88
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0xffffff03e1d770b8
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e40837d8
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
> 0xffffff03e40ac0d0::print -t mac_group_t mrg_clients | ::print mac_grp_client_t {
mgc_next = 0xffffff03e50a3a38
mgc_client = 0xffffff03e4254898
}

Here's mac_get_grp_primary():

static mac_client_impl_t *      
mac_get_grp_primary(mac_group_t *grp) {
mac_grp_client_t *mgcp = grp->mrg_clients;
mac_client_impl_t *mcip;
while (mgcp != NULL) {
mcip = mgcp->mgc_client;
if (mcip->mci_flent->fe_type & FLOW_PRIMARY_MAC)
return (mcip);
mgcp = mgcp->mgc_next;
}
return (NULL);
}

Walking the lit of mac_client_impl_t's, we see that these are of type FLOW_VNIC_MAC:

> 0xffffff03e40ac0d0::print t mac_group_t mrg_clients | ::list mac_grp_client_t mgc_next | ::print mac_grp_client_t mgc_client | ::print mac_client_impl_t mci_flent>fe_type
mci_flent->fe_type = 0x2
mci_flent->fe_type = 0x2

Here are the rx groups:

ffffff03e4256958::print -t mac_client_impl_t mci_mip | ::print -t mac_impl_t m

i_rx_groups | ::array mac_group_t 0t10 | ::print -t mac_group_t
mac_group_t {
int mrg_index = 0
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 3 (MAC_GROUP_STATE_SHARED)
mac_group_t *mrg_next = 0xffffff03e40ac0d0
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cdc08
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e40837c0
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x1
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 3 (MAC_GROUP_STATE_SHARED)
mac_group_t *mrg_next = 0xffffff03e40ac160
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cda88
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0xffffff03e1d770b8
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e40837d8
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x2
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac1f0
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd908
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e40837f0
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x3
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac280
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd788
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083808
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x4
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac310
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd608
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083820
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x5
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac3a0
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd488
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083838
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x6
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac430
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd308
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083850
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x7
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac4c0
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd188
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083868
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x8
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac550
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e41cd008
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083880
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}
mac_group_t {
int mrg_index = 0x9
mac_ring_type_t mrg_type = 0x1 (MAC_RING_TYPE_RX)
mac_group_state_t mrg_state = 1 (MAC_GROUP_STATE_REGISTERED)
mac_group_t *mrg_next = 0xffffff03e40ac5e0
mac_handle_t mrg_mh = 0xffffff03e40414f8
mac_ring_t *mrg_rings = 0xffffff03e421bd90
uint_t mrg_cur_count = 0x2
mac_grp_client_t *mrg_clients = 0
mac_group_info_t mrg_info = {
mac_group_driver_t mgi_driver = 0xffffff03e4083898
mac_group_start_t mgi_start = 0
mac_group_stop_t mgi_stop = 0
uint_t mgi_count = 0x2
mac_intr_t mgi_intr = {
mac_intr_handle_t mi_handle = 0
mac_intr_enable_t mi_enable = 0
mac_intr_disable_t mi_disable = 0
ddi_intr_handle_t mi_ddi_handle = 0
boolean_t mi_ddi_shared = 0 (0)
}
mac_add_mac_addr_t mgi_addmac = ixgbe_addmac
mac_rem_mac_addr_t mgi_remmac = ixgbe_remmac
}
}

..and their clients:

ffffff03e4256958::print -t mac_client_impl_t mci_mip | ::print -t mac_impl_t m

i_rx_groups | ::array mac_group_t 0t10 | ::print -t mac_group_t mrg_clients
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0xffffff03e1d770b8
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0
mac_grp_client_t *mrg_clients = 0

0xffffff03e1d770b8::list mac_grp_client_t mgc_next | ::print mac_grp_client_t

{
mgc_next = 0xffffff03e50a3a38
mgc_client = 0xffffff03e4254898
} {
mgc_next = 0
mgc_client = 0xffffff03e42550c8
}

#2

Updated by Rafael Vanoni about 7 years ago

Here's the suggested fix, I'll follow up with a code review request.

diff r e25397bf4bdc -r 929a9900d2fa usr/src/uts/common/io/mac/mac.c
--
a/usr/src/uts/common/io/mac/mac.c Fri Aug 15 12:21:46 2014 0700
+++ b/usr/src/uts/common/io/mac/mac.c Mon Aug 18 13:39:00 2014 -0700
@ -6110,6 +6110,10 @
DTRACE_PROBE3(rx__group__trying, char *, mip
>mi_name,
int, grp->mrg_index, mac_group_state_t, grp->mrg_state);

+ if (grp->mrg_state >= MAC_GROUP_STATE_RESERVED &&
+ mip->mi_rx_group_type != MAC_GROUP_TYPE_DYNAMIC)
+ continue;
+
/* * Check if this group could be a candidate group for * eviction if we need a group for this MAC client,
@ -6119,7 +6123,7 @ * the donor group can donate) for the new MAC * client
/
- if (grp->mrg_state >= MAC_GROUP_STATE_RESERVED && !isprimary) {
+ if (grp->mrg_state >= MAC_GROUP_STATE_RESERVED) {
/
* If the primary/donor group is not the default * group, don't bother looking for a candidate group.
@ -6204,6 +6208,9 @
/* We didn't find an exclusive group for this MAC client */
if (i >= mip->mi_rx_group_count) {

+ if (mip->mi_rx_group_type != MAC_GROUP_TYPE_DYNAMIC)
+ return (MAC_DEFAULT_RX_GROUP(mip));
+
if (!need_exclgrp)
return (NULL);

Also available in: Atom PDF