illumos gate

Encountered a situation where the ldapsearch command was crashing. Re-running with UMEM_DEBUG=default provided a useful info:

> ::status
debugging core file of ldapsearch (32-bit) from xxxx
initial argv: /usr/bin/ldapsearch -v -h xxxx -p 636 -o mech=GSSAPI -o authid
threading model: native threads
status: process terminated by SIGABRT (Abort), pid=25451 uid=0 code=-1
> ::umem_status
Status:         ready and active
Concurrency:    64
Logs:           (inactive)
Message buffer:
free(fe1714f9): invalid or corrupted buffer
stack trace:
libumem.so.1'umem_err_recoverable+0x37
libumem.so.1'process_free+0x74
libumem.so.1'umem_malloc_free+0x1a
mech_krb5.so.1'krb5_send_tgs_basic+0x11e
mech_krb5.so.1'krb5_send_tgs2+0x27b
mech_krb5.so.1'krb5_get_cred_via_tkt+0x7b6
mech_krb5.so.1'krb5_get_cred_from_kdc_opt+0x2ab
mech_krb5.so.1'krb5_get_cred_from_kdc+0x1c
mech_krb5.so.1'krb5_get_credentials+0xfd
mech_krb5.so.1'get_credentials+0xb0
mech_krb5.so.1'new_connection+0x21f
mech_krb5.so.1'krb5_gss_init_sec_context+0x446
mech_krb5.so.1'k5glue_init_sec_context+0x3e
libgss.so.1'gss_init_sec_context+0xde
gssapi.so.1'gssapi_client_mech_step+0x11f
libsasl.so.1'sasl_client_step+0x93
libsasl.so.1'sasl_client_start+0x52e
libldap.so.5'nsldapi_sasl_do_bind+0xf3
libldap.so.5'ldap_sasl_interactive_bind_s+0x1f0
ldapsearch'ldaptool_bind+0x227
ldapsearch'main+0x1fb
ldapsearch'_start_crt+0x97
ldapsearch'_start+0x1a

Looking at what is being freed:

> ::stack
libc.so.1`_lwp_kill+0x15(1, 6, 0, 1, fea92000, fea70fbd)
libc.so.1`raise+0x2b(6)
libumem.so.1`umem_do_abort+0x53()
libumem.so.1`__umem_assert_failed(fea70fbd, fea71237, fe1714f9)
libumem.so.1`process_free+0x74(fe1714f9, 1, 0)
libumem.so.1`umem_malloc_free+0x1a(fe1714f9)
mech_krb5.so.1`krb5_send_tgs_basic+0x11e(81970c8, 81f4830, 8047408, 804725c)
...
> fe1714f9::whatis
fe1714f9 is mech_krb5.so.1`encode_krb5_kdc_req_body+0x23, in /usr/lib/gss/mech_krb5.so.1 [fe110000,fe1b2000)

So the code is passing in the address in the middle of encode_krb5_kdc_req_body() – obviously wrong. Looking at the implementation of krb5_send_tgs_basic, we see a number of stack allocated structures with members that are passed to free(3C) based on various conditions. None of the structures are initialized, so it seems very likely one of these stack-allocated structures just happens to contain the return address from an earlier function call and that is getting passed into free(3C).

To figure out which call in the function (there are several calls to free(3C)), disassembling krb5_send_tgs_basic(), we see the following key pieces:

mech_krb5.so.1`krb5_send_tgs_basic+0x33:call   -0x59c0f <PLT=mech_krb5.so.1`krb5_c_make_checksum>
mech_krb5.so.1`krb5_send_tgs_basic+0x38:addl   $0x20,%esp
mech_krb5.so.1`krb5_send_tgs_basic+0x3b:testl  %eax,%eax
mech_krb5.so.1`krb5_send_tgs_basic+0x3d:jne    +0xce    <mech_krb5.so.1`krb5_send_tgs_basic+0x111>
...
mech_krb5.so.1`krb5_send_tgs_basic+0x111:       movl   %eax,%esi
mech_krb5.so.1`krb5_send_tgs_basic+0x113:       subl   $0xc,%esp
mech_krb5.so.1`krb5_send_tgs_basic+0x116:       pushl  -0x1c(%ebp)
mech_krb5.so.1`krb5_send_tgs_basic+0x119:       call   -0x5a2c5 <PLT=libumem.so.1`free>

Based on that, it suggests this is the free(3C) in question:

     /* Generate checksum */
      if ((retval = krb5_c_make_checksum(context, context->kdc_req_sumtype,
                        &in_cred->keyblock,
                         KRB5_KEYUSAGE_TGS_REQ_AUTH_CKSUM,
                         in_data, &checksum))) {
      free(checksum.contents);
      return(retval);
      }

If we look at krb5_c_make_checksum(), we see:

krb5_error_code KRB5_CALLCONV
  krb5_c_make_checksum(krb5_context context, krb5_cksumtype cksumtype,
               const krb5_keyblock *key, krb5_keyusage usage,
               const krb5_data *input, krb5_checksum *cksum)
  {
      int i, e1, e2;
      krb5_data data;
      krb5_error_code ret = 0;
      size_t cksumlen;

      KRB5_LOG0(KRB5_INFO, "krb5_c_make_checksum() start.");

      for (i=0; i<krb5_cksumtypes_length; i++) {
      if (krb5_cksumtypes_list[i].ctype == cksumtype)
          break;
      }

      if (i == krb5_cksumtypes_length)
      return(KRB5_BAD_ENCTYPE);

      if (krb5_cksumtypes_list[i].keyhash)
      cksumlen = krb5_cksumtypes_list[i].keyhash->hashsize;
      else
      cksumlen = krb5_cksumtypes_list[i].hash->hashsize;

  #ifdef _KERNEL
      context->kef_cksum_mt = krb5_cksumtypes_list[i].kef_cksum_mt;
  #endif
      cksum->length = cksumlen;

      if ((cksum->contents = (krb5_octet *) MALLOC(cksum->length)) == NULL)
      return(ENOMEM);

So the first return will leave cksum uninitialized.