illumos gate

Here is how to reproduce it:

1. cd /var/tmp ; wget http://www.cs.duke.edu/ari/fstress/download/fstress-export.tgz ; tar xf fstress-export.tgz ; cd /var/tmp/fstress
2. export FSTRESS_HOME=/var/tmp/fstress
3. replace src-testprogs/readdir-test.c with the following code:

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <rpc/rpc.h>

#include "porting.h" 
#include "nfs_constants.h" 
#include "dns.h" 
#include "msg.h" 
#include "rpc.h" 
#include "nfs.h" 

int
main(int argc, char *argv[])
{
        char *hostname = argv[1];
        int loops = 2000000000;
        struct in_addr addr;
        int sock, socktype = SOCK_STREAM;
        int i;
        int cnt_local = 0;
        int cnt_burst = 32;

        if (argc < 2) {
                fprintf(stderr, "usage: readdir-test <hostname> [loops] [cnt_burst]\n");
                return -1;
        }

        if (argc >= 3) loops = atoi(argv[2]);
        if (argc >= 4) cnt_burst = atoi(argv[3]);

        printf("loops = %d\n", loops);
        printf("bursts = %d\n", cnt_burst);

        if (dns_name2addr(hostname, &addr) != 0) {
                fprintf(stderr, "dns_name2addr(\"%s\", ...) failed\n",
                        hostname);
                return -1;
        }

        for (i = 0; i < loops; i++) {
                if ((sock = rpc_client(addr, NFS_PROG, NFS_VER3, socktype, 0)) < 0) {
                        fprintf(stderr, "rpc_client failed\n");
                        return -1;
                }

                close(sock);
                cnt_local++;
                if (cnt_local >= cnt_burst) {
                        cnt_local = 0;
                        sleep(1);
                }
        }
        return 0;
}

Run the following command to monitor the rpcbind memory usage:

# while true; do mdb -e ::umastat -p $(pgrep -f rpcbind) | grep 'umem_alloc_16 '; sleep 2; done

You will see that the 'bufs in use' keeps climbing up.

The memory leaking can be stopped by disabling libumem per thread caching. Just edit the start up script, /lib/svc/method/rpc-bind, change the startup line to be the following:

env UMEM_OPTIONS="perthread_cache=0" /usr/sbin/rpcbind > /dev/msglog 2>&1