illumos gate

From http://smartos.org/bugview/OS-2689:

As it turns out, the problem here is -- in the end -- straightforward:
POLLHUP is not generated (ever) for disconnected sockets. This
is not seen more broadly because POLLIN is generated for these
sockets -- and most will poll on POLLIN as well as POLLOUT. In haproxy's
case, however, it is (legitimately) polling only on POLLOUT; should
a socket go through a clean but disorderly shutdown (e.g., an RST),
haproxy will never be made aware of the fact. This, in turn, causes
the backend-facing socket associated with the connection to backup
with accumulated data. We will therefore hold onto kernel memory at
a rate proportional to the the rate of RSTs -- currently on the
order of tens of per day (which, at 4MB of buffered data per, amounts
to on the order of a hundred megabytes per day). The workaround is
to restart haproxy; the fix is to generate POLLHUP when the socket
is clearly disconnected and can no longer read or write.

I used the test programs from http://oss.sgi.com/cgi-bin/extract-mesg.cgi?a=netdev&m=2002-08&i=20020816010809.GA28446%40anakin.wychk.org:

Here's poll.c:

#include <poll.h>
#include <sys/socket.h>

#include <netinet/in.h>

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h> /* strdup */
#include <unistd.h> /* getopt */
#include <strings.h>

#define PORT (22)

#define BACKLOG 20

#ifndef INFTIM
#define INFTIM (-1)
#endif

extern char *optarg;

int main(argc, argv)
        int argc;
        char *argv[];
{
        struct pollfd nfd;
        struct sockaddr_in saddr, caddr;
        int fd;
        int c;
        int clientfd;
        size_t len;
        unsigned short port = PORT;

        bzero(&saddr, sizeof(struct sockaddr_in));
        bzero(&caddr, sizeof(struct sockaddr_in));

        while ((c = getopt(argc, argv, "h:p:")) != -1) {
                switch (c) {
                        case 'p':
                                port = atoi(optarg);
                                break;
                        default:
                                break;
                }
        }

        saddr.sin_family = AF_INET;
        saddr.sin_port   = htons(port);
        saddr.sin_addr.s_addr   = INADDR_ANY;

        fd = socket(AF_INET, SOCK_STREAM, 0);

        if (fd < 0) {
                printf("socket\\\\n");
                exit(1);
        }

        if (bind(fd, (struct sockaddr *)&saddr, sizeof(struct sockaddr_in)) < 0) {
                printf("bind\\\\n");
                exit(1);
        }

        if (listen(fd, BACKLOG) < 0) {
                printf("listen\\\\n");
                exit(1);
        }

        len = sizeof(struct sockaddr_in);

        if ((clientfd = accept(fd, (struct sockaddr *)&caddr, &len)) < 0) {
                printf("accept: %s\\\\n", strerror(errno)), exit(1);
        }

        printf("accept ok.\\\\n");

        /* reset */     
        nfd.fd = clientfd;
        nfd.events = 0;
        nfd.revents = 0;

        if (poll(&nfd, 1, INFTIM) < 0) 
                printf("poll: %s\\\\n", strerror(errno)), exit(1);

        if (nfd.revents & POLLERR) 
                printf("poll: POLLERR set\\\\n");

        if (nfd.revents & POLLHUP)
                printf("poll: POLLHUP set\\\\n");

        if (nfd.revents & POLLNVAL)
                printf("poll: POLLNVAL set\\\\n"); /* should not happen */

        close(clientfd);
        printf("sever terminating\\\\n");
        exit(0);
}

And linger.c:

#include <sys/socket.h>

#include <netinet/in.h>

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h> /* getopt */
#include <strings.h>

#define HOST "192.168.0.1" 
#define PORT (22)

int main(argc, argv)
        int argc;
        char *argv[];
{
        int sockfd;
        int c;
        struct linger ling;
        struct sockaddr_in servaddr;
        char *host = HOST;
        unsigned short port = PORT;

        while ((c = getopt(argc, argv, "h:p:")) != -1) {
                switch (c) {
                        case 'h':
                                host = (char *)strdup(optarg);
                                break;
                        case 'p':
                                port = atoi(optarg);
                                break;
                        default:
                                break;
                }
        }

        sockfd = socket(AF_INET, SOCK_STREAM, 0);

        if (sockfd < 0) {
                printf("socket\\\\n");
                exit(1);
        }

        bzero(&servaddr, sizeof(struct sockaddr_in));

        servaddr.sin_family = AF_INET;
        servaddr.sin_port   = htons(port);
        servaddr.sin_addr.s_addr   = inet_addr(host);

        ling.l_onoff = 1;
        ling.l_linger = 0;

        if (setsockopt(sockfd, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)) < 0) {
                printf("setsockopt\\\\n");
                exit(1);
        }

        printf("connecting ..\\\\n");
        if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(struct sockaddr_in)) < 0) {
                printf("connect: %s\\\\n", strerror(errno));
                exit(1);
        }

        sleep(10);

        printf("closing ...\\\\n");
        close(sockfd);  /* send RST */
        printf("closed.\\\\n");

        exit(0);
}

I ran them over localhost ("./poll -p 8000" and "./linger -h 127.0.0.1 -p 8000") and confirmed that Linux behavior now is what it was 12 years ago – and that our behavior now matches it with respect to POLLHUP.