illumos gate

While debugging a problem in LX, I came across some peculiar epoll behavior where existing epoll FDs were nested under one "master" epoll fd which was used to event detection.

I wrote up a test program and our behavior appears to differ:

root@native-linux-host:~# ./a.out
wait1: 0 0
wait2: 1 0
ev: 1 200

[root@smartos-host ~]# ./a.out
wait1: 0 0
wait2: 0 0

Here's the epoll-noob program I wrote up to test it:

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <strings.h>
#include <sys/epoll.h>

int main()
{
        int ep1, ep2;
        int fds[2];
        struct epoll_event ev;
        struct epoll_event out[2];
        int res, i;

        pipe(fds);

        ev.events = EPOLLIN|EPOLLERR|EPOLLHUP;
        ev.data.fd = 100;
        ep1 = epoll_create1(0);
        epoll_ctl(ep1, EPOLL_CTL_ADD, fds[0], &ev);

        ev.events = EPOLLIN|EPOLLERR|EPOLLHUP;
        ev.data.fd = 200;
        ep2 = epoll_create1(0);
        epoll_ctl(ep2, EPOLL_CTL_ADD, ep1, &ev);

        bzero(out, sizeof (out));
        res = epoll_wait(ep2, out, 2, 500);
        printf("wait1: %d %d\n", res, errno);
        for (i = 0; i < res; i++) {
                printf("ev: %X %d\n", out[i].events, out[i].data.fd);
        }

        write(fds[1], &fds[0], sizeof (int));

        bzero(out, sizeof (out));
        res = epoll_wait(ep2, out, 2, 500);
        printf("wait2: %d %d\n", res, errno);
        for (i = 0; i < res; i++) {
                printf("ev: %X %d\n", out[i].events, out[i].data.fd);
        }

        return 0;
}

Like with event ports, we need the ability to nest instances of epoll.