illumos gate

webrev: http://cr.illumos.org/~webrev/yyang/7804/

The following program can be used to reproduce the issue:

# cat rdisk.c
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

int main(int argc, char **argv)
{
        char *disk = NULL;
        int fd;
        uchar_t *buf;
        int requested_size = 512;
        int bytes;

        if (argc < 2) {
                printf("Usage: %s /dev/rdsk/... [bytes to read]\n");
                exit(1);
        }

        disk = argv[1];
        if (argc > 2) requested_size = atoi(argv[2]);
        buf = calloc(requested_size, 1);
        if (buf == NULL) {
                printf("No memory\n");
                exit(1);
        }

        fd = open(disk, O_RDONLY);
        if (fd == -1) {
                perror("open");
                free(buf);
                exit(1);
        }
        if (lseek(fd, 0, SEEK_SET) < 0) {
                perror("lseek");
                close(fd);
                free(buf);
                exit(1);
        }
        bytes = read(fd, buf, requested_size);
        if (bytes == -1) perror("read");
        printf("asked %d, got %d, buf ptr = %p, pid %d\n", requested_size, bytes, buf, getpid());
        close(fd);
        free(buf);
        return(0);
}

Build it: gcc -o rdisk rdisk.c

In one terminal window, run

# dtrace -n 'nvme_create_nvm_cmd:entry /arg1 == 2 && args[2]->x_blkno == 0 && args[2]->x_nblks == 1/ {stack();printf("blkno %d nblks %d ndmac %d dmac_size %d dmac_laddress %p pid %d",args[2]->x_blkno, args[2]->x_nblks, args[2]->x_ndmac, args[2]->x_dmac.dmac_size, args[2]->x_dmac._dmu._dmac_ll, pid);}'

In another window, run the program, it crashes

# LD_PRELOAD=libumem.so UMEM_DEBUG=audit /var/tmp/rdisk /dev/rdsk/c1t1d0p0 512
asked 512, got 512, buf ptr = 4ffbd0, pid 4124
Abort (core dumped)

and in dtrace window, you will get something like the following:

CPU     ID                    FUNCTION:NAME
  1  66976        nvme_create_nvm_cmd:entry
              nvme`nvme_bd_cmd+0x45
              nvme`nvme_bd_read+0x1a
              blkdev`bd_sched+0x96
              blkdev`bd_submit+0x57
              blkdev`bd_tg_rdwr+0xee
              cmlb`cmlb_use_efi+0xc0
              cmlb`cmlb_validate_geometry+0xcb
              cmlb`cmlb_validate+0x83
              blkdev`bd_open+0xd2
              genunix`dev_open+0x30
              specfs`spec_open+0x206
              genunix`fop_open+0x89
              genunix`vn_openat+0x234
              genunix`copen+0x1fc
              genunix`openat+0x2a
              genunix`open+0x25
              unix`sys_syscall+0x1a2
blkno 0 nblks 1 ndmac 1 dmac_size 4096 dmac_laddress 5f7653f000 pid 4124
  1  66976        nvme_create_nvm_cmd:entry
              nvme`nvme_bd_cmd+0x45
              nvme`nvme_bd_read+0x1a
              blkdev`bd_sched+0x96
              blkdev`bd_submit+0x57
              blkdev`bd_tg_rdwr+0xee
              cmlb`cmlb_read_fdisk+0x7d
              cmlb`cmlb_validate_geometry+0x1e6
              cmlb`cmlb_validate+0x83
              blkdev`bd_open+0xd2
              genunix`dev_open+0x30
              specfs`spec_open+0x206
              genunix`fop_open+0x89
              genunix`vn_openat+0x234
              genunix`copen+0x1fc
              genunix`openat+0x2a
              genunix`open+0x25
              unix`sys_syscall+0x1a2
blkno 0 nblks 1 ndmac 1 dmac_size 4096 dmac_laddress 5f6e1c8000 pid 4124
  1  66976        nvme_create_nvm_cmd:entry
              nvme`nvme_bd_cmd+0x45
              nvme`nvme_bd_read+0x1a
              blkdev`bd_sched+0x96
              blkdev`bd_submit+0x57
              blkdev`bd_strategy+0x13c
              genunix`default_physio+0x2db
              genunix`physio+0x25
              blkdev`bd_read+0x34
              genunix`cdev_read+0x2d
              specfs`spec_read+0x2b9
              genunix`fop_read+0xf3
              genunix`read+0x2a7
              unix`sys_syscall+0x1a2
blkno 0 nblks 1 ndmac 1 dmac_size 512 dmac_laddress 5e46c16bd0 pid 4124

mdb the core indicates there was memory corruption:

# mdb /zones/global/cores/core.rdisk.4124
Loading modules: [ libumem.so.1 libc.so.1 ld.so.1 ]
> $C
ffffbf7fffdff890 libc.so.1`_lwp_kill+0xa()
ffffbf7fffdff8c0 libc.so.1`raise+0x20(6)
ffffbf7fffdff8d0 libumem.so.1`umem_do_abort+0x44()
ffffbf7fffdff9d0 0xffffbf7ffec28025()
ffffbf7fffdffa70 libumem.so.1`umem_error+0x1aa(1, 48e028, 4ffbc0)
ffffbf7fffdffa90 libumem.so.1`umem_free+0xa8(4ffbc0, 210)
ffffbf7fffdffae0 libumem.so.1`process_free+0x145(4ffbd0, 1, 0)
ffffbf7fffdffb00 libumem.so.1`umem_malloc_free+0x1d(4ffbd0)
ffffbf7fffdffb40 main+0x17d()
ffffbf7fffdffb50 _start+0x6c()

Why it crashes? The user space 512 bytes buffer is DMA mapped by bd_strategy() -> bd_xfer_alloc(), and that DMA memory address will be used by NVMe driver to store the 1st block of the device. However, since the drive has been formatted to be 4k block size, so the minimum size transferred by NVMe device is 4K. So even though the application only requests 512 bytes, but NVMe driver still reads 4K data into the buffer.