Project

General

Profile

Bug #8593

zfs panic: assertion failed: ldi_strategy(dvd->vd_lh, bp) == 0, file: ../../common/fs/zfs/vdev_disk.c, line: 801

Added by Igor Kozhukhov almost 3 years ago.

Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Start date:
2017-08-29
Due date:
% Done:

0%

Estimated time:
Difficulty:
Medium
Tags:
needs-triage
Gerrit CR:

Description

iSCSI COMSTAR
one initiator + 3 targets.
10G network with ixgbe
GNU dd tests are working with one small/big files - 1G/10G/100G - as well
FIO random tests are failed

root@con3:/var/crash/con3# mdb *.0
Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc apix scsi_vhci zfs sata sd ip hook neti sockfs arp usba stmf stmf_sbd mm mpt_sas idm random crypto ptm ufs logindmux smbsrv nfs nsmb ipc ]
> $C
ffffd001ee6368c0 vpanic()
ffffd001ee636910 0xfffffffffbaac725()
ffffd001ee636950 vdev_disk_io_start+0x11d(ffffd064bae2a420)
ffffd001ee6369a0 zio_vdev_io_start+0xad(ffffd064bae2a420)
ffffd001ee6369d0 zio_execute+0x76(ffffd064bae2a420)
ffffd001ee636a10 vdev_queue_io_done+0x5d(ffffd064afa61468)
ffffd001ee636a50 zio_vdev_io_done+0xa0(ffffd064afa61468)
ffffd001ee636a80 zio_execute+0x76(ffffd064afa61468)
ffffd001ee636b30 taskq_thread+0x2a6(ffffd0632ef3fb20)
ffffd001ee636b40 thread_start+8()
> 
> ::status
debugging crash dump vmcore.0 (64-bit) from con3
operating system: 5.11 1.3.7.199 (i86pc)
image uuid: 153ad3d9-d8ad-4a5a-8fbe-ebda6206d38b
panic message: assertion failed: ldi_strategy(dvd->vd_lh, bp) == 0, file: ../../common/fs/zfs/vdev_disk.c, line: 801
dump content: kernel pages only

root@con3:/var/crash/con3# cat 01.txt 
ffffd001ee6368c0 vpanic()
ffffd001ee636910 0xfffffffffbaac725()
ffffd001ee636950 vdev_disk_io_start+0x11d(ffffd064bae2a420)
ffffd001ee6369a0 zio_vdev_io_start+0xad(ffffd064bae2a420)
ffffd001ee6369d0 zio_execute+0x76(ffffd064bae2a420)
ffffd001ee636a10 vdev_queue_io_done+0x5d(ffffd064afa61468)
ffffd001ee636a50 zio_vdev_io_done+0xa0(ffffd064afa61468)
ffffd001ee636a80 zio_execute+0x76(ffffd064afa61468)
ffffd001ee636b30 taskq_thread+0x2a6(ffffd0632ef3fb20)
ffffd001ee636b40 thread_start+8()
root@con3:/var/crash/con3# cat 02.txt 
MESSAGE                                                               
WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adeb1180000599164520004 (sd22):
        incomplete write- retrying

WARNING: /scsi_vhci/disk@g600144f07adedf00000059916fe40003 (sd15):
        incomplete write- retrying

WARNING: /scsi_vhci (scsi_vhci0):
        /scsi_vhci/disk@g600144f07adede1e000059916c570001 (sd9): Command Timeout on path iscsi0/disk@0000iqn.2010-08.org.illumos:02:5fd770c8-8b41-441c-a48f-91d6a2013a490001,0
WARNING: /scsi_vhci (scsi_vhci0):
        /scsi_vhci/disk@g600144f07adede1e000059916c630004 (sd18): Command Timeout on path iscsi0/disk@0000iqn.2010-08.org.illumos:02:5fd770c8-8b41-441c-a48f-91d6a2013a490001,3
WARNING: iscsi connection(27/3f) closing connection - target requested reason:0x7
NOTICE: iscsi session(26) iqn.2010-08.org.illumos:02:5fd770c8-8b41-441c-a48f-91d6a2013a49 offline

WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x1
/scsi_vhci/disk@g600144f07adede1e000059916c670005 (sd21) offline
/scsi_vhci/disk@g600144f07adede1e000059916c670005 (sd21) multipath status: failed: path 22 iscsi0/disk@0000iqn.2010-08.org.illumos:02:5fd770c8-8b41-441c-a48f-91d6a2013a490001,4 is offline
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: vhci_scsi_reset 0x1
WARNING: vhci_scsi_reset 0x0
WARNING: /scsi_vhci/disk@g600144f07adede1e000059916c630004 (sd18):
        SYNCHRONIZE CACHE command failed (5)

panic[cpu15]/thread=ffffd001ee636c40: 
assertion failed: ldi_strategy(dvd->vd_lh, bp) == 0, file: ../../common/fs/zfs/vdev_disk.c, line: 801

ffffd001ee636910 fffffffffbaac725 ()
ffffd001ee636950 zfs:vdev_disk_io_start+11d ()
ffffd001ee6369a0 zfs:zio_vdev_io_start+ad ()
ffffd001ee6369d0 zfs:zio_execute+76 ()
ffffd001ee636a10 zfs:vdev_queue_io_done+5d ()
ffffd001ee636a50 zfs:zio_vdev_io_done+a0 ()
ffffd001ee636a80 zfs:zio_execute+76 ()
ffffd001ee636b30 genunix:taskq_thread+2a6 ()
ffffd001ee636b40 unix:thread_start+8 ()

dumping to /dev/zvol/dsk/data/dump, offset 65536, content: kernel

Details:
Precondition:
1) create a zpool
tstpool3 - 4 disks from 3 shelves (lbsize 512 only) - (#8: SHELF8) (#13: SHELF7) (#14: SHELF7) (#18: SHELF4)

root@con3:~# zpool create -f tstpool3 c0t600144F07ADEB1180000599164520004d0 c0t600144F07ADEDE1E000059916C570001d0 c0t600144F07ADEDE1E000059916C630004d0 c0t600144F07ADEDF00000059916FE40003d0
root@con3:~# zpool list -v tstpool3
NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
tstpool3 65T 95K 65.0T - 0% 0% 1.00x ONLINE -
c0t600144F07ADEB1180000599164520004d0 16.2T 35.5K 16.2T - 0% 0%
c0t600144F07ADEDE1E000059916C570001d0 16.2T 24K 16.2T - 0% 0%
c0t600144F07ADEDE1E000059916C630004d0 16.2T 0 16.2T - 0% 0%
c0t600144F07ADEDF00000059916FE40003d0 16.2T 35.5K 16.2T - 0% 0%

root@con3:~# zdb tstpool3 | grep ashift
ashift: 9
ashift: 9
ashift: 9
ashift: 9
ashift: 9
ashift: 9
ashift: 9
ashift: 9
loading space map for vdev 3 of 4, metaslab 0 of 130 ...

2) create dataset
root@con3:~# zfs create tstpool3/space1

Steps:
Run fio write test on dataset tstpool3/space1

1.1) run the following script
./init_write_rand_1G_10G.sh tstpool3 space1

$ cat init_write_rand_1G_10G.sh
#!/bin/bash

if [ $# != 2 ]; then
 echo "Usage: $0 <tstpool> <space>" 
 exit 1
fi

POOL=$1
SPACE=$2

for file in conf/write_rand/*;
do
if [ -f $file ]; then
. $file;
echo -e "\n" 
echo "RUNPATH ${RUNPATH}" 
echo "RESPATH ${RESPATH}" 

mkdir -p ${RESPATH}

FILENAME=$(echo $RESPATH | cut -d'/' -f5-)
echo "FILENAME ${FILENAME}" 

echo -e "\n" 
echo "Started for ${RUNPATH} 1job" 
for ((i=1; i<=10; i++))
do
rm -rf /${POOL}/${SPACE}/*
wait
echo "Randwrite-J1 run $i" 2>&1 | tee ${RESPATH}/Wrand_1j_r$i.txt
wait
fio ${RUNPATH}/write_rand_1j.fio --filename=/${POOL}/${SPACE}/W${FILENAME}1j$i 2>&1 | tee ${RESPATH}/Wrand_1j_r$i.txt
grep "iops" ${RESPATH}/Wrand_1j_r$i.txt >> ${RESPATH}/res_Wrand_1j.txt
grep "clat (" ${RESPATH}/Wrand_1j_r$i.txt >> ${RESPATH}/res_Wrand_1j.txt
zfs list -H ${POOL}/${SPACE} >> ${RESPATH}/zfs_res_Wrand_1j.txt
time sync; sync
done
wait
echo "Compeleted for ${RUNPATH} 1job" 

echo -e "\n" 
echo "Started for ${RUNPATH} 10jobs" 
for ((i=1; i<=10; i++))
do
rm -rf /${POOL}/${SPACE}/*
wait
echo "Randwrite-J10 run $i" 2>&1 | tee ${RESPATH}/Wrand_10j_r$i.txt
wait
fio ${RUNPATH}/write_rand_10j.fio --filename=/${POOL}/${SPACE}/W${FILENAME}10j$i 2>&1 | tee ${RESPATH}/Wrand_10j_r$i.txt
grep "iops" ${RESPATH}/Wrand_10j_r$i.txt >> ${RESPATH}/res_Wrand_10j.txt
grep "clat (" ${RESPATH}/Wrand_10j_r$i.txt >> ${RESPATH}/res_Wrand_10j.txt
zfs list -H ${POOL}/${SPACE} >> ${RESPATH}/zfs_res_Wrand_10j.txt
time sync; sync
done
wait
echo "Compeleted for ${RUNPATH} 10jobs" 

fi
done

2) path config file

$ cat conf/write_rand/randwrite_128k1G.sh
#!/bin/bash
RUNPATH=/opt/ksenia/New_fio_v2/tests/write_rand/128k/1G
RESPATH=/opt/ksenia/New_fio_v2/res/${POOL}/${SPACE}/write_rand/128k1G
$ cat conf/write_rand/randwrite_128k10G.sh
#!/bin/bash
RUNPATH=/opt/ksenia/New_fio_v2/tests/write_rand/128k/10G
RESPATH=/opt/ksenia/New_fio_v2/res/${POOL}/${SPACE}/write_rand/128k10G

3) fio job files

$ cat tests/write_rand/128k/1G/write_rand_1j.fio
[writetest1]
bs=128k
size=1G
name=randwrite
rw=randwrite
ioengine=sync
iodepth=1
numjobs=1
runtime=20
group_reporting
fallocate=none
clocksource=clock_gettime
time_based

$ cat tests/write_rand/128k/1G/write_rand_10j.fio
[writetest1]
bs=128k
size=1G
name=randwrite
rw=randwrite
ioengine=sync
iodepth=1
numjobs=10
runtime=30
group_reporting
fallocate=none
clocksource=clock_gettime
time_based

$ cat tests/write_rand/128k/10G/write_rand_1j.fio
[writetest]
bs=128k
size=10G
name=randwrite
rw=randwrite
ioengine=sync
iodepth=1
numjobs=1
runtime=160
group_reporting
fallocate=none
clocksource=clock_gettime
time_based
cat tests/write_rand/128k/10G/write_rand_10j.fio
[writetest1]
bs=128k
size=10G
name=randwrite
rw=randwrite
ioengine=sync
iodepth=1
numjobs=10
runtime=300
group_reporting
fallocate=none
clocksource=clock_gettime
time_based

Result:
The following writes are passed: - stopped at 10G in 10jobs

  1. ls l res/tstpool3/space1/write_rand/128k10G/
    total 69
    -rw-r--r-
    1 root root 383 Aug 29 04:58 res_Wrand_10j.txt
    rw-r--r- 1 root root 1257 Aug 29 04:42 res_Wrand_1j.txt
    rw-r--r- 1 root root 2102 Aug 29 04:47 Wrand_10j_r1.txt
    rw-r--r- 1 root root 2117 Aug 29 04:52 Wrand_10j_r2.txt
    rw-r--r- 1 root root 2105 Aug 29 04:58 Wrand_10j_r3.txt
    rw-r--r- 1 root root 581 Aug 29 04:58 Wrand_10j_r4.txt
    rw-r--r- 1 root root 1574 Aug 29 04:16 Wrand_1j_r1.txt
    rw-r--r- 1 root root 1583 Aug 29 04:42 Wrand_1j_r10.txt
    rw-r--r- 1 root root 1585 Aug 29 04:19 Wrand_1j_r2.txt
    rw-r--r- 1 root root 1561 Aug 29 04:22 Wrand_1j_r3.txt
    rw-r--r- 1 root root 1586 Aug 29 04:25 Wrand_1j_r4.txt
    rw-r--r- 1 root root 1585 Aug 29 04:27 Wrand_1j_r5.txt
    rw-r--r- 1 root root 1584 Aug 29 04:30 Wrand_1j_r6.txt
    rw-r--r- 1 root root 1595 Aug 29 04:33 Wrand_1j_r7.txt
    rw-r--r- 1 root root 1586 Aug 29 04:36 Wrand_1j_r8.txt
    rw-r--r- 1 root root 1596 Aug 29 04:39 Wrand_1j_r9.txt
    rw-r--r- 1 root root 153 Aug 29 04:58 zfs_res_Wrand_10j.txt
    rw-r--r- 1 root root 510 Aug 29 04:42 zfs_res_Wrand_1j.txt
  2. ls -l res/tstpool3/space1/write_rand/128k1G/
    ls: cannot access 'res/tstpool3/space1/write_rand/128k1G/': No such file or directory #

Also available in: Atom PDF