# beegfs-ctl --storagebench --alltargets --status --verbose

Server benchmark status:
Finished:    10

Write benchmark results:
Min throughput:            1674695  KiB/s   nodeID: ddn03.hpc.usc.edu [ID: 3], targetID: 30
Max throughput:            2671112  KiB/s   nodeID: ddn01.hpc.usc.edu [ID: 1], targetID: 10
Avg throughput:            1984877  KiB/s
Aggregate throughput:     19848771  KiB/s

List of all targets:
10                         2671112  KiB/s   nodeID: ddn01.hpc.usc.edu [ID: 1]
20                         1944102  KiB/s   nodeID: ddn02.hpc.usc.edu [ID: 2]
30                         1674695  KiB/s   nodeID: ddn03.hpc.usc.edu [ID: 3]
40                         1959559  KiB/s   nodeID: ddn04.hpc.usc.edu [ID: 4]
50                         1968269  KiB/s   nodeID: ddn05.hpc.usc.edu [ID: 5]
60                         1921547  KiB/s   nodeID: ddn06.hpc.usc.edu [ID: 6]
70                         1925734  KiB/s   nodeID: ddn07.hpc.usc.edu [ID: 7]
80                         1937437  KiB/s   nodeID: ddn08.hpc.usc.edu [ID: 8]
90                         1931541  KiB/s   nodeID: ddn09.hpc.usc.edu [ID: 9]
100                        1914775  KiB/s   nodeID: ddn10.hpc.usc.edu [ID: 10]

[root@disco jimichu]# beegfs-ctl --storagebench --alltargets --status --verbose

Server benchmark status:
Finished:    10

Read benchmark results:
Min throughput:            2342299  KiB/s   nodeID: ddn03.hpc.usc.edu [ID: 3], targetID: 30
Max throughput:            2506096  KiB/s   nodeID: ddn10.hpc.usc.edu [ID: 10], targetID: 100
Avg throughput:            2445227  KiB/s
Aggregate throughput:     24452273  KiB/s

List of all targets:
10                         2375138  KiB/s   nodeID: ddn01.hpc.usc.edu [ID: 1]
20                         2501535  KiB/s   nodeID: ddn02.hpc.usc.edu [ID: 2]
30                         2342299  KiB/s   nodeID: ddn03.hpc.usc.edu [ID: 3]
40                         2495777  KiB/s   nodeID: ddn04.hpc.usc.edu [ID: 4]
50                         2468373  KiB/s   nodeID: ddn05.hpc.usc.edu [ID: 5]
60                         2481405  KiB/s   nodeID: ddn06.hpc.usc.edu [ID: 6]
70                         2426543  KiB/s   nodeID: ddn07.hpc.usc.edu [ID: 7]
80                         2384652  KiB/s   nodeID: ddn08.hpc.usc.edu [ID: 8]
90                         2470455  KiB/s   nodeID: ddn09.hpc.usc.edu [ID: 9]
100                        2506096  KiB/s   nodeID: ddn10.hpc.usc.edu [ID: 10]


[root@ddn01 ~]# time dcfldd if=/dev/zero of=/data/pool-1/output of=/data/pool-2/output bs=4M count=20k
20480 blocks (81920Mb) written.
20480+0 records in
20480+0 records out

real	1m10.401s
user	0m0.074s
sys	0m31.794s

2327MB/s, that's the limit of 3Gbs SAS

-bash-4.2$ tail slurm-212.out 

Max Write: 3842.11 MiB/sec (4028.74 MB/sec)
Max Read:  5363.50 MiB/sec (5624.04 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write        3842.11    3595.90    3695.92     101.02   88.72562 0 32 4 4 1 0 1 0 0 1 10737418240 8388608 343597383680 POSIX 0
read         5363.50    4710.50    5114.22     254.60   64.23726 0 32 4 4 1 0 1 0 0 1 10737418240 8388608 343597383680 POSIX 0

Finished: Tue Oct  8 16:59:51 2019
-bash-4.2$ beegfs-ctl --getentryinfo .
EntryID: 0-5D9C0429-2
Metadata node: hpc-10ke1.hpcc.usc.edu [ID: 1]
Stripe pattern details:
+ Type: RAID0
+ Chunksize: 8M
+ Number of storage targets: desired: 1
+ Storage Pool: 1 (Default)

-bash-4.2$ cat job.sl 
#!/bin/bash
#SBATCH -N 8
#SBATCH -n 32
srun --mpi=pmix_v2 ior -v -w -r -i 4 -F -t 8m -b 10g

-bash-4.2$ tail slurm-216_chunk512k.out 

Max Write: 3041.26 MiB/sec (3189.00 MB/sec)
Max Read:  10389.95 MiB/sec (10894.65 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write        3041.26    2705.74    2909.91     129.37   17.63093 0 5 1 4 1 0 1 0 0 1 10737418240 2097152 53687091200 POSIX 0
read        10389.95    8575.87    9106.32     746.86    5.65738 0 5 1 4 1 0 1 0 0 1 10737418240 2097152 53687091200 POSIX 0

Finished: Wed Oct  9 13:01:17 2019

-bash-4.2$ beegfs-ctl --getentryinfo .
EntryID: 0-5D9BC186-2
Metadata node: hpc-10ke2.hpcc.usc.edu [ID: 2]
Stripe pattern details:
+ Type: RAID0
+ Chunksize: 512K
+ Number of storage targets: desired: 4
+ Storage Pool: 1 (Default)

-bash-4.2$ cat job.sl 
#!/bin/bash
#SBATCH -N 5
#SBATCH -n 5
srun --mpi=pmix_v2 ior -v -w -r -i 4 -F -t 2m -b 10g

MDS zfs need different tunings, ref
pool options:
ashift=9 (for space reasons)
dataset options:
recordsize=128K
compression=off
atime=off
xattr=sa
dnodesize=auto
zfs module options:
zfs_prefetch_disable=1
metaslab_debug_unload=1
zfs_dirty_data_max=2147483648
zfs_vdev_async_write_min_active=5
zfs_vdev_async_write_max_active=15
zfs_vdev_async_write_active_min_dirty_percent=20
zfs_vdev_scheduler=deadline
zfs_arc_max=103079215104
zfs_arc_meta_limit=103079215104