Allow for random IO workloads via fio - default to sequential

Signed-Off-By: Joe Handzik <jhandzik@nvidia.com>
NVIDIA · Apr 8, 2021 · 477b218 · 477b218
1 parent 3430c23
commit 477b218
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 3 deletions.
diff --git a/bobber/bobber.py b/bobber/bobber.py
@@ -20,7 +20,9 @@
     RUN_STG_BW,
     RUN_STG_IOPS,
     RUN_STG_META,
-    SYSTEMS
+    SYSTEMS,
+    READ_PATTERNS,
+    WRITE_PATTERNS
 )
 from bobber.lib.analysis import parse_results
 from bobber.lib.system.file_handler import create_directory
@@ -131,6 +133,16 @@ def parse_args(version: str) -> Namespace:
                                  type=int)
     commands_parent.add_argument('--iops-threads', help='Maximum number of '
                                  'threads to use for iops tests', type=int)
+    commands_parent.add_argument('--read-pattern', help='Specify IO pattern '
+                                 'for fio read tests. Supported values: '
+                                 'sequential, random. Defaults to sequential.',
+                                 default='sequential',
+                                 choices=READ_PATTERNS.keys())
+    commands_parent.add_argument('--write-pattern', help='Specify IO pattern '
+                                 'for fio write tests. Supported values: '
+                                 'sequential, random. Defaults to sequential.',
+                                 default='sequential',
+                                 choices=WRITE_PATTERNS.keys())
     commands_parent.add_argument('--iterations', help='Number of iterations to'
                                  ' execute per test - a seperate log file will'
                                  ' be generated for each iteration', type=int,

diff --git a/bobber/lib/constants.py b/bobber/lib/constants.py
@@ -51,6 +51,16 @@
     'dgx-2': DGX_2
 }
 
+READ_PATTERNS = {
+    'sequential': 'read',
+    'random': 'randread'
+}
+
+WRITE_PATTERNS = {
+    'sequential': 'write',
+    'random': 'randwrite'
+}
+
 # Baseline Results
 # This is considered a minimum value that tests should hit in order to be
 # verified the system has been configured properly for HPC and AI workloads.

diff --git a/bobber/lib/tests/run_tests.py b/bobber/lib/tests/run_tests.py
@@ -83,13 +83,17 @@ def run_stg_bw(args: Namespace, bobber_version: str, iteration: int,
                               f'threads_{args.bw_threads}_'
                               f'direct_{args.direct}_'
                               f'depth_{args.io_depth}_'
+                              f'read_pattern_{args.read_pattern}_'
+                              f'write_pattern_{args.write_pattern}_'
                               f'systems_{len(hosts.split(","))}_'
                               f'version_{bobber_version}.log')
     environment = {
         'EXTRA_FLAGS': args.stg_extra_flags,
         'IO_DEPTH': args.io_depth,
         'DIRECTIO': args.direct,
         'THREADS': args.bw_threads,
+        'READ_PATTERN': args.read_pattern,
+        'WRITE_PATTERN': args.write_pattern,
         'HOSTS': hosts
     }
     manager.execute('tests/fio_multi.sh',
@@ -125,6 +129,8 @@ def run_stg_iops(args: Namespace, bobber_version: str, iteration: int,
                                 f'stg_iops_iteration_{iteration}_'
                                 f'threads_{args.iops_threads}_'
                                 f'direct_{args.direct}_'
+                                f'read_pattern_{args.read_pattern}_'
+                                f'write_pattern_{args.write_pattern}_'
                                 f'systems_{len(hosts.split(","))}_'
                                 f'version_{bobber_version}.log')
     environment = {
@@ -133,6 +139,8 @@ def run_stg_iops(args: Namespace, bobber_version: str, iteration: int,
         'DIRECTIO': args.direct,
         'THREADS': args.iops_threads,
         'IOSIZE': 4,
+        'READ_PATTERN': args.read_pattern,
+        'WRITE_PATTERN': args.write_pattern,
         'HOSTS': hosts
     }
     manager.execute('tests/fio_multi.sh',

diff --git a/bobber/test_scripts/fio_multi.sh b/bobber/test_scripts/fio_multi.sh
@@ -20,6 +20,14 @@ if [ "x$EXTRA_FLAGS" = "x" ]; then
         EXTRA_FLAGS=''
 fi
 
+if [ "x$READ_PATTERN" = "x" ]; then
+        READ_PATTERN="read"
+fi
+
+if [ "x$WRITE_PATTERN" = "x" ]; then
+        WRITE_PATTERN="write"
+fi
+
 HOSTS_WITH_SPACES=`echo $HOSTS | sed "s/,/ /g"`
 
 FSDIR=/mnt/fs_under_test
@@ -44,10 +52,10 @@ CREATEOPTS="--invalidate=${INVALIDATE} --blocksize=${CREATE_IOSIZE}k --size=${SI
 ## Run create with a large blocksize, because using a smaller blocksize will take an inordinate amount of time
 launch_fio --create_only=1 --rw=write ${IOSETTINGS} ${STDOPTS} ${CREATEOPTS}
 
-launch_fio --rw=write ${IOSETTINGS} ${STDOPTS} ${RUNOPTS} ${EXTRA_FLAGS}
+launch_fio --rw=${WRITE_PATTERN} ${IOSETTINGS} ${STDOPTS} ${RUNOPTS} ${EXTRA_FLAGS}
 drop_caches
 
-launch_fio --rw=read ${IOSETTINGS} ${STDOPTS} ${RUNOPTS} ${EXTRA_FLAGS}
+launch_fio --rw=${READ_PATTERN} ${IOSETTINGS} ${STDOPTS} ${RUNOPTS} ${EXTRA_FLAGS}
 drop_caches
 
 # Clean up the job