Merge pull request #4 from spetrovi/master

Record size options

README.md

@@ -63,6 +63,10 @@ Set a limit on how many random reads in a row are done to a file per random read
 
 Set a limit on how many random writes in a row can be done to a file per random write op.(default 2)
 
+-+s|--singleIO
+
+If true, use only one IO per request on random operations. If false, use segment approach.
+
 -Y|--fsyncs
 
 If true, allows fsync() call to be done every so often when files are written. Value is probability in percent. (default 20)

fsop.py

@@ -201,6 +201,11 @@ def try_to_close(closefd, filename):
             return False
     return True
 
+def get_recsz():
+    if opts.fix_record_size_kb:
+        return opts.fix_record_size_kb * BYTES_PER_KB
+    else:
+        return random_record_size()
 
 def read():
     global e_file_not_found, have_read, read_requests, read_bytes
@@ -216,7 +221,7 @@ def read():
         total_read = 0
         time_before = time.time()
         while total_read < stinfo.st_size:
-            rdsz = random_record_size()
+            rdsz = get_recsz()
             bytes = os.read(fd, rdsz)
             count = len(bytes)
             read_requests += 1
@@ -254,18 +259,16 @@ def random_read():
                 fn, stinfo.st_size, target_read_reqs))
         time_before = time.time()
         while total_read_reqs < target_read_reqs:
-            if opts.fix_record_size_kb:
-                recsz = opts.fix_record_size_kb * BYTES_PER_KB
-            else:
-                recsz = random_record_size()
             off = os.lseek(fd, random_seek_offset(stinfo.st_size), 0)
             if verbosity & 0x2000:
                 print('randread off %u sz %u' % (off, rdsz))            
             total_count = 0
             remaining_sz = stinfo.st_size - off
             targetsz = random_segment_size(stinfo.st_size)
+            if opts.singleIO:
+                targetsz = get_recsz()
             while total_count < targetsz:
-                recsz = random_record_size()
+                recsz = get_recsz()
                 if recsz + total_count > remaining_sz:
                     recsz = remaining_sz - total_count
                 elif recsz + total_count > targetsz:
@@ -334,7 +337,7 @@ def create():
         offset = 0
         time_before = time.time()
         while total_sz < target_sz:
-            recsz = random_record_size()
+            recsz = get_recsz()
             if recsz + total_sz > target_sz:
                 recsz = target_sz - total_sz
             count = os.write(fd, buf[offset:offset+recsz])
@@ -378,7 +381,7 @@ def append():
         offset = 0
         time_before = time.time()
         while total_appended < target_sz:
-            recsz = random_record_size()
+            recsz = get_recsz()
             if recsz + total_appended > target_sz:
                 recsz = target_sz - total_appended
             assert recsz > 0
@@ -422,19 +425,17 @@ def random_write():
             print('randwrite %s reqs %u' % (fn, target_write_reqs))
         time_before = time.time()
         while total_write_reqs < target_write_reqs:
-            if opts.fix_record_size_kb:
-                recsz = opts.fix_record_size_kb * BYTES_PER_KB
-            else:
-                recsz = random_record_size()
             off = os.lseek(fd, random_seek_offset(stinfo.st_size), 0)
             total_count = 0
-            wrsz = random_segment_size(stinfo.st_size)
+            targetsz = random_segment_size(stinfo.st_size)
+            if opts.singleIO:
+                targetsz = get_recsz() 
             if verbosity & 0x20000:
-                print('randwrite off %u sz %u' % (off, wrsz))
-            while total_count < wrsz:
-                recsz = random_record_size()
-                if recsz + total_count > wrsz:
-                    recsz = wrsz - total_count
+                print('randwrite off %u sz %u' % (off, targetsz))
+            while total_count < targetsz:
+                recsz = get_recsz()
+                if recsz + total_count > targetsz:
+                    recsz = targetsz - total_count
                 count = os.write(fd, buf[0:recsz])
                 if verbosity & 0x20000:
                     print('randwrite count=%u recsz=%u' % (count, recsz))

opts.py

@@ -22,6 +22,7 @@ def usage(msg):
     print('-+r|--fix-record-size-kb')
     print('-R|--max-random-reads')
     print('-W|--max-random-writes')
+    print('-+s|--singleIO')
     print('-Y|--fsyncs')
     print('-y|--fdatasyncs')
     print('-T|--response-times')
@@ -51,6 +52,7 @@ def usage(msg):
 fix_record_size_kb = 0
 max_random_reads = 2
 max_random_writes = 2
+singleIO = False
 fdatasync_probability_pct = 10
 fsync_probability_pct = 20
 short_stats = False
@@ -73,7 +75,7 @@ def usage(msg):
 
 def parseopts():
     global top_directory, starting_gun_file, opcount, max_files, max_file_size_kb, duration, short_stats
-    global max_record_size_kb, fix_record_size_kb, max_random_reads, max_random_writes, rsptimes, bw
+    global max_record_size_kb, fix_record_size_kb, max_random_reads, max_random_writes, singleIO, rsptimes, bw
     global fsync_probability_pct, fdatasync_probability_pct, workload_table_filename
     global stats_report_interval, levels, dirs_per_level
     global rand_distr_type, rand_distr_type_str, mean_index_velocity, gaussian_stddev, create_stddevs_ahead
@@ -109,6 +111,8 @@ def parseopts():
                 max_random_reads = int(val)
             elif nm == '--max-random-writes' or nm == '-W':
                 max_random_writes = int(val)
+            elif nm == '--singleIO' or nm == '-+s':
+                singleIO = True
             elif nm == '--fdatasync_pct' or nm == '-y':
                 fdatasync_probability_pct = int(val)
             elif nm == '--fsync_pct' or nm == '-Y':
@@ -160,6 +164,7 @@ def parseopts():
         '%11s%9d = fix record size (KB)\n'
         '%11s%9d = maximum random reads\n'
         '%11s%9d = maximum random writes\n'
+        '%11s%9d = single IO random operations\n'
         '%11s%9d = fdatasync percentage\n'
         '%11s%9d = fsync percentage\n'
         '%11s%9d = directory levels\n'
@@ -171,7 +176,7 @@ def parseopts():
         '%20s = save response times\n'
         '%20s = save bandwidth\n'
         % (top_directory, str(starting_gun_file), '', opcount, '', duration, '', max_files, '', max_file_size_kb,
-           '', max_record_size_kb, '', fix_record_size_kb, '', max_random_reads, '', max_random_writes,
+           '', max_record_size_kb, '', fix_record_size_kb, '', max_random_reads, '', max_random_writes, '', singleIO,
            '', fdatasync_probability_pct, '', fsync_probability_pct,
            '', levels, '', dirs_per_level,
            rand_distr_type_str, '', mean_index_velocity, '', gaussian_stddev, '', create_stddevs_ahead,