From 69a5e793295aa7dbebf4538566f66fb28c01b76a Mon Sep 17 00:00:00 2001 From: Samuel Petrovic Date: Wed, 17 Oct 2018 16:38:24 +0200 Subject: [PATCH 1/2] Random operations New option to run random operations either single IO or segment Signed-off-by: Samuel Petrovic --- README.md | 4 ++++ fsop.py | 31 ++++++++++++++++--------------- opts.py | 9 +++++++-- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ccd44f9..73d599a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,10 @@ Set a limit on how many random reads in a row are done to a file per random read Set a limit on how many random writes in a row can be done to a file per random write op.(default 2) +-+s|--singleIO + +If true, use only one IO per request on random operations. If false, use segment approach. + -Y|--fsyncs If true, allows fsync() call to be done every so often when files are written. Value is probability in percent. (default 20) diff --git a/fsop.py b/fsop.py index f5c9e6b..5292dbd 100644 --- a/fsop.py +++ b/fsop.py @@ -201,6 +201,11 @@ def try_to_close(closefd, filename): return False return True +def get_recsz(): + if opts.fix_record_size_kb: + return opts.fix_record_size_kb * BYTES_PER_KB + else: + return random_record_size() def read(): global e_file_not_found, have_read, read_requests, read_bytes @@ -254,18 +259,16 @@ def random_read(): fn, stinfo.st_size, target_read_reqs)) time_before = time.time() while total_read_reqs < target_read_reqs: - if opts.fix_record_size_kb: - recsz = opts.fix_record_size_kb * BYTES_PER_KB - else: - recsz = random_record_size() off = os.lseek(fd, random_seek_offset(stinfo.st_size), 0) if verbosity & 0x2000: print('randread off %u sz %u' % (off, rdsz)) total_count = 0 remaining_sz = stinfo.st_size - off targetsz = random_segment_size(stinfo.st_size) + if opts.singleIO: + targetsz = get_recsz() while total_count < targetsz: - recsz = random_record_size() + recsz = get_recsz() if recsz + total_count > remaining_sz: recsz = remaining_sz - total_count elif recsz + total_count > targetsz: @@ -422,19 +425,17 @@ def random_write(): print('randwrite %s reqs %u' % (fn, target_write_reqs)) time_before = time.time() while total_write_reqs < target_write_reqs: - if opts.fix_record_size_kb: - recsz = opts.fix_record_size_kb * BYTES_PER_KB - else: - recsz = random_record_size() off = os.lseek(fd, random_seek_offset(stinfo.st_size), 0) total_count = 0 - wrsz = random_segment_size(stinfo.st_size) + targetsz = random_segment_size(stinfo.st_size) + if opts.singleIO: + targetsz = get_recsz() if verbosity & 0x20000: - print('randwrite off %u sz %u' % (off, wrsz)) - while total_count < wrsz: - recsz = random_record_size() - if recsz + total_count > wrsz: - recsz = wrsz - total_count + print('randwrite off %u sz %u' % (off, targetsz)) + while total_count < targetsz: + recsz = get_recsz() + if recsz + total_count > targetsz: + recsz = targetsz - total_count count = os.write(fd, buf[0:recsz]) if verbosity & 0x20000: print('randwrite count=%u recsz=%u' % (count, recsz)) diff --git a/opts.py b/opts.py index 1db8cd1..d04c8cc 100644 --- a/opts.py +++ b/opts.py @@ -22,6 +22,7 @@ def usage(msg): print('-+r|--fix-record-size-kb') print('-R|--max-random-reads') print('-W|--max-random-writes') + print('-+s|--singleIO') print('-Y|--fsyncs') print('-y|--fdatasyncs') print('-T|--response-times') @@ -51,6 +52,7 @@ def usage(msg): fix_record_size_kb = 0 max_random_reads = 2 max_random_writes = 2 +singleIO = False fdatasync_probability_pct = 10 fsync_probability_pct = 20 short_stats = False @@ -73,7 +75,7 @@ def usage(msg): def parseopts(): global top_directory, starting_gun_file, opcount, max_files, max_file_size_kb, duration, short_stats - global max_record_size_kb, fix_record_size_kb, max_random_reads, max_random_writes, rsptimes, bw + global max_record_size_kb, fix_record_size_kb, max_random_reads, max_random_writes, singleIO, rsptimes, bw global fsync_probability_pct, fdatasync_probability_pct, workload_table_filename global stats_report_interval, levels, dirs_per_level global rand_distr_type, rand_distr_type_str, mean_index_velocity, gaussian_stddev, create_stddevs_ahead @@ -109,6 +111,8 @@ def parseopts(): max_random_reads = int(val) elif nm == '--max-random-writes' or nm == '-W': max_random_writes = int(val) + elif nm == '--singleIO' or nm == '-+s': + singleIO = True elif nm == '--fdatasync_pct' or nm == '-y': fdatasync_probability_pct = int(val) elif nm == '--fsync_pct' or nm == '-Y': @@ -160,6 +164,7 @@ def parseopts(): '%11s%9d = fix record size (KB)\n' '%11s%9d = maximum random reads\n' '%11s%9d = maximum random writes\n' + '%11s%9d = single IO random operations\n' '%11s%9d = fdatasync percentage\n' '%11s%9d = fsync percentage\n' '%11s%9d = directory levels\n' @@ -171,7 +176,7 @@ def parseopts(): '%20s = save response times\n' '%20s = save bandwidth\n' % (top_directory, str(starting_gun_file), '', opcount, '', duration, '', max_files, '', max_file_size_kb, - '', max_record_size_kb, '', fix_record_size_kb, '', max_random_reads, '', max_random_writes, + '', max_record_size_kb, '', fix_record_size_kb, '', max_random_reads, '', max_random_writes, '', singleIO, '', fdatasync_probability_pct, '', fsync_probability_pct, '', levels, '', dirs_per_level, rand_distr_type_str, '', mean_index_velocity, '', gaussian_stddev, '', create_stddevs_ahead, From 2c9b05dd1af6b09421e5407cc2442215fa2df387 Mon Sep 17 00:00:00 2001 From: Samuel Petrovic Date: Wed, 31 Oct 2018 13:38:46 +0100 Subject: [PATCH 2/2] Fix and random recsz for seq and random ops Signed-off-by: Samuel Petrovic --- fsop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fsop.py b/fsop.py index 5292dbd..843208c 100644 --- a/fsop.py +++ b/fsop.py @@ -221,7 +221,7 @@ def read(): total_read = 0 time_before = time.time() while total_read < stinfo.st_size: - rdsz = random_record_size() + rdsz = get_recsz() bytes = os.read(fd, rdsz) count = len(bytes) read_requests += 1 @@ -337,7 +337,7 @@ def create(): offset = 0 time_before = time.time() while total_sz < target_sz: - recsz = random_record_size() + recsz = get_recsz() if recsz + total_sz > target_sz: recsz = target_sz - total_sz count = os.write(fd, buf[offset:offset+recsz]) @@ -381,7 +381,7 @@ def append(): offset = 0 time_before = time.time() while total_appended < target_sz: - recsz = random_record_size() + recsz = get_recsz() if recsz + total_appended > target_sz: recsz = target_sz - total_appended assert recsz > 0