From f1319ee361440f8fba522500a2a16211b4ea8251 Mon Sep 17 00:00:00 2001 From: yangdsh Date: Sun, 22 Dec 2019 16:58:13 +0000 Subject: [PATCH 1/2] write script to add tunable knobs --- .../oracle/create_knob_settings.py | 38 +++++ .../knob_settings/oracle/selected_knobs.csv | 82 +++++++++++ .../website/management/commands/updateknob.py | 131 ++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100644 server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv create mode 100644 server/website/website/management/commands/updateknob.py diff --git a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py index e03cc5e3..fe287492 100644 --- a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py +++ b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py @@ -36,6 +36,9 @@ # cursor_sharing +TUNABLE_KNOB_NUM = 50 +TUNABLE_KNOB_FILE = "selected_knobs.csv" + EXTRA_KNOBS = { '_pga_max_size': { 'default': 200000000, @@ -286,10 +289,34 @@ def set_field(fields): fields['enumvals'] = 'asynch,directio,none,setall' +def update_field(fields, tunable_knob_file): + with open(tunable_knob_file, 'r') as f: + # csv file columns: knob,min,max,vartype,safe,note, + # discard the label row + lines = f.readlines()[1:-1] + for line in lines: + field_list = line.split(',') + if ('global.' + field_list[0]).upper() == fields['name'].upper(): + fields['tunable'] = True + if len(field_list[1]) > 0 and len(field_list[2]) > 0: + fields['minval'] = field_list[1] + fields['maxval'] = field_list[2] + fields['default'] = field_list[1] + if field_list[3] == 'Enum': + fields['vartype'] = 5 + enums = field_list[5].replace("\"", "") + fields['enumvals'] = enums + fields['default'] = enums.split(',')[0] + elif field_list[3] == 'Bool': + fields['vartype'] = 4 + fields['default'] = True + + COLNAMES = ("NAME", "TYPE", "DEFAULT_VALUE", "DESCRIPTION") def process_version(version, delim=','): + tunable_knob_num = 0 fields_list = [] add_fields(fields_list, version) with open('oracle{}.csv'.format(version), 'r', newline='') as f: @@ -336,10 +363,21 @@ def process_version(version, delim=','): set_field(fields) fields['name'] = ('global.' + fields['name']).lower() + if fields['tunable']: + tunable_knob_num += 1 fields_list.append(fields) ri += 1 fields_list = sorted(fields_list, key=itemgetter('name')) + add_tunable_knob_num = 0 + for fields in fields_list: + if add_tunable_knob_num < TUNABLE_KNOB_NUM - tunable_knob_num: + update_field(fields, TUNABLE_KNOB_FILE) + if fields['tunable']: + add_tunable_knob_num += 1 + else: + break + final_metrics = [dict(model='website.KnobCatalog', fields=fs) for fs in fields_list] filename = 'oracle-{}_knobs.json'.format(version) with open(filename, 'w') as f: diff --git a/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv new file mode 100644 index 00000000..eedc2b90 --- /dev/null +++ b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv @@ -0,0 +1,82 @@ +knob,min,max,vartype,safe,note, +ALLOW_GLOBAL_DBLINKS,,,Bool,yes,, +ALLOW_GROUP_ACCESS_TO_SGA,,,Bool,yes,, +ARCHIVE_LAG_TARGET,60,7200,,mostly,, +ASM_IO_PROCESSES,1,32,,maybe,, +BACKUP_TAPE_IO_SLAVES,,,Bool,mostly,, +BITMAP_MERGE_AREA_SIZE,1048576,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +BLANK_TRIMMING,,,Bool,maybe,, +COMMIT_LOGGING,,,Enum,yes,"IMMEDIATE,BATCH", +COMMIT_WAIT,,,Enum,yes,"NOWAIT,WAIT,FORCE_WAIT", +CONTAINERS_PARALLEL_DEGREE,2,65535,,maybe,, +CONTROL_FILE_RECORD_KEEP_TIME,0,365,,mostly,, +CURSOR_BIND_CAPTURE_DESTINATION,,,Enum,yes,"off,memory,memory+disk", +CURSOR_INVALIDATION,,,Enum,yes,"DEFERRED,IMMEDIATE", +CURSOR_SHARING,,,Enum,yes,"EXACT,FORCE", +CURSOR_SPACE_FOR_TIME,,,Bool,deprecated,, +DB_BLOCK_CHECKING,,,Enum,mostly,"FALSE,OFF,LOW,MEDIUM,TRUE,FULL", +DB_BLOCK_CHECKSUM,,,Enum,mostly,"OFF,FALSE,TYPICAL,TRUE,FULL", +DB_CACHE_ADVICE,,,Enum,mostly,"ON,READY,OFF", +DB_INDEX_COMPRESSION_INHERITANCE,,,Enum,mostly,"TABLESPACE,TABLE,ALL,NONE", +DB_LOST_WRITE_PROTECT,,,Enum,mostly,"NONE,TYPICAL,FULL", +DB_ULTRA_SAFE,,,Enum,mostly,"OFF,DATA_ONLY,DATA_AND_INDEX", +DB_UNRECOVERABLE_SCN_TRACKING,,,Bool,mostly,, +DB_WRITER_PROCESSES,1,100,,mostly,, +DBFIPS_140,,,Bool,mostly,, +DDL_LOCK_TIMEOUT,0,1000000,,mostly,, +DEFAULT_SHARING,,,Enum,mostly,"NONE,METADATA,DATA,EXTENDED DATA", +DEFERRED_SEGMENT_CREATION,,,Bool,mostly,, +DG_BROKER_START,,,Bool,mostly,, +DISK_ASYNCH_IO,,,Bool,mostly,, +DISTRIBUTED_LOCK_TIMEOUT,1,1000000,,mostly,, +DNFS_BATCH_SIZE,0,4096,,maybe,, +DST_UPGRADE_INSERT_CONV,,,Bool,mostly,, +ENABLE_AUTOMATIC_MAINTENANCE_PDB,,,Bool,maybe,enable or disable the running of automated maintenance tasks, +ENABLE_DDL_LOGGING,,,Bool,mostly,, +ENABLE_DNFS_DISPATCHER,,,Bool,mostly,, +ENABLE_GOLDENGATE_REPLICATION,,,Bool,maybe,, +FAST_START_MTTR_TARGET,0,3600,,no effect,, +FAST_START_PARALLEL_ROLLBACK,,,Enum,no effect,"HIGH,LOW,FALSE", +FILE_MAPPING,,,Bool,deprecated,, +FILESYSTEMIO_OPTIONS,,,Enum,mostly,"none,setall,directIO,asynch", +GCS_SERVER_PROCESSES,0,5,,maybe,, +GLOBAL_TXN_PROCESSES,1,5,,maybe,, +HASH_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +SORT_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +HEAT_MAP,,,Bool,mostly,, +HS_AUTOREGISTER,,,Bool,maybe,, +INMEMORY_ADG_ENABLED,,,Bool,mostly,, +INMEMORY_EXPRESSIONS_USAGE,,,Enum,mostly,"STATIC_ONLY,DYNAMIC_ONLY,ENABLE,DISABLE", +INMEMORY_FORCE,,,Enum,mostly,"DEFAULT,OFF", +INMEMORY_QUERY,,,Enum,mostly,"ENABLE,DISABLE", +INMEMORY_VIRTUAL_COLUMNS,,,Enum,mostly,"ENABLE,MANUAL,DISABLE", +INSTANCE_ABORT_DELAY_TIME,0,60,,no effect,, +LOCK_SGA,,,Bool,maybe,, +LOG_CHECKPOINT_INTERVAL,1,1000000,,maybe,, +LOG_CHECKPOINT_TIMEOUT,180,3600,,maybe,, +LOG_CHECKPOINTS_TO_ALERT,,,Bool,no effect,, +MAX_DATAPUMP_JOBS_PER_PDB,10,200,,no effect,, +MAX_IDLE_TIME,1000,2000,,no effect,, +OBJECT_CACHE_MAX_SIZE_PERCENT,0,50,,maybe,, +OFS_THREADS,2,64,,mostly,, +OLAP_PAGE_POOL_SIZE,0,2000000000,,no,, +OPEN_CURSORS,50,5000,,mostly,, +OPTIMIZER_ADAPTIVE_REPORTING_ONLY,,,Bool,mostly,, +OPTIMIZER_ADAPTIVE_STATISTICS,,,Bool,mostly,, +OPTIMIZER_CAPTURE_SQL_PLAN_BASELINES,,,Bool,mostly,, +OPTIMIZER_DYNAMIC_SAMPLING,0,11,,mostly,, +OPTIMIZER_INDEX_CACHING,0,100,,mostly,, +OPTIMIZER_INDEX_COST_ADJ,50,200,,mostly,, +OPTIMIZER_INMEMORY_AWARE,,,Bool,mostly,, +OPTIMIZER_MODE,,,Enum,mostly,"FIRST_ROWS_1,FIRST_ROWS_10,FIRST_ROWS_100,FIRST_ROWS_1000,ALL_ROWS", +OPTIMIZER_SECURE_VIEW_MERGING,,,Bool,mostly,, +OPTIMIZER_USE_INVISIBLE_INDEXES,,,Bool,mostly,, +OPTIMIZER_USE_PENDING_STATISTICS,,,Bool,mostly,, +OPTIMIZER_USE_SQL_PLAN_BASELINES,,,Bool,mostly,, +PARALLEL_ADAPTIVE_MULTI_USER,,,Bool,deprecated,, +PARALLEL_DEGREE_POLICY,,,Enum,mostly,"MANUAL,LIMITED,AUTO,ADAPTIVE", +PARALLEL_EXECUTION_MESSAGE_SIZE,2148,16384,,mostly,, +PARALLEL_FORCE_LOCAL,,,Bool,mostly,, +PARALLEL_MIN_PERCENT,0,100,,mostly,, +PRE_PAGE_SGA,,,Bool,mostly,, +QUERY_REWRITE_ENABLED,,,Enum,mostly,"false,true,force", diff --git a/server/website/website/management/commands/updateknob.py b/server/website/website/management/commands/updateknob.py new file mode 100644 index 00000000..f1bcaf73 --- /dev/null +++ b/server/website/website/management/commands/updateknob.py @@ -0,0 +1,131 @@ +# +# OtterTune - updateknob.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +import json +import os +from argparse import RawTextHelpFormatter + +from django.core.management.base import BaseCommand, CommandError + + +HELP = """ +Update the knobs in a json file. + +example of JSON file format: + { + "global.knob1": { + "minval": 0, + "maxval": 100, + "tunable": true + }, + "global.knob2": { + "minval": 1000000, + "maxval": 2000000, + "tunable": false + } + } +""" + + +def update_knobs(tunable_knob_file, knobs, num): + if num < 0: + return + cnt = 0 + with open(tunable_knob_file, 'r') as f: + # csv file columns: knob,min,max,vartype,safe,note, + # discard the label row + lines = f.readlines()[1:-1] + for line in lines: + field_list = line.split(',') + name = ('global.' + field_list[0]).lower() + if name in knobs: + knob = knobs[name] + knob['tunable'] = True + if len(field_list[1]) > 0 and len(field_list[2]) > 0: + knob['minval'] = field_list[1] + knob['maxval'] = field_list[2] + else: + continue + cnt += 1 + if cnt == num: + break + + +class Command(BaseCommand): + help = HELP + + def create_parser(self, prog_name, subcommand): + parser = super(Command, self).create_parser(prog_name, subcommand) + parser.formatter_class = RawTextHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + '-f', '--filename', + metavar='FILE', + default='session_knobs.json', + help='Name of the target knob file in json format. ' + 'Default: session_knobs.json') + parser.add_argument( + '-s', '--source', + metavar='SOURCE', + default='selected_knobs.csv', + help='Name of the file to read the session knob tunability from. ' + 'Default: selected_knobs.csv') + parser.add_argument( + '-d', '--file_dir', + metavar='FILEDIR', + help='Path of the directory of the target knob file. ' + 'Default: current directory') + parser.add_argument( + '-i', '--source_dir', + metavar='SOURCEDIR', + help='Path of the directory to read the session knob tunability from. ' + 'Default: current directory') + parser.add_argument( + '-n', '--num', + metavar='NUM', + default='50', + help='Total number of tunable session knobs. ' + 'Default: 50') + + def handle(self, *args, **options): + file_dir = options['file_dir'] or '' + path = os.path.join(file_dir, options['filename']) + + try: + with open(path, 'r') as f: + knobs = json.load(f) + except FileNotFoundError: + raise CommandError("ERROR: File '{}' does not exist.".format(path)) + except json.decoder.JSONDecodeError: + raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path)) + + source_dir = options['source_dir'] or '' + tunable_knob_file = os.path.join(source_dir, options['source']) + target_tunable_knobs = int(options['num']) + iters = 0 + total_tunable_knobs = 0 + for knob in knobs.values(): + if knob['tunable']: + total_tunable_knobs += 1 + while total_tunable_knobs < target_tunable_knobs and iters < 10: + iters += 1 + cur_tunable_knobs = 0 + for knob in knobs.values(): + if knob['tunable']: + cur_tunable_knobs += 1 + + update_knobs(tunable_knob_file, knobs, target_tunable_knobs - cur_tunable_knobs) + + total_tunable_knobs = 0 + for knob in knobs.values(): + if knob['tunable']: + total_tunable_knobs += 1 + + with open(path, 'w') as f: + json.dump(knobs, f, indent=4) + self.stdout.write(self.style.SUCCESS( + "After update, there are {} tunable session knobs.".format(total_tunable_knobs))) From 90c375cf3ef2c514faf5f2a16df0840e34b2db12 Mon Sep 17 00:00:00 2001 From: yangdsh Date: Mon, 23 Dec 2019 12:53:15 +0000 Subject: [PATCH 2/2] add more knobs to csv; fix bugs --- .../oracle/create_knob_settings.py | 7 +++- .../knob_settings/oracle/selected_knobs.csv | 17 +++++++- .../website/management/commands/updateknob.py | 40 +++++++++---------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py index fe287492..4a01cb6d 100644 --- a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py +++ b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py @@ -292,6 +292,7 @@ def set_field(fields): def update_field(fields, tunable_knob_file): with open(tunable_knob_file, 'r') as f: # csv file columns: knob,min,max,vartype,safe,note, + # for Enum type knobs, the note field is the enum_val # discard the label row lines = f.readlines()[1:-1] for line in lines: @@ -304,7 +305,7 @@ def update_field(fields, tunable_knob_file): fields['default'] = field_list[1] if field_list[3] == 'Enum': fields['vartype'] = 5 - enums = field_list[5].replace("\"", "") + enums = ",".join(field_list[5:-1]).replace("\"", "") fields['enumvals'] = enums fields['default'] = enums.split(',')[0] elif field_list[3] == 'Bool': @@ -369,8 +370,12 @@ def process_version(version, delim=','): ri += 1 fields_list = sorted(fields_list, key=itemgetter('name')) + + # if TUNABLE_KNOB_NUM > tunable_knob_num, we will attempt to make more knobs tunable add_tunable_knob_num = 0 for fields in fields_list: + if fields['tunable']: + continue if add_tunable_knob_num < TUNABLE_KNOB_NUM - tunable_knob_num: update_field(fields, TUNABLE_KNOB_FILE) if fields['tunable']: diff --git a/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv index eedc2b90..538a5ced 100644 --- a/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv +++ b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv @@ -42,7 +42,6 @@ FILESYSTEMIO_OPTIONS,,,Enum,mostly,"none,setall,directIO,asynch", GCS_SERVER_PROCESSES,0,5,,maybe,, GLOBAL_TXN_PROCESSES,1,5,,maybe,, HASH_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, -SORT_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, HEAT_MAP,,,Bool,mostly,, HS_AUTOREGISTER,,,Bool,maybe,, INMEMORY_ADG_ENABLED,,,Bool,mostly,, @@ -80,3 +79,19 @@ PARALLEL_FORCE_LOCAL,,,Bool,mostly,, PARALLEL_MIN_PERCENT,0,100,,mostly,, PRE_PAGE_SGA,,,Bool,mostly,, QUERY_REWRITE_ENABLED,,,Enum,mostly,"false,true,force", +QUERY_REWRITE_INTEGRITY,,,Enum,mostly,"enforced,trusted,stale_tolerated", +READ_ONLY_OPEN_DELAYED,,,Bool,mostly,, +RESULT_CACHE_MAX_RESULT,3,10,,mostly,, +SERIAL_REUSE,,,Enum,mostly,"disable,all,select,dml,plsql,force", +SHRD_DUPL_TABLE_REFRESH_RATE,20,100,,mostly,, +SORT_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +SPATIAL_VECTOR_ACCELERATION,,,Bool,mostly,, +SQL_TRACE,,,Bool,maybe,, +STANDBY_DB_PRESERVE_STATES,,,Enum,maybe,"NONE,SESSION,ALL", +STANDBY_FILE_MANAGEMENT,,,Enum,maybe,"MANUAL,AUTO", +STAR_TRANSFORMATION_ENABLED,,,Enum,mostly,"FALSE,TRUE,TEMP_DISABLE", +TAPE_ASYNCH_IO,,,Bool,mostly,, +TEMP_UNDO_ENABLED,,,Bool,mostly,, +UNDO_RETENTION,300,1500,,mostly,, +UNIFIED_AUDIT_SGA_QUEUE_SIZE,1100000,30000000,,mostly,, +WORKAREA_SIZE_POLICY,,,Enum,mostly,"AUTO,MANUAL", diff --git a/server/website/website/management/commands/updateknob.py b/server/website/website/management/commands/updateknob.py index f1bcaf73..e6947258 100644 --- a/server/website/website/management/commands/updateknob.py +++ b/server/website/website/management/commands/updateknob.py @@ -30,7 +30,7 @@ def update_knobs(tunable_knob_file, knobs, num): - if num < 0: + if num <= 0: return cnt = 0 with open(tunable_knob_file, 'r') as f: @@ -42,15 +42,15 @@ def update_knobs(tunable_knob_file, knobs, num): name = ('global.' + field_list[0]).lower() if name in knobs: knob = knobs[name] + if knob['tunable'] is True: + continue knob['tunable'] = True if len(field_list[1]) > 0 and len(field_list[2]) > 0: knob['minval'] = field_list[1] knob['maxval'] = field_list[2] - else: - continue - cnt += 1 - if cnt == num: - break + cnt += 1 + if cnt == num: + break class Command(BaseCommand): @@ -74,6 +74,12 @@ def add_arguments(self, parser): default='selected_knobs.csv', help='Name of the file to read the session knob tunability from. ' 'Default: selected_knobs.csv') + parser.add_argument( + '-o', '--output', + metavar='OUTPUT', + default='selected_knobs.csv', + help='Name of the file to write the updated session knob tunability to. ' + 'Default: selected_knobs.csv') parser.add_argument( '-d', '--file_dir', metavar='FILEDIR', @@ -106,26 +112,20 @@ def handle(self, *args, **options): source_dir = options['source_dir'] or '' tunable_knob_file = os.path.join(source_dir, options['source']) target_tunable_knobs = int(options['num']) - iters = 0 - total_tunable_knobs = 0 + + cur_tunable_knobs = 0 for knob in knobs.values(): if knob['tunable']: - total_tunable_knobs += 1 - while total_tunable_knobs < target_tunable_knobs and iters < 10: - iters += 1 + cur_tunable_knobs += 1 + if cur_tunable_knobs < target_tunable_knobs: + update_knobs(tunable_knob_file, knobs, target_tunable_knobs - cur_tunable_knobs) cur_tunable_knobs = 0 for knob in knobs.values(): if knob['tunable']: cur_tunable_knobs += 1 - update_knobs(tunable_knob_file, knobs, target_tunable_knobs - cur_tunable_knobs) - - total_tunable_knobs = 0 - for knob in knobs.values(): - if knob['tunable']: - total_tunable_knobs += 1 - - with open(path, 'w') as f: + out_path = os.path.join(file_dir, options['output']) + with open(out_path, 'w') as f: json.dump(knobs, f, indent=4) self.stdout.write(self.style.SUCCESS( - "After update, there are {} tunable session knobs.".format(total_tunable_knobs))) + "Writing {} tunable session knobs into {}.".format(cur_tunable_knobs, out_path)))