diff --git a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py index e03cc5e3..4a01cb6d 100644 --- a/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py +++ b/server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py @@ -36,6 +36,9 @@ # cursor_sharing +TUNABLE_KNOB_NUM = 50 +TUNABLE_KNOB_FILE = "selected_knobs.csv" + EXTRA_KNOBS = { '_pga_max_size': { 'default': 200000000, @@ -286,10 +289,35 @@ def set_field(fields): fields['enumvals'] = 'asynch,directio,none,setall' +def update_field(fields, tunable_knob_file): + with open(tunable_knob_file, 'r') as f: + # csv file columns: knob,min,max,vartype,safe,note, + # for Enum type knobs, the note field is the enum_val + # discard the label row + lines = f.readlines()[1:-1] + for line in lines: + field_list = line.split(',') + if ('global.' + field_list[0]).upper() == fields['name'].upper(): + fields['tunable'] = True + if len(field_list[1]) > 0 and len(field_list[2]) > 0: + fields['minval'] = field_list[1] + fields['maxval'] = field_list[2] + fields['default'] = field_list[1] + if field_list[3] == 'Enum': + fields['vartype'] = 5 + enums = ",".join(field_list[5:-1]).replace("\"", "") + fields['enumvals'] = enums + fields['default'] = enums.split(',')[0] + elif field_list[3] == 'Bool': + fields['vartype'] = 4 + fields['default'] = True + + COLNAMES = ("NAME", "TYPE", "DEFAULT_VALUE", "DESCRIPTION") def process_version(version, delim=','): + tunable_knob_num = 0 fields_list = [] add_fields(fields_list, version) with open('oracle{}.csv'.format(version), 'r', newline='') as f: @@ -336,10 +364,25 @@ def process_version(version, delim=','): set_field(fields) fields['name'] = ('global.' + fields['name']).lower() + if fields['tunable']: + tunable_knob_num += 1 fields_list.append(fields) ri += 1 fields_list = sorted(fields_list, key=itemgetter('name')) + + # if TUNABLE_KNOB_NUM > tunable_knob_num, we will attempt to make more knobs tunable + add_tunable_knob_num = 0 + for fields in fields_list: + if fields['tunable']: + continue + if add_tunable_knob_num < TUNABLE_KNOB_NUM - tunable_knob_num: + update_field(fields, TUNABLE_KNOB_FILE) + if fields['tunable']: + add_tunable_knob_num += 1 + else: + break + final_metrics = [dict(model='website.KnobCatalog', fields=fs) for fs in fields_list] filename = 'oracle-{}_knobs.json'.format(version) with open(filename, 'w') as f: diff --git a/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv new file mode 100644 index 00000000..538a5ced --- /dev/null +++ b/server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv @@ -0,0 +1,97 @@ +knob,min,max,vartype,safe,note, +ALLOW_GLOBAL_DBLINKS,,,Bool,yes,, +ALLOW_GROUP_ACCESS_TO_SGA,,,Bool,yes,, +ARCHIVE_LAG_TARGET,60,7200,,mostly,, +ASM_IO_PROCESSES,1,32,,maybe,, +BACKUP_TAPE_IO_SLAVES,,,Bool,mostly,, +BITMAP_MERGE_AREA_SIZE,1048576,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +BLANK_TRIMMING,,,Bool,maybe,, +COMMIT_LOGGING,,,Enum,yes,"IMMEDIATE,BATCH", +COMMIT_WAIT,,,Enum,yes,"NOWAIT,WAIT,FORCE_WAIT", +CONTAINERS_PARALLEL_DEGREE,2,65535,,maybe,, +CONTROL_FILE_RECORD_KEEP_TIME,0,365,,mostly,, +CURSOR_BIND_CAPTURE_DESTINATION,,,Enum,yes,"off,memory,memory+disk", +CURSOR_INVALIDATION,,,Enum,yes,"DEFERRED,IMMEDIATE", +CURSOR_SHARING,,,Enum,yes,"EXACT,FORCE", +CURSOR_SPACE_FOR_TIME,,,Bool,deprecated,, +DB_BLOCK_CHECKING,,,Enum,mostly,"FALSE,OFF,LOW,MEDIUM,TRUE,FULL", +DB_BLOCK_CHECKSUM,,,Enum,mostly,"OFF,FALSE,TYPICAL,TRUE,FULL", +DB_CACHE_ADVICE,,,Enum,mostly,"ON,READY,OFF", +DB_INDEX_COMPRESSION_INHERITANCE,,,Enum,mostly,"TABLESPACE,TABLE,ALL,NONE", +DB_LOST_WRITE_PROTECT,,,Enum,mostly,"NONE,TYPICAL,FULL", +DB_ULTRA_SAFE,,,Enum,mostly,"OFF,DATA_ONLY,DATA_AND_INDEX", +DB_UNRECOVERABLE_SCN_TRACKING,,,Bool,mostly,, +DB_WRITER_PROCESSES,1,100,,mostly,, +DBFIPS_140,,,Bool,mostly,, +DDL_LOCK_TIMEOUT,0,1000000,,mostly,, +DEFAULT_SHARING,,,Enum,mostly,"NONE,METADATA,DATA,EXTENDED DATA", +DEFERRED_SEGMENT_CREATION,,,Bool,mostly,, +DG_BROKER_START,,,Bool,mostly,, +DISK_ASYNCH_IO,,,Bool,mostly,, +DISTRIBUTED_LOCK_TIMEOUT,1,1000000,,mostly,, +DNFS_BATCH_SIZE,0,4096,,maybe,, +DST_UPGRADE_INSERT_CONV,,,Bool,mostly,, +ENABLE_AUTOMATIC_MAINTENANCE_PDB,,,Bool,maybe,enable or disable the running of automated maintenance tasks, +ENABLE_DDL_LOGGING,,,Bool,mostly,, +ENABLE_DNFS_DISPATCHER,,,Bool,mostly,, +ENABLE_GOLDENGATE_REPLICATION,,,Bool,maybe,, +FAST_START_MTTR_TARGET,0,3600,,no effect,, +FAST_START_PARALLEL_ROLLBACK,,,Enum,no effect,"HIGH,LOW,FALSE", +FILE_MAPPING,,,Bool,deprecated,, +FILESYSTEMIO_OPTIONS,,,Enum,mostly,"none,setall,directIO,asynch", +GCS_SERVER_PROCESSES,0,5,,maybe,, +GLOBAL_TXN_PROCESSES,1,5,,maybe,, +HASH_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +HEAT_MAP,,,Bool,mostly,, +HS_AUTOREGISTER,,,Bool,maybe,, +INMEMORY_ADG_ENABLED,,,Bool,mostly,, +INMEMORY_EXPRESSIONS_USAGE,,,Enum,mostly,"STATIC_ONLY,DYNAMIC_ONLY,ENABLE,DISABLE", +INMEMORY_FORCE,,,Enum,mostly,"DEFAULT,OFF", +INMEMORY_QUERY,,,Enum,mostly,"ENABLE,DISABLE", +INMEMORY_VIRTUAL_COLUMNS,,,Enum,mostly,"ENABLE,MANUAL,DISABLE", +INSTANCE_ABORT_DELAY_TIME,0,60,,no effect,, +LOCK_SGA,,,Bool,maybe,, +LOG_CHECKPOINT_INTERVAL,1,1000000,,maybe,, +LOG_CHECKPOINT_TIMEOUT,180,3600,,maybe,, +LOG_CHECKPOINTS_TO_ALERT,,,Bool,no effect,, +MAX_DATAPUMP_JOBS_PER_PDB,10,200,,no effect,, +MAX_IDLE_TIME,1000,2000,,no effect,, +OBJECT_CACHE_MAX_SIZE_PERCENT,0,50,,maybe,, +OFS_THREADS,2,64,,mostly,, +OLAP_PAGE_POOL_SIZE,0,2000000000,,no,, +OPEN_CURSORS,50,5000,,mostly,, +OPTIMIZER_ADAPTIVE_REPORTING_ONLY,,,Bool,mostly,, +OPTIMIZER_ADAPTIVE_STATISTICS,,,Bool,mostly,, +OPTIMIZER_CAPTURE_SQL_PLAN_BASELINES,,,Bool,mostly,, +OPTIMIZER_DYNAMIC_SAMPLING,0,11,,mostly,, +OPTIMIZER_INDEX_CACHING,0,100,,mostly,, +OPTIMIZER_INDEX_COST_ADJ,50,200,,mostly,, +OPTIMIZER_INMEMORY_AWARE,,,Bool,mostly,, +OPTIMIZER_MODE,,,Enum,mostly,"FIRST_ROWS_1,FIRST_ROWS_10,FIRST_ROWS_100,FIRST_ROWS_1000,ALL_ROWS", +OPTIMIZER_SECURE_VIEW_MERGING,,,Bool,mostly,, +OPTIMIZER_USE_INVISIBLE_INDEXES,,,Bool,mostly,, +OPTIMIZER_USE_PENDING_STATISTICS,,,Bool,mostly,, +OPTIMIZER_USE_SQL_PLAN_BASELINES,,,Bool,mostly,, +PARALLEL_ADAPTIVE_MULTI_USER,,,Bool,deprecated,, +PARALLEL_DEGREE_POLICY,,,Enum,mostly,"MANUAL,LIMITED,AUTO,ADAPTIVE", +PARALLEL_EXECUTION_MESSAGE_SIZE,2148,16384,,mostly,, +PARALLEL_FORCE_LOCAL,,,Bool,mostly,, +PARALLEL_MIN_PERCENT,0,100,,mostly,, +PRE_PAGE_SGA,,,Bool,mostly,, +QUERY_REWRITE_ENABLED,,,Enum,mostly,"false,true,force", +QUERY_REWRITE_INTEGRITY,,,Enum,mostly,"enforced,trusted,stale_tolerated", +READ_ONLY_OPEN_DELAYED,,,Bool,mostly,, +RESULT_CACHE_MAX_RESULT,3,10,,mostly,, +SERIAL_REUSE,,,Enum,mostly,"disable,all,select,dml,plsql,force", +SHRD_DUPL_TABLE_REFRESH_RATE,20,100,,mostly,, +SORT_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0, +SPATIAL_VECTOR_ACCELERATION,,,Bool,mostly,, +SQL_TRACE,,,Bool,maybe,, +STANDBY_DB_PRESERVE_STATES,,,Enum,maybe,"NONE,SESSION,ALL", +STANDBY_FILE_MANAGEMENT,,,Enum,maybe,"MANUAL,AUTO", +STAR_TRANSFORMATION_ENABLED,,,Enum,mostly,"FALSE,TRUE,TEMP_DISABLE", +TAPE_ASYNCH_IO,,,Bool,mostly,, +TEMP_UNDO_ENABLED,,,Bool,mostly,, +UNDO_RETENTION,300,1500,,mostly,, +UNIFIED_AUDIT_SGA_QUEUE_SIZE,1100000,30000000,,mostly,, +WORKAREA_SIZE_POLICY,,,Enum,mostly,"AUTO,MANUAL", diff --git a/server/website/website/management/commands/updateknob.py b/server/website/website/management/commands/updateknob.py new file mode 100644 index 00000000..e6947258 --- /dev/null +++ b/server/website/website/management/commands/updateknob.py @@ -0,0 +1,131 @@ +# +# OtterTune - updateknob.py +# +# Copyright (c) 2017-18, Carnegie Mellon University Database Group +# +import json +import os +from argparse import RawTextHelpFormatter + +from django.core.management.base import BaseCommand, CommandError + + +HELP = """ +Update the knobs in a json file. + +example of JSON file format: + { + "global.knob1": { + "minval": 0, + "maxval": 100, + "tunable": true + }, + "global.knob2": { + "minval": 1000000, + "maxval": 2000000, + "tunable": false + } + } +""" + + +def update_knobs(tunable_knob_file, knobs, num): + if num <= 0: + return + cnt = 0 + with open(tunable_knob_file, 'r') as f: + # csv file columns: knob,min,max,vartype,safe,note, + # discard the label row + lines = f.readlines()[1:-1] + for line in lines: + field_list = line.split(',') + name = ('global.' + field_list[0]).lower() + if name in knobs: + knob = knobs[name] + if knob['tunable'] is True: + continue + knob['tunable'] = True + if len(field_list[1]) > 0 and len(field_list[2]) > 0: + knob['minval'] = field_list[1] + knob['maxval'] = field_list[2] + cnt += 1 + if cnt == num: + break + + +class Command(BaseCommand): + help = HELP + + def create_parser(self, prog_name, subcommand): + parser = super(Command, self).create_parser(prog_name, subcommand) + parser.formatter_class = RawTextHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + '-f', '--filename', + metavar='FILE', + default='session_knobs.json', + help='Name of the target knob file in json format. ' + 'Default: session_knobs.json') + parser.add_argument( + '-s', '--source', + metavar='SOURCE', + default='selected_knobs.csv', + help='Name of the file to read the session knob tunability from. ' + 'Default: selected_knobs.csv') + parser.add_argument( + '-o', '--output', + metavar='OUTPUT', + default='selected_knobs.csv', + help='Name of the file to write the updated session knob tunability to. ' + 'Default: selected_knobs.csv') + parser.add_argument( + '-d', '--file_dir', + metavar='FILEDIR', + help='Path of the directory of the target knob file. ' + 'Default: current directory') + parser.add_argument( + '-i', '--source_dir', + metavar='SOURCEDIR', + help='Path of the directory to read the session knob tunability from. ' + 'Default: current directory') + parser.add_argument( + '-n', '--num', + metavar='NUM', + default='50', + help='Total number of tunable session knobs. ' + 'Default: 50') + + def handle(self, *args, **options): + file_dir = options['file_dir'] or '' + path = os.path.join(file_dir, options['filename']) + + try: + with open(path, 'r') as f: + knobs = json.load(f) + except FileNotFoundError: + raise CommandError("ERROR: File '{}' does not exist.".format(path)) + except json.decoder.JSONDecodeError: + raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path)) + + source_dir = options['source_dir'] or '' + tunable_knob_file = os.path.join(source_dir, options['source']) + target_tunable_knobs = int(options['num']) + + cur_tunable_knobs = 0 + for knob in knobs.values(): + if knob['tunable']: + cur_tunable_knobs += 1 + if cur_tunable_knobs < target_tunable_knobs: + update_knobs(tunable_knob_file, knobs, target_tunable_knobs - cur_tunable_knobs) + cur_tunable_knobs = 0 + for knob in knobs.values(): + if knob['tunable']: + cur_tunable_knobs += 1 + + out_path = os.path.join(file_dir, options['output']) + with open(out_path, 'w') as f: + json.dump(knobs, f, indent=4) + self.stdout.write(self.style.SUCCESS( + "Writing {} tunable session knobs into {}.".format(cur_tunable_knobs, out_path)))