Add tunable knobs and scripts

server/website/script/fixture_generators/knob_settings/oracle/create_knob_settings.py

@@ -36,6 +36,9 @@
 # cursor_sharing
 
 
+TUNABLE_KNOB_NUM = 50
+TUNABLE_KNOB_FILE = "selected_knobs.csv"
+
 EXTRA_KNOBS = {
  '_pga_max_size': {
  'default': 200000000,
@@ -286,10 +289,35 @@ def set_field(fields):
  fields['enumvals'] = 'asynch,directio,none,setall'
 
 
+def update_field(fields, tunable_knob_file):
+ with open(tunable_knob_file, 'r') as f:
+ # csv file columns: knob,min,max,vartype,safe,note,
+ # for Enum type knobs, the note field is the enum_val
+ # discard the label row
+ lines = f.readlines()[1:-1]
+ for line in lines:
+ field_list = line.split(',')
+ if ('global.' + field_list[0]).upper() == fields['name'].upper():
+ fields['tunable'] = True
+ if len(field_list[1]) > 0 and len(field_list[2]) > 0:
+ fields['minval'] = field_list[1]
+ fields['maxval'] = field_list[2]
+ fields['default'] = field_list[1]
+ if field_list[3] == 'Enum':
+ fields['vartype'] = 5
+ enums = ",".join(field_list[5:-1]).replace("\"", "")
+ fields['enumvals'] = enums
+ fields['default'] = enums.split(',')[0]
+ elif field_list[3] == 'Bool':
+ fields['vartype'] = 4
+ fields['default'] = True
+
+
 COLNAMES = ("NAME", "TYPE", "DEFAULT_VALUE", "DESCRIPTION")
 
 
 def process_version(version, delim=','):
+ tunable_knob_num = 0
  fields_list = []
  add_fields(fields_list, version)
  with open('oracle{}.csv'.format(version), 'r', newline='') as f:
@@ -336,10 +364,25 @@ def process_version(version, delim=','):
 
  set_field(fields)
  fields['name'] = ('global.' + fields['name']).lower()
+ if fields['tunable']:
+ tunable_knob_num += 1
  fields_list.append(fields)
  ri += 1
 
  fields_list = sorted(fields_list, key=itemgetter('name'))
+
+ # if TUNABLE_KNOB_NUM > tunable_knob_num, we will attempt to make more knobs tunable
+ add_tunable_knob_num = 0
+ for fields in fields_list:
+ if fields['tunable']:
+ continue
+ if add_tunable_knob_num < TUNABLE_KNOB_NUM - tunable_knob_num:
+ update_field(fields, TUNABLE_KNOB_FILE)
+ if fields['tunable']:
+ add_tunable_knob_num += 1
+ else:
+ break
+
  final_metrics = [dict(model='website.KnobCatalog', fields=fs) for fs in fields_list]
  filename = 'oracle-{}_knobs.json'.format(version)
  with open(filename, 'w') as f:

server/website/script/fixture_generators/knob_settings/oracle/selected_knobs.csv

@@ -0,0 +1,97 @@
+knob,min,max,vartype,safe,note,
+ALLOW_GLOBAL_DBLINKS,,,Bool,yes,,
+ALLOW_GROUP_ACCESS_TO_SGA,,,Bool,yes,,
+ARCHIVE_LAG_TARGET,60,7200,,mostly,,
+ASM_IO_PROCESSES,1,32,,maybe,,
+BACKUP_TAPE_IO_SLAVES,,,Bool,mostly,,
+BITMAP_MERGE_AREA_SIZE,1048576,10485760,,mostly,need PGA_AGGREGATE_TARGET=0,
+BLANK_TRIMMING,,,Bool,maybe,,
+COMMIT_LOGGING,,,Enum,yes,"IMMEDIATE,BATCH",
+COMMIT_WAIT,,,Enum,yes,"NOWAIT,WAIT,FORCE_WAIT",
+CONTAINERS_PARALLEL_DEGREE,2,65535,,maybe,,
+CONTROL_FILE_RECORD_KEEP_TIME,0,365,,mostly,,
+CURSOR_BIND_CAPTURE_DESTINATION,,,Enum,yes,"off,memory,memory+disk",
+CURSOR_INVALIDATION,,,Enum,yes,"DEFERRED,IMMEDIATE",
+CURSOR_SHARING,,,Enum,yes,"EXACT,FORCE",
+CURSOR_SPACE_FOR_TIME,,,Bool,deprecated,,
+DB_BLOCK_CHECKING,,,Enum,mostly,"FALSE,OFF,LOW,MEDIUM,TRUE,FULL",
+DB_BLOCK_CHECKSUM,,,Enum,mostly,"OFF,FALSE,TYPICAL,TRUE,FULL",
+DB_CACHE_ADVICE,,,Enum,mostly,"ON,READY,OFF",
+DB_INDEX_COMPRESSION_INHERITANCE,,,Enum,mostly,"TABLESPACE,TABLE,ALL,NONE",
+DB_LOST_WRITE_PROTECT,,,Enum,mostly,"NONE,TYPICAL,FULL",
+DB_ULTRA_SAFE,,,Enum,mostly,"OFF,DATA_ONLY,DATA_AND_INDEX",
+DB_UNRECOVERABLE_SCN_TRACKING,,,Bool,mostly,,
+DB_WRITER_PROCESSES,1,100,,mostly,,
+DBFIPS_140,,,Bool,mostly,,
+DDL_LOCK_TIMEOUT,0,1000000,,mostly,,
+DEFAULT_SHARING,,,Enum,mostly,"NONE,METADATA,DATA,EXTENDED DATA",
+DEFERRED_SEGMENT_CREATION,,,Bool,mostly,,
+DG_BROKER_START,,,Bool,mostly,,
+DISK_ASYNCH_IO,,,Bool,mostly,,
+DISTRIBUTED_LOCK_TIMEOUT,1,1000000,,mostly,,
+DNFS_BATCH_SIZE,0,4096,,maybe,,
+DST_UPGRADE_INSERT_CONV,,,Bool,mostly,,
+ENABLE_AUTOMATIC_MAINTENANCE_PDB,,,Bool,maybe,enable or disable the running of automated maintenance tasks,
+ENABLE_DDL_LOGGING,,,Bool,mostly,,
+ENABLE_DNFS_DISPATCHER,,,Bool,mostly,,
+ENABLE_GOLDENGATE_REPLICATION,,,Bool,maybe,,
+FAST_START_MTTR_TARGET,0,3600,,no effect,,
+FAST_START_PARALLEL_ROLLBACK,,,Enum,no effect,"HIGH,LOW,FALSE",
+FILE_MAPPING,,,Bool,deprecated,,
+FILESYSTEMIO_OPTIONS,,,Enum,mostly,"none,setall,directIO,asynch",
+GCS_SERVER_PROCESSES,0,5,,maybe,,
+GLOBAL_TXN_PROCESSES,1,5,,maybe,,
+HASH_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0,
+HEAT_MAP,,,Bool,mostly,,
+HS_AUTOREGISTER,,,Bool,maybe,,
+INMEMORY_ADG_ENABLED,,,Bool,mostly,,
+INMEMORY_EXPRESSIONS_USAGE,,,Enum,mostly,"STATIC_ONLY,DYNAMIC_ONLY,ENABLE,DISABLE",
+INMEMORY_FORCE,,,Enum,mostly,"DEFAULT,OFF",
+INMEMORY_QUERY,,,Enum,mostly,"ENABLE,DISABLE",
+INMEMORY_VIRTUAL_COLUMNS,,,Enum,mostly,"ENABLE,MANUAL,DISABLE",
+INSTANCE_ABORT_DELAY_TIME,0,60,,no effect,,
+LOCK_SGA,,,Bool,maybe,,
+LOG_CHECKPOINT_INTERVAL,1,1000000,,maybe,,
+LOG_CHECKPOINT_TIMEOUT,180,3600,,maybe,,
+LOG_CHECKPOINTS_TO_ALERT,,,Bool,no effect,,
+MAX_DATAPUMP_JOBS_PER_PDB,10,200,,no effect,,
+MAX_IDLE_TIME,1000,2000,,no effect,,
+OBJECT_CACHE_MAX_SIZE_PERCENT,0,50,,maybe,,
+OFS_THREADS,2,64,,mostly,,
+OLAP_PAGE_POOL_SIZE,0,2000000000,,no,,
+OPEN_CURSORS,50,5000,,mostly,,
+OPTIMIZER_ADAPTIVE_REPORTING_ONLY,,,Bool,mostly,,
+OPTIMIZER_ADAPTIVE_STATISTICS,,,Bool,mostly,,
+OPTIMIZER_CAPTURE_SQL_PLAN_BASELINES,,,Bool,mostly,,
+OPTIMIZER_DYNAMIC_SAMPLING,0,11,,mostly,,
+OPTIMIZER_INDEX_CACHING,0,100,,mostly,,
+OPTIMIZER_INDEX_COST_ADJ,50,200,,mostly,,
+OPTIMIZER_INMEMORY_AWARE,,,Bool,mostly,,
+OPTIMIZER_MODE,,,Enum,mostly,"FIRST_ROWS_1,FIRST_ROWS_10,FIRST_ROWS_100,FIRST_ROWS_1000,ALL_ROWS",
+OPTIMIZER_SECURE_VIEW_MERGING,,,Bool,mostly,,
+OPTIMIZER_USE_INVISIBLE_INDEXES,,,Bool,mostly,,
+OPTIMIZER_USE_PENDING_STATISTICS,,,Bool,mostly,,
+OPTIMIZER_USE_SQL_PLAN_BASELINES,,,Bool,mostly,,
+PARALLEL_ADAPTIVE_MULTI_USER,,,Bool,deprecated,,
+PARALLEL_DEGREE_POLICY,,,Enum,mostly,"MANUAL,LIMITED,AUTO,ADAPTIVE",
+PARALLEL_EXECUTION_MESSAGE_SIZE,2148,16384,,mostly,,
+PARALLEL_FORCE_LOCAL,,,Bool,mostly,,
+PARALLEL_MIN_PERCENT,0,100,,mostly,,
+PRE_PAGE_SGA,,,Bool,mostly,,
+QUERY_REWRITE_ENABLED,,,Enum,mostly,"false,true,force",
+QUERY_REWRITE_INTEGRITY,,,Enum,mostly,"enforced,trusted,stale_tolerated",
+READ_ONLY_OPEN_DELAYED,,,Bool,mostly,,
+RESULT_CACHE_MAX_RESULT,3,10,,mostly,,
+SERIAL_REUSE,,,Enum,mostly,"disable,all,select,dml,plsql,force",
+SHRD_DUPL_TABLE_REFRESH_RATE,20,100,,mostly,,
+SORT_AREA_SIZE,65536,10485760,,mostly,need PGA_AGGREGATE_TARGET=0,
+SPATIAL_VECTOR_ACCELERATION,,,Bool,mostly,,
+SQL_TRACE,,,Bool,maybe,,
+STANDBY_DB_PRESERVE_STATES,,,Enum,maybe,"NONE,SESSION,ALL",
+STANDBY_FILE_MANAGEMENT,,,Enum,maybe,"MANUAL,AUTO",
+STAR_TRANSFORMATION_ENABLED,,,Enum,mostly,"FALSE,TRUE,TEMP_DISABLE",
+TAPE_ASYNCH_IO,,,Bool,mostly,,
+TEMP_UNDO_ENABLED,,,Bool,mostly,,
+UNDO_RETENTION,300,1500,,mostly,,
+UNIFIED_AUDIT_SGA_QUEUE_SIZE,1100000,30000000,,mostly,,
+WORKAREA_SIZE_POLICY,,,Enum,mostly,"AUTO,MANUAL",

server/website/website/management/commands/updateknob.py

@@ -0,0 +1,131 @@
+#
+# OtterTune - updateknob.py
+#
+# Copyright (c) 2017-18, Carnegie Mellon University Database Group
+#
+import json
+import os
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand, CommandError
+
+
+HELP = """
+Update the knobs in a json file.
+
+example of JSON file format:
+ {
+ "global.knob1": {
+ "minval": 0,
+ "maxval": 100,
+ "tunable": true
+ },
+ "global.knob2": {
+ "minval": 1000000,
+ "maxval": 2000000,
+ "tunable": false
+ }
+ }
+"""
+
+
+def update_knobs(tunable_knob_file, knobs, num):
+ if num <= 0:
+ return
+ cnt = 0
+ with open(tunable_knob_file, 'r') as f:
+ # csv file columns: knob,min,max,vartype,safe,note,
+ # discard the label row
+ lines = f.readlines()[1:-1]
+ for line in lines:
+ field_list = line.split(',')
+ name = ('global.' + field_list[0]).lower()
+ if name in knobs:
+ knob = knobs[name]
+ if knob['tunable'] is True:
+ continue
+ knob['tunable'] = True
+ if len(field_list[1]) > 0 and len(field_list[2]) > 0:
+ knob['minval'] = field_list[1]
+ knob['maxval'] = field_list[2]
+ cnt += 1
+ if cnt == num:
+ break
+
+
+class Command(BaseCommand):
+ help = HELP
+
+ def create_parser(self, prog_name, subcommand):
+ parser = super(Command, self).create_parser(prog_name, subcommand)
+ parser.formatter_class = RawTextHelpFormatter
+ return parser
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ '-f', '--filename',
+ metavar='FILE',
+ default='session_knobs.json',
+ help='Name of the target knob file in json format. '
+ 'Default: session_knobs.json')
+ parser.add_argument(
+ '-s', '--source',
+ metavar='SOURCE',
+ default='selected_knobs.csv',
+ help='Name of the file to read the session knob tunability from. '
+ 'Default: selected_knobs.csv')
+ parser.add_argument(
+ '-o', '--output',
+ metavar='OUTPUT',
+ default='selected_knobs.csv',
+ help='Name of the file to write the updated session knob tunability to. '
+ 'Default: selected_knobs.csv')
+ parser.add_argument(
+ '-d', '--file_dir',
+ metavar='FILEDIR',
+ help='Path of the directory of the target knob file. '
+ 'Default: current directory')
+ parser.add_argument(
+ '-i', '--source_dir',
+ metavar='SOURCEDIR',
+ help='Path of the directory to read the session knob tunability from. '
+ 'Default: current directory')
+ parser.add_argument(
+ '-n', '--num',
+ metavar='NUM',
+ default='50',
+ help='Total number of tunable session knobs. '
+ 'Default: 50')
+
+ def handle(self, *args, **options):
+ file_dir = options['file_dir'] or ''
+ path = os.path.join(file_dir, options['filename'])
+
+ try:
+ with open(path, 'r') as f:
+ knobs = json.load(f)
+ except FileNotFoundError:
+ raise CommandError("ERROR: File '{}' does not exist.".format(path))
+ except json.decoder.JSONDecodeError:
+ raise CommandError("ERROR: Unable to decode JSON file '{}'.".format(path))
+
+ source_dir = options['source_dir'] or ''
+ tunable_knob_file = os.path.join(source_dir, options['source'])
+ target_tunable_knobs = int(options['num'])
+
+ cur_tunable_knobs = 0
+ for knob in knobs.values():
+ if knob['tunable']:
+ cur_tunable_knobs += 1
+ if cur_tunable_knobs < target_tunable_knobs:
+ update_knobs(tunable_knob_file, knobs, target_tunable_knobs - cur_tunable_knobs)
+ cur_tunable_knobs = 0
+ for knob in knobs.values():
+ if knob['tunable']:
+ cur_tunable_knobs += 1
+
+ out_path = os.path.join(file_dir, options['output'])
+ with open(out_path, 'w') as f:
+ json.dump(knobs, f, indent=4)
+ self.stdout.write(self.style.SUCCESS(
+ "Writing {} tunable session knobs into {}.".format(cur_tunable_knobs, out_path)))