From 0b2ce095fa742c01f7ed18da2e595e0d08455e1a Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 15:01:00 +0200 Subject: [PATCH 01/10] utils/schema: catch operationalerror thrown by sqlite backend Former-commit-id: 17a020b321f5f73275d136d164e2d422728a1b6c Former-commit-id: a7cbe45b16c10e21a9d7b1e10e5c909965efac7b [formerly aa69fdb70e59685c3fedae73ff89f3e5503be15c] [formerly 6b08e9334c21503f985df1c21105398c7a7138ff [formerly 7e44df2d4f27aa2e23f36d94834c905ed92c4dc8]] Former-commit-id: 1d903021c21c3532db593d9c3e2d25787ba7a62c [formerly 6e174876ff4cf79863e2ff4d16d88ea73dde57b2] Former-commit-id: c24bf2e3b95bbe64959f3f723d8d81c5740b8e1f --- benchbuild/utils/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index fb2668f27..bd95fe0bc 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -291,7 +291,11 @@ def needed_schema(connection, meta): except sa.exc.CompileError as cerr: LOG.fatal("Schema could not be created! Details: %s", str(cerr)) sys.exit(-4) + except sa.exc.OperationalError: + # SQLite throws an OperationalError + return False except sa.exc.ProgrammingError: + # PostgreSQL throws a ProgrammingError return False return True From f7367cbe0b55ded716d0967796445f8d430feec1 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 15:01:16 +0200 Subject: [PATCH 02/10] utils/schema: force correct version when a new schema is created Former-commit-id: e765b1f168dd14ef3ff142120e79b7d5576e5247 Former-commit-id: 31865be1f42ee494f0fb4724f3e91c85c9abd8cd [formerly e1929ad1b0c0a9d59aa3330a2c57d0688d6d33c9] [formerly f055e9e9f67da8affa070f65f9f9be8bc2e5be03 [formerly 051ae531b310fd99819f8b70975eac7168a17edc]] Former-commit-id: 25a6c442374e542c70e32e6ee93efee36b8d9110 [formerly a46035c6678cf3586a016fa8e16984689053640c] Former-commit-id: c20f4f06f71899fbe8ea88ff32995f70fd770425 --- benchbuild/utils/schema.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index bd95fe0bc..cd29174df 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -300,10 +300,26 @@ def needed_schema(connection, meta): return True -def setup_versioning(): +def get_version_data(): + """Retreive migration information.""" connect_str = settings.CFG["db"]["connect_string"].value() repo_url = bbpath.template_path("../db/") + return (connect_str, repo_url) + + +def enforce_versioning(): + """Install versioning on the db.""" + connect_str, repo_url = get_version_data() + LOG.warning("Your database uses an unversioned benchbuild schema.") + if not ui.ask("Should I enforce version control on your schema?"): + LOG.error("User declined schema versioning.") + return (-1, -1) + repo_version = migrate.version(repo_url, url=connect_str) + migrate.version_control(connect_str, repo_url, version=repo_version) + +def setup_versioning(): + connect_str, repo_url = get_version_data() repo_version = migrate.version(repo_url, url=connect_str) db_version = None requires_versioning = False @@ -313,11 +329,7 @@ def setup_versioning(): requires_versioning = True if requires_versioning: - LOG.warning("Your database uses an unversioned benchbuild schema.") - if not ui.ask("Should I enforce version control on your schema?"): - LOG.error("User declined schema versioning.") - return (-1, -1) - migrate.version_control(connect_str, repo_url) + enforce_versioning() return setup_versioning() return (repo_version, db_version) @@ -394,6 +406,7 @@ def __init__(self): if needed_schema(self.connection, BASE.metadata): LOG.debug("Initialized new db schema.") + enforce_versioning() repo_version, db_version = setup_versioning() maybe_update_db(repo_version, db_version) From 83122f6aaa274ccec8a984c1792ae53eb68750e4 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 15:03:06 +0200 Subject: [PATCH 03/10] utils/schema: do not log an error when we use sqlite with GUIDs Former-commit-id: 342a625cfb38d3c79450b97dc7a9d9a6e6191172 Former-commit-id: 6de8f27020749d5bd12b3fb2819cc352c9bab625 [formerly d80c75e8b5593b8d4d2f0469ebe89a34dd7fac5e] [formerly 667b54df3f7ab83d4316cecc3b01223e44c694f5 [formerly e22bc99de32d3687ddebd0efadd481e7d2977e8e]] Former-commit-id: 25beecdf7719797d2bc2fa5befaa49b6c64522b0 [formerly d67e26d7d1c6fd7477fcf3e4d91789f351500fcf] Former-commit-id: b5e9033a46c064ddf2b1bc4361425eca75d99aec --- benchbuild/utils/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index cd29174df..34d2af0bf 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -84,7 +84,6 @@ def process_result_value(self, value, dialect): if isinstance(value, uuid.UUID): return value else: - LOG.error(str(value)) return uuid.UUID(str(value)) From 171922c580434b2457a9152eefb00610fcc80974 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:14:22 +0200 Subject: [PATCH 04/10] utils/ui: track an automated response when not running with a TTY Former-commit-id: 3c6419ec5cfd6d15a92f62244d7a76deda669ede Former-commit-id: f37ae9314a2060f1a5ed2d7e1d1cbb2a100f9a88 [formerly 277ff2508aa0f1b245475bb68356412770c4a7a1] [formerly 8a8e7a2ab9f9197b0f8f750277b5a50a11283723 [formerly 9a768033e02efbc087c2af39dc1f3cedee6b7864]] Former-commit-id: 8003378e1acff6ad16e061683c98b3d2ab63ebdc [formerly ce9f02dec3d7adef1c8b12b56b9950226ebd7c89] Former-commit-id: 8b16f1bb0c2b87455ecc3cd1eced5861d921c039 --- benchbuild/utils/user_interface.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchbuild/utils/user_interface.py b/benchbuild/utils/user_interface.py index e4f5da0ff..157e734b3 100644 --- a/benchbuild/utils/user_interface.py +++ b/benchbuild/utils/user_interface.py @@ -1,8 +1,11 @@ """ User interface helpers for benchbuild. """ -import sys +import logging import os +import sys + +LOG = logging.getLogger(__name__) # Taken from the following recipe: http://code.activestate.com/recipes/577058/ @@ -76,4 +79,7 @@ def should_ignore_tty(): has_tty = sys.stdin.isatty() and not ignore_stdin_istty if has_tty: response = query_yes_no(question, default_answer_str) + else: + LOG.debug("NoTTY: %s -> %s" % (question, response)) + return response From cdb735f2e32c2a8bdf233a61d383eb1d2f04d478 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:16:53 +0200 Subject: [PATCH 05/10] cli: do not create the build directory, ConfigPath takes care of it. We provide the build_dir inside a ConfigPath object. If the user wants to rely on a directory without ConfigPath protection, it is his problem. Reminder: !create-if-needed in front of your path string makes sure that it becomes ConfigPath protected. Former-commit-id: 16fc745edb6f7ef4dbb16d933a1c1824383265d6 Former-commit-id: ee23792f64c5ef906161772006316d470d866264 [formerly ebedae4d9c0df3a45527b1c08dab15af0a425889] [formerly bd50c17e76d2866ebdc0726a5d081c3c42698312 [formerly 0e87494bd6b6763129bf161db30a8b15599d04d1]] Former-commit-id: b2009aa9fc0390cdace2906cc0f1950b7aace372 [formerly 8a4b723da8f1b018978cd2dfc2d19b14e94d76d5] Former-commit-id: 1ae756945ab9caf234b1eb4f6f83a3adad34e07b --- benchbuild/cli/bootstrap.py | 2 -- benchbuild/cli/run.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/benchbuild/cli/bootstrap.py b/benchbuild/cli/bootstrap.py index 73da22a31..e26b1cfd4 100644 --- a/benchbuild/cli/bootstrap.py +++ b/benchbuild/cli/bootstrap.py @@ -3,7 +3,6 @@ from plumbum import cli import benchbuild.utils.bootstrap as bs -import benchbuild.utils.path as p from benchbuild.cli.main import BenchBuild from benchbuild.settings import CFG @@ -27,7 +26,6 @@ class BenchBuildBootstrap(cli.Application): def main(self, *args): del args # Unused - p.mkdir_interactive(str(CFG["build_dir"])) print("Checking benchbuild binary dependencies...") provide_package("cmake") provide_package("fusermount") diff --git a/benchbuild/cli/run.py b/benchbuild/cli/run.py index 53517a158..f8c8fea03 100644 --- a/benchbuild/cli/run.py +++ b/benchbuild/cli/run.py @@ -69,9 +69,6 @@ def set_group(self, groups): pretend = cli.Flag(['p', 'pretend'], default=False) def __generate_plan(self, exps, prjs, cfg): - if prjs: - path.mkdir_interactive(str(cfg["build_dir"])) - for exp_cls in exps.values(): exp = exp_cls(projects=prjs) eactn = actions.Experiment(obj=exp, actions=exp.actions()) From 1c58a60450c6220843e1e25a343aea9cc1951cb4 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:19:02 +0200 Subject: [PATCH 06/10] utils/log: revert logging format change Former-commit-id: 9781ee829df52286ba2ada527f3ff8d52e04fd5d Former-commit-id: 9e9049418005eb76abb86cdc8dc1b38bb11da883 [formerly f6df92de91c989542e7385b4790ba48c396a2671] [formerly 78d546588ddf26f251ea4545679ab98542fba2c1 [formerly 819324cc4d40e9ea16a6e9dc93a69c69eb88c63b]] Former-commit-id: ed386fb6158caffc77dc8f0c9090c004a72cb527 [formerly 06039461b5bcbdfc99d8afb91cdf37f75be7a9db] Former-commit-id: f036dcbed72bb9fdc34d17a8da388bbd6e315aae --- benchbuild/utils/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/utils/log.py b/benchbuild/utils/log.py index 158faa02e..9f705e9b4 100644 --- a/benchbuild/utils/log.py +++ b/benchbuild/utils/log.py @@ -40,7 +40,7 @@ def configure(): details_hdl.setFormatter(details_format) root_logger.addHandler(details_hdl) else: - brief_format = logging.Formatter('%(name)s %(message)s') + brief_format = logging.Formatter('%(message)s') console_hdl = logging.StreamHandler() console_hdl.setFormatter(brief_format) root_logger.addHandler(console_hdl) From a2bac7313b8acc65a39d664f5fe59de183decf8d Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:19:22 +0200 Subject: [PATCH 07/10] utils/path: wording of mkdir_interactive was wrong. Former-commit-id: 51c0d24d966a8c9e8c4786d66d3c4cdc5561770f Former-commit-id: d3699f65dbc0c6fda22c6703fc78323756650428 [formerly 08927406ad685620d2c39b5cb91e562c2fd5fe2b] [formerly 412e34cd18e256feff645df07d6bb41a282b4cfe [formerly ebffa73c142d28ca1bf20b927ea395a6981aa91c]] Former-commit-id: eabb0d87fa092fc859491b6e821189ddfbd93c73 [formerly 358ffd23d34af951606edbb9d4f8a09bcf39260b] Former-commit-id: 52275cfe812ba33093192a3dc13c457c72ddef65 --- benchbuild/utils/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/utils/path.py b/benchbuild/utils/path.py index be27bb879..ff75cea9a 100644 --- a/benchbuild/utils/path.py +++ b/benchbuild/utils/path.py @@ -120,7 +120,7 @@ def mkdir_interactive(dirpath): return response = ui.ask( - "The build directory {dirname} does not exist yet. " + "The directory {dirname} does not exist yet. " "Should I create it?".format(dirname=dirpath), default_answer=True, default_answer_str="yes") From b43bc2b35fa417c4781011d0d491878ced62ed7f Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:20:43 +0200 Subject: [PATCH 08/10] utils/settings: do not validate ConfigPath on creation, but on access. This avoids a problem where ConfigPath would create the default values as soon as they are loaded but before they are updated with a new value. Former-commit-id: 663dcafb7013278046e0a71dfbf6212956152ab0 Former-commit-id: 5a542d627c6790c9c2281e826729b68b801edab8 [formerly ee8d287b4850c944c0a466199200e3529432d91f] [formerly a7ea08ad74038b7a508a6dda29f5e6ba0c462175 [formerly 7cac6c22888527ddcb0b521350ddd615d7a1177d]] Former-commit-id: b2dedb099df8badd7b9f15bb32599d766d2d671a [formerly 994b8a0542ef151623c60283d254527e74eb76f8] Former-commit-id: 4bec62e727bc482a34c733be8da9807ba115b0f1 --- benchbuild/utils/settings.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/benchbuild/utils/settings.py b/benchbuild/utils/settings.py index fba0f843a..7faaf52a7 100644 --- a/benchbuild/utils/settings.py +++ b/benchbuild/utils/settings.py @@ -288,8 +288,14 @@ def value(self): TEST_X_Z=2 """ + + def validate(node_value): + if hasattr(node_value, 'validate'): + node_value.validate() + return node_value + if 'value' in self.node: - return self.node['value'] + return validate(self.node['value']) else: return self @@ -404,17 +410,19 @@ class ConfigPath(object): >>> p = ConfigPath(['tmp']); str(p) '/tmp' >>> p = ConfigPath('/tmp/test/foo'); str(p) - The path '/tmp/test/foo' is required by your configuration. '/tmp/test/foo' + >>> p.validate() + The path '/tmp/test/foo' is required by your configuration. + >>> p.validate() + >>> p = ConfigPath([]); str(p) '/' """ components = attr.ib(converter=convert_components) - @components.validator - def validate_path(self, attribute, value): - del attribute - path_str = ConfigPath.path_to_str(value) + def validate(self): + """Make sure this configuration path exists.""" + path_str = ConfigPath.path_to_str(self.components) path_exists = os.path.exists(path_str) def create_dir(): From c538055b48d29e9d79437aab8d623116213f4dc9 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:37:44 +0200 Subject: [PATCH 09/10] utils/ui: provide format arguments as function arguments Former-commit-id: e2d9b804bfdfd0bae623515794ccbaadf629e510 Former-commit-id: 75d1ba8ddc39c7e2fef0f39ad6bf35c6ed924289 [formerly 01e043ca4db22c297098bd5b7ffd9347499ad3cf] [formerly eb70c98fd1731fd62f09a72abebf28de956466ea [formerly 24e3459b5435fbf508da546e5b93c1cc24cede90]] Former-commit-id: d7efd6f1666ddb6fa2a3f82ce99af411be5236d6 [formerly 248e9ca4fad35cfed3753cefe763730199a0712c] Former-commit-id: 11d6c07f2bcd0e336a9ae58d35c157a2331b12c4 --- benchbuild/utils/user_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/utils/user_interface.py b/benchbuild/utils/user_interface.py index 157e734b3..ac804b26a 100644 --- a/benchbuild/utils/user_interface.py +++ b/benchbuild/utils/user_interface.py @@ -80,6 +80,6 @@ def should_ignore_tty(): if has_tty: response = query_yes_no(question, default_answer_str) else: - LOG.debug("NoTTY: %s -> %s" % (question, response)) + LOG.debug("NoTTY: %s -> %s", question, response) return response From 275e2f605dd0d80d56136fa276069837f8d2f46b Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 11 Jul 2018 23:40:44 +0200 Subject: [PATCH 10/10] utils/schema: hotfix endless loop when user declines schema versioning. Former-commit-id: 445624e6d297dacd7da373e2ce68a0d2a9101f40 Former-commit-id: 721ea6efe523bf5eb79f6cd0e71fcac3d700d7f4 [formerly d660c8101f95d7f349c286799b053da3be9b27e0] [formerly b40f64f02d04b47ee01da2127d8dadcff2deb1cf [formerly 4e1f1fb9975cb74a5d3b553bf5236f7c03d4d507]] Former-commit-id: 5026791b405fe71d94082dff50995f8391525030 [formerly e3b9c86e7abd67c57ebce1599bf373f03118231f] Former-commit-id: 581e175c1e3c49554144c43ae72b684e73d32373 --- benchbuild/utils/schema.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index 34d2af0bf..2bced76d4 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -306,15 +306,17 @@ def get_version_data(): return (connect_str, repo_url) -def enforce_versioning(): +def enforce_versioning(force=False): """Install versioning on the db.""" connect_str, repo_url = get_version_data() LOG.warning("Your database uses an unversioned benchbuild schema.") - if not ui.ask("Should I enforce version control on your schema?"): + if not force and not ui.ask( + "Should I enforce version control on your schema?"): LOG.error("User declined schema versioning.") - return (-1, -1) + return None repo_version = migrate.version(repo_url, url=connect_str) migrate.version_control(connect_str, repo_url, version=repo_version) + return repo_version def setup_versioning(): @@ -328,15 +330,14 @@ def setup_versioning(): requires_versioning = True if requires_versioning: - enforce_versioning() - return setup_versioning() + db_version = enforce_versioning() return (repo_version, db_version) def maybe_update_db(repo_version, db_version): if db_version is None: - db_version = -1 + return if db_version == repo_version: return @@ -386,7 +387,7 @@ def configure_engine(self): try: self.connection.execution_options(isolation_level="READ COMMITTED") except sa.exc.ArgumentError: - LOG.error("Unable to set isolation level to READ COMMITTED") + LOG.warning("Unable to set isolation level to READ COMMITTED") return True def __init__(self): @@ -405,7 +406,7 @@ def __init__(self): if needed_schema(self.connection, BASE.metadata): LOG.debug("Initialized new db schema.") - enforce_versioning() + enforce_versioning(force=True) repo_version, db_version = setup_versioning() maybe_update_db(repo_version, db_version)