From 7a026c7e103bebd2adc813fd0577f51483c31330 Mon Sep 17 00:00:00 2001 From: Levi Malott Date: Wed, 31 Jul 2019 10:18:12 -0500 Subject: [PATCH 1/3] Log, but allow, failures during cleanup rollbacks. In Postgres, rollbacks can fail if the transaction was killed by the database. One common scenario is that the `idle_in_transaction_session_timeout` is enabled. If the transaction was cancelled, the connection is left open in `dbt`. `dbt` attempts to close that connection after issuing a `ROLLBACK`. But it fails since the transaction was severed. Since the cleanup is carried out in a `finally` statement, the `psycopg2.InternalDatabaseError` is thrown and prevents the test case results from ever being shown. Changes here wrap the `ROLLBACK` in a try-catch such that if there is an exception thrown, it is logged appropriately, but ultimately proceeds. --- core/dbt/adapters/base/connections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 8a29e7d9ff7..59904b07993 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -248,7 +248,10 @@ def commit(self): @classmethod def _rollback_handle(cls, connection): """Perform the actual rollback operation.""" - connection.handle.rollback() + try: + connection.handle.rollback() + except Exception: + logger.exception('Failed to rollback {}'.format(connection.name)) @classmethod def _close_handle(cls, connection): From 309911981533c504d42332ec276cb2f35edbf659 Mon Sep 17 00:00:00 2001 From: Levi Malott Date: Wed, 31 Jul 2019 14:59:14 -0500 Subject: [PATCH 2/3] Log failed rollbacks to debug logs rather than to stdout. --- core/dbt/adapters/base/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 59904b07993..dc684c1e9f5 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -251,7 +251,7 @@ def _rollback_handle(cls, connection): try: connection.handle.rollback() except Exception: - logger.exception('Failed to rollback {}'.format(connection.name)) + logger.debug('Failed to rollback {}'.format(connection.name), exc_info=True) @classmethod def _close_handle(cls, connection): From 09270933036ec8a38081f6c6eb2cf96ab20b3a63 Mon Sep 17 00:00:00 2001 From: Levi Malott Date: Wed, 31 Jul 2019 16:22:29 -0500 Subject: [PATCH 3/3] Slim down the rollback failure log to appease flake8. --- core/dbt/adapters/base/connections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index dc684c1e9f5..5ea91b42e82 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -251,7 +251,10 @@ def _rollback_handle(cls, connection): try: connection.handle.rollback() except Exception: - logger.debug('Failed to rollback {}'.format(connection.name), exc_info=True) + logger.debug( + 'Failed to rollback {}'.format(connection.name), + exc_info=True + ) @classmethod def _close_handle(cls, connection):