Merge pull request #206 from MarketSquare/parse-script-improvements

Fix #184 Parse script improvements

src/DatabaseLibrary/query.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import inspect
+import re
 import sys
 from typing import List, Optional
 
@@ -208,11 +209,17 @@ def delete_all_rows_from_table(self, tableName: str, sansTran: bool = False, ali
  if cur and not sansTran:
  db_connection.client.rollback()
 
- def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, alias: Optional[str] = None):
+ def execute_sql_script(
+ self, sqlScriptFileName: str, sansTran: bool = False, split: bool = True, alias: Optional[str] = None
+ ):
  """
  Executes the content of the `sqlScriptFileName` as SQL commands. Useful for setting the database to a known
  state before running your tests, or clearing out your test data after running each a test.
 
+ SQL commands are expected to be delimited by a semicolon (';') - they will be split and executed separately.
+ You can disable this behaviour setting the parameter `split` to _False_ -
+ in this case the entire script content will be passed to the database module for execution.
+
  Sample usage :
  | Execute Sql Script | ${EXECDIR}${/}resources${/}DDL-setup.sql |
  | Execute Sql Script | ${EXECDIR}${/}resources${/}DML-setup.sql |
@@ -221,7 +228,6 @@ def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, ali
  | Execute Sql Script | ${EXECDIR}${/}resources${/}DML-teardown.sql |
  | Execute Sql Script | ${EXECDIR}${/}resources${/}DDL-teardown.sql |
 
- SQL commands are expected to be delimited by a semicolon (';') - they will be executed separately.
 
  For example:
  DELETE FROM person_employee_table;
@@ -272,76 +278,101 @@ def execute_sql_script(self, sqlScriptFileName: str, sansTran: bool = False, ali
  with open(sqlScriptFileName, encoding="UTF-8") as sql_file:
  cur = None
  try:
- statements_to_execute = []
  cur = db_connection.client.cursor()
  logger.info(f"Executing : Execute SQL Script | {sqlScriptFileName}")
- current_statement = ""
- inside_statements_group = False
-
- for line in sql_file:
- line = line.strip()
- if line.startswith("#") or line.startswith("--") or line == "/":
- continue
- if line.lower().startswith("begin"):
- inside_statements_group = True
-
- # semicolons inside the line? use them to separate statements
- # ... but not if they are inside a begin/end block (aka. statements group)
- sqlFragments = line.split(";")
- # no semicolons
- if len(sqlFragments) == 1:
- current_statement += line + " "
- continue
- quotes = 0
- # "select * from person;" -> ["select..", ""]
- for sqlFragment in sqlFragments:
- if len(sqlFragment.strip()) == 0:
+ if not split:
+ logger.info("Statements splitting disabled - pass entire script content to the database module")
+ self.__execute_sql(cur, sql_file.read())
+ else:
+ logger.info("Splitting script file into statements...")
+ statements_to_execute = []
+ current_statement = ""
+ inside_statements_group = False
+ proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?")
+ proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
+ for line in sql_file:
+ line = line.strip()
+ if line.startswith("#") or line.startswith("--") or line == "/":
  continue
- if inside_statements_group:
- # if statements inside a begin/end block have semicolns,
- # they must persist - even with oracle
- sqlFragment += "; "
- if sqlFragment.lower() == "end; ":
- inside_statements_group = False
- elif sqlFragment.lower().startswith("begin"):
+
+ # check if the line matches the creating procedure regexp pattern
+ if proc_start_pattern.match(line.lower()):
+ inside_statements_group = True
+ elif line.lower().startswith("begin"):
  inside_statements_group = True
 
- # check if the semicolon is a part of the value (quoted string)
- quotes += sqlFragment.count("'")
- quotes -= sqlFragment.count("\\'")
- quotes -= sqlFragment.count("''")
- inside_quoted_string = quotes % 2 != 0
- if inside_quoted_string:
- sqlFragment += ";" # restore the semicolon
-
- current_statement += sqlFragment
- if not inside_statements_group and not inside_quoted_string:
- statements_to_execute.append(current_statement.strip())
- current_statement = ""
- quotes = 0
-
- current_statement = current_statement.strip()
- if len(current_statement) != 0:
- statements_to_execute.append(current_statement)
-
- for statement in statements_to_execute:
- logger.info(f"Executing statement from script file: {statement}")
- omit_semicolon = not statement.lower().endswith("end;")
- self.__execute_sql(cur, statement, omit_semicolon)
+ # semicolons inside the line? use them to separate statements
+ # ... but not if they are inside a begin/end block (aka. statements group)
+ sqlFragments = line.split(";")
+ # no semicolons
+ if len(sqlFragments) == 1:
+ current_statement += line + " "
+ continue
+ quotes = 0
+ # "select * from person;" -> ["select..", ""]
+ for sqlFragment in sqlFragments:
+ if len(sqlFragment.strip()) == 0:
+ continue
+
+ if inside_statements_group:
+ # if statements inside a begin/end block have semicolns,
+ # they must persist - even with oracle
+ sqlFragment += "; "
+
+ if proc_end_pattern.match(sqlFragment.lower()):
+ inside_statements_group = False
+ elif proc_start_pattern.match(sqlFragment.lower()):
+ inside_statements_group = True
+ elif sqlFragment.lower().startswith("begin"):
+ inside_statements_group = True
+
+ # check if the semicolon is a part of the value (quoted string)
+ quotes += sqlFragment.count("'")
+ quotes -= sqlFragment.count("\\'")
+ quotes -= sqlFragment.count("''")
+ inside_quoted_string = quotes % 2 != 0
+ if inside_quoted_string:
+ sqlFragment += ";" # restore the semicolon
+
+ current_statement += sqlFragment
+ if not inside_statements_group and not inside_quoted_string:
+ statements_to_execute.append(current_statement.strip())
+ current_statement = ""
+ quotes = 0
+
+ current_statement = current_statement.strip()
+ if len(current_statement) != 0:
+ statements_to_execute.append(current_statement)
+
+ for statement in statements_to_execute:
+ logger.info(f"Executing statement from script file: {statement}")
+ line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$")
+ omit_semicolon = not line_ends_with_proc_end.search(statement.lower())
+ self.__execute_sql(cur, statement, omit_semicolon)
  if not sansTran:
  db_connection.client.commit()
  finally:
  if cur and not sansTran:
  db_connection.client.rollback()
 
  def execute_sql_string(
- self, sqlString: str, sansTran: bool = False, alias: Optional[str] = None, parameters: Optional[List] = None
+ self,
+ sqlString: str,
+ sansTran: bool = False,
+ omitTrailingSemicolon: Optional[bool] = None,
+ alias: Optional[str] = None,
+ parameters: Optional[List] = None,
  ):
  """
- Executes the sqlString as SQL commands. Useful to pass arguments to your sql.
- SQL commands are expected to be delimited by a semicolon (';').
+ Executes the ``sqlString`` as a single SQL command.
 
- Use optional `sansTran` to run command without an explicit transaction commit or rollback:
+ Use optional ``sansTran`` to run command without an explicit transaction commit or rollback.
+
+ Use optional ``omitTrailingSemicolon`` parameter for explicit instruction,
+ if the trailing semicolon (;) at the SQL string end should be removed or not:
+ - Some database modules (e.g. Oracle) throw an exception, if you leave a semicolon at the string end
+ - However, there are exceptional cases, when you need it even for Oracle - e.g. at the end of a PL/SQL block.
+ - If not specified, it's decided based on the current database module in use. For Oracle, the semicolon is removed by default.
 
  Use optional ``alias`` parameter to specify what connection should be used for the query if you have more
  than one connection open.
@@ -353,6 +384,7 @@ def execute_sql_string(
  | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table |
  | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table | alias=my_alias |
  | Execute Sql String | DELETE FROM person_employee_table; DELETE FROM person_table | sansTran=True |
+ | Execute Sql String | CREATE PROCEDURE proc AS BEGIN DBMS_OUTPUT.PUT_LINE('Hello!'); END; | omitTrailingSemicolon=False |
  | @{parameters} | Create List | person_employee_table |
  | Execute Sql String | SELECT * FROM %s | parameters=${parameters} |
  """
@@ -361,7 +393,7 @@ def execute_sql_string(
  try:
  cur = db_connection.client.cursor()
  logger.info(f"Executing : Execute SQL String | {sqlString}")
- self.__execute_sql(cur, sqlString, parameters=parameters)
+ self.__execute_sql(cur, sqlString, omit_trailing_semicolon=omitTrailingSemicolon, parameters=parameters)
  if not sansTran:
  db_connection.client.commit()
  finally:

test/readme.md

@@ -40,7 +40,7 @@ See the folder `.github/workflows`
 
 ## Microsoft SQL Server
 - https://hub.docker.com/_/microsoft-mssql-server
-- docker run --rm --name mssql -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=MyPass1234!" -p 1433:1433 -d mcr.microsoft.com/mssql/server
+- docker run --rm --name mssql -e ACCEPT_EULA=Y -e MSSQL_SA_PASSWORD='MyPass1234!' -p 1433:1433 -d mcr.microsoft.com/mssql/server
 --> login and create DB:
  - docker exec -it mssql bash
  - /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P 'MyPass1234!'

test/resources/create_stored_procedure_mssql.sql → test/resources/create_stored_procedures_mssql.sql

@@ -33,4 +33,18 @@ BEGIN
 SELECT FIRST_NAME FROM person;
 SELECT LAST_NAME FROM person;
 RETURN;
+END;
+
+DROP PROCEDURE IF EXISTS check_condition;
+CREATE PROCEDURE check_condition
+AS
+DECLARE @v_condition BIT = 1;
+IF @v_condition = 1
+BEGIN
+PRINT 'Condition is true';
+END
+ELSE
+BEGIN
+PRINT 'Condition is false';
+END
 END;

test/resources/create_stored_procedure_mysql.sql → test/resources/create_stored_procedures_mysql.sql

@@ -27,4 +27,15 @@ CREATE PROCEDURE get_all_first_and_second_names()
 BEGIN
 SELECT FIRST_NAME FROM person;
 SELECT LAST_NAME FROM person;
-END;
+END;
+
+DROP PROCEDURE IF EXISTS check_condition;
+CREATE PROCEDURE check_condition()
+BEGIN
+ DECLARE v_condition BOOLEAN DEFAULT TRUE;
+ IF v_condition THEN
+ SELECT 'Condition is true' AS Result;
+ ELSE
+ SELECT 'Condition is false' AS Result;
+ END IF;
+END

test/resources/create_stored_procedures_oracle.sql

@@ -29,4 +29,15 @@ OPEN first_names_cursor for
 SELECT FIRST_NAME FROM person;
 OPEN second_names_cursor for
 SELECT LAST_NAME FROM person;
-END;
+END;
+
+CREATE OR REPLACE PROCEDURE 
+check_condition AS
+v_condition BOOLEAN := TRUE;
+BEGIN
+IF v_condition THEN
+DBMS_OUTPUT.PUT_LINE('Condition is true');
+ELSE
+DBMS_OUTPUT.PUT_LINE('Condition is false');
+END IF;
+END check_condition;

test/resources/create_stored_procedure_postgres.sql → test/resources/create_stored_procedures_postgres.sql

@@ -49,4 +49,23 @@ RETURN NEXT result1;
 OPEN result2 FOR SELECT LAST_NAME FROM person;
 RETURN NEXT result2;
 END
+';
+
+DROP ROUTINE IF EXISTS check_condition;
+CREATE FUNCTION
+check_condition()
+RETURNS VOID
+LANGUAGE plpgsql
+AS
+'
+DECLARE
+ v_condition BOOLEAN := TRUE;
+ v_res BOOLEAN := TRUE;
+BEGIN
+ IF v_condition THEN
+ v_res := TRUE;
+ ELSE
+ v_res := FALSE;
+ END IF;
+END
 ';

test/tests/common_tests/stored_procedures.robot

@@ -88,18 +88,21 @@ Procedure Returns Multiple Result Sets
  Should Be Equal ${second result set}[0][0] See
  Should Be Equal ${second result set}[1][0] Schneider
 
+Procedure With IF/ELSE Block
+ Call Stored Procedure check_condition
+
 
 *** Keywords ***
 Create And Fill Tables And Stored Procedures
  Create Person Table And Insert Data
  IF "${DB_MODULE}" in ["oracledb", "cx_Oracle"]
  Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_oracle.sql
  ELSE IF "${DB_MODULE}" in ["pymysql"]
- Execute SQL Script ${CURDIR}/../../resources/create_stored_procedure_mysql.sql
+ Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_mysql.sql
  ELSE IF "${DB_MODULE}" in ["psycopg2", "psycopg3"]
- Execute SQL Script ${CURDIR}/../../resources/create_stored_procedure_postgres.sql
+ Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_postgres.sql
  ELSE IF "${DB_MODULE}" in ["pymssql"]
- Execute SQL Script ${CURDIR}/../../resources/create_stored_procedure_mssql.sql
+ Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_mssql.sql
  ELSE
  Skip Don't know how to create stored procedures for '${DB_MODULE}'
  END

test/tests/custom_db_tests/oracle_omit_semicolon.robot

@@ -0,0 +1,27 @@
+*** Settings ***
+Documentation Tests for the parameter _omitTrailingSemicolon_ in the keyword
+... _Execute SQL String_ - special for the issue #184:
+... https://github.com/MarketSquare/Robotframework-Database-Library/issues/184
+... The _PLSQL BLOCK_ is most likely valid for Oracle DB only.
+
+Resource ../../resources/common.resource
+Suite Setup Connect To DB
+Suite Teardown Disconnect From Database
+Test Setup Create Person Table And Insert Data
+Test Teardown Drop Tables Person And Foobar
+
+
+*** Variables ***
+${NORMAL QUERY} SELECT * FROM person;
+${PLSQL BLOCK} DECLARE ERRCODE NUMBER; ERRMSG VARCHAR2(200); BEGIN DBMS_OUTPUT.PUT_LINE('Hello!'); END;
+
+
+*** Test Cases ***
+Explicitely Omit Semicolon
+ [Documentation] Check if it works for Oracle - explicitely omitting the semicolon
+ ... is equal to the default behaviour, otherwise oracle_db throws an error
+ Execute Sql String ${NORMAL QUERY} omitTrailingSemicolon=True
+
+Explicitely Dont't Omit Semicolon
+ [Documentation] Check if it works for Oracle - it throws an error without a semicolon
+ Execute Sql String ${PLSQL BLOCK} omitTrailingSemicolon=False

test/tests/custom_db_tests/sql_script_split_commands.robot

@@ -0,0 +1,22 @@
+*** Settings ***
+Documentation Tests for the parameter _split_ in the keyword
+... _Execute SQL Script_ - special for the issue #184:
+... https://github.com/MarketSquare/Robotframework-Database-Library/issues/184
+
+Resource ../../resources/common.resource
+Suite Setup Connect To DB
+Suite Teardown Disconnect From Database
+Test Setup Create Person Table
+Test Teardown Drop Tables Person And Foobar
+
+
+*** Test Cases ***
+Split Commands
+ [Documentation] Such a simple script works always,
+ ... just check if the logs if the parameter value was processed properly
+ Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table.sql split=True
+
+Don't Split Commands
+ [Documentation] Running such a script as a single statement works for PostgreSQL,
+ ... but fails in Oracle. Check in the logs if the splitting was disabled.
+ Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table.sql split=False