feat: FIR-30155 support of v2 engines (#63)

FIR-31110, FIR-29665, FIR-30655, FIR-31842, FIR-30155, FIR-30147, FIR-32172, FIR-31660
firebolt-db · Apr 24, 2024 · ab72d73 · ab72d73
1 parent a3cc545
commit ab72d73
Show file tree

Hide file tree

Showing 18 changed files with 742 additions and 236 deletions.
diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml
@@ -1,15 +1,26 @@
 name: Integration tests 1.0
 on:
   workflow_dispatch:
+    inputs:
       database:
         description: 'Database - a new one will be created if not provided'
         required: false
         default: ''
       engine:
         description: 'Engine - a new one will be created if not provided'
         required: false
-
   workflow_call:
+    inputs:
+      database:
+        description: 'Database - a new one will be created if not provided'
+        required: false
+        default: ''
+        type: string
+      engine:
+        description: 'Engine - a new one will be created if not provided'
+        required: false
+        type: string
+
     secrets:
       FIREBOLT_STG_USERNAME:
         required: true
@@ -80,5 +91,5 @@ jobs:
           FIREBOLT_USERNAME: ${{ secrets.FIREBOLT_STG_USERNAME }}
           FIREBOLT_PASSWORD: ${{ secrets.FIREBOLT_STG_PASSWORD }}
         run: |
-          dotnet test --filter '(Category=Integration|Category=v1)&Category!=v2' -l "console;verbosity=normal"
+          dotnet test --filter 'Category=Integration&(Category=v1|Category=general)&Name!=ExecuteLongTest' -l "console;verbosity=normal"
 
diff --git a/.github/workflows/integration-tests-v2.yml b/.github/workflows/integration-tests-v2.yml
@@ -9,7 +9,33 @@ on:
       engine:
         description: 'Engine - a new one will be created if not provided'
         required: false
+      category:
+        description: 'Version of used engine (engine-v2 means v2, infra version 2.1)'
+        required: true
+        default: 'true'
+        type: choice
+        options:
+          - 'v2'
+          - 'engine-v2'
   workflow_call:
+    inputs:
+      database:
+        description: 'Database - a new one will be created if not provided'
+        required: false
+        default: ''
+        type: string
+      engine:
+        description: 'Engine - a new one will be created if not provided'
+        required: false
+        type: string
+      account:
+        description: 'Account'
+        required: true
+        type: string
+      category:
+        description: 'Run category'
+        required: true
+        type: string
     secrets:
       FIREBOLT_CLIENT_ID_STG_NEW_IDN:
         required: true
@@ -20,6 +46,19 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     steps:
+      - name: Set account engine v2
+        id: set-account
+        run: |
+          if ! [[ -z "${{ inputs.account }}" ]]; then
+            echo "account=${{ inputs.account }}" >> $GITHUB_OUTPUT
+            echo "account=${{ inputs.account }}"
+          elif [[ "${{ inputs.account }}" == "engine-v2" ]]; then
+            echo "account=${{ vars.FIREBOLT_ACCOUNT_V2 }}" >> $GITHUB_OUTPUT
+            echo "account=${{ vars.FIREBOLT_ACCOUNT_V2 }}"
+          else
+            echo "account=${{ vars.FIREBOLT_ACCOUNT_V1 }}" >> $GITHUB_OUTPUT
+            echo "account=${{ vars.FIREBOLT_ACCOUNT_V1 }}"
+          fi
       - name: Check out code
         uses: actions/checkout@v2
         with:
@@ -43,13 +82,13 @@ jobs:
         run: echo '### Ran integration tests against ${{ inputs.environment }} ' >> $GITHUB_STEP_SUMMARY
 
       - name: Setup database and engine
-        if: ${{ inputs.database == '' }}
+        if: ${{ (inputs.database == '') && (inputs.category != 'engine-v2') }}
         id: setup
         uses: firebolt-db/integration-testing-setup@v2
         with:
           firebolt-client-id: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}
           firebolt-client-secret: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}
-          account: ${{ vars.FIREBOLT_ACCOUNT }}
+          account: ${{ steps.set-account.outputs.account }}
           api-endpoint: "api.staging.firebolt.io"
           instance-type: "B2"
 
@@ -71,13 +110,23 @@ jobs:
               echo "engine_name=${{ steps.setup.outputs.engine_name }}" >> $GITHUB_OUTPUT
           fi
 
+      - name: Setup env by build
+        id: setup-env
+        run: |
+          if [ ${{ inputs.category }} == 'engine-v2' ]; then
+              echo "run_setup=true" >> $GITHUB_OUTPUT
+          else
+              echo "run_setup=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Run integration tests
         env:
           FIREBOLT_ENV: staging
-          FIREBOLT_ACCOUNT: ${{ vars.FIREBOLT_ACCOUNT }}
+          FIREBOLT_ACCOUNT: ${{ steps.set-account.outputs.account }}
           FIREBOLT_DATABASE: ${{ steps.setup.outputs.database_name }}
           FIREBOLT_ENGINE_NAME: ${{ steps.setup.outputs.engine_name }}
           FIREBOLT_CLIENT_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN  }}
           FIREBOLT_CLIENT_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN  }}
+          FIREBOLT_RUN_SETUP: ${{ steps.setup-env.outputs.run_setup }}
         run: |
-          dotnet test --filter '(Category=Integration|Category=v2)&Category!=v1' -l "console;verbosity=normal"
+          dotnet test --filter 'Category=Integration&(Category=${{ inputs.category }}|Category=general)&Name!=ExecuteLongTest' -l "console;verbosity=normal"
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -1,6 +1,32 @@
 name: Integration tests
 on:
   workflow_dispatch:
+    inputs:
+      run-v1:
+        description: 'Run tests against Firebolt DB v1'
+        required: true
+        default: 'true'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+      run-v2:
+        description: 'Run tests against Firebolt DB v2'
+        required: true
+        default: 'true'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+      run-database-with-engine-v2:
+        description: 'Run tests against Firebolt DB v2 and Engine V2'
+        required: true
+        default: 'true'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+
   workflow_call:
     secrets:
       # V1 secrets
@@ -13,14 +39,31 @@ on:
         required: true
       FIREBOLT_CLIENT_SECRET_STG_NEW_IDN:
         required: true
+
 jobs:
   integration-test-v1:
+    if: ${{ inputs.run-v1 == 'true' }}
     uses: ./.github/workflows/integration-tests-v1.yml
     secrets:
       FIREBOLT_STG_USERNAME: ${{ secrets.FIREBOLT_STG_USERNAME }}
       FIREBOLT_STG_PASSWORD: ${{ secrets.FIREBOLT_STG_PASSWORD }}
+
   integration-test-v2:
+    if: ${{ inputs.run-v2 == 'true' }}
+    uses: ./.github/workflows/integration-tests-v2.yml
+    with:
+      account: ${{ vars.FIREBOLT_ACCOUNT_V1 }}
+      category: v2
+    secrets:
+      FIREBOLT_CLIENT_ID_STG_NEW_IDN: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}
+      FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}
+
+  integration-test-engine-v2:
+    if: ${{ inputs.run-database-with-engine-v2 == 'true' }}
     uses: ./.github/workflows/integration-tests-v2.yml
+    with:
+      account: ${{ vars.FIREBOLT_ACCOUNT_V2 }}
+      category: engine-v2
     secrets:
       FIREBOLT_CLIENT_ID_STG_NEW_IDN: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}
-      FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}
+      FIREBOLT_CLIENT_SECRET_STG_NEW_IDN: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}      
diff --git a/FireboltDotNetSdk.Tests/Integration/ConnectionTest.cs b/FireboltDotNetSdk.Tests/Integration/ConnectionTest.cs
@@ -8,14 +8,15 @@ namespace FireboltDotNetSdk.Tests
     [TestFixture]
     internal class ConnectionTest : IntegrationTest
     {
-        [TestCase(true, false, Description = "Connect without engine but with database")]
-        [TestCase(false, true, Description = "Connect with engine but without database", Category = "v2")]
-        [TestCase(true, true, Description = "Connect with both engine and database")]
-        public void SuccessfulConnectTest(bool useDatabase, bool useEngine)
+        [TestCase(true, false, true, Description = "Connect without engine but with database", Category = "v1")]
+        [TestCase(false, true, true, Description = "Connect with engine but without database; used default database", Category = "v2")]
+        [TestCase(false, false, false, Description = "Connect without engine and without database", Category = "v2,engine-v2")]
+        [TestCase(false, true, false, Description = "Connect with engine but without database", Category = "engine-v2")]
+        [TestCase(true, true, true, Description = "Connect with both engine and database", Category = "v1,v2,engine-v2")]
+        public void SuccessfulConnectTest(bool useDatabase, bool useEngine, bool expectsDatabase)
         {
             var connString = GetConnectionString(useDatabase, useEngine);
-            // if engine is specified we use default database
-            string expectedDatabase = useDatabase || useEngine ? Database : string.Empty;
+            string expectedDatabase = expectsDatabase ? Database : string.Empty;
             DbConnection connection = new FireboltConnection(connString);
             Assert.That(connection.ConnectionString, Is.EqualTo(connString));
 
@@ -27,7 +28,7 @@ public void SuccessfulConnectTest(bool useDatabase, bool useEngine)
 
             Assert.That(connection.DataSource ?? string.Empty, Is.EqualTo(expectedDatabase));
             DbCommand command = connection.CreateCommand();
-            command.CommandText = "SELECT TOP 1 * FROM information_schema.tables";
+            command.CommandText = "SELECT 1";
             // Test case for use engine
             DbDataReader reader = command.ExecuteReader();
             Assert.NotNull(reader);
@@ -44,23 +45,25 @@ public void SuccessfulConnectTest(bool useDatabase, bool useEngine)
             connection.Close();
         }
 
-        [TestCase(false, false, Description = "Connect without database and engine")]
-        [TestCase(false, true, Description = "Connect with engine but without database", Category = "v1")]
-        public void FailedConnectTest(bool useDatabase, bool useEngine)
+        [TestCase(false, false, null, "Cannot get url of default engine url from  database", Category = "v1")]
+        [TestCase(false, true, "SELECT 1", "The operation is unauthorized", Category = "v1")]
+        public void UnsuccessfulConnectTest(bool useDatabase, bool useEngine, string query, string errorMessage)
         {
             var connString = GetConnectionString(useDatabase, useEngine);
-            // if engine is specified we use default database
-            string expectedDatabase = useDatabase || useEngine ? Database : string.Empty;
             DbConnection connection = new FireboltConnection(connString);
             Assert.That(connection.ConnectionString, Is.EqualTo(connString));
 
             FireboltException? exception = Assert.Throws<FireboltException>(() =>
             {
                 connection.Open();
-                DbCommand command = connection.CreateCommand();
-                command.CommandText = "SELECT TOP 1 * FROM information_schema.tables";
-                command.ExecuteReader();
+                if (query != null)
+                {
+                    DbCommand command = connection.CreateCommand();
+                    command.CommandText = query;
+                    command.ExecuteReader();
+                }
             });
+            Assert.That(exception!.Message, Does.Contain(errorMessage));
         }
 
         private string GetConnectionString(bool useDatabase, bool useEngine)
@@ -78,7 +81,7 @@ private string GetConnectionString(bool useDatabase, bool useEngine)
         }
 
         [TestCase("Account with name non-existing-account-123 was not found", Category = "v1")]
-        [TestCase("Account 'non-existing-account-123' does not exist in this organization", Category = "v2")]
+        [TestCase("Account 'non-existing-account-123' does not exist in this organization", Category = "v2,engine-v2")]
         public void InvalidAccountConnectTest(string errorMessage)
         {
             var connString = ConnectionString(new Tuple<string, string?>(nameof(Account), "non-existing-account-123"));
@@ -89,6 +92,7 @@ public void InvalidAccountConnectTest(string errorMessage)
         }
 
         [Test]
+        [Category("general")]
         public void ChangeDatabaseToNotExistingWhenConnectionIsOpen()
         {
             var connString = ConnectionString();
@@ -99,6 +103,7 @@ public void ChangeDatabaseToNotExistingWhenConnectionIsOpen()
         }
 
         [Test]
+        [Category("general")]
         public void ChangeDatabaseToNotExistingWhenConnectionIsNotOpen()
         {
             var connString = ConnectionString();
@@ -109,6 +114,7 @@ public void ChangeDatabaseToNotExistingWhenConnectionIsNotOpen()
         }
 
         [Test]
+        [Category("general")]
         public void ChangeDatabaseToExistingWhenConnectionIsOpen()
         {
             var connString = ConnectionString(new Tuple<string, string?>(nameof(Database), "DOES_NOT_EXIST"));
@@ -123,6 +129,7 @@ public void ChangeDatabaseToExistingWhenConnectionIsOpen()
         }
 
         [Test]
+        [Category("general")]
         public void ChangeDatabaseToSameConnectionIsOpen()
         {
             var connString = ConnectionString();
@@ -136,6 +143,7 @@ public void ChangeDatabaseToSameConnectionIsOpen()
         }
 
         [Test]
+        [Category("general")]
         public void ChangeDatabaseToSameConnectionNotOpen()
         {
             var connString = ConnectionString();
@@ -151,6 +159,7 @@ public void ChangeDatabaseToSameConnectionNotOpen()
 
 
         [Test]
+        [Category("general")]
         public void SetConnectionStringSameValueNotOpen()
         {
             var connString = ConnectionString();
@@ -162,6 +171,7 @@ public void SetConnectionStringSameValueNotOpen()
         }
 
         [Test]
+        [Category("general")]
         public void SetConnectionStringSameValueOpen()
         {
             var connString = ConnectionString();
@@ -192,6 +202,7 @@ public void SetAllFieldsUsingConnectionStringFirstWrongThenGoodV1()
 
         [Test]
         [Category("v2")]
+        [Category("engine-v2")]
         public void SetAllFieldsUsingConnectionStringFirstWrongThenGoodV2()
         {
             var connString1 = ConnectionString(
@@ -216,6 +227,7 @@ public void SetAllFieldsUsingConnectionStringFirstGoodThenWrongV1()
 
         [Test]
         [Category("v2")]
+        [Category("engine-v2")]
         public void SetAllFieldsUsingConnectionStringFirstGoodThenWrongV2()
         {
             SetAllFieldsUsingConnectionStringFirstGoodThenWrong<HttpRequestException>(nameof(UserName), nameof(Password));
@@ -237,6 +249,7 @@ private void SetAllFieldsUsingConnectionStringFirstGoodThenWrong<E>(params strin
 
         [TestCase(nameof(Account))]
         [TestCase(nameof(Database))]
+        [Category("general")]
         public void SetFieldUsingConnectionStringFirstGoodThenWrong(string fieldName)
         {
             var connString1 = ConnectionString();