feat: SFI SQL Policy Changes (#469)

* Added Sfi Sql related code changes

* removing the index script module from main.bicep

* Adding managed identity authentication to scripts file

* Added code changes in copy_kp_files to resolved unzip issue

* added chnages in copy_kb_files

* update copy_kb_files

* updated post deployment scripts

* Added pyodbc changes in create_sql_tables.py

* Created new managed identity with data reader and writer role

* adding main.json changes

* Adding some changes to support pyodbc connection

* updating the odbc driver in db.py

* removed debug related code from app.py

* refactor the code and added comments

* Added except block to connect using username, password for existing deployment

* resolved the pylint issues

* Resolved pylint issues

* Updated the test_app.py and test_db.py

* Resolved pylint issues for unit tests

* Updated post deployment script

* updated main.json

Author: Abdul-Microsoft

ClientAdvisor/App/WebApp.Dockerfile

@@ -18,8 +18,12 @@ RUN apk add --no-cache --virtual .build-deps \
     libffi-dev \  
     openssl-dev \  
     curl \  
+    unixodbc-dev \ 
     && apk add --no-cache \  
-    libpq 
+    libpq \
+    && curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_amd64.apk \
+    && apk add --allow-untrusted msodbcsql18_18.4.1.1-1_amd64.apk \
+    && rm msodbcsql18_18.4.1.1-1_amd64.apk 
 
 COPY ./ClientAdvisor/App/requirements.txt /usr/src/app/  
 RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt \  

ClientAdvisor/App/app.py

@@ -24,6 +24,7 @@
                            format_stream_response, generateFilterString,
                            parse_multi_columns)
 from db import get_connection
+from db import dict_cursor
 
 bp = Blueprint("routes", __name__, static_folder="static", template_folder="static")
 
@@ -1583,7 +1584,8 @@ def get_users():
         ORDER BY NextMeeting ASC;
         """
         cursor.execute(sql_stmt)
-        rows = cursor.fetchall()
+        # Since pyodbc returns query results as a list of tuples, using `dict_cursor` function to convert these tuples into a list of dictionaries
+        rows = dict_cursor(cursor)
 
         if len(rows) <= 6:
             # update ClientMeetings,Assets,Retirement tables sample data to current date
@@ -1618,7 +1620,8 @@ def get_users():
                 FROM DaysDifference
                 """
             cursor.execute(combined_stmt)
-            date_diff_rows = cursor.fetchall()
+            # Since pyodbc returns query results as a list of tuples, using `dict_cursor` function to convert these tuples into a list of dictionaries
+            date_diff_rows = dict_cursor(cursor)
 
             client_days = (
                 date_diff_rows[0]["ClientMeetingDaysDifference"]

ClientAdvisor/App/db.py

@@ -1,20 +1,60 @@
 # db.py
 import os
 
-import pymssql
 from dotenv import load_dotenv
+from azure.identity import DefaultAzureCredential
+import pyodbc
+import struct
+import logging
+
 
 load_dotenv()
 
+driver = "{ODBC Driver 18 for SQL Server}"
 server = os.environ.get("SQLDB_SERVER")
 database = os.environ.get("SQLDB_DATABASE")
 username = os.environ.get("SQLDB_USERNAME")
 password = os.environ.get("SQLDB_PASSWORD")
+mid_id = os.environ.get("SQLDB_USER_MID")
+
+
+def dict_cursor(cursor):
+    """
+    Converts rows fetched by the cursor into a list of dictionaries.
+
+    Args:
+        cursor: A database cursor object.
+
+    Returns:
+        A list of dictionaries representing rows.
+    """
+    columns = [column[0] for column in cursor.description]
+    return [dict(zip(columns, row)) for row in cursor.fetchall()]
 
 
 def get_connection():
+    try:
+        credential = DefaultAzureCredential(managed_identity_client_id=mid_id)
+
+        token_bytes = credential.get_token(
+            "https://database.windows.net/.default"
+        ).token.encode("utf-16-LE")
+        token_struct = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
+        SQL_COPT_SS_ACCESS_TOKEN = (
+            1256  # This connection option is defined by Microsoft in msodbcsql.h
+        )
 
-    conn = pymssql.connect(
-        server=server, user=username, password=password, database=database, as_dict=True
-    )
-    return conn
+        # Set up the connection
+        connection_string = f"DRIVER={driver};SERVER={server};DATABASE={database};"
+        conn = pyodbc.connect(
+            connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}
+        )
+        return conn
+    except pyodbc.Error as e:
+        logging.error(f"Failed with Default Credential: {str(e)}")
+        conn = pyodbc.connect(
+            f"DRIVER={driver};SERVER={server};DATABASE={database};UID={username};PWD={password}",
+            timeout=5
+        )
+        logging.info("Connected using Username & Password")
+        return conn

ClientAdvisor/App/requirements.txt

@@ -23,4 +23,6 @@ isort==6.0.0
 # Testing tools
 pytest>=8.2,<9  # Compatible version for pytest-asyncio
 pytest-asyncio==0.25.3
-pytest-cov==6.0.0
+pytest-cov==6.0.0
+
+pyodbc==5.2.0

ClientAdvisor/App/tests/test_app.py

@@ -200,50 +200,60 @@ async def test_ensure_cosmos_generic_exception(mock_init_cosmosdb_client, client
 
 
 @pytest.mark.asyncio
-async def test_get_users_success(client):
+@patch("app.get_connection")
+@patch("app.dict_cursor")
+async def test_get_users_success(mock_dict_cursor, mock_get_connection, client):
+    # Mock database connection and cursor
     mock_conn = MagicMock()
     mock_cursor = MagicMock()
+    mock_get_connection.return_value = mock_conn
     mock_conn.cursor.return_value = mock_cursor
-    mock_cursor.fetchall.return_value = [
-        {
-            "ClientId": 1,
-            "ndays": 10,
-            "ClientMeetingDaysDifference": 1,
-            "AssetMonthsDifference": 1,
-            "StatusMonthsDifference": 1,
-            "DaysDifference": 1,
-            "Client": "Client A",
-            "Email": "clienta@example.com",
-            "AssetValue": "1,000,000",
-            "ClientSummary": "Summary A",
-            "LastMeetingDateFormatted": "Monday January 1, 2023",
-            "LastMeetingStartTime": "10:00 AM",
-            "LastMeetingEndTime": "10:30 AM",
-            "NextMeetingFormatted": "Monday January 8, 2023",
-            "NextMeetingStartTime": "11:00 AM",
-            "NextMeetingEndTime": "11:30 AM",
-        }
-    ]
 
-    with patch("app.get_connection", return_value=mock_conn):
-        response = await client.get("/api/users")
-        assert response.status_code == 200
-        res_text = await response.get_data(as_text=True)
-        assert json.loads(res_text) == [
+    # Mock query results
+    mock_dict_cursor.side_effect = [
+        [  # First call (client data)
             {
                 "ClientId": 1,
-                "ClientName": "Client A",
-                "ClientEmail": "clienta@example.com",
+                "Client": "Client A",
+                "Email": "clienta@example.com",
                 "AssetValue": "1,000,000",
-                "NextMeeting": "Monday January 8, 2023",
-                "NextMeetingTime": "11:00 AM",
-                "NextMeetingEndTime": "11:30 AM",
-                "LastMeeting": "Monday January 1, 2023",
+                "ClientSummary": "Summary A",
+                "LastMeetingDateFormatted": "Monday January 1, 2023",
                 "LastMeetingStartTime": "10:00 AM",
                 "LastMeetingEndTime": "10:30 AM",
-                "ClientSummary": "Summary A",
+                "NextMeetingFormatted": "Monday January 8, 2023",
+                "NextMeetingStartTime": "11:00 AM",
+                "NextMeetingEndTime": "11:30 AM",
+            }
+        ],
+        [  # Second call (date difference query)
+            {
+                "ClientMeetingDaysDifference": 5,
+                "AssetMonthsDifference": 1,
+                "StatusMonthsDifference": 1
             }
         ]
+    ]
+
+    # Call the function
+    response = await client.get("/api/users")
+    assert response.status_code == 200
+    res_text = await response.get_data(as_text=True)
+    assert json.loads(res_text) == [
+        {
+            "ClientId": 1,
+            "ClientName": "Client A",
+            "ClientEmail": "clienta@example.com",
+            "AssetValue": "1,000,000",
+            "NextMeeting": "Monday January 8, 2023",
+            "NextMeetingTime": "11:00 AM",
+            "NextMeetingEndTime": "11:30 AM",
+            "LastMeeting": "Monday January 1, 2023",
+            "LastMeetingStartTime": "10:00 AM",
+            "LastMeetingEndTime": "10:30 AM",
+            "ClientSummary": "Summary A",
+        }
+    ]
 
 
 @pytest.mark.asyncio

ClientAdvisor/App/tests/test_db.py

@@ -1,30 +1,92 @@
+import struct
 from unittest.mock import MagicMock, patch
 
 import db
+import pyodbc
 
+# Mock configuration
 db.server = "mock_server"
 db.username = "mock_user"
 db.password = "mock_password"
 db.database = "mock_database"
+db.driver = "mock_driver"
+db.mid_id = "mock_mid_id"  # Managed identity client ID if needed
 
 
-@patch("db.pymssql.connect")
-def test_get_connection(mock_connect):
+@patch("db.pyodbc.connect")  # Mock pyodbc.connect
+@patch("db.DefaultAzureCredential")  # Mock DefaultAzureCredential
+def test_get_connection(mock_credential_class, mock_connect):
+    # Mock the DefaultAzureCredential and get_token method
+    mock_credential = MagicMock()
+    mock_credential_class.return_value = mock_credential
+    mock_token = MagicMock()
+    mock_token.token = "mock_token"
+    mock_credential.get_token.return_value = mock_token
     # Create a mock connection object
     mock_conn = MagicMock()
     mock_connect.return_value = mock_conn
 
     # Call the function
     conn = db.get_connection()
 
-    # Assert that pymssql.connect was called with the correct parameters
+    # Assert that DefaultAzureCredential and get_token were called correctly
+    mock_credential_class.assert_called_once_with(managed_identity_client_id=db.mid_id)
+    mock_credential.get_token.assert_called_once_with("https://database.windows.net/.default")
+
+    # Assert that pyodbc.connect was called with the correct parameters, including the token
+    expected_attrs_before = {
+        1256: struct.pack(f"<I{len(mock_token.token.encode('utf-16-LE'))}s", len(mock_token.token.encode("utf-16-LE")), mock_token.token.encode("utf-16-LE"))
+    }
     mock_connect.assert_called_once_with(
-        server="mock_server",
-        user="mock_user",
-        password="mock_password",
-        database="mock_database",
-        as_dict=True,
+        f"DRIVER={db.driver};SERVER={db.server};DATABASE={db.database};",
+        attrs_before=expected_attrs_before
+    )
+
+    # Assert that the connection returned is the mock connection
+    assert conn == mock_conn
+
+
+@patch("db.pyodbc.connect")  # Mock pyodbc.connect
+@patch("db.DefaultAzureCredential")  # Mock DefaultAzureCredential
+def test_get_connection_token_failure(mock_credential_class, mock_connect):
+    # Mock the DefaultAzureCredential and get_token method
+    mock_credential = MagicMock()
+    mock_credential_class.return_value = mock_credential
+    mock_token = MagicMock()
+    mock_token.token = "mock_token"
+    mock_credential.get_token.return_value = mock_token
+
+    # Create a mock connection object
+    mock_conn = MagicMock()
+    mock_connect.return_value = mock_conn
+
+    # Simulate a failure in pyodbc.connect by raising pyodbc.Error on the first call
+    mock_connect.side_effect = [pyodbc.Error("pyodbc connection error"), mock_conn]
+
+    # Call the function and ensure fallback is used after the pyodbc error
+    conn = db.get_connection()
+
+    # Assert that pyodbc.connect was called with username and password as fallback
+    mock_connect.assert_any_call(
+        f"DRIVER={db.driver};SERVER={db.server};DATABASE={db.database};UID={db.username};PWD={db.password}",
+        timeout=5
     )
 
     # Assert that the connection returned is the mock connection
     assert conn == mock_conn
+
+
+def test_dict_cursor():
+    # Create a mock cursor
+    mock_cursor = MagicMock()
+
+    # Simulate the cursor.description and cursor.fetchall
+    mock_cursor.description = [("id",), ("name",), ("age",)]
+    mock_cursor.fetchall.return_value = [(1, "Alice", 30), (2, "Bob", 25)]
+
+    # Call the dict_cursor function
+    result = db.dict_cursor(mock_cursor)
+
+    # Verify the result
+    expected_result = [{'id': 1, 'name': 'Alice', 'age': 30}, {'id': 2, 'name': 'Bob', 'age': 25}]
+    assert result == expected_result

ClientAdvisor/AzureFunction/Dockerfile

@@ -2,6 +2,10 @@
 # FROM mcr.microsoft.com/azure-functions/python:4-python3.11-appservice
 FROM mcr.microsoft.com/azure-functions/python:4-python3.11
 
+# Install Microsoft ODBC Driver
+RUN apt-get update && \
+    ACCEPT_EULA=Y apt-get install -y msodbcsql18
+
 ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
     AzureFunctionsJobHost__Logging__Console__IsEnabled=true
 

ClientAdvisor/AzureFunction/function_app.py

@@ -17,7 +17,10 @@
 from semantic_kernel.functions.kernel_arguments import KernelArguments
 from semantic_kernel.functions.kernel_function_decorator import kernel_function
 from semantic_kernel.kernel import Kernel
-import pymssql
+from azure.identity import DefaultAzureCredential
+import pyodbc
+import struct
+import logging
 
 # Azure Function App
 app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
@@ -120,13 +123,7 @@ def get_SQL_Response(
             sql_query = sql_query.replace("```sql",'').replace("```",'')
             #print(sql_query)
 
-            connectionString = os.environ.get("SQLDB_CONNECTION_STRING")
-            server = os.environ.get("SQLDB_SERVER")
-            database = os.environ.get("SQLDB_DATABASE")
-            username = os.environ.get("SQLDB_USERNAME")
-            password = os.environ.get("SQLDB_PASSWORD")
-
-            conn = pymssql.connect(server, username, password, database)
+            conn = get_connection()
             # conn = pyodbc.connect(connectionString)
             cursor = conn.cursor()
             cursor.execute(sql_query)
@@ -224,6 +221,40 @@ def get_answers_from_calltranscripts(
         answer = completion.choices[0].message.content
         return answer
 
+def get_connection():
+    driver = "{ODBC Driver 18 for SQL Server}"
+    server = os.environ.get("SQLDB_SERVER")
+    database = os.environ.get("SQLDB_DATABASE")
+    username = os.environ.get("SQLDB_USERNAME")
+    password = os.environ.get("SQLDB_PASSWORD")
+    mid_id = os.environ.get("SQLDB_USER_MID")
+    try :
+        credential = DefaultAzureCredential(managed_identity_client_id=mid_id)
+
+        token_bytes = credential.get_token(
+        "https://database.windows.net/.default"
+        ).token.encode("utf-16-LE")
+        token_struct = struct.pack(f"<I{len(token_bytes)}s", len(token_bytes), token_bytes)
+        SQL_COPT_SS_ACCESS_TOKEN = (
+        1256  # This connection option is defined by microsoft in msodbcsql.h
+        )
+
+        # Set up the connection
+        connection_string = f"DRIVER={driver};SERVER={server};DATABASE={database};"
+        conn = pyodbc.connect(
+        connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}
+        )
+        return conn
+    
+    except pyodbc.Error as e:
+        logging.error(f"Failed with Default Credential: {str(e)}")
+        conn = pyodbc.connect(
+            f"DRIVER={driver};SERVER={server};DATABASE={database};UID={username};PWD={password}",
+            timeout=5
+        )
+        logging.info("Connected using Username & Password")
+        return conn
+    
 # Get data from Azure Open AI
 async def stream_processor(response):
     async for message in response:

ClientAdvisor/AzureFunction/requirements.txt

@@ -8,3 +8,6 @@ openai==1.63.0
 semantic_kernel==1.0.4
 pymssql==2.3.2
 azure-search-documents==11.6.0b9
+
+pyodbc==5.2.0
+azure-identity==1.20.0