DataMigration: Adding time based collection and support for installed-ir-path

src/datamigration/HISTORY.rst

@@ -3,6 +3,11 @@
 Release History
 ===============
 
+0.3.1
+++++++
+* [NEW PARAMETER] `az datamigration register-integration-runtime`: Added parameter `--installed-ir-path` to read the installed location of Microsoft Integration Runtime (SHIR) and use it for registering the Database Migration Service if command is unable to find the installed SHIR path. 
+* [NEW PARAMETER] `az datamigration performance-data-collection`: Added parameter `--time` to specify the amount of time(in seconds) performance data collection is to be done. After the timeout the process is terminated automatically.
+
 0.3.0
 ++++++
 * [BREAKING CHANGE] `az datamigration sql-managed-instance/sql-vm create`: Remove `--provisioing-error` and `--migration-operation-id` as they are unnecessary parameters. 

src/datamigration/README.md

@@ -26,7 +26,7 @@ az datamigration performance-data-collection --connection-string "Data Source=La
 
 ##### Get-sku-recommendation #####
 ```
-az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite
+az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks AdventureWorks2 --display-result --overwrite
 ```
 
 #### datamigration sql-managed-instance ####

src/datamigration/azext_datamigration/manual/_help.py

@@ -22,6 +22,9 @@
       - name: Run SQL Assessment on given SQL Server using assessment config file.
         text: |-
                az datamigration get-assessment --config-file-path "C:\\Users\\user\\document\\config.json"
+      - name: Run SQL Assessment on multiple SQL Servers in one call using connection string.
+        text: |-
+               az datamigration get-assessment --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\AssessmentOutput" --overwrite
 """
 
 helps['datamigration performance-data-collection'] = """
@@ -31,9 +34,15 @@
       - name: Collect performance data of a given SQL Server using connection string.
         text: |-
                az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60
+      - name: Collect performance data of multiple SQL Servers in one call using connection string.
+        text: |-
+               az datamigration performance-data-collection --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60
       - name: Collect performance data of a given SQL Server using assessment config file.
         text: |-
                az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json"
+      - name: Collect performance data of a given SQL Server by specifying a time limit. If the time limit specified is before the complition of a iteration cycle, the process will end without saving the last cycle performance data.
+        text: |-
+               az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 --time 60
 """
 
 helps['datamigration get-sku-recommendation'] = """
@@ -42,7 +51,7 @@
     examples:
       - name: Get SKU recommendation for given SQL Server using command line.
         text: |-
-               az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite
+               az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks1 AdventureWorks2 --display-result --overwrite
       - name: Get SKU recommendation for given SQL Server using assessment config file.
         text: |-
                az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json"
@@ -58,6 +67,9 @@
       - name: Install Integration Runtime and register a Sql Migration Service on it.
         text: |-
                az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --ir-path "C:\\Users\\user\\Downloads\\IntegrationRuntime.msi"
+      - name: Read the Integration Runtime from given installation location.
+        text: |-
+               az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --installed-ir-path "D:\\My Softwares\\Microsoft Integration Runtime\\5.0"
 """
 
 helps['datamigration sql-managed-instance create'] = """

src/datamigration/azext_datamigration/manual/_params.py

@@ -35,6 +35,7 @@ def load_arguments(self, _):
         c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds.')
         c.argument('number_of_iteration', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. Minimum: 2.')
         c.argument('config_file_path', type=str, help='Path of the ConfigFile')
+        c.argument('time', type=int, help='Time after which the command execution automatically stops, in seconds. If this parameter is not specified manual intervention will be required to stop the command execution.')
 
     with self.argument_context('datamigration get-sku-recommendation') as c:
         c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection')
@@ -54,6 +55,7 @@ def load_arguments(self, _):
     with self.argument_context('datamigration register-integration-runtime') as c:
         c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service')
         c.argument('ir_path', type=str, help='Path of Integration Runtime MSI')
+        c.argument('installed_ir_path', type=str, help='Version folder path in the Integration Runtime installed location. This can be provided when IR is installed but the command is failing to read it. Format: "<Parent-folder-path>\\Microsoft Integration Runtime\\<Version>"')
 
     with self.argument_context('datamigration sql-db create') as c:
         c.argument('resource_group_name', resource_group_name_type)

src/datamigration/azext_datamigration/manual/action.py

@@ -22,6 +22,8 @@
 from knack.util import CLIError
 
 
+# Adding new input type for target connection details
+# As having type as AddSourceSqlConnection overwrites one of the parameters
 class AddTargetSqlConnection(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         action = self.get_action(values, option_string)

src/datamigration/azext_datamigration/manual/custom.py

@@ -12,6 +12,7 @@
 # pylint: disable=line-too-long
 
 import os
+import signal
 import subprocess
 from azure.cli.core.azclierror import MutuallyExclusiveArgumentError
 from azure.cli.core.azclierror import RequiredArgumentMissingError
@@ -29,20 +30,31 @@ def datamigration_assessment(connection_string=None,
 
     try:
 
+        # Setup the console app
         defaultOutputFolder, exePath = helper.console_app_setup()
 
+        # Specifying both parameters is an error
         if connection_string is not None and config_file_path is not None:
             raise MutuallyExclusiveArgumentError("Both connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.")
 
+        # When Connection string.
         if connection_string is not None:
+
+            # Formating for multiple connection string
             connection_string = " ".join(f"\"{i}\"" for i in connection_string)
+
+            # Joining parameters in a string
             cmd = f'{exePath} Assess --sqlConnectionStrings {connection_string} ' if output_folder is None else f'{exePath} Assess --sqlConnectionStrings {connection_string} --outputFolder "{output_folder}" '
             cmd += '--overwrite False' if overwrite is False else ''
             subprocess.call(cmd, shell=False)
+
+        # When config file.
         elif config_file_path is not None:
             helper.validate_config_file_path(config_file_path, "assess")
             cmd = f'{exePath} --configFile "{config_file_path}"'
             subprocess.call(cmd, shell=False)
+
+        # if no parameter is provided.
         else:
             raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: connection_string, config_file_path')
 
@@ -62,32 +74,64 @@ def datamigration_performance_data_collection(connection_string=None,
                                               perf_query_interval=30,
                                               static_query_interval=3600,
                                               number_of_iteration=20,
-                                              config_file_path=None):
+                                              config_file_path=None,
+                                              time=None):
 
     try:
 
+        # Setup the console app
         defaultOutputFolder, exePath = helper.console_app_setup()
 
         if connection_string is not None and config_file_path is not None:
             raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.")
 
+        # When Connection string.
         if connection_string is not None:
+
+            # Formating for multiple connection string
             connection_string = " ".join(f"\"{i}\"" for i in connection_string)
+
+            # parameter set for Perfornace data collection
             parameterList = {
                 "--outputFolder": output_folder,
                 "--perfQueryIntervalInSec": perf_query_interval,
                 "--staticQueryIntervalInSec": static_query_interval,
                 "--numberOfIterations": number_of_iteration
             }
+
+            # joining paramaters together in a string
             cmd = f'{exePath} PerfDataCollection --sqlConnectionStrings {connection_string}'
             for param in parameterList:
                 if parameterList[param] is not None:
                     cmd += f' {param} "{parameterList[param]}"'
-            subprocess.call(cmd, shell=False)
+
+            # If time parameter is specified, catch TimeoutExpired exception and terminate the process
+            if time is None:
+                subprocess.call(cmd, shell=False)
+            else:
+                sp = subprocess.Popen(cmd, shell=False)
+                try:
+                    outs, errs = sp.communicate(timeout=time)
+                except subprocess.TimeoutExpired:
+                    sp.send_signal(signal.SIGTERM)
+                    outs, errs = sp.communicate()
+
+        # When Config file.
         elif config_file_path is not None:
             helper.validate_config_file_path(config_file_path, "perfdatacollection")
             cmd = f'{exePath} --configFile "{config_file_path}"'
-            subprocess.call(cmd, shell=False)
+
+            # If time parameter is specified, catch TimeoutExpired exception and terminate the process
+            if time is None:
+                subprocess.call(cmd, shell=False)
+            else:
+                sp = subprocess.Popen(cmd, shell=False)
+                try:
+                    outs, errs = sp.communicate(timeout=time)
+                except subprocess.TimeoutExpired:
+                    sp.send_signal(signal.SIGTERM)
+                    outs, errs = sp.communicate()
+
         else:
             raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: sql_connection_string, config_file_path')
 
@@ -117,16 +161,23 @@ def datamigration_get_sku_recommendation(output_folder=None,
                                          config_file_path=None):
 
     try:
+
+        # Setup Console app
         defaultOutputFolder, exePath = helper.console_app_setup()
 
         if output_folder is not None and config_file_path is not None:
             raise MutuallyExclusiveArgumentError("Both output_folder and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.")
 
+        # When Config file - Handling this case first to allow no parameter to be specified (runs non config file scenario)
         if config_file_path is not None:
             helper.validate_config_file_path(config_file_path, "getskurecommendation")
             cmd = f'{exePath} --configFile "{config_file_path}"'
             subprocess.call(cmd, shell=False)
+
+        # When non-config file
         else:
+
+            # parameter set for Sku recommendation
             parameterList = {
                 "--outputFolder": output_folder,
                 "--targetPlatform": target_platform,
@@ -142,9 +193,13 @@ def datamigration_get_sku_recommendation(output_folder=None,
                 "--databaseDenyList": database_deny_list
             }
             cmd = f'{exePath} GetSkuRecommendation'
+
+            # formating the parameter list into a string
             for param in parameterList:
                 if parameterList[param] is not None and not param.__contains__("List"):
                     cmd += f' {param} "{parameterList[param]}"'
+
+                # in case the parameter input is list format it accordingly
                 elif param.__contains__("List") and parameterList[param] is not None:
                     parameterList[param] = " ".join(f"\"{i}\"" for i in parameterList[param])
                     cmd += f' {param} {parameterList[param]}'
@@ -162,14 +217,19 @@ def datamigration_get_sku_recommendation(output_folder=None,
 # Register Sql Migration Service on IR command Implementation.
 # -----------------------------------------------------------------------------------------------------------------
 def datamigration_register_ir(auth_key,
-                              ir_path=None):
+                              ir_path=None,
+                              installed_ir_path=None):
 
     helper.validate_os_env()
 
+    # This command can only be run as admin and in windows
     if not helper.is_user_admin():
         raise UnclassifiedUserFault("Failed: You do not have Administrator rights to run this command. Please re-run this command as an Administrator!")
     helper.validate_input(auth_key)
+
+    # Run installation if ir_path is provided
     if ir_path is not None:
         helper.install_gateway(ir_path)
 
-    helper.register_ir(auth_key)
+    # register or re-register Dms on ir
+    helper.register_ir(auth_key, installed_ir_path)