Dev fix aml

ts/nni_manager/config/aml/amlUtil.py

@@ -35,6 +35,7 @@
     run_config.node_count = 1
     config = ScriptRunConfig(source_directory=args.script_dir, script=args.script_name, run_config=run_config)
     run = experiment.submit(config)
+    run_id = run.get_details()["runId"]
     print(run.get_details()["runId"])
     while True:
         line = sys.stdin.readline().rstrip()
@@ -44,7 +45,17 @@
             print('tracking_url:' + run.get_portal_url())
         elif line == 'stop':
             run.cancel()
-            exit(0)
+            with open('~/aml_script_result.log', 'a') as out_file:
+                out_file.write(run_id + " send stop request")
+            loop_count = 0
+            status = run.get_status()
+            while status != 'Canceled':
+                if loop_count > 5:
+                    print('stop_result:failed')
+                    exit(0)
+                loop_count += 1
+                time.sleep(500)
+            print('stop_result:success')
         elif line == 'receive':
             print('receive:' + json.dumps(run.get_metrics()))
         elif line:

ts/nni_manager/training_service/reusable/aml/amlClient.ts

@@ -60,11 +60,21 @@ export class AMLClient {
         return deferred.promise;
     }
 
-    public stop(): void {
+    public stop(): Promise<boolean> {
         if (this.pythonShellClient === undefined) {
             throw Error('python shell client not initialized!');
         }
+        const deferred: Deferred<boolean> = new Deferred<boolean>();
         this.pythonShellClient.send('stop');
+        this.pythonShellClient.on('message', (result: any) => {
+            const stopResult = this.parseContent('stop_result', result);
+            if (stopResult === 'success') {
+                deferred.resolve(true);
+            } else {
+                deferred.resolve(false);
+            }
+        });
+        return deferred.promise;
     }
 
     public getTrackingUrl(): Promise<string> {

ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts

@@ -127,6 +127,11 @@ export class AMLEnvironmentService extends EnvironmentService {
         if (!amlClient) {
             throw new Error('AML client not initialized!');
         }
-        amlClient.stop();
+        const result = await amlClient.stop();
+        if (result) {
+            this.log.info(`Stop ${environment.id} success!`);
+        } else {
+            this.log.info(`Stop ${environment.id} failed!`);
+        }
     }
 }