Fix localTrainingService cancel logic and nnictl logic

src/nni_manager/training_service/local/localTrainingService.ts

@@ -120,13 +120,16 @@ class LocalTrainingService implements TrainingService {
         while (!this.stopping) {
             while (this.jobQueue.length !== 0) {
                 const trialJobId: string = this.jobQueue[0];
-                const [success, resource] = this.tryGetAvailableResource();
-                if (!success) {
-                    break;
+                const trialJobDeatil = this.jobMap.get(trialJobId)
+                if (trialJobDeatil !== undefined && trialJobDeatil.status === 'WAITING'){
+                    const [success, resource] = this.tryGetAvailableResource();
+                    if (!success) {
+                        break;
+                    }
+                    this.occupyResource(resource);
+                    this.jobQueue.shift();
+                    await this.runTrialJob(trialJobId, resource);
                 }
-                this.occupyResource(resource);
-                this.jobQueue.shift();
-                await this.runTrialJob(trialJobId, resource);
             }
             await delay(5000);
         }
@@ -249,9 +252,14 @@ class LocalTrainingService implements TrainingService {
         if (trialJob === undefined) {
             throw new NNIError(NNIErrorNames.NOT_FOUND, 'Trial job not found');
         }
+        if (trialJob.pid === undefined){
+            this.setTrialJobStatus(trialJob, 'USER_CANCELED');
+            return;
+        }
         if (trialJob.form.jobType === 'TRIAL') {
             await tkill(trialJob.pid, 'SIGKILL');
-        } else if (trialJob.form.jobType === 'HOST') {
+        }
+        else if (trialJob.form.jobType === 'HOST') {
             await cpp.exec(`pkill -9 -P ${trialJob.pid}`);
         } else {
             throw new Error(`Job type not supported: ${trialJob.form.jobType}`);

tools/nni_cmd/constants.py

@@ -59,7 +59,7 @@
                      '%s\n' \
                      '-----------------------------------------------------------------------\n'
 
-EXPERIMENT_DETAIL_FORMAT = 'Id: %s    Status: %s    StartTime: %s    EndTime: %s    \n'
+EXPERIMENT_DETAIL_FORMAT = 'Id: %s    Status: %s    Port: %s    StartTime: %s    EndTime: %s    \n'
 
 PACKAGE_REQUIREMENTS = {
     'SMAC': 'smac_tuner'

tools/nni_cmd/launcher_utils.py

@@ -103,7 +103,9 @@ def validate_common_content(experiment_config):
                     experiment_config['machineList'][index]['port'] = 22
 
     except Exception as exception:
-        raise Exception(exception)
+        print_error('Your config file is not correct, please check your config file content!')
+        print_error('Information:\n%s' % exception)
+        exit(1)
 
 def parse_tuner_content(experiment_config):
     '''Validate whether tuner in experiment_config is valid'''

tools/nni_cmd/nnictl_utils.py

@@ -184,7 +184,10 @@ def stop_experiment(args):
             if running:
                 response = rest_delete(experiment_url(rest_port), 20)
                 if not response or not check_response(response):
-                    print_error('Stop experiment failed!')
+                    if response:
+                        print_error(response.text)
+                    else:
+                        print_error('No response from restful server!')
                     stop_rest_result = False
             #sleep to wait rest handler done
             time.sleep(3)
@@ -365,7 +368,7 @@ def experiment_list(args):
             print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
     experiment_information = ""
     for key in experiment_id_list:
-        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
+        experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
         experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
     print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)