Fix paiToken update logic (#976)

microsoft · Apr 17, 2019 · 9a3c61e · 9a3c61e
1 parent 44f5486
commit 9a3c61e
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts
@@ -529,7 +529,15 @@ class PAITrainingService implements TrainingService {
 
     private async statusCheckingLoop(): Promise<void> {
         while (!this.stopping) {
-            await this.updatePaiToken();
+            try{
+                await this.updatePaiToken();
+            }catch(error){
+                this.log.error(`${error}`);
+                //only throw error when initlize paiToken first time
+                if(!this.paiToken) {
+                    throw new Error(error);
+                }
+            }
             await this.paiJobCollector.retrieveTrialStatus(this.paiToken, this.paiClusterConfig);
             const restServer: PAIJobRestServer = component.get(PAIJobRestServer);
             if (restServer.getErrorMessage) {