fix multi-thread issue (#3899)

nni/retiarii/experiment/pytorch.py

@@ -310,6 +310,11 @@ def stop(self) -> None:
         """
         _logger.info('Stopping experiment, please wait...')
         atexit.unregister(self.stop)
+
+        # stop strategy first
+        if self._dispatcher_thread is not None:
+            self._dispatcher.stopping = True
+            self._dispatcher_thread.join(timeout=1)
 
         if self.id is not None:
             nni.runtime.log.stop_experiment_log(self.id)
@@ -326,9 +331,6 @@ def stop(self) -> None:
 
         if self._pipe is not None:
             self._pipe.close()
-        if self._dispatcher_thread is not None:
-            self._dispatcher.stopping = True
-            self._dispatcher_thread.join(timeout=1)
 
         self.id = None
         self.port = None

ts/nni_manager/core/nnimanager.ts

@@ -639,8 +639,6 @@ class NNIManager implements Manager {
                 this.trialConcurrencyChange = requestTrialNum;
             }
 
-            this.requestTrialJobs(requestTrialNum);
-
             // check maxtrialnum and maxduration here
             // NO_MORE_TRIAL is more like a subset of RUNNING, because during RUNNING tuner
             // might tell nnimanager that this is no more trials. In NO_MORE_TRIAL state, the experiment is viewed
@@ -665,6 +663,8 @@ class NNIManager implements Manager {
                     }
                 }
             } else {
+                this.requestTrialJobs(requestTrialNum);
+
                 if (this.status.status === 'DONE') {
                     delete this.experimentProfile.endTime;
                     await this.storeExperimentProfile();