merge master

README.md

@@ -4,7 +4,7 @@
 [![Issues](https://img.shields.io/github/issues-raw/Microsoft/nni.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen)
 [![Bugs](https://img.shields.io/github/issues/Microsoft/nni/bug.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen+label%3Abug)
 [![Pull Requests](https://img.shields.io/github/issues-pr-raw/Microsoft/nni.svg)](https://github.com/Microsoft/nni/pulls?q=is%3Apr+is%3Aopen)
-[![Version](https://img.shields.io/github/tag/Microsoft/nni.svg)]()
+[![Version](https://img.shields.io/github/release/Microsoft/nni.svg)](https://github.com/Microsoft/nni/releases)
 
 NNI (Neural Network Intelligence) is a toolkit to help users run automated machine learning experiments. 
 The tool dispatches and runs trial jobs that generated by tuning algorithms to search the best neural architecture and/or hyper-parameters in different environments (e.g. local machine, remote servers and cloud).

deployment/README.md

@@ -2,6 +2,8 @@ Dockerfile
 ===
 ## 1.Description  
 This is the Dockerfile of nni project, including the most kinds of deeplearning frameworks and nni source code.  You can run your nni experiment in this docker container directly.
+Dockerfile.build.base could build the base Docker image, users can get a docker image with Ubuntu and NNI environment after building this file. 
+Dockerfile could build the customized docker image, users could build their customized docker image using this file.
 ## 2.Including Libraries  
 
 ```

src/webui/src/components/Para.tsx

@@ -99,7 +99,7 @@ class Para extends React.Component<{}, ParaState> {
                                 speDimName.push(tem);
                             }
                             if (accParaData[item].status === 'SUCCEEDED') {
-                                if (accParaData[item].finalMetricData !== undefined) {
+                                if (accParaData[item].finalMetricData && accParaData[item].hyperParameters) {
                                     // get acc array
                                     accPara.push(parseFloat(accParaData[item].finalMetricData.data));
                                     // get dim and every line specific number
@@ -445,7 +445,7 @@ class Para extends React.Component<{}, ParaState> {
                                 className="changeBtu"
                                 onClick={this.swapBtn}
                             >
-                                sure
+                                Confirm
                             </Button>
                         </div>
                     </div>

src/webui/src/components/Sessionpro.tsx

@@ -7,6 +7,10 @@ import JSONTree from 'react-json-tree';
 require('../style/sessionpro.css');
 require('../style/logPath.css');
 
+interface ErrorPara {
+    error?: string;
+}
+
 interface TableObj {
     key: number;
     id: string;
@@ -19,7 +23,7 @@ interface TableObj {
 }
 
 interface Parameters {
-    parameters: object;
+    parameters: ErrorPara;
     logPath?: string;
     isLink?: boolean;
 }
@@ -189,7 +193,11 @@ class Sessionpro extends React.Component<{}, SessionState> {
                             if (tableData[item].finalMetricData) {
                                 acc = parseFloat(tableData[item].finalMetricData.data);
                             }
-                            desJobDetail.parameters = JSON.parse(tableData[item].hyperParameters).parameters;
+                            if (tableData[item].hyperParameters) {
+                                desJobDetail.parameters = JSON.parse(tableData[item].hyperParameters).parameters;
+                            } else {
+                                desJobDetail.parameters = { error: 'This trial\'s parameters are not available.' };
+                            }
                             if (tableData[item].logPath !== undefined) {
                                 desJobDetail.logPath = tableData[item].logPath;
                                 const isSessionLink = /^http/gi.test(tableData[item].logPath);
@@ -237,6 +245,23 @@ class Sessionpro extends React.Component<{}, SessionState> {
         }
     }
 
+    // trial's duration, accurate to seconds
+    convertDuration = (num: number) => {
+        const hour = Math.floor(num / 3600);
+        const min = Math.floor(num / 60 % 60);
+        const second = Math.floor(num % 60);
+        const result = hour > 0 ? `${hour} h ${min} min ${second}s` : `${min} min ${second}s`;
+        if (hour <= 0 && min === 0 && second !== 0) {
+            return `${second}s`;
+        } else if (hour === 0 && min !== 0 && second === 0) {
+            return `${min}min`;
+        } else if (hour === 0 && min !== 0 && second !== 0) {
+            return `${min}min ${second}s`;
+        } else {
+            return result;
+        }
+    }
+
     downExperimentContent = () => {
         axios
             .all([
@@ -313,7 +338,17 @@ class Sessionpro extends React.Component<{}, SessionState> {
             title: 'Duration/s',
             dataIndex: 'duration',
             key: 'duration',
-            width: '9%'
+            width: '9%',
+            sorter: (a: TableObj, b: TableObj) => (a.duration as number) - (b.duration as number),
+            render: (text: string, record: TableObj) => {
+                let duration;
+                if (record.duration) {
+                    duration = this.convertDuration(record.duration);
+                }
+                return (
+                    <span>{duration}</span>
+                );
+            },
         }, {
             title: 'Start',
             dataIndex: 'start',
@@ -344,6 +379,10 @@ class Sessionpro extends React.Component<{}, SessionState> {
         }];
 
         const openRow = (record: TableObj) => {
+            let isHasParameters = true;
+            if (record.description.parameters.error) {
+                isHasParameters = false;
+            }
             const openRowDataSource = {
                 parameters: record.description.parameters
             };
@@ -354,12 +393,21 @@ class Sessionpro extends React.Component<{}, SessionState> {
             }
             return (
                 <pre id="description" className="jsontree">
-                    <JSONTree
-                        hideRoot={true}
-                        shouldExpandNode={() => true}  // default expandNode
-                        getItemString={() => (<span />)}  // remove the {} items
-                        data={openRowDataSource}
-                    />
+                    {
+                        isHasParameters
+                            ?
+                            <JSONTree
+                                hideRoot={true}
+                                shouldExpandNode={() => true}  // default expandNode
+                                getItemString={() => (<span />)}  // remove the {} items
+                                data={openRowDataSource}
+                            />
+                            :
+                            <div className="logpath">
+                                <span className="logName">Error: </span>
+                                <span className="error">'This trial's parameters are not available.'</span>
+                            </div>
+                    }
                     {
                         isLogLink
                             ?

src/webui/src/components/TrialStatus.tsx

@@ -17,8 +17,12 @@ echarts.registerTheme('my_theme', {
     color: '#3c8dbc'
 });
 
+interface ErrorPara {
+    error?: string;
+}
+
 interface DescObj {
-    parameters: Object;
+    parameters: ErrorPara;
     logPath?: string;
     isLink?: boolean;
 }
@@ -237,6 +241,8 @@ class TrialStatus extends React.Component<{}, TabState> {
                             : '';
                         if (trialJobs[item].hyperParameters !== undefined) {
                             desc.parameters = JSON.parse(trialJobs[item].hyperParameters).parameters;
+                        } else {
+                            desc.parameters = { error: 'This trial\'s parameters are not available.' };
                         }
                         if (trialJobs[item].logPath !== undefined) {
                             desc.logPath = trialJobs[item].logPath;
@@ -346,6 +352,22 @@ class TrialStatus extends React.Component<{}, TabState> {
         };
     }
 
+    convertTime = (num: number) => {
+        const hour = Math.floor(num / 3600);
+        const min = Math.floor(num / 60 % 60);
+        const second = Math.floor(num % 60);
+        const result = hour > 0 ? `${hour} h ${min} min ${second}s` : `${min} min ${second}s`;
+        if (hour <= 0 && min === 0 && second !== 0) {
+            return `${second}s`;
+        } else if (hour === 0 && min !== 0 && second === 0) {
+            return `${min}min`;
+        } else if (hour === 0 && min !== 0 && second !== 0) {
+            return `${min}min ${second}s`;
+        } else {
+            return result;
+        }
+    }
+
     componentDidMount() {
 
         this._isMounted = true;
@@ -388,7 +410,16 @@ class TrialStatus extends React.Component<{}, TabState> {
             key: 'duration',
             width: '10%',
             // the sort of number
-            sorter: (a: TableObj, b: TableObj) => (a.duration as number) - (b.duration as number)
+            sorter: (a: TableObj, b: TableObj) => (a.duration as number) - (b.duration as number),
+            render: (text: string, record: TableObj) => {
+                let duration;
+                if (record.duration) {
+                    duration = this.convertTime(record.duration);
+                }
+                return (
+                    <span>{duration}</span>
+                );
+            },
         }, {
             title: 'Start',
             dataIndex: 'start',
@@ -478,6 +509,10 @@ class TrialStatus extends React.Component<{}, TabState> {
         ];
 
         const openRow = (record: TableObj) => {
+            let isHasParameters = true;
+            if (record.description.parameters.error) {
+                isHasParameters = false;
+            }
             const parametersRow = {
                 parameters: record.description.parameters
             };
@@ -488,12 +523,21 @@ class TrialStatus extends React.Component<{}, TabState> {
             }
             return (
                 <pre className="hyperpar">
-                    <JSONTree
-                        hideRoot={true}
-                        shouldExpandNode={() => true}  // default expandNode
-                        getItemString={() => (<span />)}  // remove the {} items
-                        data={parametersRow}
-                    />
+                    {
+                        isHasParameters
+                            ?
+                            < JSONTree
+                                hideRoot={true}
+                                shouldExpandNode={() => true}  // default expandNode
+                                getItemString={() => (<span />)}  // remove the {} items
+                                data={parametersRow}
+                            />
+                            :
+                            <div className="logpath">
+                                <span className="logName">Error: </span>
+                                <span className="error">'This trial's parameters are not available.'</span>
+                            </div>
+                    }
                     {
                         isLogLink
                             ?

src/webui/src/style/logPath.css

@@ -12,3 +12,6 @@
     color: blue;
     text-decoration: underline;
 }
+.error{
+    color: #CB4B16;
+}

test/naive/run.py

@@ -82,4 +82,4 @@ def run():
         traceback.print_exc()
         raise error
 
-    subprocess.run(['nnictl', 'stop', '--port', '51188'])
+    subprocess.run(['nnictl', 'stop'])

tools/nnicmd/common_utils.py

@@ -21,6 +21,7 @@
 import json
 import yaml
 import psutil
+import socket
 from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO, COLOR_RED_FORMAT, COLOR_YELLOW_FORMAT
 
 def get_yml_content(file_path):
@@ -60,3 +61,13 @@ def detect_process(pid):
         return process.is_running()
     except:
         return False
+
+def detect_port(port):
+    '''Detect if the port is used'''
+    socket_test = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
+    try:
+        socket_test.connect(('127.0.0.1', int(port)))
+        socket_test.shutdown(2)
+        return True
+    except:
+        return False

tools/nnicmd/config_utils.py

@@ -22,12 +22,12 @@
 import os
 import json
 import shutil
-from .constants import HOME_DIR
+from .constants import NNICTL_HOME_DIR
 
 class Config:
     '''a util class to load and save config'''
     def __init__(self, port):
-        config_path = os.path.join(HOME_DIR, str(port))
+        config_path = os.path.join(NNICTL_HOME_DIR, str(port))
         os.makedirs(config_path, exist_ok=True)
         self.config_file = os.path.join(config_path, '.config')
         self.config = self.read_file()
@@ -46,12 +46,6 @@ def get_config(self, key):
         '''get a value according to key'''
         return self.config.get(key)
 
-    def copy_metadata_to_new_path(self, path):
-        '''copy metadata to a new path'''
-        if not os.path.exists(path):
-            os.mkdir(path)
-        shutil.copy(self.config_file, path)
-
     def write_file(self):
         '''save config to local file'''
         if self.config:
@@ -71,3 +65,44 @@ def read_file(self):
             except ValueError:
                 return {}
         return {}
+
+class Experiments:
+    '''Maintain experiment list'''
+    def __init__(self):
+        os.makedirs(NNICTL_HOME_DIR, exist_ok=True)
+        self.experiment_file = os.path.join(NNICTL_HOME_DIR, '.experiment')
+        self.experiments = self.read_file()
+
+    def add_experiment(self, id, port, time):
+        '''set {key:value} paris to self.experiment'''
+        self.experiments[id] = [port, time]
+        self.write_file()
+
+    def remove_experiment(self, id):
+        '''remove an experiment by id'''
+        if id in self.experiments:
+            self.experiments.pop(id)
+        self.write_file()
+
+    def get_all_experiments(self):
+        '''return all of experiments'''
+        return self.experiments
+
+    def write_file(self):
+        '''save config to local file'''
+        try:
+            with open(self.experiment_file, 'w') as file:
+                json.dump(self.experiments, file)
+        except IOError as error:
+            print('Error:', error)
+            return
+
+    def read_file(self):
+        '''load config from local file'''
+        if os.path.exists(self.experiment_file):
+            try:
+                with open(self.experiment_file, 'r') as file:
+                    return json.load(file)
+            except ValueError:
+                return {}
+        return {}