Merge latest code changes into Github Master (#54)

deployment/Dockerfile

@@ -0,0 +1,13 @@
+FROM nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04
+
+LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
+
+#
+#Tensorflow 1.10.0
+#
+RUN pip3 --no-cache-dir install tensorflow-gpu==1.10.0
+
+#
+#Keras 2.1.6
+#
+RUN pip3 --no-cache-dir install Keras==2.1.6

deployment/Dockerfile.build.base

@@ -0,0 +1,48 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+
+LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    sudo apt-utils git curl vim unzip openssh-client wget \
+    build-essential cmake \
+    libopenblas-dev
+
+#
+# Python 3.5
+#
+RUN apt-get install -y --no-install-recommends python3.5 python3.5-dev python3-pip python3-tk && \
+    pip3 install --no-cache-dir --upgrade pip setuptools && \
+    echo "alias python='python3'" >> /root/.bash_aliases && \
+    echo "alias pip='pip3'" >> /root/.bash_aliases
+
+# numpy 1.14.3  scipy 1.1.0 
+RUN pip3 --no-cache-dir install \
+    numpy==1.14.3 scipy==1.1.0 
+
+#
+#Install node 10.9.0, yarn 1.9.4, NNI v0.1
+#
+RUN git clone -b v0.1 https://github.com/Microsoft/nni.git
+RUN cd nni && sh install.sh
+RUN echo 'PATH=~/.local/node/bin:~/.local/yarn/bin:~/.local/bin:$PATH' >> ~/.bashrc
+RUN cd .. && rm -rf nni

deployment/README.md

@@ -0,0 +1,20 @@
+Dockerfile
+===
+## 1.Description  
+This is the Dockerfile of nni project, including the most kinds of deeplearning frameworks and nni source code.  You can run your nni experiment in this docker container directly.
+## 2.Including Libraries  
+
+```
+Ubuntu 16.04 LTS
+CUDA 9.0, CuDNN 7.0
+numpy 1.14.3,scipy 1.1.0
+TensorFlow 1.5.0
+Keras 2.1.6
+NNI v0.1
+```
+
+## 3 How to run  
+
+    docker build -f Dockerfile.build.base -t nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04 .
+    docker build -t nni/nni .
+    nvidia-docker run -it nni/nni

setup.py

@@ -80,10 +80,17 @@ def run(self):
         'psutil',
         'pyyaml',
         'requests',
-        'scipy'
+        'scipy',
+        'schema'        
+    ],
+    dependency_links = [
+        'git+https://github.com/hyperopt/hyperopt.git'
     ],
 
     cmdclass={
         'install': CustomInstallCommand
+    },
+    entry_points={
+        'console_scripts': ['nnictl = nnicmd.nnictl:parse_args']
     }
 )

src/nni_manager/common/datastore.ts

@@ -26,14 +26,14 @@ type TrialJobEvent = TrialJobStatus | 'USER_TO_CANCEL' | 'ADD_CUSTOMIZED';
 type MetricType = 'PERIODICAL' | 'FINAL' | 'CUSTOM';
 
 interface ExperimentProfileRecord {
-    readonly timestamp: Date;
+    readonly timestamp: number;
     readonly experimentId: number;
     readonly revision: number;
     readonly data: ExperimentProfile;
 }
 
 interface TrialJobEventRecord {
-    readonly timestamp: Date;
+    readonly timestamp: number;
     readonly trialJobId: string;
     readonly event: TrialJobEvent;
     readonly data?: string;
@@ -49,7 +49,7 @@ interface MetricData {
 }
 
 interface MetricDataRecord {
-    readonly timestamp: Date;
+    readonly timestamp: number;
     readonly trialJobId: string;
     readonly parameterId: string;
     readonly type: MetricType;
@@ -60,8 +60,8 @@ interface MetricDataRecord {
 interface TrialJobInfo {
     id: string;
     status: TrialJobStatus;
-    startTime?: Date;
-    endTime?: Date;
+    startTime?: number;
+    endTime?: number;
     hyperParameters?: string;
     logPath?: string;
     finalMetricData?: string;
@@ -96,4 +96,4 @@ abstract class Database {
 export {
     DataStore, Database, TrialJobEvent, MetricType, MetricData, TrialJobInfo,
     ExperimentProfileRecord, TrialJobEventRecord, MetricDataRecord
-}
+};

src/nni_manager/common/manager.ts

@@ -59,8 +59,8 @@ interface ExperimentProfile {
     params: ExperimentParams;
     id: string;
     execDuration: number;
-    startTime?: Date;
-    endTime?: Date;
+    startTime?: number;
+    endTime?: number;
     revision: number;
 }
 
@@ -69,6 +69,11 @@ interface TrialJobStatistics {
     trialJobNumber: number;
 }
 
+interface NNIManagerStatus {
+    status: 'INITIALIZED' | 'EXPERIMENT_RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED';
+    errors: string[];
+}
+
 abstract class Manager {
     public abstract startExperiment(experimentParams: ExperimentParams): Promise<string>;
     public abstract resumeExperiment(): Promise<void>;
@@ -86,6 +91,7 @@ abstract class Manager {
 
     public abstract getMetricData(trialJobId: string, metricType: MetricType): Promise<MetricDataRecord[]>;
     public abstract getTrialJobStatistics(): Promise<TrialJobStatistics[]>;
+    public abstract getStatus(): NNIManagerStatus;
 }
 
-export { Manager, ExperimentParams, ExperimentProfile, TrialJobStatistics, ProfileUpdateType };
+export { Manager, ExperimentParams, ExperimentProfile, TrialJobStatistics, ProfileUpdateType, NNIManagerStatus };

src/nni_manager/rest_server/server.ts → src/nni_manager/common/restServer.ts

@@ -19,28 +19,27 @@
 
 'use strict';
 
-import * as bodyParser from 'body-parser';
 import * as express from 'express';
 import * as http from 'http';
 import { Deferred } from 'ts-deferred';
+import { getLogger, Logger } from './log';
 
-import * as component from '../common/component';
-import { getLogger, Logger } from '../common/log';
-import { Manager } from '../common/manager';
-import { createRestHandler } from './restHandler';
-
-@component.Singleton
-export class RestServer {
-    public static readonly DEFAULT_PORT: number = 51188;
-    private readonly API_ROOT_URL: string = '/api/v1/nni';
-    private hostName: string = '0.0.0.0';
-    private port: number = RestServer.DEFAULT_PORT;
+/**
+ * Abstraction class to create a RestServer
+ * The module who wants to use a RestServer could <b>extends</b> this abstract class 
+ * And implement its own registerRestHandler() function to register routers
+ */
+export abstract class RestServer {
     private startTask!: Deferred<void>;
     private stopTask!: Deferred<void>;
-    private app: express.Application = express();
     private server!: http.Server;
-    private log: Logger = getLogger();
 
+    /** The fields can be inherited by subclass */
+    protected hostName: string = '0.0.0.0';
+    protected port?: number;
+    protected app: express.Application = express();
+    protected log: Logger = getLogger();
+
     get endPoint(): string {
         // tslint:disable-next-line:no-http-string
         return `http://${this.hostName}:${this.port}`;
@@ -61,7 +60,7 @@ export class RestServer {
             this.port = port;
         }
 
-        this.server = this.app.listen(this.port, this.hostName).on('listening', () => {
+        this.server = this.app.listen(this.port as number, this.hostName).on('listening', () => {
             this.startTask.resolve();
         }).on('error', (e: Error) => {
             this.startTask.reject(e);
@@ -100,8 +99,8 @@ export class RestServer {
         return this.stopTask.promise;
     }
 
-    private registerRestHandler(): void {
-        this.app.use(bodyParser.json());
-        this.app.use(this.API_ROOT_URL, createRestHandler(this));
-    }
+    /**
+     * Register REST handler, which is left for subclass to implement
+     */
+    protected abstract registerRestHandler(): void;
 }

src/nni_manager/common/trainingService.ts

@@ -58,9 +58,9 @@ interface HostJobApplicationForm extends JobApplicationForm {
 interface TrialJobDetail {
     readonly id: string;
     readonly status: TrialJobStatus;
-    readonly submitTime: Date;
-    readonly startTime?: Date;
-    readonly endTime?: Date;
+    readonly submitTime: number;
+    readonly startTime?: number;
+    readonly endTime?: number;
     readonly tags?: string[];
     readonly url?: string;
     readonly workingDirectory: string;

src/nni_manager/common/utils.ts

@@ -19,6 +19,7 @@
 
 'use strict';
 
+import * as assert from 'assert';
 import { randomBytes } from 'crypto';
 import * as fs from 'fs';
 import * as os from 'os';
@@ -32,7 +33,7 @@ import { ExperimentStartupInfo, getExperimentId, setExperimentStartupInfo } from
 import { Manager } from './manager';
 import { TrainingService } from './trainingService';
 
-function getExperimentRootDir(): string{
+function getExperimentRootDir(): string {
     return path.join(os.homedir(), 'nni', 'experiments', getExperimentId());
 }
 
@@ -115,6 +116,12 @@ function uniqueString(len: number): string {
     return String.fromCharCode(...codes);
 }
 
+function randomSelect<T>(a: T[]): T {
+    assert(a !== undefined);
+
+    // tslint:disable-next-line:insecure-random
+    return a[Math.floor(Math.random() * a.length)];
+}
 function parseArg(names: string[]): string {
     if (process.argv.length >= 4) {
         for (let i: number = 2; i < process.argv.length - 1; i++) {
@@ -223,4 +230,4 @@ function cleanupUnitTest(): void {
 }
 
 export { getMsgDispatcherCommand, getLogDir, getExperimentRootDir, getDefaultDatabaseDir, mkDirP, delay, prepareUnitTest,
-    parseArg, cleanupUnitTest, uniqueString };
+    parseArg, cleanupUnitTest, uniqueString, randomSelect };

src/nni_manager/core/nniDataStore.ts

@@ -126,7 +126,7 @@ class NNIDataStore implements DataStore {
             type: metrics.type,
             sequence: metrics.sequence,
             data: metrics.value,
-            timestamp: new Date()
+            timestamp: Date.now()
         }));
     }