Refactor: remote machine isolate OS commands phase 1

src/nni_manager/training_service/common/extends/linuxCommands.ts

@@ -0,0 +1,87 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+'use strict';
+
+import { OsCommands } from "../osCommands";
+import { RemoteCommandResult } from "../remoteMachineData";
+
+class LinuxCommands extends OsCommands {
+    public createFolder(folderName: string, sharedFolder: boolean = false): string {
+        let command;
+        if (sharedFolder) {
+            command = `umask 0; mkdir -p '${folderName}'`;
+        } else {
+            command = `mkdir -p '${folderName}'`;
+        }
+        return command;
+    }
+
+    public allowPermission(isRecursive: boolean = false, ...folders: string[]): string {
+        const folderString = folders.join("' '");
+        let command;
+
+        if (isRecursive) {
+            command = `chmod 777 -R '${folderString}'`;
+        } else {
+            command = `chmod 777 '${folderString}'`;
+        }
+        return command;
+    }
+
+    public removeFolder(folderName: string, isRecursive: boolean = false, isForce: boolean = true): string {
+        let flags = '';
+        if (isForce || isRecursive) {
+            flags = `-${isRecursive ? 'r' : 'd'}${isForce ? 'f' : ''} `;
+        }
+
+        const command = `rm ${flags}'${folderName}'`;
+        return command;
+    }
+
+    public removeFiles(folderName: string, filePattern: string): string {
+        const files = this.joinPath(folderName, filePattern);
+        const command = `rm ${files}`;
+        return command;
+    }
+
+    public readLastLines(fileName: string, lineCount: number = 1): string {
+        const command = `tail -n ${lineCount} ${fileName}`;
+        return command;
+    }
+
+    public isProcessAliveCommand(pidFileName: string): string {
+        const command = `kill -0 \`cat ${pidFileName}\``;
+        return command;
+    }
+
+    public isProcessAliveProcessOutput(commandResult: RemoteCommandResult): boolean {
+        let result = true;
+        if (commandResult.exitCode > 0) {
+            result = false;
+        }
+        return result;
+    }
+
+    public killChildProcesses(pidFileName: string): string {
+        const command = `pkill -P \`cat ${pidFileName}\``;
+        return command;
+    }
+
+    public extractFile(tarFileName: string, targetFolder: string): string {
+        const command = `tar -oxzf ${tarFileName} -C ${targetFolder}`;
+        return command;
+    }
+
+    public executeScript(script: string, isFile: boolean): string {
+        let command: string;
+        if (isFile) {
+            command = `bash ${script}`;
+        } else {
+            command = `bash -c ${script}`;
+        }
+        return command;
+    }
+}
+
+export { LinuxCommands };

src/nni_manager/training_service/common/osCommands.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+'use strict';
+
+import { RemoteCommandResult } from "./remoteMachineData";
+
+abstract class OsCommands {
+
+    protected pathSpliter: string = '/';
+    protected multiplePathSpliter: RegExp = new RegExp(`\\${this.pathSpliter}{2,}`);
+
+    public abstract createFolder(folderName: string, sharedFolder: boolean): string;
+    public abstract allowPermission(isRecursive: boolean, ...folders: string[]): string;
+    public abstract removeFolder(folderName: string, isRecursive: boolean, isForce: boolean): string;
+    public abstract removeFiles(folderOrFileName: string, filePattern: string): string;
+    public abstract readLastLines(fileName: string, lineCount: number): string;
+    public abstract isProcessAliveCommand(pidFileName: string): string;
+    public abstract isProcessAliveProcessOutput(result: RemoteCommandResult): boolean;
+    public abstract killChildProcesses(pidFileName: string): string;
+    public abstract extractFile(tarFileName: string, targetFolder: string): string;
+    public abstract executeScript(script: string, isFile: boolean): string;
+
+    public joinPath(...paths: string[]): string {
+        let dir: string = paths.filter((path: any) => path !== '').join(this.pathSpliter);
+        if (dir === '') {
+            dir = '.';
+        } else {
+            dir = dir.replace(this.multiplePathSpliter, this.pathSpliter);
+        }
+        return dir;
+    }
+}
+
+export { OsCommands };

src/nni_manager/training_service/remote_machine/remoteMachineData.ts → src/nni_manager/training_service/common/remoteMachineData.ts

@@ -7,7 +7,8 @@ import * as fs from 'fs';
 import { Client, ConnectConfig } from 'ssh2';
 import { Deferred } from 'ts-deferred';
 import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
-import { GPUInfo, GPUSummary } from '../common/gpuData';
+import { GPUInfo, GPUSummary } from './gpuData';
+import { ShellExecutor } from './shellExecutor';
 
 /**
  * Metadata of remote machine for configuration and statuc query
@@ -83,101 +84,71 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail {
     }
 }
 
-/**
- * The remote machine ssh client used for trial and gpu detector
- */
-export class SSHClient {
-    private readonly sshClient: Client;
-    private usedConnectionNumber: number; //count the connection number of every client
-    constructor(sshClient: Client, usedConnectionNumber: number) {
-        this.sshClient = sshClient;
-        this.usedConnectionNumber = usedConnectionNumber;
-    }
-
-    public get getSSHClientInstance(): Client {
-        return this.sshClient;
-    }
-
-    public get getUsedConnectionNumber(): number {
-        return this.usedConnectionNumber;
-    }
-
-    public addUsedConnectionNumber(): void {
-        this.usedConnectionNumber += 1;
-    }
-
-    public minusUsedConnectionNumber(): void {
-        this.usedConnectionNumber -= 1;
-    }
-}
-
 /**
  * The remote machine ssh client manager
  */
 export class SSHClientManager {
-    private readonly sshClientArray: SSHClient[];
+    private readonly sshExecutorArray: ShellExecutor[];
     private readonly maxTrialNumberPerConnection: number;
     private readonly rmMeta: RemoteMachineMeta;
-    constructor(sshClientArray: SSHClient[], maxTrialNumberPerConnection: number, rmMeta: RemoteMachineMeta) {
+    constructor(sshExecutorArray: ShellExecutor[], maxTrialNumberPerConnection: number, rmMeta: RemoteMachineMeta) {
         this.rmMeta = rmMeta;
-        this.sshClientArray = sshClientArray;
+        this.sshExecutorArray = sshExecutorArray;
         this.maxTrialNumberPerConnection = maxTrialNumberPerConnection;
     }
 
     /**
      * find a available ssh client in ssh array, if no ssh client available, return undefined
      */
-    public async getAvailableSSHClient(): Promise<Client> {
-        const deferred: Deferred<Client> = new Deferred<Client>();
-        for (const index of this.sshClientArray.keys()) {
-            const connectionNumber: number = this.sshClientArray[index].getUsedConnectionNumber;
+    public async getAvailableSshExecutor(): Promise<ShellExecutor> {
+        for (const index of this.sshExecutorArray.keys()) {
+            const connectionNumber: number = this.sshExecutorArray[index].getUsedConnectionNumber;
             if (connectionNumber < this.maxTrialNumberPerConnection) {
-                this.sshClientArray[index].addUsedConnectionNumber();
-                deferred.resolve(this.sshClientArray[index].getSSHClientInstance);
+                this.sshExecutorArray[index].addUsedConnectionNumber();
 
-                return deferred.promise;
+                return this.sshExecutorArray[index];
             }
         }
 
         //init a new ssh client if could not get an available one
-        return this.initNewSSHClient();
+        return await this.initNewSSHClient();
     }
 
     /**
      * add a new ssh client to sshClientArray
      * @param sshClient SSH Client
      */
-    public addNewSSHClient(client: Client): void {
-        this.sshClientArray.push(new SSHClient(client, 1));
+    public addNewSSHClient(executor: ShellExecutor): void {
+        this.sshExecutorArray.push(executor);
     }
 
     /**
      * first ssh client instance is used for gpu collector and host job
      */
-    public getFirstSSHClient(): Client {
-        return this.sshClientArray[0].getSSHClientInstance;
+    public getFirstSshExecutor(): ShellExecutor {
+        return this.sshExecutorArray[0];
     }
 
     /**
      * close all of ssh client
      */
     public closeAllSSHClient(): void {
-        for (const sshClient of this.sshClientArray) {
+        for (const sshClient of this.sshExecutorArray) {
             sshClient.getSSHClientInstance.end();
         }
     }
 
     /**
      * retrieve resource, minus a number for given ssh client
-     * @param client SSH Client
+     * @param executor SSH Client
      */
-    public releaseConnection(client: Client | undefined): void {
-        if (client === undefined) {
+    public releaseConnection(executor: ShellExecutor | undefined): void {
+        if (executor === undefined) {
             throw new Error(`could not release a undefined ssh client`);
         }
-        for (const index of this.sshClientArray.keys()) {
-            if (this.sshClientArray[index].getSSHClientInstance === client) {
-                this.sshClientArray[index].minusUsedConnectionNumber();
+        for (const index of this.sshExecutorArray.keys()) {
+            if (this.sshExecutorArray[index] === executor) {
+                this.sshExecutorArray[index].minusUsedConnectionNumber();
                 break;
             }
         }
@@ -186,8 +157,8 @@ export class SSHClientManager {
     /**
      * Create a new ssh connection client and initialize it
      */
-    private initNewSSHClient(): Promise<Client> {
-        const deferred: Deferred<Client> = new Deferred<Client>();
+    private async initNewSSHClient(): Promise<ShellExecutor> {
+        const deferred: Deferred<ShellExecutor> = new Deferred<ShellExecutor>();
         const conn: Client = new Client();
         const connectConfig: ConnectConfig = {
             host: this.rmMeta.ip,
@@ -208,9 +179,11 @@ export class SSHClientManager {
         } else {
             deferred.reject(new Error(`No valid passwd or sshKeyPath is configed.`));
         }
-        conn.on('ready', () => {
-            this.addNewSSHClient(conn);
-            deferred.resolve(conn);
+        conn.on('ready', async () => {
+            const executor = new ShellExecutor(conn);
+            await executor.initialize();
+            this.addNewSSHClient(executor);
+            deferred.resolve(executor);
         })
           .on('error', (err: Error) => {
             // SSH connection error, reject with error message