Merge pull request #1 from qnib/gpu

GPU enabled proxy

Author: ChristianKniep

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.2
+current_version = 0.2.0
 files = main.go
 commit = True
 tag = True

.gitignore

@@ -14,4 +14,5 @@
 .glide/
 
 # binaries
-doxy_*
+**/doxy_*
+.idea/

Dockerfile

@@ -0,0 +1,14 @@
+ARG DOCKER_REG=docker.io
+FROM ${DOCKER_REG}/qnib/alplain-golang AS build
+
+WORKDIR /usr/local/src/github.com/qnib/doxy
+COPY main.go ./main.go
+COPY proxy ./proxy
+COPY vendor/ vendor/
+RUN govendor install
+
+## Build final image
+FROM alpine:3.5
+
+COPY --from=build /usr/local/bin/doxy /usr/local/bin/
+CMD ["/usr/local/bin/doxy"]

Dockerfile.ubuntu

@@ -0,0 +1,14 @@
+ARG DOCKER_REG=docker.io
+FROM ${DOCKER_REG}/qnib/uplain-golang:1.9.2 AS build
+
+WORKDIR /usr/local/src/github.com/qnib/doxy
+COPY main.go ./main.go
+COPY proxy ./proxy
+COPY vendor/ vendor/
+RUN govendor install
+
+## Build final image
+FROM ${DOCKER_REG}/qnib/uplain-init
+
+COPY --from=build /usr/local/bin/doxy /usr/local/bin/
+CMD ["/usr/local/bin/doxy"]

GPU.md

@@ -0,0 +1,23 @@
+# GPU Support
+
+As a Proof-of-Concept how to implement GPU (and InfiniBand) support, `doxy` was extended to allow for injection of payload to the `docker create` call.
+
+On an `p2.xlarge` instance with CUDA support, `doxy:gpu` was started like that.
+
+```bash
+$ docker run -v /var/run:/var/run/ -ti --rm qnib/doxy:gpu doxy --pattern-key=hpc --debug --proxy-socket=/var/run/hpc.sock --gpu
+```
+
+The pattern `hpc` allows for read/write endpoints to be used and `--gpu` injects bind-mounts, device-mappings and environment variables to make it work.
+
+```bash
+root@ip-172-31-28-27:~# docker -H unix:///var/run/hpc.sock create nvidia/cuda nvidia-smi -L
+6b708d7cda36b9c37e325893108839f3b02f172e40ab97182fa77d770cc219fb
+root@ip-172-31-28-27:~# docker start -a 6b708d7cda36b9c37e325893108839f3b02f172e40ab97182fa77d770cc219fb
+GPU 0: Tesla K80 (UUID: GPU-234d5537-ea27-68cd-7337-ee21b2f34bf1)
+root@ip-172-31-28-27:~#
+```
+
+When `start`ing the container the default socket is used; as I had issue to pass `stdin`/`stdout` through.
+The command hangs with the `doxy`-socket.
+I suspect issues with `stdin`/`stdout` passthrough when proxying the two unix-sockets. :/

README.md

@@ -16,7 +16,7 @@ GLOBAL OPTIONS:
 $ ./doxy_darwin
 2017/08/18 11:37:43 [II] Start Version: 0.1.0
 2017/08/18 11:37:43 Error reading patterns file (open /etc/doxy.pattern: no such file or directory), using default patterns
-2017/08/18 11:37:43 [gk-soxy] Listening on /tmp/doxy.sock
+2017/08/18 11:37:43 [doxy] Listening on /tmp/doxy.sock
 ```
 
 ## Filter mechanism
@@ -79,7 +79,7 @@ $ ./doxy_darwin -debug
 2017/08/18 11:44:50 4  : ^/(v\d\.\d+/)?nodes(/\w+)?$
 2017/08/18 11:44:50 5  : ^/(v\d\.\d+/)?info$
 2017/08/18 11:44:50 6  : ^/_ping$
-2017/08/18 11:44:50 [gk-soxy] Listening on /tmp/doxy.sock
+2017/08/18 11:44:50 [doxy] Listening on /tmp/doxy.sock
 [negroni] 2017-08-18T11:45:00+02:00 | 200 | 	 3.800713ms | docker | GET /_ping
 [negroni] 2017-08-18T11:45:00+02:00 | 403 | 	 34.067µs | docker | GET /v1.31/containers/a62250e0890a/export
 [negroni] 2017-08-18T11:45:04+02:00 | 200 | 	 1.800044ms | docker | GET /_ping

build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+COMMIT_TAG=$(git describe --tags)
+TEMP_TAG=$(base64 /dev/urandom | tr -d '/+' | dd bs=32 count=1 2>/dev/null)
+mkdir -p bin/
+
+echo ">> Build temp-image 'qnib/$(basename $(pwd)):${TEMP_TAG}'"
+docker build -t qnib/$(basename $(pwd)):${TEMP_TAG} -f Dockerfile.ubuntu .
+echo ">> Start image as ${TEMP_TAG} to copy binary"
+ID=$(docker run -d --name ${TEMP_TAG} qnib/$(basename $(pwd)):${TEMP_TAG} tail -f /dev/null)
+echo ">> CONTAINER_ID=${ID}"
+docker cp ${ID}:/usr/local/bin/doxy bin/doxy_x86_${COMMIT_TAG}
+echo ">> Remove container and image"
+docker rm -f ${ID}
+docker rmi qnib/$(basename $(pwd)):${TEMP_TAG}

main.go

@@ -6,6 +6,7 @@ import (
 	"github.com/zpatrick/go-config"
 	"github.com/codegangsta/cli"
 	"github.com/qnib/doxy/proxy"
+	"strings"
 )
 
 var (
@@ -21,11 +22,37 @@ var (
 		Usage: "Proxy socket to be created",
 		EnvVar: "DOXY_PROXY_SOCKET",
 	}
+	proxyPatternKey = cli.StringFlag{
+		Name:  "pattern-key",
+		Value: "default",
+		Usage: "pattern key predefined",
+		EnvVar: "DOXY_PATTERN_KEY",
+	}
+	gpuEnabled = cli.BoolFlag{
+		Name: "gpu",
+		Usage: "Map devices, bind-mounts and environment into each container to allow GPU usage",
+		EnvVar: "DOXY_GPU_ENABLED",
+	}
+	constrainUser = cli.BoolFlag{
+		Name: "user-pinning",
+		Usage: "Pin user within container to the UID calling the command",
+		EnvVar: "DOXY_USER_PINNING_ENABLED",
+	}
 	debugFlag = cli.BoolFlag{
 		Name: "debug",
 		Usage: "Print proxy requests",
 		EnvVar: "DOXY_DEBUG",
 	}
+	bindAddFlag = cli.StringFlag{
+		Name: "add-binds",
+		Usage: "Comma separated list of bind-mounts to add",
+		EnvVar: "DOXY_ADDITIONAL_BINDS",
+	}
+	devMapFlag = cli.StringFlag{
+		Name: "device-mappings",
+		Usage: "Comma separated list of device mappings",
+		EnvVar: "DOXY_DEVICE_MAPPINGS",
+	}
 	patternFileFlag = cli.StringFlag{
 		Name:  "pattern-file",
 		Value: proxy.PATTERN_FILE,
@@ -41,6 +68,10 @@ func EvalOptions(cfg *config.Config) (po []proxy.ProxyOption) {
 	po = append(po, proxy.WithDockerSocket(dockerSock))
 	debug, _ := cfg.Bool("debug")
 	po = append(po, proxy.WithDebugValue(debug))
+	devMaps, _ := cfg.String("device-mappings")
+	gpu, _ := cfg.Bool("gpu")
+	po = append(po, proxy.WithGpuValue(gpu))
+	po = append(po, proxy.WithDevMappings(strings.Split(devMaps,",")))
 	return
 }
 
@@ -50,18 +81,32 @@ func EvalPatternOpts(cfg *config.Config) (proxy.ProxyOption) {
 	defer reader.Close()
 	patterns := []string{}
 	if err != nil {
-		log.Printf("Error reading patterns file (%s), using default patterns\n", err.Error())
-		return proxy.WithPatterns(proxy.DEFAULT_PATTERNS)
+		patternsKey, _ := cfg.String("pattern-key")
+		if patterns, ok := proxy.PATTERNS[patternsKey]; ok {
+			log.Printf("Error reading patterns file '%s', using %s patterns\n", err.Error(), patternsKey)
+			return proxy.WithPatterns(patterns)
+		}
+		log.Printf("Could not find pattern-key '%s'\n", patternsKey)
+		os.Exit(1)
+
 	}
 	patterns, err  = proxy.ReadPatterns(reader)
 	return proxy.WithPatterns(patterns)
 }
 
+func EvalBindMountOpts(cfg *config.Config) (proxy.ProxyOption) {
+	bindStr, _ := cfg.String("add-binds")
+	bindMounts := strings.Split(bindStr,",")
+	return proxy.WithBindMounts(bindMounts)
+}
+
+
 func RunApp(ctx *cli.Context) {
 	log.Printf("[II] Start Version: %s", ctx.App.Version)
 	cfg := config.NewConfig([]config.Provider{config.NewCLI(ctx, true)})
 	po := EvalOptions(cfg)
 	po = append(po, EvalPatternOpts(cfg))
+	po = append(po, EvalBindMountOpts(cfg))
 	p := proxy.NewProxy(po...)
 	p.Run()
 }
@@ -70,12 +115,15 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "Proxy Docker unix socket to filter out insecure, harmful requests."
 	app.Usage = "doxy [options]"
-	app.Version = "0.1.2"
+	app.Version = "0.2.0"
 	app.Flags = []cli.Flag{
 		dockerSocketFlag,
 		proxySocketFlag,
 		debugFlag,
+		gpuEnabled,
 		patternFileFlag,
+		proxyPatternKey,
+		bindAddFlag,
 	}
 	app.Action = RunApp
 	app.Run(os.Args)

proxy/helper.go

@@ -0,0 +1,64 @@
+package proxy
+
+import (
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"io"
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"fmt"
+)
+
+type Headers map[string][]string
+
+
+
+type configWrapper struct {
+	*container.Config
+	HostConfig       *container.HostConfig
+	NetworkingConfig *network.NetworkingConfig
+}
+
+func createDevMapping(dev string) (dm container.DeviceMapping, err error) {
+	spl := strings.Split(dev, ":")
+	switch len(spl) {
+	case 2:
+		dm = container.DeviceMapping{
+			PathOnHost: spl[0],
+			PathInContainer: spl[1],
+		}
+	case 3:
+		dm = container.DeviceMapping{
+			PathOnHost: spl[0],
+			PathInContainer: spl[1],
+			CgroupPermissions: spl[2],
+		}
+	default:
+		return dm, fmt.Errorf("string needs to specify <src>:<dst>[:permission]")
+		}
+ 	return
+}
+
+func encodeBody(obj interface{}, header http.Header) (io.Reader, http.Header, error) {
+	if obj == nil {
+		return nil, header, nil
+	}
+
+	body, err := encodeData(obj)
+	if err != nil {
+		return nil, header, err
+	}
+	return body, header, nil
+}
+
+func encodeData(data interface{}) (*bytes.Buffer, error) {
+	params := bytes.NewBuffer(nil)
+	if data != nil {
+		if err := json.NewEncoder(params).Encode(data); err != nil {
+			return nil, err
+		}
+	}
+	return params, nil
+}

proxy/main.go

@@ -16,7 +16,7 @@ const (
 )
 
 var (
-	DEFAULT_PATTERNS = []string{
+	DEF_PAT = []string{
 		`^/(v\d\.\d+/)?containers(/\w+)?/(json|stats|top)$`,
 		`^/(v\d\.\d+/)?services(/[0-9a-f]+)?$`,
 		`^/(v\d\.\d+/)?tasks(/\w+)?$`,
@@ -27,12 +27,28 @@ var (
 		`^/(v\d\.\d+/)?version$`,
 		"^/_ping$",
 	}
+	HPC_PAT = []string{
+		`^/(v\d\.\d+/)?containers(/\w+)?/(json|stats|top|create|start|run|kill)$`,
+		`^/(v\d\.\d+/)?services(/[0-9a-f]+)?$`,
+		`^/(v\d\.\d+/)?tasks(/\w+)?$`,
+		`^/(v\d\.\d+/)?networks(/\w+)?$`,
+		`^/(v\d\.\d+/)?volumes(/\w+)?$`,
+		`^/(v\d\.\d+/)?nodes(/\w+)?$`,
+		`^/(v\d\.\d+/)?info$`,
+		`^/(v\d\.\d+/)?version$`,
+		"^/_ping$",
+	}
+	PATTERNS = map[string][]string{
+		"default": DEF_PAT,
+		"hpc": HPC_PAT,
+	}
 )
 
 type Proxy struct {
 	dockerSocket, newSocket string
-	debug 					bool
+	debug, gpu 				bool
 	patterns 				[]string
+	bindMounts,devMappings	[]string
 }
 
 func NewProxy(opts ...ProxyOption) Proxy {
@@ -44,7 +60,10 @@ func NewProxy(opts ...ProxyOption) Proxy {
 		dockerSocket: options.DockerSocket,
 		newSocket: options.ProxySocket,
 		debug: options.Debug,
+		gpu: options.Gpu,
 		patterns: options.Patterns,
+		bindMounts: options.BindMounts,
+		devMappings: options.DevMappings,
 	}
 }
 
@@ -59,7 +78,7 @@ func (p *Proxy) GetOptions() map[string]interface{} {
 }
 
 func (p *Proxy) Run() {
-	upstream := NewUpstream(p.dockerSocket, p.patterns)
+	upstream := NewUpstream(p.dockerSocket, p.patterns, p.bindMounts, p.devMappings, p.gpu)
 	sigc := make(chan os.Signal, 1)
 	signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM)
 	l, err := ListenToNewSock(p.newSocket, sigc)