add pfsserver dockerfile

.travis.yml

@@ -12,7 +12,7 @@ matrix:
         - | 
           bash .tools/check_style.sh
           RESULT=$?; if [ $RESULT -eq 0 ]; then true; else false; fi;
-        - bash .tools/gen_config.sh && cd go && glide install && go test $(glide novendor)
+        - cd go && bash .tools/gen_config.sh && glide install && go test $(glide novendor)
     - language: python
       python: 2.7
       sudo: required

docker/pfs/build.sh

@@ -0,0 +1,19 @@
+cat > ./Dockerfile << EOF
+FROM ubuntu:16.04
+
+RUN apt-get update && \
+    apt-get install -y wget git && \
+    wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go.tgz && \
+    mkdir /root/gopath && \
+    rm go.tgz
+
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath 
+ENV PATH=${PATH}:${GOROOT}/bin
+
+CMD ["sh", "-c", "cd /root/gopath/src/github.com/PaddlePaddle/cloud/go/cmd/pfsserver && go get ./... && go build"]
+EOF
+
+docker build .  -t  pfsserver:dev
+
+rm -f Dockerfile

.tools/gen_config.sh → go/.tools/gen_config.sh

@@ -3,8 +3,8 @@ mkdir -p ~/.paddle
 cat > ~/.paddle/config << EOF
 datacenters:
 - name: datacenter1
-  username: user@baidu.com
-  password: T123
+  username: your-user-name
+  password: your-secret
   endpoint: http://127.0.0.1:8080
 current-datacenter: datacenter1
 EOF

go/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:16.04
+
+ADD .tools    /pfsserver/.tools
+RUN bash /pfsserver/.tools/gen_config.sh
+
+ADD ./cmd/pfsserver/pfsserver /pfsserver/
+RUN mkdir /pfsserver/log

go/README_CN.md

@@ -0,0 +1,62 @@
+1. 如何构建PFSServer的DockerImage
+  - 构建PFSServer的编译环境
+
+  ```
+  cd cloud/docker/pfs
+  bash build.sh
+  ```
+
+  - 编译PFSServer
+
+  ```
+  cd cloud/go
+  docker run  --rm -v  $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev
+  ```
+
+  - 构建PFSServer的DockerImage
+
+  ```
+  cd cloud/go
+  docker build . -t pfsserver:latest
+  ```
+  - PFSServer启动命令
+
+  ```
+  docker run pfsserver:latest /pfsserver/pfsserver -tokenuri http://cloud.paddlepaddle.org -logtostderr=true -v=3
+  ```
+
+2. 如何部署PFSServer
+
+	```
+	cd ../k8s
+	kuberctl create -f cloud_pfsserver.yaml
+	```
+
+3. 如何使用PFSClient
+	- cp
+
+	```
+	upload:
+		paddlecloud cp ./file /pfs/$DATACENTER/home/$USER/file
+
+	download:
+		paddlecloud cp /pfs/$DATACENTER/home/$USER/file ./file
+	```
+	- ls
+
+	```
+	paddlecloud ls  /pfs/$DATACENTER/home/$USER/folder
+	```
+
+	- rm
+
+	```
+	paddlecloud rm /pfs/$DATACENTER/home/$USER/file
+	paddlecloud rm -r /pfs/$DATACENTER/home/$USER/folder
+	```
+
+	- mkdir
+
+	```
+	paddlecloud mkdir  /pfs/$DATACENTER/home/$USER/folder
+	```

go/filemanager/pfsmodules/command.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"io"
 	"net/url"
+	"path"
 	"strings"
 
 	log "github.com/golang/glog"
@@ -33,13 +34,16 @@ type Command interface {
 
 // CheckUser checks if a user has authority to access a path.
 // path example:/pfs/$datacenter/home/$user
-func checkUser(path string, user string) error {
-	a := strings.Split(path, "/")
-	if len(a) < 3 {
+func checkUser(pathStr string, user string) error {
+	pathStr = path.Clean(strings.TrimSpace(pathStr))
+	a := strings.Split(pathStr, "/")
+	// the first / is convert to " "
+	if len(a) < 5 {
 		return errors.New(StatusBadPath)
 	}
 
-	if a[3] != user {
+	if a[4] != user {
+		log.V(4).Infof("request path:%s user:%s split_path:%s\n", pathStr, user, a[4])
 		return errors.New(StatusUnAuthorized)
 	}
 	return nil
@@ -57,7 +61,7 @@ func ValidatePfsPath(paths []string, userName string) error {
 		}
 
 		if err := checkUser(path, userName); err != nil {
-			return errors.New(StatusShouldBePfsPath + ":" + path)
+			return err
 		}
 	}
 	return nil

go/filemanager/pfsmodules/download.go

@@ -21,7 +21,7 @@ func remoteChunkMeta(path string,
 		ChunkSize: chunkSize,
 	}
 
-	t := fmt.Sprintf("%s/api/v1/chunks", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/chunks", Config.ActiveConfig.Endpoint)
 	ret, err := restclient.GetCall(t, cmd.ToURLParam())
 	if err != nil {
 		return nil, err
@@ -89,7 +89,7 @@ func downloadChunks(src string,
 		return nil
 	}
 
-	t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/storage/chunks", Config.ActiveConfig.Endpoint)
 	for _, meta := range diffMeta {
 		chunk := Chunk{
 			Path:   src,

go/filemanager/pfsmodules/ls.go

@@ -249,7 +249,7 @@ func formatPrint(result []LsResult) {
 
 // RemoteLs gets LsCmd result from cloud.
 func RemoteLs(cmd *LsCmd) ([]LsResult, error) {
-	t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint)
 	body, err := restclient.GetCall(t, cmd.ToURLParam())
 	if err != nil {
 		return nil, err

go/filemanager/pfsmodules/mkdir.go

@@ -126,7 +126,7 @@ func RemoteMkdir(cmd *MkdirCmd) ([]MkdirResult, error) {
 		return nil, err
 	}
 
-	t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint)
 	log.V(2).Infoln(t)
 	body, err := restclient.PostCall(t, j)
 	if err != nil {

go/filemanager/pfsmodules/rm.go

@@ -162,7 +162,7 @@ func RemoteRm(cmd *RmCmd) ([]RmResult, error) {
 		return nil, err
 	}
 
-	t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint)
 	body, err := restclient.DeleteCall(t, j)
 	if err != nil {
 		return nil, err

go/filemanager/pfsmodules/upload.go

@@ -17,7 +17,7 @@ import (
 var Config = config.ParseDefaultConfig()
 
 func remoteStat(cmd *StatCmd) (*LsResult, error) {
-	t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint)
 	log.V(3).Infoln(t)
 	body, err := restclient.GetCall(t, cmd.ToURLParam())
 	if err != nil {
@@ -49,7 +49,7 @@ func remoteTouch(cmd *TouchCmd) error {
 		return err
 	}
 
-	t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint)
 	body, err := restclient.PostCall(t, j)
 	if err != nil {
 		return err
@@ -108,7 +108,7 @@ func postChunk(src *Chunk, dst string) ([]byte, error) {
 	}
 	defer Close(f)
 
-	t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.Endpoint)
+	t := fmt.Sprintf("%s/api/v1/pfs/storage/chunks", Config.ActiveConfig.Endpoint)
 	log.V(4).Infoln(t)
 
 	return restclient.PostChunk(t, getDstParam(src, dst),

go/filemanager/pfsserver/handler.go

@@ -3,6 +3,7 @@ package pfsserver
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
@@ -24,25 +25,32 @@ var TokenURI = ""
 
 func getUserName(uri string, token string) (string, error) {
 	authHeader := make(map[string]string)
-	authHeader["Authorization"] = "Token " + token
+	authHeader["Authorization"] = token
+
+	str := fmt.Sprintf("get uri with token error uri:%s token:%s\n", uri, token)
+
 	req, err := restclient.MakeRequest(uri, "GET", nil, "", nil, authHeader)
 	if err != nil {
+		log.Errorln(str)
 		return "", err
 	}
 
 	body, err := restclient.GetResponse(req)
 	if err != nil {
+		log.Errorln(str)
 		return "", err
 	}
 
 	log.V(4).Infoln("get token2user resp:" + string(body[:]))
 	var resp interface{}
 	if err := json.Unmarshal(body, &resp); err != nil {
+		log.Errorln(string(body[:]))
 		return "", err
 	}
 
 	user := resp.(map[string]interface{})["user"].(string)
 	if len(user) < 1 {
+		log.Errorln(resp)
 		return "", errors.New("can't get username")
 	}
 

go/filemanager/pfsserver/routes.go

@@ -19,39 +19,39 @@ var routes = Routes{
 	Route{
 		"GetFiles",
 		"GET",
-		"/api/v1/files",
+		"/api/v1/pfs/files",
 		GetFilesHandler,
 	},
 	Route{
 		"PostFiles",
 		"POST",
-		"/api/v1/files",
+		"/api/v1/pfs/files",
 		PostFilesHandler,
 	},
 	Route{
 		"DeleteFiles",
 		"DELETE",
-		"/api/v1/files",
+		"/api/v1/pfs/files",
 		DeleteFilesHandler,
 	},
 
 	Route{
 		"GetChunksMeta",
 		"GET",
-		"/api/v1/chunks",
+		"/api/v1/pfs/chunks",
 		GetChunkMetaHandler,
 	},
 	Route{
 		"GetChunksData",
 		"GET",
-		"/api/v1/storage/chunks",
+		"/api/v1/pfs/storage/chunks",
 		GetChunkHandler,
 	},
 
 	Route{
 		"PostChunksData",
 		"POST",
-		"/api/v1/storage/chunks",
+		"/api/v1/pfs/storage/chunks",
 		PostChunkHandler,
 	},
 }

k8s/cloud_deployment.yaml

@@ -39,7 +39,7 @@ spec:
       containers:
       - name: paddle-cloud
         imagePullPolicy: Always
-        image: paddlepaddle/cloud
+        image: gongweibao/pcloud
         volumeMounts:
         - name: cert-volume
           mountPath: /certs

k8s/cloud_ingress.yaml

@@ -4,10 +4,15 @@ metadata:
   name: paddle-cloud-ingress
 spec:
   rules:
-  - host: cloud.paddlepaddle.org
+  - host: cloud1.paddlepaddle.org
     http:
       paths:
+      - path: /api/v1/files
+        backend:
+          serviceName: pfs-service
+          servicePort: 8080
       - path: /
         backend:
           serviceName: paddle-cloud-service
           servicePort: 8000
+

k8s/pfs_deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: pfsserver
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: pfsserver
+    spec:
+      volumes:
+      - name: data-storage
+        hostPath:
+          path: /home/gongwb/ceph
+      containers:
+      - name: pfs
+        imagePullPolicy: Always
+        image: gongweibao/pfsserver:latest
+        env:
+          - name: CURRENT_DATACENTER
+            value: "meiyan"
+        command: ["/pfsserver/pfsserver", "-tokenuri", "http://paddle-cloud-service:8000", "-logtostderr=true", "-log_dir=./log", "-v=4"]
+        volumeMounts:
+        - name: data-storage
+          mountPath: /pfs/datacenter1/home/
+        ports:
+        - containerPort: 8080
+      nodeSelector:
+        kubernetes.io/hostname: k8s-node1
+

k8s/pfs_service.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: pfs-service
+spec:
+  selector:
+    app: pfsserver 
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+  type: NodePort

paddlecloud/paddlecloud/urls.py

@@ -33,6 +33,7 @@
     url(r"^api/v1/workers/", paddlejob.views.WorkersView.as_view()),
     url(r"^api/v1/quota/", paddlejob.views.QuotaView.as_view()),
     url(r"^api/v1/file/", paddlejob.views.SimpleFileView.as_view()),
+    url(r"^api/v1/token2user/", paddlejob.views.GetUserView.as_view()),
     url(r"^api/v1/filelist/", paddlejob.views.SimpleFileList.as_view()),
     url(r"^api/v1/registry/", paddlejob.registry.RegistryView.as_view()),
 ]