From 765d0d84310ebf1d4161e095504fc76711c0148c Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 13 Jun 2017 17:57:35 +0800 Subject: [PATCH 01/23] add dockerfile --- go/Dockerfile | 5 +++++ go/README_CN.md | 19 +++++++++++++++++++ go/docker/build.sh | 17 +++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 go/Dockerfile create mode 100644 go/README_CN.md create mode 100755 go/docker/build.sh diff --git a/go/Dockerfile b/go/Dockerfile new file mode 100644 index 00000000..e4b80df8 --- /dev/null +++ b/go/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:16.04 + +ADD ./cmd/pfsserver/pfsserver /pfsserver/ + +CMD ["sh", "-c", "cd /pfsserver && ./pfsserver"] diff --git a/go/README_CN.md b/go/README_CN.md new file mode 100644 index 00000000..0f6466e2 --- /dev/null +++ b/go/README_CN.md @@ -0,0 +1,19 @@ +1. 如何创建pfsserver的DockerImage + - 创建pfsserver的编译环境 + + ``` + cd docker + bash build.sh + ``` + + - 编译pfsserver + + ``` + docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev + ``` + + - 创建pfsserver的DockerImage + - + ``` + docker build . -t pfsserver:latest + ``` \ No newline at end of file diff --git a/go/docker/build.sh b/go/docker/build.sh new file mode 100755 index 00000000..37d5e264 --- /dev/null +++ b/go/docker/build.sh @@ -0,0 +1,17 @@ +cat > ./Dockerfile << EOF +FROM ubuntu:16.04 + +RUN apt-get update && \ + apt-get install -y wget git && \ + wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go.tgz && \ + mkdir /root/gopath && \ + rm go.tgz + +ENV GOROOT=/usr/local/go GOPATH=/root/gopath +ENV PATH=${PATH}:${GOROOT}/bin + +CMD ["sh", "-c", "cd /root/gopath/src/github.com/PaddlePaddle/cloud/go/cmd/pfsserver && go get ./... && go build"] +EOF + +docker build . -t pfsserver:dev From e760269a0958ae07ff256e573fb71b75a3418c51 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 13 Jun 2017 17:59:02 +0800 Subject: [PATCH 02/23] modify README_CN.md --- go/README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/README_CN.md b/go/README_CN.md index 0f6466e2..3b2aad6e 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -13,7 +13,7 @@ ``` - 创建pfsserver的DockerImage - - + ``` docker build . -t pfsserver:latest ``` \ No newline at end of file From b0de56337dcdd7fcf170400e0abcd52c1425c0a1 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 13 Jun 2017 18:02:33 +0800 Subject: [PATCH 03/23] fix README_CN.md --- go/README_CN.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/README_CN.md b/go/README_CN.md index 3b2aad6e..b8b5541d 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -9,6 +9,7 @@ - 编译pfsserver ``` + cd .. docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev ``` @@ -16,4 +17,4 @@ ``` docker build . -t pfsserver:latest - ``` \ No newline at end of file + ``` From eac9ffccc52b0dc03dc549a91cef4863df232b7a Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 20 Jun 2017 10:11:49 +0800 Subject: [PATCH 04/23] modify readme --- go/Dockerfile | 5 ----- go/README_CN.md | 17 ++++++++++++----- go/cmd/pfsserver/Dockerfile | 2 ++ 3 files changed, 14 insertions(+), 10 deletions(-) delete mode 100644 go/Dockerfile create mode 100644 go/cmd/pfsserver/Dockerfile diff --git a/go/Dockerfile b/go/Dockerfile deleted file mode 100644 index e4b80df8..00000000 --- a/go/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ubuntu:16.04 - -ADD ./cmd/pfsserver/pfsserver /pfsserver/ - -CMD ["sh", "-c", "cd /pfsserver && ./pfsserver"] diff --git a/go/README_CN.md b/go/README_CN.md index b8b5541d..a8e3dba5 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -1,20 +1,27 @@ -1. 如何创建pfsserver的DockerImage - - 创建pfsserver的编译环境 +1. 如何构建PFSserver的DockerImage + - 构建PFSserver的编译环境 ``` cd docker bash build.sh ``` - - 编译pfsserver + - 编译PFSserver ``` cd .. - docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev + docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud PFSserver:dev ``` - - 创建pfsserver的DockerImage + - 创建PFSserver的DockerImage ``` docker build . -t pfsserver:latest ``` + +2. 如何部署PFSserver +3. 如何使用PFSclient + - cp + - ls + - rm + - mkdir diff --git a/go/cmd/pfsserver/Dockerfile b/go/cmd/pfsserver/Dockerfile new file mode 100644 index 00000000..8f3f77c7 --- /dev/null +++ b/go/cmd/pfsserver/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu:16.04 +ADD pfsserver /pfsserver/ From 22d0a8d8fb37958804a525118e0f520f5564847a Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 20 Jun 2017 12:20:21 +0800 Subject: [PATCH 05/23] fix pfsserver bugs --- .travis.yml | 2 +- go/.tools/gen_config.sh | 10 ++++++++++ go/Dockerfile | 5 +++++ go/README_CN.md | 19 +++++++++++-------- go/cmd/pfsserver/Dockerfile | 2 -- 5 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 go/.tools/gen_config.sh create mode 100644 go/Dockerfile delete mode 100644 go/cmd/pfsserver/Dockerfile diff --git a/.travis.yml b/.travis.yml index 6703194e..9b85a95a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ matrix: include: - language: go go: 1.8.x - script: cd go && go test ./... + script: cd go && bash .tool/gen_config.sh && go test ./... - language: python python: 2.7 sudo: required diff --git a/go/.tools/gen_config.sh b/go/.tools/gen_config.sh new file mode 100644 index 00000000..af2479e2 --- /dev/null +++ b/go/.tools/gen_config.sh @@ -0,0 +1,10 @@ +#/bin/bash +mkdir -p ~/.paddle +cat > ~/.paddle/config << EOF +datacenters: +- name: datacenter1 + username: user@baidu.com + password: T123 + endpoint: http://127.0.0.1:8080 +current-datacenter: datacenter1 +EOF diff --git a/go/Dockerfile b/go/Dockerfile new file mode 100644 index 00000000..61eb9fe1 --- /dev/null +++ b/go/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:16.04 +ADD .tools /pfsserver/.tools +RUN bash /pfsserver/.tools/gen_config.sh + +ADD ./cmd/pfsserver/pfsserver /pfsserver/ diff --git a/go/README_CN.md b/go/README_CN.md index a8e3dba5..e4f69ced 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -1,26 +1,29 @@ -1. 如何构建PFSserver的DockerImage - - 构建PFSserver的编译环境 +1. 如何构建PFSServer的DockerImage + - 构建PFSServer的编译环境 ``` cd docker bash build.sh ``` - - 编译PFSserver + - 编译PFSServer ``` - cd .. - docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud PFSserver:dev + docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev ``` - - 创建PFSserver的DockerImage + - 构建PFSServer的DockerImage ``` docker build . -t pfsserver:latest ``` + - PFSServer启动命令 + ``` + docker run pfsserver:latest /pfsserver/pfsserver -tokenuri http://cloud.paddlepaddle.org -logtostderr=false -log_dir=./log -v=3 + ``` -2. 如何部署PFSserver -3. 如何使用PFSclient +2. 如何部署PFSServer +3. 如何使用PFSClient - cp - ls - rm diff --git a/go/cmd/pfsserver/Dockerfile b/go/cmd/pfsserver/Dockerfile deleted file mode 100644 index 8f3f77c7..00000000 --- a/go/cmd/pfsserver/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM ubuntu:16.04 -ADD pfsserver /pfsserver/ From 122ee259bff669de0954198dda9c4aec28fe8560 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 20 Jun 2017 17:30:19 +0800 Subject: [PATCH 06/23] add cloud_pfsserver.yaml --- go/Dockerfile | 1 + k8s/cloud_pfsserver.yaml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 k8s/cloud_pfsserver.yaml diff --git a/go/Dockerfile b/go/Dockerfile index 61eb9fe1..20490bd2 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu:16.04 + ADD .tools /pfsserver/.tools RUN bash /pfsserver/.tools/gen_config.sh diff --git a/k8s/cloud_pfsserver.yaml b/k8s/cloud_pfsserver.yaml new file mode 100644 index 00000000..8b0f43dc --- /dev/null +++ b/k8s/cloud_pfsserver.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: pfsserver-deployment +spec: + replicas: 1 + template: + metadata: + labels: + app: pfsserver + spec: + containers: + - name: pfsserver + image: gongweibao/pfsserver:latest + ports: + - containerPort: 8080 + command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-logtostderr=false", "-log_dir=./log", "-v=3"] From a52096770f1b859c7fd3c6aa0f96a79b9ea8e325 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 20 Jun 2017 18:04:21 +0800 Subject: [PATCH 07/23] modify yaml --- k8s/cloud_ingress.yaml | 5 +++++ k8s/cloud_pfsserver.yaml | 2 +- k8s/pfsserver_service.yaml | 11 +++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 k8s/pfsserver_service.yaml diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index bfe8ae7e..b4b19e9a 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -7,7 +7,12 @@ spec: - host: cloud.paddlepaddle.org http: paths: + - path: /storage + backend: + serviceName: pfsserver-service + servicePort: 8080 - path: / backend: serviceName: paddle-cloud-service servicePort: 8000 + diff --git a/k8s/cloud_pfsserver.yaml b/k8s/cloud_pfsserver.yaml index 8b0f43dc..28b3c2cc 100644 --- a/k8s/cloud_pfsserver.yaml +++ b/k8s/cloud_pfsserver.yaml @@ -14,4 +14,4 @@ spec: image: gongweibao/pfsserver:latest ports: - containerPort: 8080 - command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-logtostderr=false", "-log_dir=./log", "-v=3"] + command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] diff --git a/k8s/pfsserver_service.yaml b/k8s/pfsserver_service.yaml new file mode 100644 index 00000000..85728637 --- /dev/null +++ b/k8s/pfsserver_service.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + name: pfsserver-service +spec: + selector: + app: pfsserver + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 From 612f53c96920cdbcf8bb9fbebc6134eac0e722b0 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 20 Jun 2017 18:11:51 +0800 Subject: [PATCH 08/23] modify docker file --- go/Dockerfile | 1 + go/README_CN.md | 30 ++++++++++++++++++++++++++++++ k8s/cloud_ingress.yaml | 16 ++++++++++++++-- k8s/pfsserver_service.yaml | 2 +- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/go/Dockerfile b/go/Dockerfile index 20490bd2..6e959fa4 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -4,3 +4,4 @@ ADD .tools /pfsserver/.tools RUN bash /pfsserver/.tools/gen_config.sh ADD ./cmd/pfsserver/pfsserver /pfsserver/ +RUN mkdir /pfsserver/log diff --git a/go/README_CN.md b/go/README_CN.md index e4f69ced..4021f42f 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -18,13 +18,43 @@ docker build . -t pfsserver:latest ``` - PFSServer启动命令 + ``` docker run pfsserver:latest /pfsserver/pfsserver -tokenuri http://cloud.paddlepaddle.org -logtostderr=false -log_dir=./log -v=3 ``` 2. 如何部署PFSServer + + ``` + cd ../k8s + kuberctl create -f cloud_pfsserver.yaml + ``` + 3. 如何使用PFSClient - cp + + ``` + upload: + paddlecloud cp ./file /pfs/$DATACENTER/home/$USER/file + + download: + paddlecloud cp /pfs/$DATACENTER/home/$USER/file ./file + ``` - ls + + ``` + paddlecloud ls /pfs/$DATACENTER/home/$USER/folder + ``` + - rm + + ``` + paddlecloud rm /pfs/$DATACENTER/home/$USER/file + paddlecloud rm -r /pfs/$DATACENTER/home/$USER/folder + ``` + - mkdir + + ``` + paddlecloud mkdir /pfs/$DATACENTER/home/$USER/folder + ``` diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index b4b19e9a..1dc0b988 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -7,10 +7,22 @@ spec: - host: cloud.paddlepaddle.org http: paths: - - path: /storage + - path: /api/v1/files backend: serviceName: pfsserver-service - servicePort: 8080 + servicePort: 8000 + - path: /api/v1/chunks + backend: + serviceName: pfsserver-service + servicePort: 8000 + - path: /api/v1/storage/files + backend: + serviceName: pfsserver-service + servicePort: 8000 + - path: /api/v1/storage/chunks + backend: + serviceName: pfsserver-service + servicePort: 8000 - path: / backend: serviceName: paddle-cloud-service diff --git a/k8s/pfsserver_service.yaml b/k8s/pfsserver_service.yaml index 85728637..ebaf5519 100644 --- a/k8s/pfsserver_service.yaml +++ b/k8s/pfsserver_service.yaml @@ -7,5 +7,5 @@ spec: app: pfsserver ports: - protocol: TCP - port: 8080 + port: 8000 targetPort: 8080 From 03f244dbc28f3bf0ca1e359e8960b5f143ce4cba Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 21 Jun 2017 10:19:32 +0800 Subject: [PATCH 09/23] add pfs config --- k8s/cloud_ingress.yaml | 16 ---------------- k8s/pfs_app.yaml | 18 ++++++++++++++++++ k8s/pfs_ingress.yaml | 19 +++++++++++++++++++ k8s/pfs_svc.yaml | 13 +++++++++++++ 4 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 k8s/pfs_app.yaml create mode 100644 k8s/pfs_ingress.yaml create mode 100644 k8s/pfs_svc.yaml diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index 1dc0b988..dd1fcd67 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -7,22 +7,6 @@ spec: - host: cloud.paddlepaddle.org http: paths: - - path: /api/v1/files - backend: - serviceName: pfsserver-service - servicePort: 8000 - - path: /api/v1/chunks - backend: - serviceName: pfsserver-service - servicePort: 8000 - - path: /api/v1/storage/files - backend: - serviceName: pfsserver-service - servicePort: 8000 - - path: /api/v1/storage/chunks - backend: - serviceName: pfsserver-service - servicePort: 8000 - path: / backend: serviceName: paddle-cloud-service diff --git a/k8s/pfs_app.yaml b/k8s/pfs_app.yaml new file mode 100644 index 00000000..888ac728 --- /dev/null +++ b/k8s/pfs_app.yaml @@ -0,0 +1,18 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: pfsserver + namespace: pfs +spec: + replicas: 1 + template: + metadata: + labels: + app: pfsserver + spec: + containers: + - name: pfs + imagePullPolicy: Always + image: gongweibao/pfsserver:latest + ports: + - containerPort: 8080 diff --git a/k8s/pfs_ingress.yaml b/k8s/pfs_ingress.yaml new file mode 100644 index 00000000..0ae96588 --- /dev/null +++ b/k8s/pfs_ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: pfs-ingress + namespace: pfs +spec: + tls: + - hosts: + - pfs.paddlpaddle.org + secretName: pfs-secret + rules: + - host: pfs.paddlepaddle.org + http: + paths: + - backend: + serviceName: pfs-service + servicePort: 8080 + path: / + diff --git a/k8s/pfs_svc.yaml b/k8s/pfs_svc.yaml new file mode 100644 index 00000000..f9e35103 --- /dev/null +++ b/k8s/pfs_svc.yaml @@ -0,0 +1,13 @@ +kind: Service +apiVersion: v1 +metadata: + name: pfs-service + namespace: pfs +spec: + selector: + app: pfsserver + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: NodePort From 91b44263042ccc04e55e028abe169fa85ac455c0 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 21 Jun 2017 10:30:15 +0800 Subject: [PATCH 10/23] modify app --- k8s/pfs_app.yaml | 1 + k8s/pfsserver_service.yaml | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 k8s/pfsserver_service.yaml diff --git a/k8s/pfs_app.yaml b/k8s/pfs_app.yaml index 888ac728..8e69b03f 100644 --- a/k8s/pfs_app.yaml +++ b/k8s/pfs_app.yaml @@ -16,3 +16,4 @@ spec: image: gongweibao/pfsserver:latest ports: - containerPort: 8080 + command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] diff --git a/k8s/pfsserver_service.yaml b/k8s/pfsserver_service.yaml deleted file mode 100644 index ebaf5519..00000000 --- a/k8s/pfsserver_service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: pfsserver-service -spec: - selector: - app: pfsserver - ports: - - protocol: TCP - port: 8000 - targetPort: 8080 From 64fbb8446f6f939546cb00fd02d27f7b2e81cbe3 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 21 Jun 2017 11:09:49 +0800 Subject: [PATCH 11/23] modify pfs_app.yaml --- k8s/cloud_pfsserver.yaml | 17 ----------------- k8s/pfs_app.yaml | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 k8s/cloud_pfsserver.yaml diff --git a/k8s/cloud_pfsserver.yaml b/k8s/cloud_pfsserver.yaml deleted file mode 100644 index 28b3c2cc..00000000 --- a/k8s/cloud_pfsserver.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: pfsserver-deployment -spec: - replicas: 1 - template: - metadata: - labels: - app: pfsserver - spec: - containers: - - name: pfsserver - image: gongweibao/pfsserver:latest - ports: - - containerPort: 8080 - command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] diff --git a/k8s/pfs_app.yaml b/k8s/pfs_app.yaml index 8e69b03f..5f49281d 100644 --- a/k8s/pfs_app.yaml +++ b/k8s/pfs_app.yaml @@ -16,4 +16,4 @@ spec: image: gongweibao/pfsserver:latest ports: - containerPort: 8080 - command: ["/pfsserver/pfsserver", "-tokenuri", "http://10.100.0.137", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] + command: ["/pfsserver/pfsserver", "-tokenuri", "http://paddle-cloud-service", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] From d896844068f475958cf9a0fff286103b49a4d473 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 21 Jun 2017 11:33:22 +0800 Subject: [PATCH 12/23] modify travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9b85a95a..f7d6aa65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ matrix: include: - language: go go: 1.8.x - script: cd go && bash .tool/gen_config.sh && go test ./... + script: cd go && bash .tools/gen_config.sh && go test ./... - language: python python: 2.7 sudo: required From 79862db0b063f2517b08cdfed01f4827bbb9c2a7 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Thu, 22 Jun 2017 11:17:43 +0800 Subject: [PATCH 13/23] add cephfs support --- k8s/pfs_app.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/k8s/pfs_app.yaml b/k8s/pfs_app.yaml index 5f49281d..86f15d4a 100644 --- a/k8s/pfs_app.yaml +++ b/k8s/pfs_app.yaml @@ -10,10 +10,23 @@ spec: labels: app: pfsserver spec: + volumes: + - name: data-storage + cephfs: + name: cephfs + monitors: ["172.19.32.166:6789"] + path: "/" + user: "admin" + secretRef: + name: ceph-secret containers: - name: pfs imagePullPolicy: Always image: gongweibao/pfsserver:latest + volumeMounts: + - name: data-storage + mountPath: /pfs/datacenter1/home/ ports: - containerPort: 8080 command: ["/pfsserver/pfsserver", "-tokenuri", "http://paddle-cloud-service", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] + From 473f7abfb434d87fd28b8e94946cb64936891f41 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 19 Jul 2017 14:31:27 +0800 Subject: [PATCH 14/23] mofidy --- k8s/cloud_deployment.yaml | 8 ++++---- k8s/{pfs_app.yaml => pfs_deployment.yaml} | 14 ++++++++++---- k8s/pfs_ingress.yaml | 1 - k8s/{pfs_svc.yaml => pfs_service.yaml} | 1 - 4 files changed, 14 insertions(+), 10 deletions(-) rename k8s/{pfs_app.yaml => pfs_deployment.yaml} (59%) rename k8s/{pfs_svc.yaml => pfs_service.yaml} (91%) diff --git a/k8s/cloud_deployment.yaml b/k8s/cloud_deployment.yaml index b87ec725..b8645311 100644 --- a/k8s/cloud_deployment.yaml +++ b/k8s/cloud_deployment.yaml @@ -12,13 +12,13 @@ spec: volumes: - name: cert-volume hostPath: - path: /home/pcloud/data/certs + path: /home/pcloud_gongwb/data/certs - name: mysql-data hostPath: - path: /home/pcloud/data/mysql + path: /home/pcloud_gongwb/data/mysql - name: settings hostPath: - path: /home/pcloud/data/settings.py + path: /home/pcloud_gongwb/data/settings.py #- name: email-tmp-msg: # hostPath: # path: /home/pcloud/templates/email_confirmation_message.txt @@ -39,7 +39,7 @@ spec: containers: - name: paddle-cloud imagePullPolicy: Always - image: paddlepaddle/cloud + image: gongweibao/pcloud volumeMounts: - name: cert-volume mountPath: /certs diff --git a/k8s/pfs_app.yaml b/k8s/pfs_deployment.yaml similarity index 59% rename from k8s/pfs_app.yaml rename to k8s/pfs_deployment.yaml index 86f15d4a..da6b82a9 100644 --- a/k8s/pfs_app.yaml +++ b/k8s/pfs_deployment.yaml @@ -2,7 +2,6 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: pfsserver - namespace: pfs spec: replicas: 1 template: @@ -11,10 +10,13 @@ spec: app: pfsserver spec: volumes: + - name: cert-volume + hostPath: + path: /home/pcloud_gongwb/data/certs - name: data-storage cephfs: - name: cephfs - monitors: ["172.19.32.166:6789"] + monitors: + - 172.19.32.166:6789 path: "/" user: "admin" secretRef: @@ -24,9 +26,13 @@ spec: imagePullPolicy: Always image: gongweibao/pfsserver:latest volumeMounts: + - name: cert-volume + mountPath: /certs - name: data-storage mountPath: /pfs/datacenter1/home/ ports: - containerPort: 8080 - command: ["/pfsserver/pfsserver", "-tokenuri", "http://paddle-cloud-service", "-alsologtostderr=true", "-log_dir=./log", "-v=3"] + command: ["/pfsserver/pfsserver", "-tokenuri", "http://cloud1.paddlepaddle.org", "-logtostderr=true", "-log_dir=./log", "-v=3"] + nodeSelector: + kubernetes.io/hostname: k8s-node1 diff --git a/k8s/pfs_ingress.yaml b/k8s/pfs_ingress.yaml index 0ae96588..500fc9a2 100644 --- a/k8s/pfs_ingress.yaml +++ b/k8s/pfs_ingress.yaml @@ -2,7 +2,6 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: pfs-ingress - namespace: pfs spec: tls: - hosts: diff --git a/k8s/pfs_svc.yaml b/k8s/pfs_service.yaml similarity index 91% rename from k8s/pfs_svc.yaml rename to k8s/pfs_service.yaml index f9e35103..4e23103a 100644 --- a/k8s/pfs_svc.yaml +++ b/k8s/pfs_service.yaml @@ -2,7 +2,6 @@ kind: Service apiVersion: v1 metadata: name: pfs-service - namespace: pfs spec: selector: app: pfsserver From 2f6e9aa19e605f3301323dcf7d5867ad692ec8a5 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 21 Jul 2017 16:59:45 +0800 Subject: [PATCH 15/23] modify --- k8s/cloud_ingress.yaml | 2 +- k8s/pfs_deployment.yaml | 19 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index dd1fcd67..fbdf68ab 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -4,7 +4,7 @@ metadata: name: paddle-cloud-ingress spec: rules: - - host: cloud.paddlepaddle.org + - host: cloud1.paddlepaddle.org http: paths: - path: / diff --git a/k8s/pfs_deployment.yaml b/k8s/pfs_deployment.yaml index da6b82a9..c8186d13 100644 --- a/k8s/pfs_deployment.yaml +++ b/k8s/pfs_deployment.yaml @@ -10,29 +10,22 @@ spec: app: pfsserver spec: volumes: - - name: cert-volume - hostPath: - path: /home/pcloud_gongwb/data/certs - name: data-storage - cephfs: - monitors: - - 172.19.32.166:6789 - path: "/" - user: "admin" - secretRef: - name: ceph-secret + hostPath: + path: /home/gongwb/ceph containers: - name: pfs imagePullPolicy: Always image: gongweibao/pfsserver:latest + env: + - name: CURRENT_DATACENTER + value: "meiyan" + command: ["/pfsserver/pfsserver", "-tokenuri", "http://paddle-cloud-service:8000", "-logtostderr=true", "-log_dir=./log", "-v=4"] volumeMounts: - - name: cert-volume - mountPath: /certs - name: data-storage mountPath: /pfs/datacenter1/home/ ports: - containerPort: 8080 - command: ["/pfsserver/pfsserver", "-tokenuri", "http://cloud1.paddlepaddle.org", "-logtostderr=true", "-log_dir=./log", "-v=3"] nodeSelector: kubernetes.io/hostname: k8s-node1 From a7db6e7d41490018feeac44268ee1b3dadc7dd0a Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 21 Jul 2017 09:03:32 +0000 Subject: [PATCH 16/23] modify --- go/README_CN.md | 5 +++-- go/docker/build.sh | 2 ++ go/filemanager/pfsserver/handler.go | 8 ++++++++ paddlecloud/paddlecloud/urls.py | 1 + paddlecloud/paddlejob/views.py | 11 +++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/go/README_CN.md b/go/README_CN.md index 4021f42f..92d5d01a 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -2,13 +2,14 @@ - 构建PFSServer的编译环境 ``` - cd docker + cd cloud/go/docker bash build.sh ``` - 编译PFSServer ``` + cd cloud/go docker run --rm -v $(pwd):/root/gopath/src/github.com/PaddlePaddle/cloud/go pfsserver:dev ``` @@ -20,7 +21,7 @@ - PFSServer启动命令 ``` - docker run pfsserver:latest /pfsserver/pfsserver -tokenuri http://cloud.paddlepaddle.org -logtostderr=false -log_dir=./log -v=3 + docker run pfsserver:latest /pfsserver/pfsserver -tokenuri http://cloud.paddlepaddle.org -logtostderr=true -v=3 ``` 2. 如何部署PFSServer diff --git a/go/docker/build.sh b/go/docker/build.sh index 37d5e264..c0b8525f 100755 --- a/go/docker/build.sh +++ b/go/docker/build.sh @@ -15,3 +15,5 @@ CMD ["sh", "-c", "cd /root/gopath/src/github.com/PaddlePaddle/cloud/go/cmd/pfsse EOF docker build . -t pfsserver:dev + +rm -f Dockerfile diff --git a/go/filemanager/pfsserver/handler.go b/go/filemanager/pfsserver/handler.go index 98adf240..34dec01f 100644 --- a/go/filemanager/pfsserver/handler.go +++ b/go/filemanager/pfsserver/handler.go @@ -3,6 +3,7 @@ package pfsserver import ( "encoding/json" "errors" + "fmt" "io" "io/ioutil" "mime/multipart" @@ -25,24 +26,31 @@ var TokenURI = "" func getUserName(uri string, token string) (string, error) { authHeader := make(map[string]string) authHeader["Authorization"] = "Token " + token + + str := fmt.Sprintf("get uri with token error uri:%s token:%s\n", uri, token) + req, err := restclient.MakeRequest(uri, "GET", nil, "", nil, authHeader) if err != nil { + log.Errorln(str) return "", err } body, err := restclient.GetResponse(req) if err != nil { + log.Errorln(str) return "", err } log.V(4).Infoln("get token2user resp:" + string(body[:])) var resp interface{} if err := json.Unmarshal(body, &resp); err != nil { + log.Errorln(string(body[:])) return "", err } user := resp.(map[string]interface{})["user"].(string) if len(user) < 1 { + log.Errorln(resp) return "", errors.New("can't get username") } diff --git a/paddlecloud/paddlecloud/urls.py b/paddlecloud/paddlecloud/urls.py index 021e81c4..202ef716 100644 --- a/paddlecloud/paddlecloud/urls.py +++ b/paddlecloud/paddlecloud/urls.py @@ -33,6 +33,7 @@ url(r"^api/v1/workers/", paddlejob.views.WorkersView.as_view()), url(r"^api/v1/quota/", paddlejob.views.QuotaView.as_view()), url(r"^api/v1/file/", paddlejob.views.SimpleFileView.as_view()), + url(r"^api/v1/token2user/", paddlejob.views.GetUserView.as_view()), url(r"^api/v1/filelist/", paddlejob.views.SimpleFileList.as_view()), url(r"^api/v1/registry/", paddlejob.registry.RegistryView.as_view()), ] diff --git a/paddlecloud/paddlejob/views.py b/paddlecloud/paddlejob/views.py index 70413757..737260a3 100644 --- a/paddlecloud/paddlejob/views.py +++ b/paddlecloud/paddlejob/views.py @@ -315,6 +315,17 @@ def get(self, request, format=None): .list_namespaced_resource_quota(namespace) return Response(quota_list.to_dict()) +class GetUserView(APIView): + permission_classes = (permissions.IsAuthenticated,) + + def get(self, request, format=None): + """ + Get user name + """ + content = { + 'user': request.user.username, # `django.contrib.auth.User` instance. + } + return Response(content) class SimpleFileView(APIView): permission_classes = (permissions.IsAuthenticated,) From b88dfe520b1e8d2f34e0cc208f164769a3c5f316 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 21 Jul 2017 18:05:20 +0800 Subject: [PATCH 17/23] local test ok --- go/filemanager/pfsmodules/command.go | 14 +++++++++----- go/filemanager/pfsmodules/download.go | 4 ++-- go/filemanager/pfsmodules/ls.go | 2 +- go/filemanager/pfsmodules/mkdir.go | 2 +- go/filemanager/pfsmodules/rm.go | 2 +- go/filemanager/pfsmodules/upload.go | 6 +++--- go/filemanager/pfsserver/handler.go | 2 +- go/utils/config/config.go | 13 +++++++------ 8 files changed, 25 insertions(+), 20 deletions(-) diff --git a/go/filemanager/pfsmodules/command.go b/go/filemanager/pfsmodules/command.go index 1255f540..ba49bca0 100644 --- a/go/filemanager/pfsmodules/command.go +++ b/go/filemanager/pfsmodules/command.go @@ -4,6 +4,7 @@ import ( "errors" "io" "net/url" + "path" "strings" log "github.com/golang/glog" @@ -33,13 +34,16 @@ type Command interface { // CheckUser checks if a user has authority to access a path. // path example:/pfs/$datacenter/home/$user -func checkUser(path string, user string) error { - a := strings.Split(path, "/") - if len(a) < 3 { +func checkUser(pathStr string, user string) error { + pathStr = path.Clean(strings.TrimSpace(pathStr)) + a := strings.Split(pathStr, "/") + // the first / is convert to " " + if len(a) < 5 { return errors.New(StatusBadPath) } - if a[3] != user { + if a[4] != user { + log.V(4).Infof("request path:%s user:%s split_path:%s\n", pathStr, user, a[4]) return errors.New(StatusUnAuthorized) } return nil @@ -57,7 +61,7 @@ func ValidatePfsPath(paths []string, userName string) error { } if err := checkUser(path, userName); err != nil { - return errors.New(StatusShouldBePfsPath + ":" + path) + return err } } return nil diff --git a/go/filemanager/pfsmodules/download.go b/go/filemanager/pfsmodules/download.go index 230cce81..0eb88eb0 100644 --- a/go/filemanager/pfsmodules/download.go +++ b/go/filemanager/pfsmodules/download.go @@ -21,7 +21,7 @@ func remoteChunkMeta(path string, ChunkSize: chunkSize, } - t := fmt.Sprintf("%s/api/v1/chunks", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/chunks", Config.ActiveConfig.PfsEndpoint) ret, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { return nil, err @@ -89,7 +89,7 @@ func downloadChunks(src string, return nil } - t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.PfsEndpoint) for _, meta := range diffMeta { chunk := Chunk{ Path: src, diff --git a/go/filemanager/pfsmodules/ls.go b/go/filemanager/pfsmodules/ls.go index 45b8d6b8..d250d565 100644 --- a/go/filemanager/pfsmodules/ls.go +++ b/go/filemanager/pfsmodules/ls.go @@ -249,7 +249,7 @@ func formatPrint(result []LsResult) { // RemoteLs gets LsCmd result from cloud. func RemoteLs(cmd *LsCmd) ([]LsResult, error) { - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) body, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { return nil, err diff --git a/go/filemanager/pfsmodules/mkdir.go b/go/filemanager/pfsmodules/mkdir.go index 1c89671a..1a27ec9c 100644 --- a/go/filemanager/pfsmodules/mkdir.go +++ b/go/filemanager/pfsmodules/mkdir.go @@ -126,7 +126,7 @@ func RemoteMkdir(cmd *MkdirCmd) ([]MkdirResult, error) { return nil, err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) log.V(2).Infoln(t) body, err := restclient.PostCall(t, j) if err != nil { diff --git a/go/filemanager/pfsmodules/rm.go b/go/filemanager/pfsmodules/rm.go index b2f6a761..42d371a2 100644 --- a/go/filemanager/pfsmodules/rm.go +++ b/go/filemanager/pfsmodules/rm.go @@ -162,7 +162,7 @@ func RemoteRm(cmd *RmCmd) ([]RmResult, error) { return nil, err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) body, err := restclient.DeleteCall(t, j) if err != nil { return nil, err diff --git a/go/filemanager/pfsmodules/upload.go b/go/filemanager/pfsmodules/upload.go index ef6d698a..8d1310fb 100644 --- a/go/filemanager/pfsmodules/upload.go +++ b/go/filemanager/pfsmodules/upload.go @@ -17,7 +17,7 @@ import ( var Config = config.ParseDefaultConfig() func remoteStat(cmd *StatCmd) (*LsResult, error) { - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) log.V(3).Infoln(t) body, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { @@ -49,7 +49,7 @@ func remoteTouch(cmd *TouchCmd) error { return err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) body, err := restclient.PostCall(t, j) if err != nil { return err @@ -108,7 +108,7 @@ func postChunk(src *Chunk, dst string) ([]byte, error) { } defer Close(f) - t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.Endpoint) + t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.PfsEndpoint) log.V(4).Infoln(t) return restclient.PostChunk(t, getDstParam(src, dst), diff --git a/go/filemanager/pfsserver/handler.go b/go/filemanager/pfsserver/handler.go index 34dec01f..f3e24b3e 100644 --- a/go/filemanager/pfsserver/handler.go +++ b/go/filemanager/pfsserver/handler.go @@ -25,7 +25,7 @@ var TokenURI = "" func getUserName(uri string, token string) (string, error) { authHeader := make(map[string]string) - authHeader["Authorization"] = "Token " + token + authHeader["Authorization"] = token str := fmt.Sprintf("get uri with token error uri:%s token:%s\n", uri, token) diff --git a/go/utils/config/config.go b/go/utils/config/config.go index 68c0a017..229435ce 100644 --- a/go/utils/config/config.go +++ b/go/utils/config/config.go @@ -12,12 +12,13 @@ import ( // SubmitConfigDataCenter is inner conf for paddlecloud type SubmitConfigDataCenter struct { - Name string `yaml:"name"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Usercert string `yaml:"usercert"` - Userkey string `yaml:"userkey"` - Endpoint string `yaml:"endpoint"` + Name string `yaml:"name"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Usercert string `yaml:"usercert"` + Userkey string `yaml:"userkey"` + Endpoint string `yaml:"endpoint"` + PfsEndpoint string `yaml:"pfsendpoint"` } // SubmitConfig is configuration load from user config yaml files From bef6072c35078f1dbce09079cca1fe1261e96d93 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Fri, 21 Jul 2017 18:14:49 +0800 Subject: [PATCH 18/23] rm not need --- go/.tools/gen_config.sh | 1 + k8s/cloud_deployment.yaml | 6 +++--- k8s/cloud_ingress.yaml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go/.tools/gen_config.sh b/go/.tools/gen_config.sh index af2479e2..581692d6 100644 --- a/go/.tools/gen_config.sh +++ b/go/.tools/gen_config.sh @@ -6,5 +6,6 @@ datacenters: username: user@baidu.com password: T123 endpoint: http://127.0.0.1:8080 + pfsendpoint: http://127.0.0.1:8080 current-datacenter: datacenter1 EOF diff --git a/k8s/cloud_deployment.yaml b/k8s/cloud_deployment.yaml index b8645311..33a1bd84 100644 --- a/k8s/cloud_deployment.yaml +++ b/k8s/cloud_deployment.yaml @@ -12,13 +12,13 @@ spec: volumes: - name: cert-volume hostPath: - path: /home/pcloud_gongwb/data/certs + path: /home/pcloud/data/certs - name: mysql-data hostPath: - path: /home/pcloud_gongwb/data/mysql + path: /home/pcloud/data/mysql - name: settings hostPath: - path: /home/pcloud_gongwb/data/settings.py + path: /home/pcloud/data/settings.py #- name: email-tmp-msg: # hostPath: # path: /home/pcloud/templates/email_confirmation_message.txt diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index fbdf68ab..dd1fcd67 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -4,7 +4,7 @@ metadata: name: paddle-cloud-ingress spec: rules: - - host: cloud1.paddlepaddle.org + - host: cloud.paddlepaddle.org http: paths: - path: / From 2633ad47ec00e0d103840ff0f93a27c83d611458 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 25 Jul 2017 10:35:11 +0800 Subject: [PATCH 19/23] fix --- go/.tools/gen_config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/.tools/gen_config.sh b/go/.tools/gen_config.sh index 581692d6..af2429e8 100644 --- a/go/.tools/gen_config.sh +++ b/go/.tools/gen_config.sh @@ -3,8 +3,8 @@ mkdir -p ~/.paddle cat > ~/.paddle/config << EOF datacenters: - name: datacenter1 - username: user@baidu.com - password: T123 + username: your-user-name + password: your-secret endpoint: http://127.0.0.1:8080 pfsendpoint: http://127.0.0.1:8080 current-datacenter: datacenter1 From e121c537bd6d0bf4216c14cfacbcdfa93a697eeb Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 25 Jul 2017 17:57:20 +0800 Subject: [PATCH 20/23] fix comments --- {go/docker => docker/pfs}/build.sh | 0 go/.tools/gen_config.sh | 1 - go/README_CN.md | 3 ++- go/filemanager/pfsmodules/download.go | 4 ++-- go/filemanager/pfsmodules/ls.go | 2 +- go/filemanager/pfsmodules/mkdir.go | 2 +- go/filemanager/pfsmodules/rm.go | 2 +- go/filemanager/pfsmodules/upload.go | 6 +++--- go/filemanager/pfsserver/routes.go | 12 ++++++------ go/utils/config/config.go | 13 ++++++------- k8s/cloud_ingress.yaml | 6 +++++- 11 files changed, 27 insertions(+), 24 deletions(-) rename {go/docker => docker/pfs}/build.sh (100%) diff --git a/go/docker/build.sh b/docker/pfs/build.sh similarity index 100% rename from go/docker/build.sh rename to docker/pfs/build.sh diff --git a/go/.tools/gen_config.sh b/go/.tools/gen_config.sh index af2429e8..f42d951b 100644 --- a/go/.tools/gen_config.sh +++ b/go/.tools/gen_config.sh @@ -6,6 +6,5 @@ datacenters: username: your-user-name password: your-secret endpoint: http://127.0.0.1:8080 - pfsendpoint: http://127.0.0.1:8080 current-datacenter: datacenter1 EOF diff --git a/go/README_CN.md b/go/README_CN.md index 92d5d01a..0dff235b 100644 --- a/go/README_CN.md +++ b/go/README_CN.md @@ -2,7 +2,7 @@ - 构建PFSServer的编译环境 ``` - cd cloud/go/docker + cd cloud/docker/pfs bash build.sh ``` @@ -16,6 +16,7 @@ - 构建PFSServer的DockerImage ``` + cd cloud/go docker build . -t pfsserver:latest ``` - PFSServer启动命令 diff --git a/go/filemanager/pfsmodules/download.go b/go/filemanager/pfsmodules/download.go index 0eb88eb0..24ad38b8 100644 --- a/go/filemanager/pfsmodules/download.go +++ b/go/filemanager/pfsmodules/download.go @@ -21,7 +21,7 @@ func remoteChunkMeta(path string, ChunkSize: chunkSize, } - t := fmt.Sprintf("%s/api/v1/chunks", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/chunks", Config.ActiveConfig.Endpoint) ret, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { return nil, err @@ -89,7 +89,7 @@ func downloadChunks(src string, return nil } - t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/storage/chunks", Config.ActiveConfig.Endpoint) for _, meta := range diffMeta { chunk := Chunk{ Path: src, diff --git a/go/filemanager/pfsmodules/ls.go b/go/filemanager/pfsmodules/ls.go index d250d565..7eafdf28 100644 --- a/go/filemanager/pfsmodules/ls.go +++ b/go/filemanager/pfsmodules/ls.go @@ -249,7 +249,7 @@ func formatPrint(result []LsResult) { // RemoteLs gets LsCmd result from cloud. func RemoteLs(cmd *LsCmd) ([]LsResult, error) { - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint) body, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { return nil, err diff --git a/go/filemanager/pfsmodules/mkdir.go b/go/filemanager/pfsmodules/mkdir.go index 1a27ec9c..5c8689a7 100644 --- a/go/filemanager/pfsmodules/mkdir.go +++ b/go/filemanager/pfsmodules/mkdir.go @@ -126,7 +126,7 @@ func RemoteMkdir(cmd *MkdirCmd) ([]MkdirResult, error) { return nil, err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint) log.V(2).Infoln(t) body, err := restclient.PostCall(t, j) if err != nil { diff --git a/go/filemanager/pfsmodules/rm.go b/go/filemanager/pfsmodules/rm.go index 42d371a2..b2393f6c 100644 --- a/go/filemanager/pfsmodules/rm.go +++ b/go/filemanager/pfsmodules/rm.go @@ -162,7 +162,7 @@ func RemoteRm(cmd *RmCmd) ([]RmResult, error) { return nil, err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint) body, err := restclient.DeleteCall(t, j) if err != nil { return nil, err diff --git a/go/filemanager/pfsmodules/upload.go b/go/filemanager/pfsmodules/upload.go index 8d1310fb..effb5497 100644 --- a/go/filemanager/pfsmodules/upload.go +++ b/go/filemanager/pfsmodules/upload.go @@ -17,7 +17,7 @@ import ( var Config = config.ParseDefaultConfig() func remoteStat(cmd *StatCmd) (*LsResult, error) { - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint) log.V(3).Infoln(t) body, err := restclient.GetCall(t, cmd.ToURLParam()) if err != nil { @@ -49,7 +49,7 @@ func remoteTouch(cmd *TouchCmd) error { return err } - t := fmt.Sprintf("%s/api/v1/files", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/files", Config.ActiveConfig.Endpoint) body, err := restclient.PostCall(t, j) if err != nil { return err @@ -108,7 +108,7 @@ func postChunk(src *Chunk, dst string) ([]byte, error) { } defer Close(f) - t := fmt.Sprintf("%s/api/v1/storage/chunks", Config.ActiveConfig.PfsEndpoint) + t := fmt.Sprintf("%s/api/v1/pfs/storage/chunks", Config.ActiveConfig.Endpoint) log.V(4).Infoln(t) return restclient.PostChunk(t, getDstParam(src, dst), diff --git a/go/filemanager/pfsserver/routes.go b/go/filemanager/pfsserver/routes.go index 62aa2f4e..544e6ef0 100644 --- a/go/filemanager/pfsserver/routes.go +++ b/go/filemanager/pfsserver/routes.go @@ -19,39 +19,39 @@ var routes = Routes{ Route{ "GetFiles", "GET", - "/api/v1/files", + "/api/v1/pfs/files", GetFilesHandler, }, Route{ "PostFiles", "POST", - "/api/v1/files", + "/api/v1/pfs/files", PostFilesHandler, }, Route{ "DeleteFiles", "DELETE", - "/api/v1/files", + "/api/v1/pfs/files", DeleteFilesHandler, }, Route{ "GetChunksMeta", "GET", - "/api/v1/chunks", + "/api/v1/pfs/chunks", GetChunkMetaHandler, }, Route{ "GetChunksData", "GET", - "/api/v1/storage/chunks", + "/api/v1/pfs/storage/chunks", GetChunkHandler, }, Route{ "PostChunksData", "POST", - "/api/v1/storage/chunks", + "/api/v1/pfs/storage/chunks", PostChunkHandler, }, } diff --git a/go/utils/config/config.go b/go/utils/config/config.go index 229435ce..68c0a017 100644 --- a/go/utils/config/config.go +++ b/go/utils/config/config.go @@ -12,13 +12,12 @@ import ( // SubmitConfigDataCenter is inner conf for paddlecloud type SubmitConfigDataCenter struct { - Name string `yaml:"name"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Usercert string `yaml:"usercert"` - Userkey string `yaml:"userkey"` - Endpoint string `yaml:"endpoint"` - PfsEndpoint string `yaml:"pfsendpoint"` + Name string `yaml:"name"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Usercert string `yaml:"usercert"` + Userkey string `yaml:"userkey"` + Endpoint string `yaml:"endpoint"` } // SubmitConfig is configuration load from user config yaml files diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index dd1fcd67..d675878c 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -4,9 +4,13 @@ metadata: name: paddle-cloud-ingress spec: rules: - - host: cloud.paddlepaddle.org + - host: cloud1.paddlepaddle.org http: paths: + - path: /api/v1/files + backend: + serviceName: pfs-service + servicePort: 8080 - path: / backend: serviceName: paddle-cloud-service From 1a9d15e60b8bdc087136280096a18c66c2cc9182 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 25 Jul 2017 18:01:56 +0800 Subject: [PATCH 21/23] rm not need --- .tools/gen_config.sh | 10 ---------- .travis.yml | 2 +- k8s/pfs_ingress.yaml | 18 ------------------ 3 files changed, 1 insertion(+), 29 deletions(-) delete mode 100755 .tools/gen_config.sh delete mode 100644 k8s/pfs_ingress.yaml diff --git a/.tools/gen_config.sh b/.tools/gen_config.sh deleted file mode 100755 index af2479e2..00000000 --- a/.tools/gen_config.sh +++ /dev/null @@ -1,10 +0,0 @@ -#/bin/bash -mkdir -p ~/.paddle -cat > ~/.paddle/config << EOF -datacenters: -- name: datacenter1 - username: user@baidu.com - password: T123 - endpoint: http://127.0.0.1:8080 -current-datacenter: datacenter1 -EOF diff --git a/.travis.yml b/.travis.yml index e5cd1b84..68b32b8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: - | bash .tools/check_style.sh RESULT=$?; if [ $RESULT -eq 0 ]; then true; else false; fi; - - bash .tools/gen_config.sh && cd go && glide install && go test $(glide novendor) + - cd go && bash .tools/gen_config.sh && glide install && go test $(glide novendor) - language: python python: 2.7 sudo: required diff --git a/k8s/pfs_ingress.yaml b/k8s/pfs_ingress.yaml deleted file mode 100644 index 500fc9a2..00000000 --- a/k8s/pfs_ingress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: pfs-ingress -spec: - tls: - - hosts: - - pfs.paddlpaddle.org - secretName: pfs-secret - rules: - - host: pfs.paddlepaddle.org - http: - paths: - - backend: - serviceName: pfs-service - servicePort: 8080 - path: / - From d4ce648a645968e8945833f0d92df09d99d4fbb8 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 25 Jul 2017 20:27:50 +0800 Subject: [PATCH 22/23] fix --- k8s/cloud_deployment.yaml | 2 +- k8s/cloud_ingress.yaml | 2 +- k8s/pfs_deployment.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/cloud_deployment.yaml b/k8s/cloud_deployment.yaml index 33a1bd84..8608c72e 100644 --- a/k8s/cloud_deployment.yaml +++ b/k8s/cloud_deployment.yaml @@ -39,7 +39,7 @@ spec: containers: - name: paddle-cloud imagePullPolicy: Always - image: gongweibao/pcloud + image: paddlepaddle/pcloud volumeMounts: - name: cert-volume mountPath: /certs diff --git a/k8s/cloud_ingress.yaml b/k8s/cloud_ingress.yaml index d675878c..20bbaa21 100644 --- a/k8s/cloud_ingress.yaml +++ b/k8s/cloud_ingress.yaml @@ -4,7 +4,7 @@ metadata: name: paddle-cloud-ingress spec: rules: - - host: cloud1.paddlepaddle.org + - host: cloud.paddlepaddle.org http: paths: - path: /api/v1/files diff --git a/k8s/pfs_deployment.yaml b/k8s/pfs_deployment.yaml index c8186d13..6b24b6c0 100644 --- a/k8s/pfs_deployment.yaml +++ b/k8s/pfs_deployment.yaml @@ -16,7 +16,7 @@ spec: containers: - name: pfs imagePullPolicy: Always - image: gongweibao/pfsserver:latest + image: paddlepaddle/pfsserver:latest env: - name: CURRENT_DATACENTER value: "meiyan" From 1ccc7484ec5009059437833309c7196bd37ed039 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 25 Jul 2017 20:30:31 +0800 Subject: [PATCH 23/23] fix --- k8s/cloud_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/cloud_deployment.yaml b/k8s/cloud_deployment.yaml index 8608c72e..b87ec725 100644 --- a/k8s/cloud_deployment.yaml +++ b/k8s/cloud_deployment.yaml @@ -39,7 +39,7 @@ spec: containers: - name: paddle-cloud imagePullPolicy: Always - image: paddlepaddle/pcloud + image: paddlepaddle/cloud volumeMounts: - name: cert-volume mountPath: /certs