Merge pull request #125 from opensds/development

Merge from Development to Master to prepare for m-3
sodafoundation · May 10, 2019 · c4ef374 · c4ef374
2 parents 146a9d8 + d17c4f5
commit c4ef374
Show file tree

Hide file tree

Showing 63 changed files with 1,888 additions and 817 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,25 @@
+<!-- This form is for bug reports and feature requests! -->
+
+**Is this a BUG REPORT or FEATURE REQUEST?**:
+
+> Uncomment only one, leave it on its own line: 
+>
+> /kind bug
+> /kind feature
+
+
+**What happened**:
+
+**What you expected to happen**:
+
+**How to reproduce it (as minimally and precisely as possible)**:
+
+
+**Anything else we need to know?**:
+
+**Environment**:
+- Gelato(release/branch) version:
+- OS (e.g. from /etc/os-release):
+- Kernel (e.g. `uname -a`):
+- Install tools:
+- Others:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,15 @@
+<!-- Thanks for sending a pull request! -->
+
+**What this PR does / why we need it**:
+
+**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #
+
+**Special notes for your reviewer**:
+
+**Release note**:
+<!--  Steps to write your release note:
+1. Use the release-note-* labels to set the release note state (if you have access)
+2. Enter your extended release note in the below block; leaving it blank means using the PR title as the release note. If no release note is required, just write `NONE`.
+-->
+```release-note
+```
diff --git a/Makefile b/Makefile
@@ -30,19 +30,19 @@ prebuild:
 	mkdir -p  $(BUILD_DIR)
 
 api: prebuild
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o $(BUILD_DIR)/api github.com/opensds/multi-cloud/api/cmd
+	CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s -extldflags "-static"' -o $(BUILD_DIR)/api github.com/opensds/multi-cloud/api/cmd
 
 backend: prebuild
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o $(BUILD_DIR)/backend github.com/opensds/multi-cloud/backend/cmd
+	CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s -extldflags "-static"' -o $(BUILD_DIR)/backend github.com/opensds/multi-cloud/backend/cmd
 
 s3: prebuild
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o $(BUILD_DIR)/s3 github.com/opensds/multi-cloud/s3/cmd
+	CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s -extldflags "-static"' -o $(BUILD_DIR)/s3 github.com/opensds/multi-cloud/s3/cmd
 
 dataflow: prebuild
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o $(BUILD_DIR)/dataflow github.com/opensds/multi-cloud/dataflow/cmd
+	CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s -extldflags "-static"' -o $(BUILD_DIR)/dataflow github.com/opensds/multi-cloud/dataflow/cmd
 
 datamover: prebuild
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o $(BUILD_DIR)/datamover github.com/opensds/multi-cloud/datamover/cmd
+	CGO_ENABLED=0 GOOS=linux go build -ldflags '-w -s -extldflags "-static"' -o $(BUILD_DIR)/datamover github.com/opensds/multi-cloud/datamover/cmd
 
 docker: build
 
@@ -66,8 +66,11 @@ docker: build
 	chmod 755 datamover/datamover
 	docker build datamover -t opensdsio/multi-cloud-datamover:latest
 
+goimports:
+	goimports -w $(shell go list -f {{.Dir}} ./... |grep -v /vendor/)
+
 clean:
-	rm -rf $(BUILD_DIR)
+	rm -rf $(BUILD_DIR) api/api backend/backend dataflow/dataflow datamover/datamover s3/s3
 
 version:
 	@echo ${VERSION}

diff --git a/api/cmd/main.go b/api/cmd/main.go
@@ -22,6 +22,7 @@ import (
 	"github.com/opensds/multi-cloud/api/pkg/dataflow"
 	"github.com/opensds/multi-cloud/api/pkg/filters/context"
 	"github.com/opensds/multi-cloud/api/pkg/filters/signature/signer"
+
 	//	_ "github.com/micro/go-plugins/client/grpc"
 	"github.com/opensds/multi-cloud/api/pkg/filters/auth"
 	"github.com/opensds/multi-cloud/api/pkg/filters/logging"
@@ -45,14 +46,14 @@ func main() {
 	ws.Doc("OpenSDS Multi-Cloud API")
 	ws.Consumes(restful.MIME_JSON)
 	ws.Produces(restful.MIME_JSON)
+	ws.Filter(auth.FilterFactory())
 
 	backend.RegisterRouter(ws)
 	//s3.RegisterRouter(ws)
 	dataflow.RegisterRouter(ws)
 	// add filter for authentication context
 	wc.Filter(logging.FilterFactory())
 	wc.Filter(context.FilterFactory())
-	wc.Filter(auth.FilterFactory())
 
 	s3ws := new(restful.WebService)
 	s3ws.Path("/v1/s3")

diff --git a/api/pkg/backend/router.go b/api/pkg/backend/router.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2018 Huawei Technologies Co., Ltd. All Rights Reserved.
+// Copyright 2019 The OpenSDS Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -32,5 +32,6 @@ func RegisterRouter(ws *restful.WebService) {
 	ws.Route(ws.DELETE("/{tenantId}/backends/{id}").To(handler.DeleteBackend)).
 		Doc("Delete backend")
 	ws.Route(ws.GET("/{tenantId}/types").To(handler.ListType)).
-		Doc("Get backend list")
+		Doc("Get backend types")
 }
+
diff --git a/api/pkg/backend/service.go b/api/pkg/backend/service.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2018 Huawei Technologies Co., Ltd. All Rights Reserved.
+// Copyright 2019 The OpenSDS Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,17 +15,20 @@
 package backend
 
 import (
+	"errors"
 	"net/http"
+	"strconv"
 
 	"github.com/emicklei/go-restful"
 	"github.com/micro/go-log"
 	"github.com/micro/go-micro/client"
 	"github.com/opensds/multi-cloud/api/pkg/common"
 	"github.com/opensds/multi-cloud/api/pkg/policy"
-	"github.com/opensds/multi-cloud/backend/proto"
-	"github.com/opensds/multi-cloud/dataflow/proto"
+	backend "github.com/opensds/multi-cloud/backend/proto"
+	dataflow "github.com/opensds/multi-cloud/dataflow/proto"
 	. "github.com/opensds/multi-cloud/s3/pkg/exception"
-	"github.com/opensds/multi-cloud/s3/proto"
+	pb "github.com/opensds/multi-cloud/s3/proto"
+	s3 "github.com/opensds/multi-cloud/s3/proto"
 	"golang.org/x/net/context"
 )
 
@@ -53,12 +56,12 @@ func (s *APIService) GetBackend(request *restful.Request, response *restful.Resp
 	if !policy.Authorize(request, response, "backend:get") {
 		return
 	}
-	log.Logf("Received request for backend details: %s", request.PathParameter("id"))
+	log.Logf("Received request for backend details: %s\n", request.PathParameter("id"))
 	id := request.PathParameter("id")
 	ctx := context.Background()
 	res, err := s.backendClient.GetBackend(ctx, &backend.GetBackendRequest{Id: id})
 	if err != nil {
-		log.Logf("Failed to get backend details: %v", err)
+		log.Logf("failed to get backend details: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -67,16 +70,12 @@ func (s *APIService) GetBackend(request *restful.Request, response *restful.Resp
 	response.WriteEntity(res.Backend)
 }
 
-func (s *APIService) ListBackend(request *restful.Request, response *restful.Response) {
-	if !policy.Authorize(request, response, "backend:list") {
-		return
-	}
-	log.Log("Received request for backend list.")
+func (s *APIService) listBackendDefault(request *restful.Request, response *restful.Response) {
 	listBackendRequest := &backend.ListBackendRequest{}
 
 	limit, offset, err := common.GetPaginationParam(request)
 	if err != nil {
-		log.Logf("Get pagination parameters failed: %v", err)
+		log.Logf("get pagination parameters failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -85,7 +84,7 @@ func (s *APIService) ListBackend(request *restful.Request, response *restful.Res
 
 	sortKeys, sortDirs, err := common.GetSortParam(request)
 	if err != nil {
-		log.Logf("Get sort parameters failed: %v", err)
+		log.Logf("get sort parameters failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -95,7 +94,7 @@ func (s *APIService) ListBackend(request *restful.Request, response *restful.Res
 	filterOpts := []string{"name", "type", "region"}
 	filter, err := common.GetFilter(request, filterOpts)
 	if err != nil {
-		log.Logf("Get filter failed: %v", err)
+		log.Logf("get filter failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -104,7 +103,7 @@ func (s *APIService) ListBackend(request *restful.Request, response *restful.Res
 	ctx := context.Background()
 	res, err := s.backendClient.ListBackend(ctx, listBackendRequest)
 	if err != nil {
-		log.Logf("Failed to list backends: %v", err)
+		log.Logf("failed to list backends: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -113,6 +112,52 @@ func (s *APIService) ListBackend(request *restful.Request, response *restful.Res
 	response.WriteEntity(res)
 }
 
+func (s *APIService) FilterBackendByTier(request *restful.Request, response *restful.Response, tier int32) {
+	// Get those backend type which supporte the specific tier.
+	req := pb.GetBackendTypeByTierRequest{Tier: tier}
+	res, _ := s.s3Client.GetBackendTypeByTier(context.Background(), &req)
+	req1 := &backend.ListBackendRequest{}
+	resp := &backend.ListBackendResponse{}
+	for _, v := range res.Types {
+		// Get backends with specific backend type.
+		filter := make(map[string]string)
+		filter["type"] = v
+		req1.Filter = filter
+		res1, err := s.backendClient.ListBackend(context.Background(), req1)
+		if err != nil {
+			log.Logf("failed to list backends of type[%s]: %v\n", v, err)
+			response.WriteError(http.StatusInternalServerError, err)
+		}
+		if len(res1.Backends) != 0 {
+			resp.Backends = append(resp.Backends, res1.Backends...)
+		}
+	}
+	//TODO: Need to consider pagination
+
+	log.Log("fiterBackendByTier backends successfully.")
+	response.WriteEntity(resp)
+}
+
+func (s *APIService) ListBackend(request *restful.Request, response *restful.Response) {
+	if !policy.Authorize(request, response, "backend:list") {
+		return
+	}
+	log.Log("Received request for backend list.")
+
+	para := request.QueryParameter("tier")
+	if para != "" { //List those backends which support the specific tier.
+		tier, err := strconv.Atoi(para)
+		if err != nil {
+			log.Logf("list backends with tier as filter, but tier[%s] is invalid\n", tier)
+			response.WriteError(http.StatusBadRequest, errors.New("invalid tier"))
+			return
+		}
+		s.FilterBackendByTier(request, response, int32(tier))
+	} else {
+		s.listBackendDefault(request, response)
+	}
+}
+
 func (s *APIService) CreateBackend(request *restful.Request, response *restful.Response) {
 	if !policy.Authorize(request, response, "backend:create") {
 		return
@@ -121,15 +166,15 @@ func (s *APIService) CreateBackend(request *restful.Request, response *restful.R
 	backendDetail := &backend.BackendDetail{}
 	err := request.ReadEntity(&backendDetail)
 	if err != nil {
-		log.Logf("Failed to read request body: %v", err)
+		log.Logf("failed to read request body: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
 
 	ctx := context.Background()
 	res, err := s.backendClient.CreateBackend(ctx, &backend.CreateBackendRequest{Backend: backendDetail})
 	if err != nil {
-		log.Logf("Failed to create backend: %v", err)
+		log.Logf("failed to create backend: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -142,19 +187,19 @@ func (s *APIService) UpdateBackend(request *restful.Request, response *restful.R
 	if !policy.Authorize(request, response, "backend:update") {
 		return
 	}
-	log.Logf("Received request for updating backend: %v", request.PathParameter("id"))
+	log.Logf("Received request for updating backend: %v\n", request.PathParameter("id"))
 	updateBackendRequest := backend.UpdateBackendRequest{Id: request.PathParameter("id")}
 	err := request.ReadEntity(&updateBackendRequest)
 	if err != nil {
-		log.Logf("Failed to read request body: %v", err)
+		log.Logf("failed to read request body: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
 
 	ctx := context.Background()
 	res, err := s.backendClient.UpdateBackend(ctx, &updateBackendRequest)
 	if err != nil {
-		log.Logf("Failed to update backend: %v", err)
+		log.Logf("failed to update backend: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -168,15 +213,15 @@ func (s *APIService) DeleteBackend(request *restful.Request, response *restful.R
 		return
 	}
 	id := request.PathParameter("id")
-	log.Logf("Received request for deleting backend: %s", id)
+	log.Logf("Received request for deleting backend: %s\n", id)
 	ctx := context.Background()
 	owner := "test"
 	res, err := s.s3Client.ListBuckets(ctx, &s3.BaseRequest{Id: owner})
 	count := 0
 	for _, v := range res.Buckets {
 		res, err := s.backendClient.GetBackend(ctx, &backend.GetBackendRequest{Id: id})
 		if err != nil {
-			log.Logf("Failed to get backend details: %v", err)
+			log.Logf("failed to get backend details: %v\n", err)
 			response.WriteError(http.StatusInternalServerError, err)
 			return
 		}
@@ -192,15 +237,15 @@ func (s *APIService) DeleteBackend(request *restful.Request, response *restful.R
 	if count == 0 {
 		_, err := s.backendClient.DeleteBackend(ctx, &backend.DeleteBackendRequest{Id: id})
 		if err != nil {
-			log.Logf("Failed to delete backend: %v", err)
+			log.Logf("failed to delete backend: %v\n", err)
 			response.WriteError(http.StatusInternalServerError, err)
 			return
 		}
 		log.Log("Delete backend successfully.")
 		response.WriteHeader(http.StatusOK)
 		return
 	} else {
-		log.Log("The backend can not be deleted. please delete bucket first.")
+		log.Log("the backend can not be deleted, need to delete bucket first.")
 		response.WriteError(http.StatusInternalServerError, BackendDeleteError.Error())
 		return
 	}
@@ -215,7 +260,7 @@ func (s *APIService) ListType(request *restful.Request, response *restful.Respon
 
 	limit, offset, err := common.GetPaginationParam(request)
 	if err != nil {
-		log.Logf("Get pagination parameters failed: %v", err)
+		log.Logf("get pagination parameters failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -224,7 +269,7 @@ func (s *APIService) ListType(request *restful.Request, response *restful.Respon
 
 	sortKeys, sortDirs, err := common.GetSortParam(request)
 	if err != nil {
-		log.Logf("Get sort parameters failed: %v", err)
+		log.Logf("get sort parameters failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -234,7 +279,7 @@ func (s *APIService) ListType(request *restful.Request, response *restful.Respon
 	filterOpts := []string{"name"}
 	filter, err := common.GetFilter(request, filterOpts)
 	if err != nil {
-		log.Logf("Get filter failed: %v", err)
+		log.Logf("get filter failed: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}
@@ -243,7 +288,7 @@ func (s *APIService) ListType(request *restful.Request, response *restful.Respon
 	ctx := context.Background()
 	res, err := s.backendClient.ListType(ctx, listTypeRequest)
 	if err != nil {
-		log.Logf("Failed to list types: %v", err)
+		log.Logf("failed to list types: %v\n", err)
 		response.WriteError(http.StatusInternalServerError, err)
 		return
 	}