Skip to content

Commit

Permalink
test: remove gorilla/mux from test apps (#678)
Browse files Browse the repository at this point in the history
The test apps for CSB mostly rely on gorilla/mux which is now an
archived project. This is a demonstration of a test app that does not
use gorilla/mux any more.

This removes gorilla/mux from the apps that were easy to change.
The PostgreSQL app is a little more complicated and will be done later.

[#184006000](https://www.pivotaltracker.com/story/show/184006000)

Co-authored-by: Andrea Zucchini <zandrea@vmware.com>
  • Loading branch information
blgm and zucchinidev authored Feb 22, 2023
1 parent 2769edf commit 17be0a0
Show file tree
Hide file tree
Showing 30 changed files with 376 additions and 457 deletions.
25 changes: 17 additions & 8 deletions acceptance-tests/apps/dataprocapp/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ import (
"fmt"
"log"
"net/http"
"strings"

dataproc "cloud.google.com/go/dataproc/apiv1"
"github.com/gorilla/mux"
"google.golang.org/api/option"
)

func App(creds credentials.DataprocCredentials) *mux.Router {
func App(creds credentials.DataprocCredentials) http.HandlerFunc {
endpoint := fmt.Sprintf("%s-dataproc.googleapis.com:443", creds.Region)
client, err := dataproc.NewJobControllerClient(
context.Background(),
Expand All @@ -21,13 +21,22 @@ func App(creds credentials.DataprocCredentials) *mux.Router {
if err != nil {
log.Fatalf("error creating the cluster client: %s\n", err)
}
r := mux.NewRouter()

r.HandleFunc("/", aliveness).Methods("HEAD", "GET")
r.HandleFunc("/{job}", handleRunJob(*client, creds)).Methods("PUT")
r.HandleFunc("/{job}", handleGetJob(*client, creds)).Methods("GET")
r.HandleFunc("/{job}", handleDeleteJob(*client, creds)).Methods("DELETE")
return r
return func(w http.ResponseWriter, r *http.Request) {
job := strings.Trim(r.URL.Path, "/")
switch r.Method {
case http.MethodHead:
aliveness(w, r)
case http.MethodGet:
handleGetJob(w, job, client, creds)
case http.MethodPut:
handleRunJob(w, job, client, creds)
case http.MethodDelete:
handleDeleteJob(w, job, client, creds)
default:
fail(w, http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed))
}
}
}

func aliveness(w http.ResponseWriter, r *http.Request) {
Expand Down
42 changes: 16 additions & 26 deletions acceptance-tests/apps/dataprocapp/app/delete_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,30 @@ package app

import (
"context"
"dataprocapp/credentials"
"log"
"net/http"

dataproc "cloud.google.com/go/dataproc/apiv1"
"cloud.google.com/go/dataproc/apiv1/dataprocpb"
"github.com/gorilla/mux"

"dataprocapp/credentials"
)

func handleDeleteJob(jobClient dataproc.JobControllerClient, creds credentials.DataprocCredentials) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Printf("Handling deletion of job")

jobName, ok := mux.Vars(r)["job"]
if !ok {
fail(w, http.StatusBadRequest, "job missing.")
return
}
func handleDeleteJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) {
log.Printf("Handling deletion of job")

jobReq := &dataprocpb.DeleteJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
JobId: jobName,
}

ctx := context.Background()
err := jobClient.DeleteJob(ctx, jobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to delete job: %v", err)
return
}
jobReq := &dataprocpb.DeleteJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
JobId: jobName,
}

w.WriteHeader(http.StatusGone)
log.Printf("Job deleted")
ctx := context.Background()
err := jobClient.DeleteJob(ctx, jobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to delete job: %v", err)
return
}

w.WriteHeader(http.StatusGone)
log.Printf("Job deleted")
}
46 changes: 18 additions & 28 deletions acceptance-tests/apps/dataprocapp/app/get_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,33 @@ package app

import (
"context"
"dataprocapp/credentials"
"log"
"net/http"

dataproc "cloud.google.com/go/dataproc/apiv1"
"cloud.google.com/go/dataproc/apiv1/dataprocpb"
"github.com/gorilla/mux"

"dataprocapp/credentials"
)

func handleGetJob(jobClient dataproc.JobControllerClient, creds credentials.DataprocCredentials) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Printf("Handling getting job")

jobName, ok := mux.Vars(r)["job"]
if !ok {
fail(w, http.StatusBadRequest, "job missing.")
return
}
func handleGetJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) {
log.Printf("Handling getting job")

jobReq := &dataprocpb.GetJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
JobId: jobName,
}
jobReq := &dataprocpb.GetJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
JobId: jobName,
}

ctx := context.Background()
getJobOp, err := jobClient.GetJob(ctx, jobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to getting job: %v", err)
return
}
ctx := context.Background()
getJobOp, err := jobClient.GetJob(ctx, jobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to getting job: %v", err)
return
}

w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/html")
_, err = w.Write([]byte((getJobOp.Status.State.String())))
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "text/html")
_, err = w.Write([]byte((getJobOp.Status.State.String())))

log.Printf("Job finished with status: %s", getJobOp.Status.State.String())
}
log.Printf("Job finished with status: %s", getJobOp.Status.State.String())
}
124 changes: 57 additions & 67 deletions acceptance-tests/apps/dataprocapp/app/run_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package app

import (
"context"
"dataprocapp/credentials"
"fmt"
"io"
"log"
Expand All @@ -11,86 +12,75 @@ import (
dataproc "cloud.google.com/go/dataproc/apiv1"
"cloud.google.com/go/dataproc/apiv1/dataprocpb"
"cloud.google.com/go/storage"
"github.com/gorilla/mux"
"google.golang.org/api/option"

"dataprocapp/credentials"
)

func handleRunJob(jobClient dataproc.JobControllerClient, creds credentials.DataprocCredentials) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Printf("Handling job run")

jobName, ok := mux.Vars(r)["job"]
if !ok {
fail(w, http.StatusBadRequest, "job missing.")
return
}
func handleRunJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) {
log.Printf("Handling job run")

submitJobReq := &dataprocpb.SubmitJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
Job: &dataprocpb.Job{
Reference: &dataprocpb.JobReference{
JobId: jobName,
},
Placement: &dataprocpb.JobPlacement{
ClusterName: creds.ClusterName,
},
TypeJob: &dataprocpb.Job_PysparkJob{
PysparkJob: &dataprocpb.PySparkJob{
MainPythonFileUri: fmt.Sprintf("gs://dataproc_input_for_test_do_not_delete/script.py"),
},
submitJobReq := &dataprocpb.SubmitJobRequest{
ProjectId: creds.ProjectID,
Region: creds.Region,
Job: &dataprocpb.Job{
Reference: &dataprocpb.JobReference{
JobId: jobName,
},
Placement: &dataprocpb.JobPlacement{
ClusterName: creds.ClusterName,
},
TypeJob: &dataprocpb.Job_PysparkJob{
PysparkJob: &dataprocpb.PySparkJob{
MainPythonFileUri: fmt.Sprintf("gs://dataproc_input_for_test_do_not_delete/script.py"),
},
},
}
},
}

ctx := context.Background()
submitJobOp, err := jobClient.SubmitJobAsOperation(ctx, submitJobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to submitting job: %v", err)
return
}
ctx := context.Background()
submitJobOp, err := jobClient.SubmitJobAsOperation(ctx, submitJobReq)
if err != nil {
fail(w, http.StatusFailedDependency, "error with request to submitting job: %v", err)
return
}

submitJobResp, err := submitJobOp.Wait(ctx)
if err != nil {
fail(w, http.StatusFailedDependency, "error submitting job: %v", err)
return
}
re := regexp.MustCompile("gs://(.+?)/(.+)")
matches := re.FindStringSubmatch(submitJobResp.DriverOutputResourceUri)
log.Printf("Job Response: %v", submitJobResp)
log.Printf("Job Response DriverOutputResourceUri: %s", submitJobResp.DriverOutputResourceUri)
submitJobResp, err := submitJobOp.Wait(ctx)
if err != nil {
fail(w, http.StatusFailedDependency, "error submitting job: %v", err)
return
}
re := regexp.MustCompile("gs://(.+?)/(.+)")
matches := re.FindStringSubmatch(submitJobResp.DriverOutputResourceUri)
log.Printf("Job Response: %v", submitJobResp)
log.Printf("Job Response DriverOutputResourceUri: %s", submitJobResp.DriverOutputResourceUri)

if len(matches) < 3 {
fail(w, http.StatusInternalServerError, "regex error: %s", submitJobResp.DriverOutputResourceUri)
return
}
if len(matches) < 3 {
fail(w, http.StatusInternalServerError, "regex error: %s", submitJobResp.DriverOutputResourceUri)
return
}

// Dataproc job outget gets saved to a GCS bucket allocated to it.
storageClient, err := storage.NewClient(context.Background(), option.WithCredentialsJSON(creds.Credentials))
if err != nil {
fail(w, http.StatusFailedDependency, "error creating storage jobClient: %v", err)
return
}
// Dataproc job outget gets saved to a GCS bucket allocated to it.
storageClient, err := storage.NewClient(context.Background(), option.WithCredentialsJSON(creds.Credentials))
if err != nil {
fail(w, http.StatusFailedDependency, "error creating storage jobClient: %v", err)
return
}

obj := fmt.Sprintf("%s.000000000", matches[2])
reader, err := storageClient.Bucket(creds.BucketName).Object(obj).NewReader(ctx)
if err != nil {
fail(w, http.StatusFailedDependency, "error reading job output: %v", err)
return
}
obj := fmt.Sprintf("%s.000000000", matches[2])
reader, err := storageClient.Bucket(creds.BucketName).Object(obj).NewReader(ctx)
if err != nil {
fail(w, http.StatusFailedDependency, "error reading job output: %v", err)
return
}

defer reader.Close()
defer reader.Close()

body, err := io.ReadAll(reader)
if err != nil {
fail(w, http.StatusFailedDependency, "could not read output from Dataproc Job: %v", err)
return
}
body, err := io.ReadAll(reader)
if err != nil {
fail(w, http.StatusFailedDependency, "could not read output from Dataproc Job: %v", err)
return
}

log.Printf("Job finished successfully: %s", body)
log.Printf("Job finished successfully: %s", body)

w.WriteHeader(http.StatusOK)
}
w.WriteHeader(http.StatusOK)
}
1 change: 0 additions & 1 deletion acceptance-tests/apps/dataprocapp/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ require (
cloud.google.com/go/dataproc v1.12.0
cloud.google.com/go/storage v1.29.0
github.com/cloudfoundry-community/go-cfenv v1.18.0
github.com/gorilla/mux v1.8.0
github.com/mitchellh/mapstructure v1.5.0
google.golang.org/api v0.110.0
)
Expand Down
2 changes: 0 additions & 2 deletions acceptance-tests/apps/dataprocapp/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo=
github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
Expand Down
1 change: 0 additions & 1 deletion acceptance-tests/apps/mysqlapp/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ go 1.20
require (
github.com/cloudfoundry-community/go-cfenv v1.18.0
github.com/go-sql-driver/mysql v1.7.0
github.com/gorilla/mux v1.8.0
github.com/mitchellh/mapstructure v1.5.0
)

Expand Down
2 changes: 0 additions & 2 deletions acceptance-tests/apps/mysqlapp/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo=
github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
Expand Down
Loading

0 comments on commit 17be0a0

Please sign in to comment.